wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
foundationdb/fdbclient/NativeAPI.actor.cpp

6966 lines
268 KiB
C++
Raw Blame History

/*
* NativeAPI.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2026 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbclient/NativeAPI.actor.h"
#include <algorithm>
#include <cstdio>
#include <iterator>
#include <limits>
#include <memory>
#include <random>
#include <regex>
#include <string>
#include <unordered_set>
#include <tuple>
#include <utility>
#include <vector>
#include "boost/algorithm/string.hpp"
#include "fdbclient/Knobs.h"
#include "flow/CodeProbe.h"
#include "fmt/format.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/FDBTypes.h"
#include "fdbrpc/FailureMonitor.h"
#include "fdbrpc/MultiInterface.h"
#include "fdbclient/ActorLineageProfiler.h"
#include "fdbclient/AnnotateActor.h"
#include "fdbclient/Atomic.h"
#include "fdbclient/ClusterInterface.h"
#include "fdbclient/ClusterConnectionFile.h"
#include "fdbclient/ClusterConnectionMemoryRecord.h"
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/CommitTransaction.h"
#include "fdbclient/DatabaseContext.h"
#include "fdbclient/GlobalConfig.h"
#include "fdbclient/JsonBuilder.h"
#include "fdbclient/KeyBackedTypes.h"
#include "fdbclient/KeyRangeMap.h"
#include "fdbclient/ManagementAPI.h"
#include "NameLineage.h"
#include "fdbclient/CommitProxyInterface.h"
#include "fdbclient/MonitorLeader.h"
#include "fdbclient/MutationList.h"
#include "fdbclient/ReadYourWrites.h"
#include "fdbclient/SpecialKeySpace.h"
#include "fdbclient/StorageServerInterface.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/TransactionLineage.h"
#include "fdbclient/versions.h"
#include "fdbrpc/WellKnownEndpoints.h"
#include "fdbrpc/LoadBalance.h"
#include "fdbrpc/Net2FileSystem.h"
#include "fdbrpc/simulator.h"
#include "fdbrpc/sim_validation.h"
#include "flow/Arena.h"
#include "flow/ActorCollection.h"
#include "flow/CoroUtils.h"
#include "flow/DeterministicRandom.h"
#include "flow/Error.h"
#include "flow/FastRef.h"
#include "flow/GetSourceVersion.h"
#include "flow/IRandom.h"
#include "flow/Trace.h"
#include "flow/ProtocolVersion.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "flow/Knobs.h"
#include "flow/Platform.h"
#include "flow/SystemMonitor.h"
#include "flow/TLSConfig.h"
#include "fdbclient/Tracing.h"
#include "flow/UnitTest.h"
#include "flow/network.h"
#include "flow/serialize.h"
#include "ProxyLoadBalance.h"
#ifdef ADDRESS_SANITIZER
#include <sanitizer/lsan_interface.h>
#endif
#ifdef WIN32
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#undef min
#undef max
#else
#include <time.h>
#endif
#include "flow/actorcompiler.h" // This must be the last #include.
template class RequestStream<OpenDatabaseRequest, false>;
template struct NetNotifiedQueue<OpenDatabaseRequest, false>;
namespace {
TransactionLineageCollector transactionLineageCollector;
NameLineageCollector nameLineageCollector;
} // namespace
FDB_BOOLEAN_PARAM(TransactionRecordLogInfo);
NetworkOptions networkOptions;
TLSConfig tlsConfig(TLSEndpointType::CLIENT);
// The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h.
NetworkOptions::NetworkOptions()
: traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"),
traceFormat("xml"), traceClockSource("now"), traceInitializeOnSetup(false),
supportedVersions(new ReferencedObject<Standalone<VectorRef<ClientVersionRef>>>()), runLoopProfilingEnabled(false),
primaryClient(true) {}
template <>
void addref(DatabaseContext* ptr) {
ptr->addref();
}
template <>
void delref(DatabaseContext* ptr) {
ptr->delref();
}
Future<Void> refreshTransaction(DatabaseContext* self, Transaction* tr) {
*tr = Transaction();
co_await delay(0); // Give ourselves the chance to get cancelled if self was destroyed
*tr = Transaction(Database(Reference<DatabaseContext>::addRef(self)));
}
Optional<KeyRangeLocationInfo> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) {
Arena arena;
auto range = isBackward ? locationCache.rangeContainingKeyBefore(key) : locationCache.rangeContaining(key);
if (range->value()) {
return KeyRangeLocationInfo(range->range(), range->value());
}
return Optional<KeyRangeLocationInfo>();
}
bool DatabaseContext::getCachedLocations(const KeyRangeRef& range,
std::vector<KeyRangeLocationInfo>& result,
int limit,
Reverse reverse) {
result.clear();
Arena arena;
auto begin = locationCache.rangeContaining(range.begin);
auto end = locationCache.rangeContainingKeyBefore(range.end);
loop {
auto r = reverse ? end : begin;
if (!r->value()) {
CODE_PROBE(result.size(), "had some but not all cached locations");
result.clear();
return false;
}
result.emplace_back((r->range() & range), r->value());
if (result.size() == limit || begin == end) {
break;
}
if (reverse)
--end;
else
++begin;
}
return true;
}
Reference<LocationInfo> DatabaseContext::setCachedLocation(const KeyRangeRef& absoluteKeys,
const std::vector<StorageServerInterface>& servers) {
std::vector<Reference<ReferencedInterface<StorageServerInterface>>> serverRefs;
serverRefs.reserve(servers.size());
for (const auto& interf : servers) {
serverRefs.push_back(StorageServerInfo::getInterface(this, interf, clientLocality));
}
int maxEvictionAttempts = 100, attempts = 0;
auto loc = makeReference<LocationInfo>(serverRefs);
while (locationCache.size() > locationCacheSize && attempts < maxEvictionAttempts) {
CODE_PROBE(true, "NativeAPI storage server locationCache entry evicted");
attempts++;
auto r = locationCache.randomRange();
Key begin = r.begin(), end = r.end(); // insert invalidates r, so can't be passed a mere reference into it
locationCache.insert(KeyRangeRef(begin, end), Reference<LocationInfo>());
}
locationCache.insert(absoluteKeys, loc);
return loc;
}
void DatabaseContext::invalidateCache(const KeyRef& key, Reverse isBackward) {
Arena arena;
KeyRef resolvedKey = key;
if (isBackward) {
locationCache.rangeContainingKeyBefore(resolvedKey)->value() = Reference<LocationInfo>();
} else {
locationCache.rangeContaining(resolvedKey)->value() = Reference<LocationInfo>();
}
}
void DatabaseContext::invalidateCache(const KeyRangeRef& keys) {
Arena arena;
auto rs = locationCache.intersectingRanges(keys);
Key begin = rs.begin().begin(),
end = rs.end().begin(); // insert invalidates rs, so can't be passed a mere reference into it
locationCache.insert(KeyRangeRef(begin, end), Reference<LocationInfo>());
}
void DatabaseContext::setFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
failedEndpointsOnHealthyServersInfo[endpoint] =
EndpointFailureInfo{ .startTime = now(), .lastRefreshTime = now() };
}
}
void DatabaseContext::updateFailedEndpointRefreshTime(const Endpoint& endpoint) {
if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
// The endpoint is not failed. Nothing to update.
return;
}
failedEndpointsOnHealthyServersInfo[endpoint].lastRefreshTime = now();
}
Optional<EndpointFailureInfo> DatabaseContext::getEndpointFailureInfo(const Endpoint& endpoint) {
if (failedEndpointsOnHealthyServersInfo.find(endpoint) == failedEndpointsOnHealthyServersInfo.end()) {
return Optional<EndpointFailureInfo>();
}
return failedEndpointsOnHealthyServersInfo[endpoint];
}
void DatabaseContext::clearFailedEndpointOnHealthyServer(const Endpoint& endpoint) {
failedEndpointsOnHealthyServersInfo.erase(endpoint);
}
Future<Void> DatabaseContext::onProxiesChanged() {
backoffDelay = 0.0;
return this->proxiesChangeTrigger.onTrigger();
}
bool DatabaseContext::sampleReadTags() const {
double sampleRate = globalConfig->get(transactionTagSampleRate, CLIENT_KNOBS->READ_TAG_SAMPLE_RATE);
return sampleRate > 0 && deterministicRandom()->random01() <= sampleRate;
}
bool DatabaseContext::sampleOnCost(uint64_t cost) const {
double sampleCost = globalConfig->get<double>(transactionTagSampleCost, CLIENT_KNOBS->COMMIT_SAMPLE_COST);
if (sampleCost <= 0)
return false;
return deterministicRandom()->random01() <= (double)cost / sampleCost;
}
void validateOptionValuePresent(Optional<StringRef> value) {
if (!value.present()) {
throw invalid_option_value();
}
}
void validateOptionValueNotPresent(Optional<StringRef> value) {
if (value.present() && value.get().size() > 0) {
throw invalid_option_value();
}
}
int64_t extractIntOption(Optional<StringRef> value, int64_t minValue, int64_t maxValue) {
validateOptionValuePresent(value);
if (value.get().size() != 8) {
throw invalid_option_value();
}
int64_t passed = *((int64_t*)(value.get().begin()));
if (passed > maxValue || passed < minValue) {
throw invalid_option_value();
}
return passed;
}
void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional<StringRef> value) {
int defaultFor = FDBDatabaseOptions::optionInfo.getMustExist(option).defaultFor;
if (defaultFor >= 0) {
ASSERT(FDBTransactionOptions::optionInfo.find((FDBTransactionOptions::Option)defaultFor) !=
FDBTransactionOptions::optionInfo.end());
TraceEvent(SevDebug, "DatabaseContextSetPersistentOption").detail("Option", option).detail("Value", value);
transactionDefaults.addOption((FDBTransactionOptions::Option)defaultFor, value.castTo<Standalone<StringRef>>());
} else {
switch (option) {
case FDBDatabaseOptions::LOCATION_CACHE_SIZE:
locationCacheSize = (int)extractIntOption(value, 0, std::numeric_limits<int>::max());
break;
case FDBDatabaseOptions::MACHINE_ID:
clientLocality =
LocalityData(clientLocality.processId(),
value.present() ? Standalone<StringRef>(value.get()) : Optional<Standalone<StringRef>>(),
clientLocality.machineId(),
clientLocality.dcId());
if (clientInfo->get().commitProxies.size())
commitProxies = makeReference<CommitProxyInfo>(clientInfo->get().commitProxies);
if (clientInfo->get().grvProxies.size())
grvProxies = makeReference<GrvProxyInfo>(clientInfo->get().grvProxies, BalanceOnRequests::True);
server_interf.clear();
locationCache.insert(allKeys, Reference<LocationInfo>());
break;
case FDBDatabaseOptions::MAX_WATCHES:
maxOutstandingWatches = (int)extractIntOption(value, 0, CLIENT_KNOBS->ABSOLUTE_MAX_WATCHES);
break;
case FDBDatabaseOptions::DATACENTER_ID:
clientLocality =
LocalityData(clientLocality.processId(),
clientLocality.zoneId(),
clientLocality.machineId(),
value.present() ? Standalone<StringRef>(value.get()) : Optional<Standalone<StringRef>>());
if (clientInfo->get().commitProxies.size())
commitProxies = makeReference<CommitProxyInfo>(clientInfo->get().commitProxies);
if (clientInfo->get().grvProxies.size())
grvProxies = makeReference<GrvProxyInfo>(clientInfo->get().grvProxies, BalanceOnRequests::True);
server_interf.clear();
locationCache.insert(allKeys, Reference<LocationInfo>());
break;
case FDBDatabaseOptions::SNAPSHOT_RYW_ENABLE:
validateOptionValueNotPresent(value);
snapshotRywEnabled++;
break;
case FDBDatabaseOptions::SNAPSHOT_RYW_DISABLE:
validateOptionValueNotPresent(value);
snapshotRywEnabled--;
break;
case FDBDatabaseOptions::TEST_CAUSAL_READ_RISKY:
verifyCausalReadsProp = double(extractIntOption(value, 0, 100)) / 100.0;
break;
default:
break;
}
}
}
void DatabaseContext::increaseWatchCounter() {
if (outstandingWatches >= maxOutstandingWatches)
throw too_many_watches();
++outstandingWatches;
}
void DatabaseContext::decreaseWatchCounter() {
--outstandingWatches;
ASSERT(outstandingWatches >= 0);
}
Future<Void> DatabaseContext::onConnected() const {
return connected;
}
static Future<Void> switchConnectionRecordImpl(Reference<IClusterConnectionRecord> connRecord, DatabaseContext* self) {
CODE_PROBE(true, "Switch connection file");
TraceEvent("SwitchConnectionRecord")
.detail("ClusterFile", connRecord->toString())
.detail("ConnectionString", connRecord->getConnectionString().toString());
// Reset state from former cluster.
self->commitProxies.clear();
self->grvProxies.clear();
self->minAcceptableReadVersion = std::numeric_limits<Version>::max();
self->invalidateCache(allKeys);
self->ssVersionVectorCache.clear();
auto clearedClientInfo = self->clientInfo->get();
clearedClientInfo.commitProxies.clear();
clearedClientInfo.grvProxies.clear();
clearedClientInfo.id = deterministicRandom()->randomUniqueID();
self->clientInfo->set(clearedClientInfo);
self->connectionRecord->set(connRecord);
Database db(Reference<DatabaseContext>::addRef(self));
Transaction tr(db);
while (true) {
tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
Error err;
try {
TraceEvent("SwitchConnectionRecordAttemptingGRV").log();
Version v = co_await tr.getReadVersion();
TraceEvent("SwitchConnectionRecordGotRV")
.detail("ReadVersion", v)
.detail("MinAcceptableReadVersion", self->minAcceptableReadVersion);
ASSERT(self->minAcceptableReadVersion != std::numeric_limits<Version>::max());
self->connectionFileChangedTrigger.trigger();
co_return;
} catch (Error& e) {
err = e;
}
TraceEvent("SwitchConnectionRecordError").detail("Error", err.what());
co_await tr.onError(err);
}
}
Reference<IClusterConnectionRecord> DatabaseContext::getConnectionRecord() {
if (connectionRecord) {
return connectionRecord->get();
}
return Reference<IClusterConnectionRecord>();
}
Future<Void> DatabaseContext::switchConnectionRecord(Reference<IClusterConnectionRecord> standby) {
ASSERT(switchable);
return switchConnectionRecordImpl(standby, this);
}
Future<Void> DatabaseContext::connectionFileChanged() {
return connectionFileChangedTrigger.onTrigger();
}
void DatabaseContext::expireThrottles() {
for (auto& priorityItr : throttledTags) {
for (auto tagItr = priorityItr.second.begin(); tagItr != priorityItr.second.end();) {
if (tagItr->second.expired()) {
CODE_PROBE(true, "Expiring client throttle");
tagItr = priorityItr.second.erase(tagItr);
} else {
++tagItr;
}
}
}
}
// Initialize tracing for FDB client
//
// connRecord is necessary for determining the local IP, which is then included in the trace
// file name, and also used to annotate all trace events.
//
// If trace_initialize_on_setup is not set, tracing is initialized when opening a database.
// In that case we can immediately determine the IP. Thus, we can use the IP in the
// trace file name and annotate all events with it.
//
// If trace_initialize_on_setup network option is set, tracing is at first initialized without
// connRecord and thus without the local IP. In that case we cannot use the local IP in the
// trace file names. The IP is then provided by a repeated call to initializeClientTracing
// when opening a database. All tracing events from this point are annotated with the local IP
//
// If tracing initialization is completed, further calls to initializeClientTracing are ignored
void initializeClientTracing(Reference<IClusterConnectionRecord> connRecord, Optional<int> apiVersion) {
if (!networkOptions.traceDirectory.present()) {
return;
}
bool initialized = traceFileIsOpen();
if (initialized && (isTraceLocalAddressSet() || !connRecord)) {
// Tracing initialization is completed
return;
}
// Network must be created before initializing tracing
ASSERT(g_network);
Optional<NetworkAddress> localAddress;
if (connRecord) {
IPAddress traceIP;
if (networkOptions.traceIP.present()) {
traceIP = networkOptions.traceIP.get();
} else {
// Automatically determine public IP if not provided
traceIP = connRecord->getConnectionString().determineLocalSourceIP();
}
localAddress = NetworkAddress(traceIP, ::getpid());
}
platform::ImageInfo imageInfo = platform::getImageInfo();
if (initialized) {
// Tracing already initialized, just need to update the IP address
setTraceLocalAddress(localAddress.get());
TraceEvent("ClientStart")
.detail("SourceVersion", getSourceVersion())
.detail("Version", FDB_VT_VERSION)
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
.detail("ApiVersion", apiVersion)
.detail("ClientLibrary", imageInfo.fileName)
.detailf("ImageOffset", "%p", imageInfo.offset)
.detail("Primary", networkOptions.primaryClient)
.trackLatest("ClientStart");
} else {
// Initialize tracing
selectTraceFormatter(networkOptions.traceFormat);
selectTraceClockSource(networkOptions.traceClockSource);
addUniversalTraceField("ClientDescription",
format("%s-%s-%" PRIu64,
networkOptions.primaryClient ? "primary" : "external",
FDB_VT_VERSION,
deterministicRandom()->randomUInt64()));
std::string identifier = networkOptions.traceFileIdentifier;
openTraceFile(localAddress,
networkOptions.traceRollSize,
networkOptions.traceMaxLogsSize,
networkOptions.traceDirectory.get(),
"trace",
networkOptions.traceLogGroup,
identifier,
networkOptions.tracePartialFileSuffix,
InitializeTraceMetrics::True);
TraceEvent("ClientStart")
.detail("SourceVersion", getSourceVersion())
.detail("Version", FDB_VT_VERSION)
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
.detail("ApiVersion", apiVersion)
.detail("ClientLibrary", imageInfo.fileName)
.detailf("ImageOffset", "%p", imageInfo.offset)
.detail("Primary", networkOptions.primaryClient)
.trackLatest("ClientStart");
g_network->initMetrics();
FlowTransport::transport().initMetrics();
}
// Initialize system monitoring once the local IP is available
if (localAddress.present()) {
initializeSystemMonitorMachineState(SystemMonitorMachineState(IPAddress(localAddress.get().ip)));
systemMonitor();
uncancellable(recurring(&systemMonitor, CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace));
}
}
// Creates a database object that represents a connection to a cluster
// This constructor uses a preallocated DatabaseContext that may have been created
// on another thread
Database Database::createDatabase(Reference<IClusterConnectionRecord> connRecord,
int apiVersion,
IsInternal internal,
LocalityData const& clientLocality,
DatabaseContext* preallocatedDb) {
if (!g_network)
throw network_not_setup();
ASSERT(TraceEvent::isNetworkThread());
initializeClientTracing(connRecord, apiVersion);
g_network->initTLS();
auto clientInfo = makeReference<AsyncVar<ClientDBInfo>>();
auto coordinator = makeReference<AsyncVar<Optional<ClientLeaderRegInterface>>>();
auto connectionRecord = makeReference<AsyncVar<Reference<IClusterConnectionRecord>>>();
connectionRecord->set(connRecord);
Future<Void> clientInfoMonitor = monitorProxies(connectionRecord,
clientInfo,
coordinator,
networkOptions.supportedVersions,
StringRef(networkOptions.traceLogGroup),
internal);
DatabaseContext* db;
if (preallocatedDb) {
db = new (preallocatedDb) DatabaseContext(connectionRecord,
clientInfo,
coordinator,
clientInfoMonitor,
TaskPriority::DefaultEndpoint,
clientLocality,
EnableLocalityLoadBalance::True,
LockAware::False,
internal,
apiVersion,
IsSwitchable::True);
} else {
db = new DatabaseContext(connectionRecord,
clientInfo,
coordinator,
clientInfoMonitor,
TaskPriority::DefaultEndpoint,
clientLocality,
EnableLocalityLoadBalance::True,
LockAware::False,
internal,
apiVersion,
IsSwitchable::True);
}
auto database = Database(db);
database->globalConfig->init(Reference<AsyncVar<ClientDBInfo> const>(clientInfo),
std::addressof(clientInfo->get()));
database->globalConfig->trigger(samplingFrequency, samplingProfilerUpdateFrequency);
database->globalConfig->trigger(samplingWindow, samplingProfilerUpdateWindow);
TraceEvent("ConnectToDatabase", database->dbId)
.detail("Version", FDB_VT_VERSION)
.detail("ClusterFile", connRecord ? connRecord->toString() : "None")
.detail("ConnectionString", connRecord ? connRecord->getConnectionString().toString() : "None")
.detail("ClientLibrary", platform::getImageInfo().fileName)
.detail("Primary", networkOptions.primaryClient)
.detail("Internal", internal)
.trackLatest(database->connectToDatabaseEventCacheHolder.trackingKey);
return database;
}
Database Database::createDatabase(std::string connFileName,
int apiVersion,
IsInternal internal,
LocalityData const& clientLocality) {
Reference<IClusterConnectionRecord> rccr = ClusterConnectionFile::openOrDefault(connFileName);
return Database::createDatabase(rccr, apiVersion, internal, clientLocality);
}
Database Database::createSimulatedExtraDatabase(std::string connectionString) {
auto extraFile = makeReference<ClusterConnectionMemoryRecord>(ClusterConnectionString(connectionString));
Database db = Database::createDatabase(extraFile, ApiVersion::LATEST_VERSION);
return db;
}
const UniqueOrderedOptionList<FDBTransactionOptions>& Database::getTransactionDefaults() const {
ASSERT(db);
return db->transactionDefaults;
}
void setNetworkOption(FDBNetworkOptions::Option option, Optional<StringRef> value) {
std::regex identifierRegex("^[a-zA-Z0-9_]*$");
switch (option) {
// SOMEDAY: If the network is already started, should these five throw an error?
case FDBNetworkOptions::TRACE_ENABLE:
networkOptions.traceDirectory = value.present() ? value.get().toString() : "";
break;
case FDBNetworkOptions::TRACE_ROLL_SIZE:
validateOptionValuePresent(value);
networkOptions.traceRollSize = extractIntOption(value, 0, std::numeric_limits<int64_t>::max());
break;
case FDBNetworkOptions::TRACE_MAX_LOGS_SIZE:
validateOptionValuePresent(value);
networkOptions.traceMaxLogsSize = extractIntOption(value, 0, std::numeric_limits<int64_t>::max());
break;
case FDBNetworkOptions::TRACE_FORMAT:
validateOptionValuePresent(value);
networkOptions.traceFormat = value.get().toString();
if (!validateTraceFormat(networkOptions.traceFormat)) {
fprintf(stderr, "Unrecognized trace format: `%s'\n", networkOptions.traceFormat.c_str());
throw invalid_option_value();
}
break;
case FDBNetworkOptions::TRACE_FILE_IDENTIFIER:
validateOptionValuePresent(value);
networkOptions.traceFileIdentifier = value.get().toString();
if (networkOptions.traceFileIdentifier.length() > CLIENT_KNOBS->TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH) {
fprintf(stderr, "Trace file identifier provided is too long.\n");
throw invalid_option_value();
} else if (!std::regex_match(networkOptions.traceFileIdentifier, identifierRegex)) {
fprintf(stderr, "Trace file identifier should only contain alphanumerics and underscores.\n");
throw invalid_option_value();
}
break;
case FDBNetworkOptions::TRACE_LOG_GROUP:
if (value.present()) {
if (traceFileIsOpen()) {
setTraceLogGroup(value.get().toString());
} else {
networkOptions.traceLogGroup = value.get().toString();
}
}
break;
case FDBNetworkOptions::TRACE_CLOCK_SOURCE:
validateOptionValuePresent(value);
networkOptions.traceClockSource = value.get().toString();
if (!validateTraceClockSource(networkOptions.traceClockSource)) {
fprintf(stderr, "Unrecognized trace clock source: `%s'\n", networkOptions.traceClockSource.c_str());
throw invalid_option_value();
}
break;
case FDBNetworkOptions::TRACE_PARTIAL_FILE_SUFFIX:
validateOptionValuePresent(value);
networkOptions.tracePartialFileSuffix = value.get().toString();
break;
case FDBNetworkOptions::TRACE_INITIALIZE_ON_SETUP:
networkOptions.traceInitializeOnSetup = true;
break;
case FDBNetworkOptions::TRACE_IP: {
validateOptionValuePresent(value);
auto parsedIP = IPAddress::parse(value.get().toString());
if (!parsedIP.present()) {
fprintf(stderr,
"Invalid format for trace IP: `%s', only IPv4 or IPv6 format is supported.\n",
value.get().toString().c_str());
throw invalid_option_value();
}
networkOptions.traceIP = parsedIP;
break;
}
case FDBNetworkOptions::KNOB: {
validateOptionValuePresent(value);
std::string optionValue = value.get().toString();
TraceEvent("SetKnob").detail("KnobString", optionValue);
size_t eq = optionValue.find_first_of('=');
if (eq == optionValue.npos) {
TraceEvent(SevWarnAlways, "InvalidKnobString").detail("KnobString", optionValue);
throw invalid_option_value();
}
std::string knobName = optionValue.substr(0, eq);
std::string knobValueString = optionValue.substr(eq + 1);
try {
auto knobValue = parseClientKnobValue(knobName, knobValueString);
if (g_network) {
setClientKnob(knobName, knobValue);
} else {
networkOptions.knobs[knobName] = knobValue;
}
} catch (Error& e) {
TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str());
fprintf(stderr, "FoundationDB client ignoring unrecognized knob option '%s'\n", knobName.c_str());
}
break;
}
case FDBNetworkOptions::TLS_PLUGIN:
validateOptionValuePresent(value);
break;
case FDBNetworkOptions::TLS_CERT_PATH:
validateOptionValuePresent(value);
tlsConfig.setCertificatePath(value.get().toString());
break;
case FDBNetworkOptions::TLS_CERT_BYTES: {
validateOptionValuePresent(value);
tlsConfig.setCertificateBytes(value.get().toString());
break;
}
case FDBNetworkOptions::TLS_CA_PATH: {
validateOptionValuePresent(value);
tlsConfig.setCAPath(value.get().toString());
break;
}
case FDBNetworkOptions::TLS_CA_BYTES: {
validateOptionValuePresent(value);
tlsConfig.setCABytes(value.get().toString());
break;
}
case FDBNetworkOptions::TLS_PASSWORD:
validateOptionValuePresent(value);
tlsConfig.setPassword(value.get().toString());
break;
case FDBNetworkOptions::TLS_KEY_PATH:
validateOptionValuePresent(value);
tlsConfig.setKeyPath(value.get().toString());
break;
case FDBNetworkOptions::TLS_KEY_BYTES: {
validateOptionValuePresent(value);
tlsConfig.setKeyBytes(value.get().toString());
break;
}
case FDBNetworkOptions::TLS_VERIFY_PEERS:
validateOptionValuePresent(value);
tlsConfig.clearVerifyPeers();
tlsConfig.addVerifyPeers(value.get().toString());
break;
case FDBNetworkOptions::TLS_DISABLE_PLAINTEXT_CONNECTION:
tlsConfig.setDisablePlainTextConnection(true);
break;
case FDBNetworkOptions::CLIENT_BUGGIFY_ENABLE:
enableClientBuggify();
break;
case FDBNetworkOptions::CLIENT_BUGGIFY_DISABLE:
disableClientBuggify();
break;
case FDBNetworkOptions::CLIENT_BUGGIFY_SECTION_ACTIVATED_PROBABILITY:
validateOptionValuePresent(value);
clearClientBuggifySections();
P_CLIENT_BUGGIFIED_SECTION_ACTIVATED = double(extractIntOption(value, 0, 100)) / 100.0;
break;
case FDBNetworkOptions::CLIENT_BUGGIFY_SECTION_FIRED_PROBABILITY:
validateOptionValuePresent(value);
P_CLIENT_BUGGIFIED_SECTION_FIRES = double(extractIntOption(value, 0, 100)) / 100.0;
break;
case FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING:
validateOptionValueNotPresent(value);
networkOptions.logClientInfo = false;
break;
case FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS: {
// The multi-version API should be providing us these guarantees
ASSERT(g_network);
ASSERT(value.present());
Standalone<VectorRef<ClientVersionRef>> supportedVersions;
std::vector<StringRef> supportedVersionsStrings = value.get().splitAny(";"_sr);
for (StringRef versionString : supportedVersionsStrings) {
#ifdef ADDRESS_SANITIZER
__lsan_disable();
#endif
// LSAN reports that we leak this allocation in client
// tests, but I cannot seem to figure out why. AFAICT
// it's not actually leaking. If it is a leak, it's only a few bytes.
supportedVersions.push_back_deep(supportedVersions.arena(), ClientVersionRef(versionString));
#ifdef ADDRESS_SANITIZER
__lsan_enable();
#endif
}
ASSERT(supportedVersions.size() > 0);
networkOptions.supportedVersions->set(supportedVersions);
break;
}
case FDBNetworkOptions::ENABLE_RUN_LOOP_PROFILING: // Same as ENABLE_SLOW_TASK_PROFILING
validateOptionValueNotPresent(value);
networkOptions.runLoopProfilingEnabled = true;
break;
case FDBNetworkOptions::DISTRIBUTED_CLIENT_TRACER: {
validateOptionValuePresent(value);
std::string tracer = value.get().toString();
if (tracer == "none" || tracer == "disabled") {
openTracer(TracerType::DISABLED);
} else if (tracer == "logfile" || tracer == "file" || tracer == "log_file") {
openTracer(TracerType::LOG_FILE);
} else if (tracer == "network_lossy") {
openTracer(TracerType::NETWORK_LOSSY);
} else {
fprintf(stderr, "ERROR: Unknown or unsupported tracer: `%s'", tracer.c_str());
throw invalid_option_value();
}
break;
}
case FDBNetworkOptions::EXTERNAL_CLIENT:
networkOptions.primaryClient = false;
break;
default:
break;
}
}
// update the network busyness on a 1s cadence
Future<Void> monitorNetworkBusyness() {
double prevTime = now();
while (true) {
co_await delay(CLIENT_KNOBS->NETWORK_BUSYNESS_MONITOR_INTERVAL, TaskPriority::FlushTrace);
double elapsed = now() - prevTime; // get elapsed time from last execution
prevTime = now();
struct NetworkMetrics::PriorityStats& tracker = g_network->networkInfo.metrics.starvationTrackerNetworkBusyness;
if (tracker.active) { // update metrics
tracker.duration += now() - tracker.windowedTimer;
tracker.maxDuration = std::max(tracker.maxDuration, now() - tracker.timer);
tracker.windowedTimer = now();
}
double busyFraction = std::min(elapsed, tracker.duration) / elapsed;
// The burstiness score is an indicator of the maximum busyness spike over the measurement interval.
// It scales linearly from 0 to 1 as the largest burst goes from the start to the saturation threshold.
// This allows us to account for saturation that happens in smaller bursts than the measurement interval.
//
// Burstiness will not be calculated if the saturation threshold is smaller than the start threshold or
// if either value is negative.
double burstiness = 0;
if (CLIENT_KNOBS->BUSYNESS_SPIKE_START_THRESHOLD >= 0 &&
CLIENT_KNOBS->BUSYNESS_SPIKE_SATURATED_THRESHOLD >= CLIENT_KNOBS->BUSYNESS_SPIKE_START_THRESHOLD) {
burstiness = std::min(1.0,
std::max(0.0, tracker.maxDuration - CLIENT_KNOBS->BUSYNESS_SPIKE_START_THRESHOLD) /
std::max(1e-6,
CLIENT_KNOBS->BUSYNESS_SPIKE_SATURATED_THRESHOLD -
CLIENT_KNOBS->BUSYNESS_SPIKE_START_THRESHOLD));
}
g_network->networkInfo.metrics.networkBusyness = std::max(busyFraction, burstiness);
tracker.duration = 0;
tracker.maxDuration = 0;
}
}
static void setupGlobalKnobs() {
resetClientKnobs(Randomize::False, IsSimulated::False);
for (const auto& [knobName, knobValue] : networkOptions.knobs) {
setClientKnob(knobName, knobValue);
}
}
// Setup g_network and start monitoring for network busyness
void setupNetwork(uint64_t transportId, UseMetrics useMetrics) {
if (g_network)
throw network_already_setup();
if (!networkOptions.logClientInfo.present())
networkOptions.logClientInfo = true;
setupGlobalKnobs();
g_network = newNet2(tlsConfig, false, useMetrics || networkOptions.traceDirectory.present());
g_network->addStopCallback(Net2FileSystem::stop);
FlowTransport::createInstance(true, transportId, WLTOKEN_RESERVED_COUNT);
Net2FileSystem::newFileSystem();
if (networkOptions.traceInitializeOnSetup) {
::initializeClientTracing({}, {});
}
uncancellable(monitorNetworkBusyness());
}
void runNetwork() {
if (!g_network) {
throw network_not_setup();
}
if (!g_network->checkRunnable()) {
throw network_cannot_be_restarted();
}
if (networkOptions.traceDirectory.present() && networkOptions.runLoopProfilingEnabled) {
setupRunLoopProfiler();
}
g_network->run();
if (networkOptions.traceDirectory.present())
systemMonitor();
}
void stopNetwork() {
if (!g_network)
throw network_not_setup();
TraceEvent("ClientStopNetwork").log();
if (networkOptions.traceDirectory.present() && networkOptions.runLoopProfilingEnabled) {
stopRunLoopProfiler();
}
g_network->stop();
}
void DatabaseContext::updateProxies() {
if (proxiesLastChange == clientInfo->get().id)
return;
proxiesLastChange = clientInfo->get().id;
commitProxies.clear();
grvProxies.clear();
bool commitProxyProvisional = false, grvProxyProvisional = false;
if (clientInfo->get().commitProxies.size()) {
commitProxies = makeReference<CommitProxyInfo>(clientInfo->get().commitProxies);
commitProxyProvisional = clientInfo->get().commitProxies[0].provisional;
}
if (clientInfo->get().grvProxies.size()) {
grvProxies = makeReference<GrvProxyInfo>(clientInfo->get().grvProxies, BalanceOnRequests::True);
grvProxyProvisional = clientInfo->get().grvProxies[0].provisional;
}
if (clientInfo->get().commitProxies.size() && clientInfo->get().grvProxies.size()) {
ASSERT(commitProxyProvisional == grvProxyProvisional);
proxyProvisional = commitProxyProvisional;
}
}
Reference<CommitProxyInfo> DatabaseContext::getCommitProxies(UseProvisionalProxies useProvisionalProxies) {
updateProxies();
if (proxyProvisional && !useProvisionalProxies) {
return Reference<CommitProxyInfo>();
}
return commitProxies;
}
Reference<GrvProxyInfo> DatabaseContext::getGrvProxies(UseProvisionalProxies useProvisionalProxies) {
updateProxies();
if (proxyProvisional && !useProvisionalProxies) {
return Reference<GrvProxyInfo>();
}
return grvProxies;
}
bool DatabaseContext::isCurrentGrvProxy(UID proxyId) const {
for (const auto& proxy : clientInfo->get().grvProxies) {
if (proxy.id() == proxyId)
return true;
}
CODE_PROBE(true, "stale GRV proxy detected", probe::decoration::rare);
return false;
}
// Actor which will wait until the MultiInterface<CommitProxyInterface> returned by the DatabaseContext cx is not
// nullptr
Future<Reference<CommitProxyInfo>> getCommitProxiesFuture(DatabaseContext* cx,
UseProvisionalProxies useProvisionalProxies) {
while (true) {
Reference<CommitProxyInfo> commitProxies = cx->getCommitProxies(useProvisionalProxies);
if (commitProxies)
co_return commitProxies;
co_await cx->onProxiesChanged();
}
}
// Returns a future which will not be set until the CommitProxyInfo of this DatabaseContext is not nullptr
Future<Reference<CommitProxyInfo>> DatabaseContext::getCommitProxiesFuture(
UseProvisionalProxies useProvisionalProxies) {
return ::getCommitProxiesFuture(this, useProvisionalProxies);
}
void GetRangeLimits::decrement(VectorRef<KeyValueRef> const& data) {
if (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED) {
ASSERT(data.size() <= rows);
rows -= data.size();
}
minRows = std::max(0, minRows - data.size());
if (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
bytes = std::max(0, bytes - (int)data.expectedSize() - (8 - (int)sizeof(KeyValueRef)) * data.size());
}
void GetRangeLimits::decrement(KeyValueRef const& data) {
minRows = std::max(0, minRows - 1);
if (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED)
rows--;
if (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
bytes = std::max(0, bytes - (int)8 - (int)data.expectedSize());
}
void GetRangeLimits::decrement(VectorRef<MappedKeyValueRef> const& data) {
if (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED) {
ASSERT(data.size() <= rows);
rows -= data.size();
}
minRows = std::max(0, minRows - data.size());
// TODO: For now, expectedSize only considers the size of the original key values, but not the underlying queries or
// results. Also, double check it is correct when dealing with sizeof(MappedKeyValueRef).
if (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
bytes = std::max(0, bytes - (int)data.expectedSize() - (8 - (int)sizeof(MappedKeyValueRef)) * data.size());
}
void GetRangeLimits::decrement(MappedKeyValueRef const& data) {
minRows = std::max(0, minRows - 1);
if (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED)
rows--;
// TODO: For now, expectedSize only considers the size of the original key values, but not the underlying queries or
// results. Also, double check it is correct when dealing with sizeof(MappedKeyValueRef).
if (bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED)
bytes = std::max(0, bytes - (int)8 - (int)data.expectedSize());
}
// True if either the row or byte limit has been reached
bool GetRangeLimits::isReached() const {
return rows == 0 || (bytes == 0 && minRows == 0);
}
// True if data would cause the row or byte limit to be reached
bool GetRangeLimits::reachedBy(VectorRef<KeyValueRef> const& data) const {
return (rows != GetRangeLimits::ROW_LIMIT_UNLIMITED && data.size() >= rows) ||
(bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED &&
(int)data.expectedSize() + (8 - (int)sizeof(KeyValueRef)) * data.size() >= bytes && data.size() >= minRows);
}
bool GetRangeLimits::hasByteLimit() const {
return bytes != GetRangeLimits::BYTE_LIMIT_UNLIMITED;
}
bool GetRangeLimits::hasRowLimit() const {
return rows != GetRangeLimits::ROW_LIMIT_UNLIMITED;
}
bool GetRangeLimits::hasSatisfiedMinRows() const {
return hasByteLimit() && minRows == 0;
}
AddressExclusion AddressExclusion::parse(StringRef const& key) {
// Must not change: serialized to the database!
auto parsedIp = IPAddress::parse(key.toString());
if (parsedIp.present()) {
return AddressExclusion(parsedIp.get());
}
// Not a whole machine, includes `port'.
try {
auto addr = NetworkAddress::parse(key.toString());
if (addr.isTLS()) {
TraceEvent(SevWarnAlways, "AddressExclusionParseError")
.detail("String", key)
.detail("Description", "Address inclusion string should not include `:tls' suffix.");
return AddressExclusion();
}
return AddressExclusion(addr.ip, addr.port);
} catch (Error&) {
TraceEvent(SevWarnAlways, "AddressExclusionParseError").detail("String", key);
return AddressExclusion();
}
}
Future<Optional<Value>> getValue(Reference<TransactionState> const& trState,
Key const& key,
TransactionRecordLogInfo const& recordLogInfo = TransactionRecordLogInfo::True);
Future<RangeResult> getRange(Reference<TransactionState> const& trState,
KeySelector const& begin,
KeySelector const& end,
GetRangeLimits const& limits,
Reverse const& reverse);
Future<Optional<StorageServerInterface>> fetchServerInterface(Reference<TransactionState> trState, UID id) {
Optional<Value> val = co_await getValue(trState, serverListKeyFor(id), TransactionRecordLogInfo::False);
if (!val.present()) {
// A storage server has been removed from serverList since we read keyServers
co_return Optional<StorageServerInterface>();
}
co_return decodeServerListValue(val.get());
}
Future<Optional<std::vector<StorageServerInterface>>> transactionalGetServerInterfaces(
Reference<TransactionState> trState,
std::vector<UID> ids) {
std::vector<Future<Optional<StorageServerInterface>>> serverListEntries;
serverListEntries.reserve(ids.size());
for (const auto& id : ids) {
serverListEntries.push_back(fetchServerInterface(trState, id));
}
std::vector<Optional<StorageServerInterface>> serverListValues = co_await getAll(serverListEntries);
std::vector<StorageServerInterface> serverInterfaces;
for (const auto& serverListValue : serverListValues) {
if (!serverListValue.present()) {
// A storage server has been removed from ServerList since we read keyServers
co_return Optional<std::vector<StorageServerInterface>>();
}
serverInterfaces.push_back(serverListValue.get());
}
co_return serverInterfaces;
}
void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) {
// Since a ss -> tss mapping is included in resultsTssMapping iff that SS is in results and has a tss pair,
// all SS in results that do not have a mapping present must not have a tss pair.
std::unordered_map<UID, const StorageServerInterface*> ssiById;
for (const auto& [_, shard] : reply.results) {
for (auto& ssi : shard) {
ssiById[ssi.id()] = &ssi;
}
}
for (const auto& [storageServerId, tss] : reply.resultsTssMapping) {
auto ssi = ssiById.find(storageServerId);
ASSERT(ssi != ssiById.end());
cx->addTssMapping(*ssi->second, tss);
ssiById.erase(storageServerId);
}
// if SS didn't have a mapping above, it's still in the ssiById map, so remove its tss mapping
for (const auto& it : ssiById) {
cx->removeTssMapping(*it.second);
}
}
void updateTagMappings(Database cx, const GetKeyServerLocationsReply& reply) {
for (const auto& [storageServerId, tag] : reply.resultsTagMapping) {
cx->addSSIdTagMapping(storageServerId, tag);
}
}
// If isBackward == true, returns the shard containing the key before 'key' (an infinitely long, inexpressible key).
// Otherwise returns the shard containing key
Future<KeyRangeLocationInfo> getKeyLocation_internal(Database cx,
Key key,
SpanContext spanContext,
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies,
Reverse isBackward,
Version version) {
Span span("NAPI:getKeyLocation"_loc, spanContext);
if (isBackward) {
ASSERT(key != allKeys.begin && key <= allKeys.end);
} else {
ASSERT(key < allKeys.end);
}
if (debugID.present())
g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.Before");
while (true) {
try {
co_await cx->getBackoff();
++cx->transactionKeyServerLocationRequests;
GetKeyServerLocationsReply rep = co_await commitProxyLoadBalance(
cx,
makeReqBuilder<GetKeyServerLocationsRequest>(
span.context, key, Optional<KeyRef>(), /*limit=*/100, isBackward, version, key.arena()),
&CommitProxyInterface::getKeyServersLocations,
useProvisionalProxies,
TaskPriority::DefaultPromiseEndpoint);
++cx->transactionKeyServerLocationRequestsCompleted;
if (debugID.present())
g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocation.After");
ASSERT(rep.results.size() == 1);
auto locationInfo = cx->setCachedLocation(rep.results[0].first, rep.results[0].second);
updateTssMappings(cx, rep);
updateTagMappings(cx, rep);
cx->updateBackoff(success());
co_return KeyRangeLocationInfo(KeyRange(rep.results[0].first, rep.arena), locationInfo);
} catch (Error& e) {
if (e.code() == error_code_commit_proxy_memory_limit_exceeded) {
// Eats commit_proxy_memory_limit_exceeded error from commit proxies
TraceEvent(SevWarnAlways, "CommitProxyOverloadedForKeyLocation").suppressFor(5);
cx->updateBackoff(e);
continue;
}
throw;
}
}
}
// Checks if `endpoint` is failed on a healthy server or not. Returns true if we need to refresh the location cache for
// the endpoint.
bool checkOnlyEndpointFailed(const Database& cx, const Endpoint& endpoint) {
if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
// This endpoint is failed, but the server is still healthy. There are two cases this can happen:
// - There is a recent bounce in the cluster where the endpoints in SSes get updated.
// - The SS is failed and terminated on a server, but the server is kept running.
// To account for the first case, we invalidate the cache and issue GetKeyLocation requests to the proxy to
// update the cache with the new SS points. However, if the failure is caused by the second case, the
// requested key location will continue to be the failed endpoint until the data movement is finished. But
// every read will generate a GetKeyLocation request to the proxies (and still getting the failed endpoint
// back), which may overload the proxy and affect data movement speed. Therefore, we only refresh the
// location cache for short period of time, and after the initial grace period that we keep retrying
// resolving key location, we will slow it down to resolve it only once every
// `LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL`.
cx->setFailedEndpointOnHealthyServer(endpoint);
const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
ASSERT(failureInfo.present());
if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
now() - failureInfo.get().lastRefreshTime > CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
cx->updateFailedEndpointRefreshTime(endpoint);
return true;
}
} else {
cx->clearFailedEndpointOnHealthyServer(endpoint);
}
return false;
}
template <class F>
Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,
Key const& key,
F StorageServerInterface::* member,
SpanContext spanContext,
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies,
Reverse isBackward,
Version version) {
// we first check whether this range is cached
Optional<KeyRangeLocationInfo> locationInfo = cx->getCachedLocation(key, isBackward);
if (!locationInfo.present()) {
return getKeyLocation_internal(cx, key, spanContext, debugID, useProvisionalProxies, isBackward, version);
}
bool onlyEndpointFailedAndNeedRefresh = false;
for (int i = 0; i < locationInfo.get().locations->size(); i++) {
if (checkOnlyEndpointFailed(cx, locationInfo.get().locations->get(i, member).getEndpoint())) {
onlyEndpointFailedAndNeedRefresh = true;
}
}
if (onlyEndpointFailedAndNeedRefresh) {
cx->invalidateCache(key);
// Refresh the cache with a new getKeyLocations made to proxies.
return getKeyLocation_internal(cx, key, spanContext, debugID, useProvisionalProxies, isBackward, version);
}
return locationInfo.get();
}
template <class F>
Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
Key const& key,
F StorageServerInterface::* member,
Reverse isBackward) {
return getKeyLocation(trState->cx,
key,
member,
trState->spanContext,
trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
trState->useProvisionalProxies,
isBackward,
trState->readVersionFuture.isValid() && trState->readVersionFuture.isReady()
? trState->readVersion()
: latestVersion);
}
void DatabaseContext::updateBackoff(const Error& err) {
switch (err.code()) {
case error_code_success:
backoffDelay = backoffDelay / CLIENT_KNOBS->BACKOFF_GROWTH_RATE;
if (backoffDelay < CLIENT_KNOBS->DEFAULT_BACKOFF) {
backoffDelay = 0.0;
}
break;
case error_code_commit_proxy_memory_limit_exceeded:
++transactionsResourceConstrained;
if (backoffDelay == 0.0) {
backoffDelay = CLIENT_KNOBS->DEFAULT_BACKOFF;
} else {
backoffDelay = std::min(backoffDelay * CLIENT_KNOBS->BACKOFF_GROWTH_RATE,
CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF);
}
break;
default:
ASSERT_WE_THINK(false);
}
}
Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations_internal(Database cx,
KeyRange keys,
int limit,
Reverse reverse,
SpanContext spanContext,
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies,
Version version) {
Span span("NAPI:getKeyRangeLocations"_loc, spanContext);
if (debugID.present())
g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.Before");
while (true) {
try {
co_await cx->getBackoff();
++cx->transactionKeyServerLocationRequests;
GetKeyServerLocationsReply rep = co_await commitProxyLoadBalance(
cx,
makeReqBuilder<GetKeyServerLocationsRequest>(
span.context, keys.begin, keys.end, limit, reverse, version, keys.arena()),
&CommitProxyInterface::getKeyServersLocations,
useProvisionalProxies,
TaskPriority::DefaultPromiseEndpoint);
++cx->transactionKeyServerLocationRequestsCompleted;
if (debugID.present())
g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getKeyLocations.After");
ASSERT(rep.results.size());
std::vector<KeyRangeLocationInfo> results;
for (int shard = 0; shard < rep.results.size(); ++shard) {
// FIXME: these shards are being inserted into the map sequentially, it would be much more CPU
// efficient to save the map pairs and insert them all at once.
results.emplace_back((rep.results[shard].first & keys),
cx->setCachedLocation(rep.results[shard].first, rep.results[shard].second));
co_await yield();
}
updateTssMappings(cx, rep);
updateTagMappings(cx, rep);
cx->updateBackoff(success());
co_return results;
} catch (Error& e) {
if (e.code() == error_code_commit_proxy_memory_limit_exceeded) {
// Eats commit_proxy_memory_limit_exceeded error from commit proxies
TraceEvent(SevWarnAlways, "CommitProxyOverloadedForRangeLocation").suppressFor(5);
cx->updateBackoff(e);
continue;
}
throw;
}
}
}
// Get the SS locations for each shard in the 'keys' key-range;
// Returned vector size is the number of shards in the input keys key-range.
// Returned vector element is <ShardRange, storage server location info> pairs, where
// ShardRange is the whole shard key-range, not a part of the given key range.
// Example: If query the function with key range (b, d), the returned list of pairs could be something like:
// [([a, b1), locationInfo), ([b1, c), locationInfo), ([c, d1), locationInfo)].
template <class F>
Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Database const& cx,
KeyRange const& keys,
int limit,
Reverse reverse,
F StorageServerInterface::* member,
SpanContext const& spanContext,
Optional<UID> const& debugID,
UseProvisionalProxies useProvisionalProxies,
Version version) {
ASSERT(!keys.empty());
std::vector<KeyRangeLocationInfo> locations;
if (!cx->getCachedLocations(keys, locations, limit, reverse)) {
return getKeyRangeLocations_internal(
cx, keys, limit, reverse, spanContext, debugID, useProvisionalProxies, version);
}
bool foundFailed = false;
for (const auto& locationInfo : locations) {
bool onlyEndpointFailedAndNeedRefresh = false;
for (int i = 0; i < locationInfo.locations->size(); i++) {
if (checkOnlyEndpointFailed(cx, locationInfo.locations->get(i, member).getEndpoint())) {
onlyEndpointFailedAndNeedRefresh = true;
}
}
if (onlyEndpointFailedAndNeedRefresh) {
cx->invalidateCache(locationInfo.range.begin);
foundFailed = true;
}
}
if (foundFailed) {
// Refresh the cache with a new getKeyRangeLocations made to proxies.
return getKeyRangeLocations_internal(
cx, keys, limit, reverse, spanContext, debugID, useProvisionalProxies, version);
}
return locations;
}
template <class F>
Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<TransactionState> trState,
KeyRange const& keys,
int limit,
Reverse reverse,
F StorageServerInterface::* member) {
return getKeyRangeLocations(trState->cx,
keys,
limit,
reverse,
member,
trState->spanContext,
trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
trState->useProvisionalProxies,
trState->readVersionFuture.isValid() && trState->readVersionFuture.isReady()
? trState->readVersion()
: latestVersion);
}
Future<Void> warmRange_impl(Reference<TransactionState> trState, KeyRange keys) {
int totalRanges = 0;
int totalRequests = 0;
co_await trState->startTransaction();
while (true) {
std::vector<KeyRangeLocationInfo> locations = co_await getKeyRangeLocations_internal(
trState->cx,
keys,
CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT,
Reverse::False,
trState->spanContext,
trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
trState->useProvisionalProxies,
trState->readVersion());
totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT;
totalRequests++;
if (locations.size() == 0 || totalRanges >= trState->cx->locationCacheSize ||
locations[locations.size() - 1].range.end >= keys.end)
break;
keys = KeyRangeRef(locations[locations.size() - 1].range.end, keys.end);
if (totalRequests % 20 == 0) {
// To avoid blocking the proxies from starting other transactions, occasionally get a read version.
Transaction tr(trState->cx);
while (true) {
Error err;
try {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.setOption(FDBTransactionOptions::CAUSAL_READ_RISKY);
co_await tr.getReadVersion();
break;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
}
}
}
SpanContext generateSpanID(bool transactionTracingSample, SpanContext parentContext = SpanContext()) {
if (parentContext.isValid()) {
return SpanContext(parentContext.traceID, deterministicRandom()->randomUInt64(), parentContext.m_Flags);
}
if (transactionTracingSample) {
return SpanContext(deterministicRandom()->randomUniqueID(),
deterministicRandom()->randomUInt64(),
deterministicRandom()->random01() <= FLOW_KNOBS->TRACING_SAMPLE_RATE
? TraceFlags::sampled
: TraceFlags::unsampled);
}
return SpanContext(
deterministicRandom()->randomUniqueID(), deterministicRandom()->randomUInt64(), TraceFlags::unsampled);
}
TransactionState::TransactionState(Database cx,
TaskPriority taskID,
SpanContext spanContext,
Reference<TransactionLogInfo> trLogInfo)
: cx(cx), trLogInfo(trLogInfo), options(cx), taskID(taskID), spanContext(spanContext),
readVersionObtainedFromGrvProxy(true) {}
Reference<TransactionState> TransactionState::cloneAndReset(Reference<TransactionLogInfo> newTrLogInfo,
bool generateNewSpan) const {
SpanContext newSpanContext = generateNewSpan ? generateSpanID(cx->transactionTracingSample) : spanContext;
auto newState = makeReference<TransactionState>(cx, cx->taskID, newSpanContext, newTrLogInfo);
if (!cx->apiVersionAtLeast(16)) {
newState->options = options;
}
newState->readVersionFuture = Future<Version>();
newState->metadataVersion = Promise<Optional<Key>>();
newState->numErrors = numErrors;
newState->startTime = startTime;
newState->committedVersion = committedVersion;
newState->conflictingKeys = conflictingKeys;
return newState;
}
ACTOR Future<Void> startTransaction(Reference<TransactionState> trState) {
wait(success(trState->readVersionFuture));
return Void();
}
Future<Void> TransactionState::startTransaction(uint32_t readVersionFlags) {
if (!startFuture.isValid()) {
if (!readVersionFuture.isValid()) {
readVersionFuture = getReadVersion(readVersionFlags);
}
if (readVersionFuture.isReady()) {
startFuture = Void();
} else {
startFuture = ::startTransaction(Reference<TransactionState>::addRef(this));
}
}
return startFuture;
}
Future<Void> Transaction::warmRange(KeyRange keys) {
return warmRange_impl(trState, keys);
}
ACTOR Future<Optional<Value>> getValue(Reference<TransactionState> trState,
Key key,
TransactionRecordLogInfo recordLogInfo) {
wait(trState->startTransaction());
state Span span("NAPI:getValue"_loc, trState->spanContext);
trState->cx->validateVersion(trState->readVersion());
loop {
state KeyRangeLocationInfo locationInfo =
wait(getKeyLocation(trState, key, &StorageServerInterface::getValue, Reverse::False));
state Optional<UID> getValueID = Optional<UID>();
state uint64_t startTime;
state double startTimeD;
state VersionVector ssLatestCommitVersions;
state Optional<ReadOptions> readOptions = trState->readOptions;
trState->cx->getLatestCommitVersions(locationInfo.locations, trState, ssLatestCommitVersions);
try {
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
getValueID = nondeterministicRandom()->randomUniqueID();
readOptions.get().debugID = getValueID;
g_traceBatch.addAttach(
"GetValueAttachID", trState->readOptions.get().debugID.get().first(), getValueID.get().first());
g_traceBatch.addEvent("GetValueDebug",
getValueID.get().first(),
"NativeAPI.getValue.Before"); //.detail("TaskID", g_network->getCurrentTask());
/*TraceEvent("TransactionDebugGetValueInfo", getValueID.get())
.detail("Key", key)
.detail("ReqVersion", ver)
.detail("Servers", describe(ssi.second->get()));*/
}
++trState->cx->getValueSubmitted;
startTime = timer_int();
startTimeD = now();
++trState->cx->transactionPhysicalReads;
state GetValueReply reply;
try {
if (CLIENT_BUGGIFY_WITH_PROB(.01)) {
throw deterministicRandom()->randomChoice(
std::vector<Error>{ transaction_too_old(), future_version() });
}
choose {
when(wait(trState->cx->connectionFileChanged())) {
throw transaction_too_old();
}
when(GetValueReply _reply = wait(
loadBalance(locationInfo.locations->locations(),
&StorageServerInterface::getValue,
GetValueRequest(span.context,
key,
trState->readVersion(),
trState->cx->sampleReadTags() ? trState->options.readTags
: Optional<TagSet>(),
readOptions,
ssLatestCommitVersions),
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::False,
trState->cx->enableLocalityLoadBalance ? &trState->cx->queueModel : nullptr,
trState->options.enableReplicaConsistencyCheck,
trState->options.requiredReplicas))) {
reply = _reply;
}
}
++trState->cx->transactionPhysicalReadsCompleted;
} catch (Error&) {
++trState->cx->transactionPhysicalReadsCompleted;
throw;
}
double latency = now() - startTimeD;
trState->cx->readLatencies.addSample(latency);
if (trState->trLogInfo && recordLogInfo) {
int valueSize = reply.value.present() ? reply.value.get().size() : 0;
trState->trLogInfo->addLog(FdbClientLogEvents::EventGet(
startTimeD, trState->cx->clientLocality.dcId(), latency, valueSize, key));
}
trState->cx->getValueCompleted->latency = timer_int() - startTime;
trState->cx->getValueCompleted->log();
trState->totalCost +=
getReadOperationCost(key.size() + (reply.value.present() ? reply.value.get().size() : 0));
if (getValueID.present()) {
g_traceBatch.addEvent("GetValueDebug",
getValueID.get().first(),
"NativeAPI.getValue.After"); //.detail("TaskID", g_network->getCurrentTask());
/*TraceEvent("TransactionDebugGetValueDone", getValueID.get())
.detail("Key", key)
.detail("ReqVersion", ver)
.detail("ReplySize", reply.value.present() ? reply.value.get().size() : -1);*/
}
trState->cx->transactionBytesRead += reply.value.present() ? reply.value.get().size() : 0;
++trState->cx->transactionKeysRead;
return reply.value;
} catch (Error& e) {
trState->cx->getValueCompleted->latency = timer_int() - startTime;
trState->cx->getValueCompleted->log();
if (getValueID.present()) {
g_traceBatch.addEvent("GetValueDebug", getValueID.get().first(), "NativeAPI.getValue.Error");
}
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
trState->cx->invalidateCache(key);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
} else {
if (trState->trLogInfo && recordLogInfo)
trState->trLogInfo->addLog(FdbClientLogEvents::EventGetError(
startTimeD, trState->cx->clientLocality.dcId(), static_cast<int>(e.code()), key));
throw e;
}
}
}
}
ACTOR Future<Key> getKey(Reference<TransactionState> trState, KeySelector k) {
wait(trState->startTransaction());
state Optional<UID> getKeyID;
state Optional<ReadOptions> readOptions = trState->readOptions;
state Span span("NAPI:getKey"_loc, trState->spanContext);
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
getKeyID = nondeterministicRandom()->randomUniqueID();
readOptions.get().debugID = getKeyID;
g_traceBatch.addAttach(
"GetKeyAttachID", trState->readOptions.get().debugID.get().first(), getKeyID.get().first());
g_traceBatch.addEvent(
"GetKeyDebug",
getKeyID.get().first(),
"NativeAPI.getKey.AfterVersion"); //.detail("StartKey",
// k.getKey()).detail("Offset",k.offset).detail("OrEqual",k.orEqual);
}
loop {
if (k.getKey() == allKeys.end) {
if (k.offset > 0) {
return allKeys.end;
}
k.orEqual = false;
} else if (k.getKey() == allKeys.begin && k.offset <= 0) {
return Key();
}
Key locationKey(k.getKey(), k.arena());
state KeyRangeLocationInfo locationInfo =
wait(getKeyLocation(trState, locationKey, &StorageServerInterface::getKey, Reverse{ k.isBackward() }));
state VersionVector ssLatestCommitVersions;
trState->cx->getLatestCommitVersions(locationInfo.locations, trState, ssLatestCommitVersions);
try {
if (getKeyID.present())
g_traceBatch.addEvent(
"GetKeyDebug",
getKeyID.get().first(),
"NativeAPI.getKey.Before"); //.detail("StartKey",
// k.getKey()).detail("Offset",k.offset).detail("OrEqual",k.orEqual);
++trState->cx->transactionPhysicalReads;
GetKeyRequest req(span.context,
k,
trState->readVersion(),
trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>(),
readOptions,
ssLatestCommitVersions);
req.arena.dependsOn(k.arena());
state GetKeyReply reply;
try {
choose {
when(wait(trState->cx->connectionFileChanged())) {
throw transaction_too_old();
}
when(GetKeyReply _reply = wait(
loadBalance(locationInfo.locations->locations(),
&StorageServerInterface::getKey,
req,
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::False,
trState->cx->enableLocalityLoadBalance ? &trState->cx->queueModel : nullptr,
trState->options.enableReplicaConsistencyCheck,
trState->options.requiredReplicas))) {
reply = _reply;
}
}
++trState->cx->transactionPhysicalReadsCompleted;
} catch (Error&) {
++trState->cx->transactionPhysicalReadsCompleted;
throw;
}
if (getKeyID.present())
g_traceBatch.addEvent("GetKeyDebug",
getKeyID.get().first(),
"NativeAPI.getKey.After"); //.detail("NextKey",reply.sel.key).detail("Offset",
// reply.sel.offset).detail("OrEqual", k.orEqual);
k = reply.sel;
if (!k.offset && k.orEqual) {
return k.getKey();
}
} catch (Error& e) {
if (getKeyID.present())
g_traceBatch.addEvent("GetKeyDebug", getKeyID.get().first(), "NativeAPI.getKey.Error");
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
trState->cx->invalidateCache(k.getKey(), Reverse{ k.isBackward() });
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
} else {
TraceEvent(SevInfo, "GetKeyError").error(e).detail("AtKey", k.getKey()).detail("Offset", k.offset);
throw e;
}
}
}
}
ACTOR Future<Version> waitForCommittedVersion(Database cx, Version version, SpanContext spanContext) {
state Span span("NAPI:waitForCommittedVersion"_loc, spanContext);
loop {
try {
choose {
when(wait(cx->onProxiesChanged())) {}
when(GetReadVersionReply v = wait(basicLoadBalance(
cx->getGrvProxies(UseProvisionalProxies::False),
&GrvProxyInterface::getConsistentReadVersion,
GetReadVersionRequest(
span.context, 0, TransactionPriority::IMMEDIATE, cx->ssVersionVectorCache.getMaxVersion()),
cx->taskID))) {
cx->minAcceptableReadVersion = std::min(cx->minAcceptableReadVersion, v.version);
if (v.midShardSize > 0)
cx->smoothMidShardSize.setTotal(v.midShardSize);
if (cx->versionVectorCacheActive(v.ssVersionVectorDelta)) {
if (cx->isCurrentGrvProxy(v.proxyId)) {
cx->ssVersionVectorCache.applyDelta(v.ssVersionVectorDelta);
} else {
cx->ssVersionVectorCache.clear();
}
}
if (v.version >= version)
return v.version;
// SOMEDAY: Do the wait on the server side, possibly use less expensive source of committed version
// (causal consistency is not needed for this purpose)
wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, cx->taskID));
}
}
} catch (Error& e) {
if (e.code() == error_code_batch_transaction_throttled ||
e.code() == error_code_grv_proxy_memory_limit_exceeded) {
// GRV Proxy returns an error
wait(delayJittered(CLIENT_KNOBS->GRV_ERROR_RETRY_DELAY));
} else {
TraceEvent(SevError, "WaitForCommittedVersionError").error(e);
throw;
}
}
}
}
ACTOR Future<Version> getRawVersion(Reference<TransactionState> trState) {
state Span span("NAPI:getRawVersion"_loc, trState->spanContext);
loop {
choose {
when(wait(trState->cx->onProxiesChanged())) {}
when(GetReadVersionReply v =
wait(basicLoadBalance(trState->cx->getGrvProxies(UseProvisionalProxies::False),
&GrvProxyInterface::getConsistentReadVersion,
GetReadVersionRequest(trState->spanContext,
0,
TransactionPriority::IMMEDIATE,
trState->cx->ssVersionVectorCache.getMaxVersion()),
trState->cx->taskID))) {
if (trState->cx->versionVectorCacheActive(v.ssVersionVectorDelta)) {
if (trState->cx->isCurrentGrvProxy(v.proxyId)) {
trState->cx->ssVersionVectorCache.applyDelta(v.ssVersionVectorDelta);
} else {
trState->cx->ssVersionVectorCache.clear();
}
}
return v.version;
}
}
}
}
ACTOR Future<Void> readVersionBatcher(
DatabaseContext* cx,
FutureStream<std::pair<Promise<GetReadVersionReply>, Optional<UID>>> versionStream,
uint32_t flags);
ACTOR Future<Version> watchValue(Database cx, Reference<const WatchParameters> parameters) {
state Span span("NAPI:watchValue"_loc, parameters->spanContext);
state Version ver = parameters->version;
cx->validateVersion(parameters->version);
ASSERT(parameters->version != latestVersion);
loop {
state KeyRangeLocationInfo locationInfo = wait(getKeyLocation(cx,
parameters->key,
&StorageServerInterface::watchValue,
parameters->spanContext,
parameters->debugID,
parameters->useProvisionalProxies,
Reverse::False,
parameters->version));
try {
state Optional<UID> watchValueID = Optional<UID>();
if (parameters->debugID.present()) {
watchValueID = nondeterministicRandom()->randomUniqueID();
g_traceBatch.addAttach(
"WatchValueAttachID", parameters->debugID.get().first(), watchValueID.get().first());
g_traceBatch.addEvent("WatchValueDebug",
watchValueID.get().first(),
"NativeAPI.watchValue.Before"); //.detail("TaskID", g_network->getCurrentTask());
}
state WatchValueReply resp;
choose {
when(WatchValueReply r = wait(
loadBalance(locationInfo.locations->locations(),
&StorageServerInterface::watchValue,
WatchValueRequest(span.context,
parameters->key,
parameters->value,
ver,
cx->sampleReadTags() ? parameters->tags : Optional<TagSet>(),
watchValueID),
TaskPriority::DefaultPromiseEndpoint))) {
resp = r;
}
when(wait(cx->connectionRecord ? cx->connectionRecord->onChange() : Never())) {
wait(Never());
}
}
if (watchValueID.present()) {
g_traceBatch.addEvent("WatchValueDebug", watchValueID.get().first(), "NativeAPI.watchValue.After");
}
// FIXME: wait for known committed version on the storage server before replying,
// cannot do this until the storage server is notified on knownCommittedVersion changes from tlog (faster
// than the current update loop)
Version v = wait(waitForCommittedVersion(cx, resp.version, span.context));
// False if there is a master failure between getting the response
// and getting the committed version, Dependent on
// SERVER_KNOBS->MAX_VERSIONS_IN_FLIGHT. Set to around half of the
// max versions in flight in an attempt to reliably recognize when
// a recovery has occurred, but avoid triggering if it just takes a
// little while to get the committed version.
bool buggifyRetry = g_network->isSimulated() && !g_simulator->speedUpSimulation && buggify(0.1);
CODE_PROBE(buggifyRetry, "Watch buggifying version gap retry");
if (v - resp.version < 50'000'000 && !buggifyRetry) {
return resp.version;
}
ver = v;
if (watchValueID.present()) {
g_traceBatch.addEvent("WatchValueDebug", watchValueID.get().first(), "NativeAPI.watchValue.Retry");
}
} catch (Error& e) {
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
cx->invalidateCache(parameters->key);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, parameters->taskID));
} else if (e.code() == error_code_watch_cancelled || e.code() == error_code_process_behind) {
// clang-format off
CODE_PROBE(e.code() == error_code_watch_cancelled, "Too many watches on the storage server, poll for changes instead");
CODE_PROBE(e.code() == error_code_process_behind, "The storage servers are all behind", probe::decoration::rare);
// clang-format on
wait(delay(CLIENT_KNOBS->WATCH_POLLING_TIME, parameters->taskID));
} else if (e.code() == error_code_timed_out) { // The storage server occasionally times out watches in case
// it was cancelled
CODE_PROBE(true, "A watch timed out");
wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, parameters->taskID));
} else {
state Error err = e;
wait(delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, parameters->taskID));
throw err;
}
}
}
}
Future<Void> watchStorageServerResp(Key key, Database cx) {
while (true) {
try {
Reference<WatchMetadata> metadata = cx->getWatchMetadata(key);
if (!metadata.isValid()) {
co_return;
}
Version watchVersion = co_await watchValue(cx, metadata->parameters);
metadata = cx->getWatchMetadata(key);
if (!metadata.isValid())
co_return;
// case 1: version_1 (SS) >= version_2 (map)
if (watchVersion >= metadata->parameters->version) {
cx->deleteWatchMetadata(key);
if (metadata->watchPromise.canBeSet())
metadata->watchPromise.send(watchVersion);
}
// ABA happens
else {
CODE_PROBE(true,
"ABA issue where the version returned from the server is less than the version in the map");
// case 2: version_1 < version_2 and future_count == 1
if (metadata->watchPromise.getFutureReferenceCount() == 1) {
cx->deleteWatchMetadata(key);
}
}
} catch (Error& e) {
if (e.code() == error_code_operation_cancelled) {
throw e;
}
Reference<WatchMetadata> metadata = cx->getWatchMetadata(key);
if (!metadata.isValid()) {
co_return;
} else if (metadata->watchPromise.getFutureReferenceCount() == 1) {
cx->deleteWatchMetadata(key);
co_return;
} else if (e.code() == error_code_future_version) {
continue;
}
cx->deleteWatchMetadata(key);
metadata->watchPromise.sendError(e);
throw e;
}
}
}
Future<Void> sameVersionDiffValue(Database cx, Reference<WatchParameters> parameters) {
ReadYourWritesTransaction tr(cx);
while (true) {
Error err;
try {
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
Optional<Value> valSS = co_await tr.get(parameters->key);
Reference<WatchMetadata> metadata = cx->getWatchMetadata(parameters->key);
// val_3 != val_1 (storage server value doesn't match value in map)
if (metadata.isValid() && valSS != metadata->parameters->value) {
cx->deleteWatchMetadata(parameters->key);
metadata->watchPromise.send(parameters->version);
metadata->watchFutureSS.cancel();
}
// val_3 == val_2 (storage server value matches value passed into the function -> new watch)
if (valSS == parameters->value) {
metadata = makeReference<WatchMetadata>(parameters);
cx->setWatchMetadata(metadata);
metadata->watchFutureSS = watchStorageServerResp(parameters->key, cx);
co_await metadata->watchPromise.getFuture();
}
co_return;
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
}
Future<Void> getWatchFuture(Database cx, Reference<WatchParameters> parameters) {
Reference<WatchMetadata> metadata = cx->getWatchMetadata(parameters->key);
// case 1: key not in map
if (!metadata.isValid()) {
metadata = makeReference<WatchMetadata>(parameters);
cx->setWatchMetadata(metadata);
metadata->watchFutureSS = watchStorageServerResp(parameters->key, cx);
return success(metadata->watchPromise.getFuture());
}
// case 2: val_1 == val_2 (received watch with same value as key already in the map so just update)
else if (metadata->parameters->value == parameters->value) {
if (parameters->version > metadata->parameters->version) {
metadata->parameters = parameters;
}
return success(metadata->watchPromise.getFuture());
}
// case 3: val_1 != val_2 && version_2 > version_1 (received watch with different value and a higher version so
// recreate in SS)
else if (parameters->version > metadata->parameters->version) {
CODE_PROBE(true,
"Setting a watch that has a different value than the one in the map but a higher version (newer)");
cx->deleteWatchMetadata(parameters->key);
metadata->watchPromise.send(parameters->version);
metadata->watchFutureSS.cancel();
metadata = makeReference<WatchMetadata>(parameters);
cx->setWatchMetadata(metadata);
metadata->watchFutureSS = watchStorageServerResp(parameters->key, cx);
return success(metadata->watchPromise.getFuture());
}
// case 5: val_1 != val_2 && version_1 == version_2 (received watch with different value but same version)
else if (metadata->parameters->version == parameters->version) {
CODE_PROBE(true, "Setting a watch which has a different value than the one in the map but the same version");
return sameVersionDiffValue(cx, parameters);
}
CODE_PROBE(true, "Setting a watch which has a different value than the one in the map but a lower version (older)");
// case 4: val_1 != val_2 && version_2 < version_1
return Void();
}
namespace {
// NOTE: Since an ACTOR could receive multiple exceptions for a single catch clause, e.g. broken promise together with
// operation cancelled, If the decreaseWatchRefCount is placed at the catch clause, it might be triggered for multiple
// times. One could check if the SAV isSet, but seems a more intuitive way is to use RAII-style constructor/destructor
// pair. Yet the object has to be constructed after a wait statement, so it must be trivially-constructible. This
// requires move-assignment operator implemented.
class WatchRefCountUpdater {
Database cx;
KeyRef key;
Version version;
public:
WatchRefCountUpdater() = default;
WatchRefCountUpdater(const Database& cx_, KeyRef key_, const Version& ver) : cx(cx_), key(key_), version(ver) {}
WatchRefCountUpdater& operator=(WatchRefCountUpdater&& other) {
if (cx.getReference()) {
cx->decreaseWatchRefCount(key, version);
}
cx = std::move(other.cx);
key = std::move(other.key);
version = std::move(other.version);
cx->increaseWatchRefCount(key, version);
return *this;
}
~WatchRefCountUpdater() {
if (cx.getReference()) {
cx->decreaseWatchRefCount(key, version);
}
}
};
} // namespace
ACTOR Future<Void> watchValueMap(Future<Version> version,
Key key,
Optional<Value> value,
Database cx,
TagSet tags,
SpanContext spanContext,
TaskPriority taskID,
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies) {
state Version ver = wait(version);
state WatchRefCountUpdater watchRefCountUpdater(cx, key, ver);
wait(getWatchFuture(
cx,
makeReference<WatchParameters>(key, value, ver, tags, spanContext, taskID, debugID, useProvisionalProxies)));
return Void();
}
template <class GetKeyValuesFamilyRequest>
void transformRangeLimits(GetRangeLimits limits, Reverse reverse, GetKeyValuesFamilyRequest& req) {
if (limits.bytes != 0) {
if (!limits.hasRowLimit())
req.limit = CLIENT_KNOBS->REPLY_BYTE_LIMIT; // Can't get more than this many rows anyway
else
req.limit = std::min(CLIENT_KNOBS->REPLY_BYTE_LIMIT, limits.rows);
if (reverse)
req.limit *= -1;
if (!limits.hasByteLimit())
req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
else
req.limitBytes = std::min(CLIENT_KNOBS->REPLY_BYTE_LIMIT, limits.bytes);
} else {
req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT;
req.limit = reverse ? -limits.minRows : limits.minRows;
}
}
template <class GetKeyValuesFamilyRequest>
PublicRequestStream<GetKeyValuesFamilyRequest> StorageServerInterface::* getRangeRequestStream() {
if constexpr (std::is_same<GetKeyValuesFamilyRequest, GetKeyValuesRequest>::value) {
return &StorageServerInterface::getKeyValues;
} else if (std::is_same<GetKeyValuesFamilyRequest, GetMappedKeyValuesRequest>::value) {
return &StorageServerInterface::getMappedKeyValues;
} else {
UNREACHABLE();
}
}
ACTOR template <class GetKeyValuesFamilyRequest, class GetKeyValuesFamilyReply, class RangeResultFamily>
Future<RangeResultFamily> getExactRange(Reference<TransactionState> trState,
KeyRange keys,
Key mapper,
GetRangeLimits limits,
Reverse reverse) {
state RangeResultFamily output;
state Span span("NAPI:getExactRange"_loc, trState->spanContext);
loop {
state std::vector<KeyRangeLocationInfo> locations =
wait(getKeyRangeLocations(trState,
keys,
CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT,
reverse,
getRangeRequestStream<GetKeyValuesFamilyRequest>()));
ASSERT(locations.size());
state int shard = 0;
loop {
const KeyRangeRef& range = locations[shard].range;
GetKeyValuesFamilyRequest req;
req.mapper = mapper;
req.arena.dependsOn(mapper.arena());
req.version = trState->readVersion();
req.begin = firstGreaterOrEqual(range.begin);
req.end = firstGreaterOrEqual(range.end);
req.spanContext = span.context;
trState->cx->getLatestCommitVersions(locations[shard].locations, trState, req.ssLatestCommitVersions);
// keep shard's arena around in case of async tss comparison
req.arena.dependsOn(locations[shard].range.arena());
transformRangeLimits(limits, reverse, req);
ASSERT(req.limitBytes > 0 && req.limit != 0 && req.limit < 0 == reverse);
// FIXME: buggify byte limits on internal functions that use them, instead of globally
req.tags = trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>();
req.options = trState->readOptions;
try {
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
g_traceBatch.addEvent("TransactionDebug",
trState->readOptions.get().debugID.get().first(),
"NativeAPI.getExactRange.Before");
/*TraceEvent("TransactionDebugGetExactRangeInfo", trState->readOptions.get().debugID.get())
.detail("ReqBeginKey", req.begin.getKey())
.detail("ReqEndKey", req.end.getKey())
.detail("ReqLimit", req.limit)
.detail("ReqLimitBytes", req.limitBytes)
.detail("ReqVersion", req.version)
.detail("Reverse", reverse)
.detail("Servers", locations[shard].locations->locations()->description());*/
}
++trState->cx->transactionPhysicalReads;
state GetKeyValuesFamilyReply rep;
try {
choose {
when(wait(trState->cx->connectionFileChanged())) {
throw transaction_too_old();
}
when(GetKeyValuesFamilyReply _rep = wait(loadBalance(
locations[shard].locations->locations(),
getRangeRequestStream<GetKeyValuesFamilyRequest>(),
req,
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::False,
trState->cx->enableLocalityLoadBalance ? &trState->cx->queueModel : nullptr,
trState->options.enableReplicaConsistencyCheck,
trState->options.requiredReplicas))) {
rep = _rep;
}
}
++trState->cx->transactionPhysicalReadsCompleted;
} catch (Error&) {
++trState->cx->transactionPhysicalReadsCompleted;
throw;
}
if (trState->readOptions.present() && trState->readOptions.get().debugID.present())
g_traceBatch.addEvent("TransactionDebug",
trState->readOptions.get().debugID.get().first(),
"NativeAPI.getExactRange.After");
output.arena().dependsOn(rep.arena);
output.append(output.arena(), rep.data.begin(), rep.data.size());
if (limits.hasRowLimit() && rep.data.size() > limits.rows) {
TraceEvent(SevError, "GetExactRangeTooManyRows")
.detail("RowLimit", limits.rows)
.detail("DeliveredRows", output.size());
ASSERT(false);
}
limits.decrement(rep.data);
if (limits.isReached()) {
output.more = true;
return output;
}
bool more = rep.more;
// If the reply says there is more but we know that we finished the shard, then fix rep.more
if (reverse && more && rep.data.size() > 0 &&
output[output.size() - 1].key == locations[shard].range.begin)
more = false;
if (more) {
if (!rep.data.size()) {
TraceEvent(SevError, "GetExactRangeError")
.detail("Reason", "More data indicated but no rows present")
.detail("LimitBytes", limits.bytes)
.detail("LimitRows", limits.rows)
.detail("OutputSize", output.size())
.detail("OutputBytes", output.expectedSize())
.detail("BlockSize", rep.data.size())
.detail("BlockBytes", rep.data.expectedSize());
ASSERT(false);
}
CODE_PROBE(true, "GetKeyValuesFamilyReply.more in getExactRange");
// Make next request to the same shard with a beginning key just after the last key returned
if (reverse)
locations[shard].range =
KeyRangeRef(locations[shard].range.begin, output[output.size() - 1].key);
else
locations[shard].range =
KeyRangeRef(keyAfter(output[output.size() - 1].key), locations[shard].range.end);
}
bool redoKeyLocationRequest = false;
if (!more || locations[shard].range.empty()) {
CODE_PROBE(true, "getExactrange (!more || locations[shard].first.empty())");
if (shard == locations.size() - 1) {
const KeyRangeRef& range = locations[shard].range;
KeyRef begin = reverse ? keys.begin : range.end;
KeyRef end = reverse ? range.begin : keys.end;
if (begin >= end) {
output.more = false;
return output;
}
keys = KeyRangeRef(begin, end);
redoKeyLocationRequest = true;
}
++shard;
}
// Soft byte limit - return results early if the user specified a byte limit and we got results
// This can prevent problems where the desired range spans many shards and would be too slow to
// fetch entirely.
if (limits.hasSatisfiedMinRows() && output.size() > 0) {
output.more = true;
return output;
}
if (redoKeyLocationRequest) {
CODE_PROBE(true, "Multiple requests of key locations");
break;
}
} catch (Error& e) {
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
const KeyRangeRef& range = locations[shard].range;
if (reverse)
keys = KeyRangeRef(keys.begin, range.end);
else
keys = KeyRangeRef(range.begin, keys.end);
trState->cx->invalidateCache(keys);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
break;
} else {
TraceEvent(SevInfo, "GetExactRangeError")
.error(e)
.detail("ShardBegin", locations[shard].range.begin)
.detail("ShardEnd", locations[shard].range.end);
throw;
}
}
}
}
}
Future<Key> resolveKey(Reference<TransactionState> trState, KeySelector const& key) {
if (key.isFirstGreaterOrEqual())
return Future<Key>(key.getKey());
if (key.isFirstGreaterThan())
return Future<Key>(keyAfter(key.getKey()));
return getKey(trState, key);
}
ACTOR template <class GetKeyValuesFamilyRequest, class GetKeyValuesFamilyReply, class RangeResultFamily>
Future<RangeResultFamily> getRangeFallback(Reference<TransactionState> trState,
KeySelector begin,
KeySelector end,
Key mapper,
GetRangeLimits limits,
Reverse reverse) {
Future<Key> fb = resolveKey(trState, begin);
state Future<Key> fe = resolveKey(trState, end);
state Key b = wait(fb);
state Key e = wait(fe);
if (b >= e) {
return RangeResultFamily();
}
// if e is allKeys.end, we have read through the end of the database
// if b is allKeys.begin, we have either read through the beginning of the database
// or allKeys.begin exists in the database and will be part of the conflict range anyways
RangeResultFamily _r = wait(getExactRange<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
trState, KeyRangeRef(b, e), mapper, limits, reverse));
RangeResultFamily r = _r;
if (b == allKeys.begin && ((reverse && !r.more) || !reverse))
r.readToBegin = true;
// TODO: this currently causes us to have a conflict range that is too large if our end key resolves to the
// key after the last key in the database. In that case, we don't need a conflict between the last key and
// the end of the database.
//
// If fixed, the ConflictRange test can be updated to stop checking for this condition.
if (e == allKeys.end && ((!reverse && !r.more) || reverse))
r.readThroughEnd = true;
ASSERT(!limits.hasRowLimit() || r.size() <= limits.rows);
// If we were limiting bytes and the returned range is twice the request (plus 10K) log a warning
if (limits.hasByteLimit() &&
r.expectedSize() >
size_t(limits.bytes + CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT + CLIENT_KNOBS->VALUE_SIZE_LIMIT + 1) &&
limits.minRows == 0) {
TraceEvent(SevWarnAlways, "GetRangeFallbackTooMuchData")
.detail("LimitBytes", limits.bytes)
.detail("DeliveredBytes", r.expectedSize())
.detail("LimitRows", limits.rows)
.detail("DeliveredRows", r.size());
}
return r;
}
int64_t inline getRangeResultFamilyBytes(RangeResultRef result) {
return result.expectedSize();
}
int64_t inline getRangeResultFamilyBytes(MappedRangeResultRef result) {
int64_t bytes = 0;
for (const MappedKeyValueRef& mappedKeyValue : result) {
bytes += mappedKeyValue.key.size() + mappedKeyValue.value.size();
auto& reqAndResult = mappedKeyValue.reqAndResult;
if (std::holds_alternative<GetValueReqAndResultRef>(reqAndResult)) {
auto getValue = std::get<GetValueReqAndResultRef>(reqAndResult);
bytes += getValue.expectedSize();
} else if (std::holds_alternative<GetRangeReqAndResultRef>(reqAndResult)) {
auto getRange = std::get<GetRangeReqAndResultRef>(reqAndResult);
bytes += getRange.result.expectedSize();
} else {
throw internal_error();
}
}
return bytes;
}
// TODO: Client should add mapped keys to conflict ranges.
template <class RangeResultFamily> // RangeResult or MappedRangeResult
void getRangeFinished(Reference<TransactionState> trState,
double startTime,
KeySelector begin,
KeySelector end,
Snapshot snapshot,
Promise<std::pair<Key, Key>> conflictRange,
Reverse reverse,
RangeResultFamily result) {
int64_t bytes = getRangeResultFamilyBytes(result);
trState->totalCost += getReadOperationCost(bytes);
trState->cx->transactionBytesRead += bytes;
trState->cx->transactionKeysRead += result.size();
if (trState->trLogInfo) {
trState->trLogInfo->addLog(FdbClientLogEvents::EventGetRange(
startTime, trState->cx->clientLocality.dcId(), now() - startTime, bytes, begin.getKey(), end.getKey()));
}
if (!snapshot) {
Key rangeBegin;
Key rangeEnd;
if (result.readToBegin) {
rangeBegin = allKeys.begin;
} else if (((!reverse || !result.more || begin.offset > 1) && begin.offset > 0) || result.size() == 0) {
rangeBegin = Key(begin.getKey(), begin.arena());
} else {
rangeBegin = reverse ? result.end()[-1].key : result[0].key;
}
if (end.offset > begin.offset && end.getKey() < rangeBegin) {
rangeBegin = Key(end.getKey(), end.arena());
}
if (result.readThroughEnd) {
rangeEnd = allKeys.end;
} else if (((reverse || !result.more || end.offset <= 0) && end.offset <= 1) || result.size() == 0) {
rangeEnd = Key(end.getKey(), end.arena());
} else {
rangeEnd = keyAfter(reverse ? result[0].key : result.end()[-1].key);
}
if (begin.offset < end.offset && begin.getKey() > rangeEnd) {
rangeEnd = Key(begin.getKey(), begin.arena());
}
conflictRange.send(std::make_pair(rangeBegin, rangeEnd));
}
}
ACTOR template <class GetKeyValuesFamilyRequest, // GetKeyValuesRequest or GetMappedKeyValuesRequest
class GetKeyValuesFamilyReply, // GetKeyValuesReply or GetMappedKeyValuesReply (It would be nice if
// we could use REPLY_TYPE(GetKeyValuesFamilyRequest) instead of specify
// it as a separate template element)
class RangeResultFamily // RangeResult or MappedRangeResult
>
Future<RangeResultFamily> getRange(Reference<TransactionState> trState,
KeySelector begin,
KeySelector end,
Key mapper,
GetRangeLimits limits,
Promise<std::pair<Key, Key>> conflictRange,
Snapshot snapshot,
Reverse reverse) {
// state using RangeResultRefFamily = typename RangeResultFamily::RefType;
state GetRangeLimits originalLimits(limits);
state KeySelector originalBegin = begin;
state KeySelector originalEnd = end;
state RangeResultFamily output;
state Span span("NAPI:getRange"_loc, trState->spanContext);
state Optional<UID> getRangeID = Optional<UID>();
try {
wait(trState->startTransaction());
trState->cx->validateVersion(trState->readVersion());
state double startTime = now();
if (begin.getKey() == allKeys.begin && begin.offset < 1) {
output.readToBegin = true;
begin = KeySelector(firstGreaterOrEqual(begin.getKey()), begin.arena());
}
ASSERT(!limits.isReached());
ASSERT((!limits.hasRowLimit() || limits.rows >= limits.minRows) && limits.minRows >= 0);
loop {
if (end.getKey() == allKeys.begin && (end.offset < 1 || end.isFirstGreaterOrEqual())) {
getRangeFinished(
trState, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, output);
return output;
}
Key locationKey = reverse ? Key(end.getKey(), end.arena()) : Key(begin.getKey(), begin.arena());
Reverse locationBackward{ reverse ? (end - 1).isBackward() : begin.isBackward() };
state KeyRangeLocationInfo beginServer = wait(getKeyLocation(
trState, locationKey, getRangeRequestStream<GetKeyValuesFamilyRequest>(), locationBackward));
state KeyRange shard = beginServer.range;
state bool modifiedSelectors = false;
state GetKeyValuesFamilyRequest req;
req.mapper = mapper;
req.arena.dependsOn(mapper.arena());
req.options = trState->readOptions;
req.version = trState->readVersion();
req.taskID = trState->taskID;
trState->cx->getLatestCommitVersions(beginServer.locations, trState, req.ssLatestCommitVersions);
// In case of async tss comparison, also make req arena depend on begin, end, and/or shard's arena depending
// on which is used
bool dependOnShard = false;
if (reverse && (begin - 1).isDefinitelyLess(shard.begin) &&
(!begin.isFirstGreaterOrEqual() ||
begin.getKey() != shard.begin)) { // In this case we would be setting modifiedSelectors to true, but
// not modifying anything
req.begin = firstGreaterOrEqual(shard.begin);
modifiedSelectors = true;
req.arena.dependsOn(shard.arena());
dependOnShard = true;
} else {
req.begin = begin;
req.arena.dependsOn(begin.arena());
}
if (!reverse && end.isDefinitelyGreater(shard.end)) {
req.end = firstGreaterOrEqual(shard.end);
modifiedSelectors = true;
if (!dependOnShard) {
req.arena.dependsOn(shard.arena());
}
} else {
req.end = end;
req.arena.dependsOn(end.arena());
}
transformRangeLimits(limits, reverse, req);
ASSERT(req.limitBytes > 0 && req.limit != 0 && req.limit < 0 == reverse);
req.tags = trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>();
req.spanContext = span.context;
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
getRangeID = nondeterministicRandom()->randomUniqueID();
g_traceBatch.addAttach(
"TransactionAttachID", trState->readOptions.get().debugID.get().first(), getRangeID.get().first());
}
try {
if (getRangeID.present()) {
g_traceBatch.addEvent("TransactionDebug", getRangeID.get().first(), "NativeAPI.getRange.Before");
/*
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
TraceEvent("TransactionDebugGetRangeInfo", trState->readOptions.get().debugID.get())
.detail("ReqBeginKey", req.begin.getKey())
.detail("ReqEndKey", req.end.getKey())
.detail("OriginalBegin", originalBegin.toString())
.detail("OriginalEnd", originalEnd.toString())
.detail("Begin", begin.toString())
.detail("End", end.toString())
.detail("Shard", shard)
.detail("ReqLimit", req.limit)
.detail("ReqLimitBytes", req.limitBytes)
.detail("ReqVersion", req.version)
.detail("Reverse", reverse)
.detail("ModifiedSelectors", modifiedSelectors)
.detail("Servers", beginServer.locations->locations()->description());
}*/
}
++trState->cx->transactionPhysicalReads;
state GetKeyValuesFamilyReply rep;
try {
if (CLIENT_BUGGIFY_WITH_PROB(.01)) {
throw deterministicRandom()->randomChoice(
std::vector<Error>{ transaction_too_old(), future_version() });
}
// state AnnotateActor annotation(currentLineage);
GetKeyValuesFamilyReply _rep =
wait(loadBalance(beginServer.locations->locations(),
getRangeRequestStream<GetKeyValuesFamilyRequest>(),
req,
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::False,
trState->cx->enableLocalityLoadBalance ? &trState->cx->queueModel : nullptr,
trState->options.enableReplicaConsistencyCheck,
trState->options.requiredReplicas));
rep = _rep;
++trState->cx->transactionPhysicalReadsCompleted;
} catch (Error&) {
++trState->cx->transactionPhysicalReadsCompleted;
throw;
}
if (getRangeID.present()) {
g_traceBatch.addEvent("TransactionDebug",
getRangeID.get().first(),
"NativeAPI.getRange.After"); //.detail("SizeOf", rep.data.size());
/*
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
TraceEvent("TransactionDebugGetRangeDone", trState->readOptions.get().debugID.get())
.detail("ReqBeginKey", req.begin.getKey())
.detail("ReqEndKey", req.end.getKey())
.detail("RepIsMore", rep.more)
.detail("VersionReturned", rep.version)
.detail("RowsReturned", rep.data.size());
}*/
}
ASSERT(!rep.more || rep.data.size());
ASSERT(!limits.hasRowLimit() || rep.data.size() <= limits.rows);
limits.decrement(rep.data);
if (reverse && begin.isLastLessOrEqual() && rep.data.size() &&
rep.data.end()[-1].key == begin.getKey()) {
modifiedSelectors = false;
}
bool finished = limits.isReached() || (!modifiedSelectors && !rep.more) || limits.hasSatisfiedMinRows();
bool readThrough = modifiedSelectors && !rep.more;
// optimization: first request got all data--just return it
if (finished && !output.size()) {
bool readToBegin = output.readToBegin;
bool readThroughEnd = output.readThroughEnd;
using RangeResultRefFamily = typename RangeResultFamily::RefType;
output = RangeResultFamily(
RangeResultRefFamily(rep.data, modifiedSelectors || limits.isReached() || rep.more), rep.arena);
output.readToBegin = readToBegin;
output.readThroughEnd = readThroughEnd;
if (buggify() && limits.hasByteLimit() && output.size() > std::max(1, originalLimits.minRows) &&
(!std::is_same<GetKeyValuesFamilyRequest, GetMappedKeyValuesRequest>::value)) {
// Copy instead of resizing because TSS maybe be using output's arena for comparison. This only
// happens in simulation so it's fine
// disable it on prefetch, because boundary entries serve as continuations
RangeResultFamily copy;
int newSize =
deterministicRandom()->randomInt(std::max(1, originalLimits.minRows), output.size());
for (int i = 0; i < newSize; i++) {
copy.push_back_deep(copy.arena(), output[i]);
}
output = copy;
output.more = true;
getRangeFinished(
trState, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, output);
return output;
}
if (readThrough) {
output.arena().dependsOn(shard.arena());
// As modifiedSelectors is true, more is also true. Then set readThrough to the shard boundary.
ASSERT(modifiedSelectors);
output.more = true;
output.setReadThrough(reverse ? shard.begin : shard.end);
}
getRangeFinished(
trState, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, output);
if (!output.more) {
ASSERT(!output.readThrough.present());
}
return output;
}
output.arena().dependsOn(rep.arena);
output.append(output.arena(), rep.data.begin(), rep.data.size());
if (finished) {
output.more = modifiedSelectors || limits.isReached() || rep.more;
if (readThrough) {
output.arena().dependsOn(shard.arena());
output.setReadThrough(reverse ? shard.begin : shard.end);
}
getRangeFinished(
trState, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, output);
if (!output.more) {
ASSERT(!output.readThrough.present());
}
return output;
}
if (!rep.more) {
ASSERT(modifiedSelectors);
CODE_PROBE(true, "!GetKeyValuesFamilyReply.more and modifiedSelectors in getRange");
if (!rep.data.size()) {
RangeResultFamily result = wait(
getRangeFallback<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
trState, originalBegin, originalEnd, mapper, originalLimits, reverse));
getRangeFinished(
trState, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, result);
return result;
}
if (reverse)
end = firstGreaterOrEqual(shard.begin);
else
begin = firstGreaterOrEqual(shard.end);
} else {
CODE_PROBE(true, "GetKeyValuesFamilyReply.more in getRange");
if (reverse)
end = firstGreaterOrEqual(output[output.size() - 1].key);
else
begin = firstGreaterThan(output[output.size() - 1].key);
}
} catch (Error& e) {
if (getRangeID.present()) {
g_traceBatch.addEvent("TransactionDebug", getRangeID.get().first(), "NativeAPI.getRange.Error");
TraceEvent("TransactionDebugError", getRangeID.get()).error(e);
}
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
trState->cx->invalidateCache(reverse ? end.getKey() : begin.getKey(),
Reverse{ reverse ? (end - 1).isBackward() : begin.isBackward() });
if (e.code() == error_code_wrong_shard_server) {
RangeResultFamily result = wait(
getRangeFallback<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
trState, originalBegin, originalEnd, mapper, originalLimits, reverse));
getRangeFinished(
trState, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, result);
return result;
}
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
} else {
if (trState->trLogInfo)
trState->trLogInfo->addLog(
FdbClientLogEvents::EventGetRangeError(startTime,
trState->cx->clientLocality.dcId(),
static_cast<int>(e.code()),
begin.getKey(),
end.getKey()));
throw e;
}
}
}
} catch (Error& e) {
if (conflictRange.canBeSet()) {
conflictRange.send(std::make_pair(Key(), Key()));
}
throw;
}
}
template <class StreamReply>
struct TSSDuplicateStreamData {
PromiseStream<StreamReply> stream;
Promise<Void> tssComparisonDone;
// empty constructor for optional?
TSSDuplicateStreamData() = default;
explicit TSSDuplicateStreamData(PromiseStream<StreamReply> stream) : stream(stream) {}
bool done() { return tssComparisonDone.getFuture().isReady(); }
void setDone() {
if (tssComparisonDone.canBeSet()) {
tssComparisonDone.send(Void());
}
}
~TSSDuplicateStreamData() = default;
};
// Error tracking here is weird, and latency doesn't really mean the same thing here as it does with normal tss
// comparisons, so this is pretty much just counting mismatches
ACTOR template <class Request>
static Future<Void> tssStreamComparison(Request request,
TSSDuplicateStreamData<REPLYSTREAM_TYPE(Request)> streamData,
ReplyPromiseStream<REPLYSTREAM_TYPE(Request)> tssReplyStream,
TSSEndpointData tssData) {
state bool ssEndOfStream = false;
state bool tssEndOfStream = false;
state Optional<REPLYSTREAM_TYPE(Request)> ssReply = Optional<REPLYSTREAM_TYPE(Request)>();
state Optional<REPLYSTREAM_TYPE(Request)> tssReply = Optional<REPLYSTREAM_TYPE(Request)>();
loop {
// reset replies
ssReply = Optional<REPLYSTREAM_TYPE(Request)>();
tssReply = Optional<REPLYSTREAM_TYPE(Request)>();
state double startTime = now();
// wait for ss response
try {
REPLYSTREAM_TYPE(Request) _ssReply = waitNext(streamData.stream.getFuture());
ssReply = _ssReply;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
streamData.setDone();
throw;
}
if (e.code() == error_code_end_of_stream) {
// ss response will be set to empty, to compare to the SS response if it wasn't empty and cause a
// mismatch
ssEndOfStream = true;
} else {
tssData.metrics->ssError(e.code());
}
CODE_PROBE(e.code() != error_code_end_of_stream, "SS got error in TSS stream comparison");
}
state double sleepTime = std::max(startTime + FLOW_KNOBS->LOAD_BALANCE_TSS_TIMEOUT - now(), 0.0);
// wait for tss response
try {
choose {
when(REPLYSTREAM_TYPE(Request) _tssReply = waitNext(tssReplyStream.getFuture())) {
tssReply = _tssReply;
}
when(wait(delay(sleepTime))) {
++tssData.metrics->tssTimeouts;
CODE_PROBE(true, "Got TSS timeout in stream comparison", probe::decoration::rare);
}
}
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
streamData.setDone();
throw;
}
if (e.code() == error_code_end_of_stream) {
// tss response will be set to empty, to compare to the SS response if it wasn't empty and cause a
// mismatch
tssEndOfStream = true;
} else {
tssData.metrics->tssError(e.code());
}
CODE_PROBE(e.code() != error_code_end_of_stream, "TSS got error in TSS stream comparison");
}
if (!ssEndOfStream || !tssEndOfStream) {
++tssData.metrics->streamComparisons;
}
// if both are successful, compare
if (ssReply.present() && tssReply.present()) {
// compare results
// FIXME: this code is pretty much identical to LoadBalance.h
// TODO could add team check logic in if we added synchronous way to turn this into a fixed getRange request
// and send it to the whole team and compare? I think it's fine to skip that for streaming though
// skip tss comparison if both are end of stream
if ((!ssEndOfStream || !tssEndOfStream) && !TSS_doCompare(ssReply.get(), tssReply.get())) {
CODE_PROBE(true, "TSS mismatch in stream comparison");
TraceEvent mismatchEvent(
(simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch))
? SevWarnAlways
: SevError,
LB_mismatchTraceName(request, TSS_COMPARISON));
mismatchEvent.setMaxEventLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE);
mismatchEvent.detail("TSSID", tssData.tssId);
if (tssData.metrics->shouldRecordDetailedMismatch()) {
TSS_traceMismatch(mismatchEvent, request, ssReply.get(), tssReply.get(), TSS_COMPARISON);
CODE_PROBE(FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL,
"Tracing Full TSS Mismatch in stream comparison",
probe::decoration::rare);
CODE_PROBE(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL,
"Tracing Partial TSS Mismatch in stream comparison and storing the rest in FDB");
if (!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL) {
mismatchEvent.disable();
UID mismatchUID = deterministicRandom()->randomUniqueID();
tssData.metrics->recordDetailedMismatchData(mismatchUID, mismatchEvent.getFields().toString());
// record a summarized trace event instead
TraceEvent summaryEvent(
(g_network->isSimulated() &&
simulationPolicyHasCapability(ISimulationPolicy::Capability::WarnOnStorageMismatch))
? SevWarnAlways
: SevError,
LB_mismatchTraceName(request, TSS_COMPARISON));
summaryEvent.detail("TSSID", tssData.tssId).detail("MismatchId", mismatchUID);
}
} else {
// don't record trace event
mismatchEvent.disable();
}
streamData.setDone();
return Void();
}
}
if (!ssReply.present() || !tssReply.present() || ssEndOfStream || tssEndOfStream) {
// if both streams don't still have more data, stop comparison
streamData.setDone();
return Void();
}
}
}
// Currently only used for GetKeyValuesStream but could easily be plugged for other stream types
// User of the stream has to forward the SS's responses to the returned promise stream, if it is set
template <class Request, bool P>
Optional<TSSDuplicateStreamData<REPLYSTREAM_TYPE(Request)>>
maybeDuplicateTSSStreamFragment(Request& req, QueueModel* model, RequestStream<Request, P> const* ssStream) {
if (model) {
Optional<TSSEndpointData> tssData = model->getTssData(ssStream->getEndpoint().token.first());
if (tssData.present()) {
CODE_PROBE(true, "duplicating stream to TSS");
resetReply(req);
// FIXME: optimize to avoid creating new netNotifiedQueueWithAcknowledgements for each stream duplication
RequestStream<Request> tssRequestStream(tssData.get().endpoint);
ReplyPromiseStream<REPLYSTREAM_TYPE(Request)> tssReplyStream = tssRequestStream.getReplyStream(req);
PromiseStream<REPLYSTREAM_TYPE(Request)> ssDuplicateReplyStream;
TSSDuplicateStreamData<REPLYSTREAM_TYPE(Request)> streamData(ssDuplicateReplyStream);
model->addActor.send(tssStreamComparison(req, streamData, tssReplyStream, tssData.get()));
return Optional<TSSDuplicateStreamData<REPLYSTREAM_TYPE(Request)>>(streamData);
}
}
return Optional<TSSDuplicateStreamData<REPLYSTREAM_TYPE(Request)>>();
}
// Streams all of the KV pairs in a target key range directly to the client in order.
ACTOR Future<Void> getRangeStreamImpl(Reference<TransactionState> trState,
PromiseStream<RangeResult> results,
KeyRange keys,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse,
SpanContext spanContext) {
loop {
state std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(
trState, keys, CLIENT_KNOBS->GET_RANGE_SHARD_LIMIT, reverse, &StorageServerInterface::getKeyValuesStream));
ASSERT(locations.size());
state int shard = 0;
loop {
const KeyRange& range = locations[shard].range;
state Optional<TSSDuplicateStreamData<GetKeyValuesStreamReply>> tssDuplicateStream;
state GetKeyValuesStreamRequest req;
req.version = trState->readVersion();
req.begin = firstGreaterOrEqual(range.begin);
req.end = firstGreaterOrEqual(range.end);
req.spanContext = spanContext;
req.limit = reverse ? -CLIENT_KNOBS->REPLY_BYTE_LIMIT : CLIENT_KNOBS->REPLY_BYTE_LIMIT;
req.limitBytes = std::numeric_limits<int>::max();
req.options = trState->readOptions;
trState->cx->getLatestCommitVersions(locations[shard].locations, trState, req.ssLatestCommitVersions);
// keep shard's arena around in case of async tss comparison
req.arena.dependsOn(range.arena());
ASSERT(req.limitBytes > 0 && req.limit != 0 && req.limit < 0 == reverse);
// FIXME: buggify byte limits on internal functions that use them, instead of globally
req.tags = trState->cx->sampleReadTags() ? trState->options.readTags : Optional<TagSet>();
try {
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
g_traceBatch.addEvent("TransactionDebug",
trState->readOptions.get().debugID.get().first(),
"NativeAPI.RangeStream.Before");
}
++trState->cx->transactionPhysicalReads;
state GetKeyValuesStreamReply rep;
if (locations[shard].locations->size() == 0) {
wait(trState->cx->connectionFileChanged());
results.sendError(transaction_too_old());
return Void();
}
state int useIdx = -1;
loop {
// FIXME: create a load balance function for this code so future users of reply streams do not have
// to duplicate this code
int count = 0;
for (int i = 0; i < locations[shard].locations->size(); i++) {
if (!IFailureMonitor::failureMonitor()
.getState(locations[shard]
.locations->get(i, &StorageServerInterface::getKeyValuesStream)
.getEndpoint())
.failed) {
if (deterministicRandom()->random01() <= 1.0 / ++count) {
useIdx = i;
}
}
}
if (useIdx >= 0) {
break;
}
std::vector<Future<Void>> ok(locations[shard].locations->size());
for (int i = 0; i < ok.size(); i++) {
ok[i] = IFailureMonitor::failureMonitor().onStateEqual(
locations[shard]
.locations->get(i, &StorageServerInterface::getKeyValuesStream)
.getEndpoint(),
FailureStatus(false));
}
// Making this SevWarn means a lot of clutter
if (now() - g_network->networkInfo.newestAlternativesFailure > 1 ||
deterministicRandom()->random01() < 0.01) {
TraceEvent("AllAlternativesFailed")
.detail("Alternatives", locations[shard].locations->description());
}
wait(allAlternativesFailedDelay(quorum(ok, 1)));
}
state ReplyPromiseStream<GetKeyValuesStreamReply> replyStream =
locations[shard]
.locations->get(useIdx, &StorageServerInterface::getKeyValuesStream)
.getReplyStream(req);
tssDuplicateStream = maybeDuplicateTSSStreamFragment(
req,
trState->cx->enableLocalityLoadBalance ? &trState->cx->queueModel : nullptr,
&locations[shard].locations->get(useIdx, &StorageServerInterface::getKeyValuesStream));
state bool breakAgain = false;
loop {
wait(results.onEmpty());
try {
choose {
when(wait(trState->cx->connectionFileChanged())) {
results.sendError(transaction_too_old());
if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
tssDuplicateStream.get().stream.sendError(transaction_too_old());
}
return Void();
}
when(GetKeyValuesStreamReply _rep = waitNext(replyStream.getFuture())) {
rep = _rep;
}
}
++trState->cx->transactionPhysicalReadsCompleted;
} catch (Error& e) {
++trState->cx->transactionPhysicalReadsCompleted;
if (e.code() == error_code_broken_promise) {
if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
tssDuplicateStream.get().stream.sendError(connection_failed());
}
throw connection_failed();
}
if (e.code() != error_code_end_of_stream) {
if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
tssDuplicateStream.get().stream.sendError(e);
}
throw;
}
rep = GetKeyValuesStreamReply();
}
if (trState->readOptions.present() && trState->readOptions.get().debugID.present())
g_traceBatch.addEvent("TransactionDebug",
trState->readOptions.get().debugID.get().first(),
"NativeAPI.getExactRange.After");
RangeResult output(RangeResultRef(rep.data, rep.more), rep.arena);
if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
// shallow copy the reply with an arena depends, and send it to the duplicate stream for TSS
GetKeyValuesStreamReply replyCopy;
replyCopy.version = rep.version;
replyCopy.more = rep.more;
replyCopy.cached = rep.cached;
replyCopy.arena.dependsOn(rep.arena);
replyCopy.data.append(replyCopy.arena, rep.data.begin(), rep.data.size());
tssDuplicateStream.get().stream.send(replyCopy);
}
int64_t bytes = 0;
for (const KeyValueRef& kv : output) {
bytes += kv.key.size() + kv.value.size();
}
trState->cx->transactionBytesRead += bytes;
trState->cx->transactionKeysRead += output.size();
// If the reply says there is more but we know that we finished the shard, then fix rep.more
if (reverse && output.more && rep.data.size() > 0 &&
output[output.size() - 1].key == locations[shard].range.begin) {
output.more = false;
}
if (output.more) {
if (!rep.data.size()) {
TraceEvent(SevError, "GetRangeStreamError")
.detail("Reason", "More data indicated but no rows present")
.detail("LimitBytes", limits.bytes)
.detail("LimitRows", limits.rows)
.detail("OutputSize", output.size())
.detail("OutputBytes", output.expectedSize())
.detail("BlockSize", rep.data.size())
.detail("BlockBytes", rep.data.expectedSize());
ASSERT(false);
}
CODE_PROBE(true, "GetKeyValuesStreamReply.more in getRangeStream");
// Make next request to the same shard with a beginning key just after the last key returned
if (reverse)
locations[shard].range =
KeyRangeRef(locations[shard].range.begin, output[output.size() - 1].key);
else
locations[shard].range =
KeyRangeRef(keyAfter(output[output.size() - 1].key), locations[shard].range.end);
}
if (locations[shard].range.empty()) {
output.more = false;
}
if (!output.more) {
const KeyRange& range = locations[shard].range;
if (shard == locations.size() - 1) {
KeyRef begin = reverse ? keys.begin : range.end;
KeyRef end = reverse ? range.begin : keys.end;
if (begin >= end) {
if (range.begin == allKeys.begin) {
output.readToBegin = true;
}
if (range.end == allKeys.end) {
output.readThroughEnd = true;
}
output.arena().dependsOn(keys.arena());
// getRangeStream() uses end_of_stream to indicate exhaustion, so keep 'more' true.
output.more = true;
output.setReadThrough(reverse ? keys.begin : keys.end);
results.send(std::move(output));
results.sendError(end_of_stream());
if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
tssDuplicateStream.get().stream.sendError(end_of_stream());
}
return Void();
}
keys = KeyRangeRef(begin, end);
breakAgain = true;
} else {
++shard;
}
output.arena().dependsOn(range.arena());
// If it's not the last shard, set more to true and readThrough to the shard boundary.
output.more = true;
output.setReadThrough(reverse ? range.begin : range.end);
results.send(std::move(output));
break;
}
ASSERT(output.size());
if (keys.begin == allKeys.begin && !reverse) {
output.readToBegin = true;
}
if (keys.end == allKeys.end && reverse) {
output.readThroughEnd = true;
}
results.send(std::move(output));
}
if (breakAgain) {
break;
}
} catch (Error& e) {
// send errors to tss duplicate stream, including actor_cancelled
if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) {
tssDuplicateStream.get().stream.sendError(e);
}
if (e.code() == error_code_actor_cancelled) {
throw;
}
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
e.code() == error_code_connection_failed || e.code() == error_code_request_maybe_delivered) {
const KeyRangeRef& range = locations[shard].range;
if (reverse)
keys = KeyRangeRef(keys.begin, range.end);
else
keys = KeyRangeRef(range.begin, keys.end);
trState->cx->invalidateCache(keys);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, trState->taskID));
break;
} else {
results.sendError(e);
return Void();
}
}
}
}
}
ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<TransactionState> trState,
KeyRange keys,
int64_t chunkSize,
int limit);
// Streams the requested key range directly from storage servers without fragment-level parallelism.
ACTOR Future<Void> getRangeStream(Reference<TransactionState> trState,
PromiseStream<RangeResult> _results,
KeySelector begin,
KeySelector end,
GetRangeLimits limits,
Promise<std::pair<Key, Key>> conflictRange,
Snapshot snapshot,
Reverse reverse) {
// FIXME: better handling to disable row limits
ASSERT(!limits.hasRowLimit());
state Span span("NAPI:getRangeStream"_loc, trState->spanContext);
wait(trState->startTransaction());
trState->cx->validateVersion(trState->readVersion());
Future<Key> fb = resolveKey(trState, begin);
state Future<Key> fe = resolveKey(trState, end);
state Key b = wait(fb);
state Key e = wait(fe);
if (!snapshot) {
// FIXME: this conflict range is too large, and should be updated continuously as results are returned
conflictRange.send(std::make_pair(std::min(b, Key(begin.getKey(), begin.arena())),
std::max(e, Key(end.getKey(), end.arena()))));
}
if (b >= e) {
_results.sendError(end_of_stream());
return Void();
}
wait(getRangeStreamImpl(trState, _results, KeyRange(KeyRangeRef(b, e)), limits, snapshot, reverse, span.context));
return Void();
}
Future<RangeResult> getRange(Reference<TransactionState> const& trState,
KeySelector const& begin,
KeySelector const& end,
GetRangeLimits const& limits,
Reverse const& reverse) {
return getRange<GetKeyValuesRequest, GetKeyValuesReply, RangeResult>(
trState, begin, end, ""_sr, limits, Promise<std::pair<Key, Key>>(), Snapshot::True, reverse);
}
bool DatabaseContext::debugUseTags = false;
const std::vector<std::string> DatabaseContext::debugTransactionTagChoices = { "a", "b", "c", "d", "e", "f", "g",
"h", "i", "j", "k", "l", "m", "n",
"o", "p", "q", "r", "s", "t" };
void debugAddTags(Reference<TransactionState> trState) {
int numTags = deterministicRandom()->randomInt(0, CLIENT_KNOBS->MAX_TAGS_PER_TRANSACTION + 1);
for (int i = 0; i < numTags; ++i) {
TransactionTag tag;
if (deterministicRandom()->random01() < 0.7) {
tag = TransactionTagRef(deterministicRandom()->randomChoice(DatabaseContext::debugTransactionTagChoices));
} else {
int length = deterministicRandom()->randomInt(1, CLIENT_KNOBS->MAX_TRANSACTION_TAG_LENGTH + 1);
uint8_t* s = new (tag.arena()) uint8_t[length];
for (int j = 0; j < length; ++j) {
s[j] = (uint8_t)deterministicRandom()->randomInt(0, 256);
}
tag.contents() = TransactionTagRef(s, length);
}
if (deterministicRandom()->coinflip()) {
trState->options.readTags.addTag(tag);
}
trState->options.tags.addTag(tag);
}
}
Transaction::Transaction()
: trState(makeReference<TransactionState>(TaskPriority::DefaultEndpoint, generateSpanID(false))) {}
Transaction::Transaction(Database const& cx)
: trState(makeReference<TransactionState>(cx,
cx->taskID,
generateSpanID(cx->transactionTracingSample),
createTrLogInfoProbabilistically(cx))),
span(trState->spanContext, "Transaction"_loc), backoff(CLIENT_KNOBS->DEFAULT_BACKOFF), tr(trState->spanContext) {
if (DatabaseContext::debugUseTags) {
debugAddTags(trState);
}
}
Transaction::~Transaction() {
flushTrLogsIfEnabled();
cancelWatches();
}
void Transaction::operator=(Transaction&& r) noexcept {
flushTrLogsIfEnabled();
tr = std::move(r.tr);
trState = std::move(r.trState);
extraConflictRanges = std::move(r.extraConflictRanges);
commitResult = std::move(r.commitResult);
committing = std::move(r.committing);
backoff = r.backoff;
watches = r.watches;
}
void Transaction::flushTrLogsIfEnabled() {
if (trState && trState->trLogInfo && trState->trLogInfo->logsAdded && trState->trLogInfo->trLogWriter.getData()) {
ASSERT(trState->trLogInfo->flushed == false);
trState->cx->clientStatusUpdater.inStatusQ.push_back(
{ trState->trLogInfo->identifier, std::move(trState->trLogInfo->trLogWriter) });
trState->trLogInfo->flushed = true;
}
}
VersionVector Transaction::getVersionVector() const {
return trState->cx->ssVersionVectorCache;
}
void Transaction::setVersion(Version v) {
trState->startTime = now();
if (trState->readVersionFuture.isValid())
throw read_version_already_set();
if (v <= 0)
throw version_invalid();
trState->readVersionFuture = v;
trState->readVersionObtainedFromGrvProxy = false;
}
Future<Optional<Value>> Transaction::get(const Key& key, Snapshot snapshot) {
++trState->cx->transactionLogicalReads;
++trState->cx->transactionGetValueRequests;
// ASSERT (key < allKeys.end);
// There are no keys in the database with size greater than the max key size
if (key.size() > getMaxReadKeySize(key)) {
return Optional<Value>();
}
auto ver = getReadVersion();
/* if (!systemKeys.contains(key))
return Optional<Value>(Value()); */
if (!snapshot)
tr.transaction.read_conflict_ranges.push_back(tr.arena, singleKeyRange(key, tr.arena));
if (key == metadataVersionKey) {
++trState->cx->transactionMetadataVersionReads;
if (!ver.isReady() || trState->metadataVersion.isSet()) {
return trState->metadataVersion.getFuture();
} else {
if (ver.isError()) {
return ver.getError();
}
if (ver.get() == trState->cx->metadataVersionCache[trState->cx->mvCacheInsertLocation].first) {
return trState->cx->metadataVersionCache[trState->cx->mvCacheInsertLocation].second;
}
Version v = ver.get();
int hi = trState->cx->mvCacheInsertLocation;
int lo = (trState->cx->mvCacheInsertLocation + 1) % trState->cx->metadataVersionCache.size();
while (hi != lo) {
int cu = hi > lo ? (hi + lo) / 2
: ((hi + trState->cx->metadataVersionCache.size() + lo) / 2) %
trState->cx->metadataVersionCache.size();
if (v == trState->cx->metadataVersionCache[cu].first) {
return trState->cx->metadataVersionCache[cu].second;
}
if (cu == lo) {
break;
}
if (v < trState->cx->metadataVersionCache[cu].first) {
hi = cu;
} else {
lo = (cu + 1) % trState->cx->metadataVersionCache.size();
}
}
}
}
return getValue(trState, key);
}
void Watch::setWatch(Future<Void> watchFuture) {
this->watchFuture = watchFuture;
// Cause the watch loop to go around and start waiting on watchFuture
onSetWatchTrigger.send(Void());
}
// Restarts a watch after a database switch
Future<Void> restartWatch(Database cx,
Key key,
Optional<Value> value,
TagSet tags,
SpanContext spanContext,
TaskPriority taskID,
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies) {
// Remove the reference count as the old watches should be all dropped when switching connectionFile.
cx->deleteWatchMetadata(key, /* removeReferenceCount */ true);
co_await watchValueMap(
cx->minAcceptableReadVersion, key, value, cx, tags, spanContext, taskID, debugID, useProvisionalProxies);
}
// FIXME: This seems pretty horrible. Now a Database can't die until all of its watches do...
ACTOR Future<Void> watch(Reference<Watch> watch,
Database cx,
TagSet tags,
SpanContext spanContext,
TaskPriority taskID,
Optional<UID> debugID,
UseProvisionalProxies useProvisionalProxies) {
try {
choose {
// RYOW write to value that is being watched (if applicable)
// Errors
when(wait(watch->onChangeTrigger.getFuture())) {}
// NativeAPI finished commit and updated watchFuture
when(wait(watch->onSetWatchTrigger.getFuture())) {
loop {
choose {
// NativeAPI watchValue future finishes or errors
when(wait(watch->watchFuture)) {
break;
}
when(wait(cx->connectionFileChanged())) {
CODE_PROBE(true, "Recreated a watch after switch");
watch->watchFuture = restartWatch(cx,
watch->key,
watch->value,
tags,
spanContext,
taskID,
debugID,
useProvisionalProxies);
}
}
}
}
}
} catch (Error& e) {
cx->decreaseWatchCounter();
throw;
}
cx->decreaseWatchCounter();
return Void();
}
Future<Version> Transaction::getRawReadVersion() {
return ::getRawVersion(trState);
}
Future<Void> Transaction::watch(Reference<Watch> watch) {
++trState->cx->transactionWatchRequests;
trState->cx->increaseWatchCounter();
watch->readOptions = trState->readOptions;
watches.push_back(watch);
return ::watch(watch,
trState->cx,
trState->options.readTags,
trState->spanContext,
trState->taskID,
trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
trState->useProvisionalProxies);
}
Future<Standalone<VectorRef<const char*>>> getAddressesForKeyActor(Reference<TransactionState> trState, Key key) {
std::vector<StorageServerInterface> ssi;
co_await trState->startTransaction();
Key resolvedKey = key;
// If key >= allKeys.end, then getRange will return a kv-pair with an empty value. This will result in our
// serverInterfaces vector being empty, which will cause us to return an empty addresses list.
Key ksKey = keyServersKey(resolvedKey);
RangeResult serverTagResult = co_await getRange(trState,
lastLessOrEqual(serverTagKeys.begin),
firstGreaterThan(serverTagKeys.end),
GetRangeLimits(CLIENT_KNOBS->TOO_MANY),
Reverse::False);
ASSERT(!serverTagResult.more && serverTagResult.size() < CLIENT_KNOBS->TOO_MANY);
Future<RangeResult> futureServerUids =
getRange(trState, lastLessOrEqual(ksKey), firstGreaterThan(ksKey), GetRangeLimits(1), Reverse::False);
RangeResult serverUids = co_await futureServerUids;
ASSERT(serverUids.size()); // every shard needs to have a team
std::vector<UID> src;
std::vector<UID> ignore; // 'ignore' is so named because it is the vector into which we decode the 'dest' servers in
// the case where this key is being relocated. But 'src' is the canonical location until
// the move is finished, because it could be cancelled at any time.
decodeKeyServersValue(serverTagResult, serverUids[0].value, src, ignore);
Optional<std::vector<StorageServerInterface>> serverInterfaces =
co_await transactionalGetServerInterfaces(trState, src);
ASSERT(serverInterfaces.present()); // since this is happening transactionally, /FF/keyServers and /FF/serverList
// need to be consistent with one another
ssi = serverInterfaces.get();
Standalone<VectorRef<const char*>> addresses;
for (auto i : ssi) {
std::string ipString = trState->options.includePort ? i.address().toString() : i.address().ip.toString();
char* c_string = new (addresses.arena()) char[ipString.length() + 1];
strcpy(c_string, ipString.c_str());
addresses.push_back(addresses.arena(), c_string);
}
co_return addresses;
}
Future<Standalone<VectorRef<const char*>>> Transaction::getAddressesForKey(const Key& key) {
++trState->cx->transactionLogicalReads;
++trState->cx->transactionGetAddressesForKeyRequests;
return getAddressesForKeyActor(trState, key);
}
ACTOR Future<Key> getKeyAndConflictRange(Reference<TransactionState> trState,
KeySelector k,
Promise<std::pair<Key, Key>> conflictRange) {
try {
Key rep = wait(getKey(trState, k));
if (k.offset <= 0)
conflictRange.send(std::make_pair(rep, k.orEqual ? keyAfter(k.getKey()) : Key(k.getKey(), k.arena())));
else
conflictRange.send(
std::make_pair(k.orEqual ? keyAfter(k.getKey()) : Key(k.getKey(), k.arena()), keyAfter(rep)));
return rep;
} catch (Error& e) {
conflictRange.send(std::make_pair(Key(), Key()));
throw;
}
}
Future<Key> Transaction::getKey(const KeySelector& key, Snapshot snapshot) {
++trState->cx->transactionLogicalReads;
++trState->cx->transactionGetKeyRequests;
if (snapshot)
return ::getKey(trState, key);
Promise<std::pair<Key, Key>> conflictRange;
extraConflictRanges.push_back(conflictRange.getFuture());
return getKeyAndConflictRange(trState, key, conflictRange);
}
template <class GetKeyValuesFamilyRequest>
void increaseCounterForRequest(Database cx) {
if constexpr (std::is_same<GetKeyValuesFamilyRequest, GetKeyValuesRequest>::value) {
++cx->transactionGetRangeRequests;
} else if (std::is_same<GetKeyValuesFamilyRequest, GetMappedKeyValuesRequest>::value) {
++cx->transactionGetMappedRangeRequests;
} else {
UNREACHABLE();
}
}
template <class GetKeyValuesFamilyRequest, class GetKeyValuesFamilyReply, class RangeResultFamily>
Future<RangeResultFamily> Transaction::getRangeInternal(const KeySelector& begin,
const KeySelector& end,
const Key& mapper,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
++trState->cx->transactionLogicalReads;
increaseCounterForRequest<GetKeyValuesFamilyRequest>(trState->cx);
if (limits.isReached())
return RangeResultFamily();
if (!limits.isValid())
return range_limits_invalid();
ASSERT(limits.rows != 0);
KeySelector b = begin;
if (b.orEqual) {
CODE_PROBE(true, "Native begin orEqual==true");
b.removeOrEqual(b.arena());
}
KeySelector e = end;
if (e.orEqual) {
CODE_PROBE(true, "Native end orEqual==true");
e.removeOrEqual(e.arena());
}
if (b.offset >= e.offset && b.getKey() >= e.getKey()) {
CODE_PROBE(true, "Native range inverted");
return RangeResultFamily();
}
if (!snapshot && !std::is_same_v<GetKeyValuesFamilyRequest, GetKeyValuesRequest>) {
// Currently, NativeAPI does not support serialization for getMappedRange. You should consider use
// ReadYourWrites APIs which wraps around NativeAPI and provides serialization for getMappedRange. (Even if
// you don't want RYW, you may use ReadYourWrites APIs with RYW disabled.)
throw unsupported_operation();
}
Promise<std::pair<Key, Key>> conflictRange;
if (!snapshot) {
extraConflictRanges.push_back(conflictRange.getFuture());
}
return ::getRange<GetKeyValuesFamilyRequest, GetKeyValuesFamilyReply, RangeResultFamily>(
trState, b, e, mapper, limits, conflictRange, snapshot, reverse);
}
Future<RangeResult> Transaction::getRange(const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
return getRangeInternal<GetKeyValuesRequest, GetKeyValuesReply, RangeResult>(
begin, end, ""_sr, limits, snapshot, reverse);
}
Future<MappedRangeResult> Transaction::getMappedRange(const KeySelector& begin,
const KeySelector& end,
const Key& mapper,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
return getRangeInternal<GetMappedKeyValuesRequest, GetMappedKeyValuesReply, MappedRangeResult>(
begin, end, mapper, limits, snapshot, reverse);
}
Future<RangeResult> Transaction::getRange(const KeySelector& begin,
const KeySelector& end,
int limit,
Snapshot snapshot,
Reverse reverse) {
return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse);
}
// A method for streaming data from the storage server that is more efficient than getRange when reading large amounts
// of data
Future<Void> Transaction::getRangeStream(PromiseStream<RangeResult>& results,
const KeySelector& begin,
const KeySelector& end,
GetRangeLimits limits,
Snapshot snapshot,
Reverse reverse) {
++trState->cx->transactionLogicalReads;
++trState->cx->transactionGetRangeStreamRequests;
// FIXME: limits are not implemented yet, and this code has not be tested with reverse=true
ASSERT(!limits.hasByteLimit() && !limits.hasRowLimit() && !reverse);
KeySelector b = begin;
if (b.orEqual) {
CODE_PROBE(true, "Native stream begin orEqual==true", probe::decoration::rare);
b.removeOrEqual(b.arena());
}
KeySelector e = end;
if (e.orEqual) {
CODE_PROBE(true, "Native stream end orEqual==true", probe::decoration::rare);
e.removeOrEqual(e.arena());
}
if (b.offset >= e.offset && b.getKey() >= e.getKey()) {
CODE_PROBE(true, "Native stream range inverted", probe::decoration::rare);
results.sendError(end_of_stream());
return Void();
}
Promise<std::pair<Key, Key>> conflictRange;
if (!snapshot) {
extraConflictRanges.push_back(conflictRange.getFuture());
}
return forwardErrors(::getRangeStream(trState, results, b, e, limits, conflictRange, snapshot, reverse), results);
}
Future<Void> Transaction::getRangeStream(PromiseStream<RangeResult>& results,
const KeySelector& begin,
const KeySelector& end,
int limit,
Snapshot snapshot,
Reverse reverse) {
return getRangeStream(results, begin, end, GetRangeLimits(limit), snapshot, reverse);
}
void Transaction::addReadConflictRange(KeyRangeRef const& keys) {
ASSERT(!keys.empty());
// There aren't any keys in the database with size larger than the max key size, so if range contains large keys
// we can translate it to an equivalent one with smaller keys
KeyRef begin = keys.begin;
KeyRef end = keys.end;
int64_t beginMaxSize = getMaxReadKeySize(begin);
int64_t endMaxSize = getMaxReadKeySize(end);
if (begin.size() > beginMaxSize) {
begin = begin.substr(0, beginMaxSize + 1);
}
if (end.size() > endMaxSize) {
end = end.substr(0, endMaxSize + 1);
}
KeyRangeRef r = KeyRangeRef(begin, end);
if (r.empty()) {
return;
}
tr.transaction.read_conflict_ranges.push_back_deep(tr.arena, r);
}
void Transaction::makeSelfConflicting() {
BinaryWriter wr(Unversioned());
wr.serializeBytes("\xFF/SC/"_sr);
wr << deterministicRandom()->randomUniqueID();
auto r = singleKeyRange(wr.toValue(), tr.arena);
tr.transaction.read_conflict_ranges.push_back(tr.arena, r);
tr.transaction.write_conflict_ranges.push_back(tr.arena, r);
}
void Transaction::set(const KeyRef& key, const ValueRef& value, AddConflictRange addConflictRange) {
++trState->cx->transactionSetMutations;
if (key.size() > getMaxWriteKeySize(key, trState->options.rawAccess))
throw key_too_large();
if (value.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT)
throw value_too_large();
auto& req = tr;
auto& t = req.transaction;
auto r = singleKeyRange(key, req.arena);
auto v = ValueRef(req.arena, value);
t.mutations.emplace_back(req.arena, MutationRef::SetValue, r.begin, v);
trState->totalCost += getWriteOperationCost(key.expectedSize() + value.expectedSize());
if (addConflictRange) {
t.write_conflict_ranges.push_back(req.arena, r);
}
}
void Transaction::atomicOp(const KeyRef& key,
const ValueRef& operand,
MutationRef::Type operationType,
AddConflictRange addConflictRange) {
++trState->cx->transactionAtomicMutations;
if (key.size() > getMaxWriteKeySize(key, trState->options.rawAccess))
throw key_too_large();
if (operand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT)
throw value_too_large();
if (apiVersionAtLeast(510)) {
if (operationType == MutationRef::Min)
operationType = MutationRef::MinV2;
else if (operationType == MutationRef::And)
operationType = MutationRef::AndV2;
}
auto& req = tr;
auto& t = req.transaction;
auto r = singleKeyRange(key, req.arena);
auto v = ValueRef(req.arena, operand);
t.mutations.emplace_back(req.arena, operationType, r.begin, v);
trState->totalCost += getWriteOperationCost(key.expectedSize());
if (addConflictRange && operationType != MutationRef::SetVersionstampedKey)
t.write_conflict_ranges.push_back(req.arena, r);
CODE_PROBE(true, "NativeAPI atomic operation");
}
void TransactionState::addClearCost() {
// NOTE: The throttling cost of each clear is assumed to be one page.
// This makes computation fast, but can be inaccurate and may
// underestimate the cost of large clears.
totalCost += CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE;
}
void Transaction::clear(const KeyRangeRef& range, AddConflictRange addConflictRange) {
++trState->cx->transactionClearMutations;
auto& req = tr;
auto& t = req.transaction;
KeyRef begin = range.begin;
KeyRef end = range.end;
// There aren't any keys in the database with size larger than the max key size, so if range contains large keys
// we can translate it to an equivalent one with smaller keys
int64_t beginMaxSize = getMaxClearKeySize(begin);
int64_t endMaxSize = getMaxClearKeySize(end);
if (begin.size() > beginMaxSize) {
begin = begin.substr(0, beginMaxSize + 1);
}
if (end.size() > endMaxSize) {
end = end.substr(0, endMaxSize + 1);
}
auto r = KeyRangeRef(req.arena, KeyRangeRef(begin, end));
if (r.empty())
return;
t.mutations.emplace_back(req.arena, MutationRef::ClearRange, r.begin, r.end);
trState->addClearCost();
if (addConflictRange)
t.write_conflict_ranges.push_back(req.arena, r);
}
void Transaction::clear(const KeyRef& key, AddConflictRange addConflictRange) {
++trState->cx->transactionClearMutations;
// There aren't any keys in the database with size larger than the max key size
if (key.size() > getMaxClearKeySize(key)) {
return;
}
auto& req = tr;
auto& t = req.transaction;
// efficient single key range clear range mutation, see singleKeyRange
uint8_t* data = new (req.arena) uint8_t[key.size() + 1];
memcpy(data, key.begin(), key.size());
data[key.size()] = 0;
t.mutations.emplace_back(
req.arena, MutationRef::ClearRange, KeyRef(data, key.size()), KeyRef(data, key.size() + 1));
trState->addClearCost();
if (addConflictRange)
t.write_conflict_ranges.emplace_back(req.arena, KeyRef(data, key.size()), KeyRef(data, key.size() + 1));
}
void Transaction::addWriteConflictRange(const KeyRangeRef& keys) {
ASSERT(!keys.empty());
auto& req = tr;
auto& t = req.transaction;
// There aren't any keys in the database with size larger than the max key size, so if range contains large keys
// we can translate it to an equivalent one with smaller keys
KeyRef begin = keys.begin;
KeyRef end = keys.end;
int64_t beginMaxSize = getMaxKeySize(begin);
int64_t endMaxSize = getMaxKeySize(end);
if (begin.size() > beginMaxSize) {
begin = begin.substr(0, beginMaxSize + 1);
}
if (end.size() > endMaxSize) {
end = end.substr(0, endMaxSize + 1);
}
KeyRangeRef r = KeyRangeRef(begin, end);
if (r.empty()) {
return;
}
t.write_conflict_ranges.push_back_deep(req.arena, r);
}
double Transaction::getBackoff(int errCode) {
double returnedBackoff = backoff;
if (errCode == error_code_tag_throttled) {
auto priorityItr = trState->cx->throttledTags.find(trState->options.priority);
for (auto& tag : trState->options.tags) {
if (priorityItr != trState->cx->throttledTags.end()) {
auto tagItr = priorityItr->second.find(tag);
if (tagItr != priorityItr->second.end()) {
CODE_PROBE(true, "Returning throttle backoff");
returnedBackoff = std::max(
returnedBackoff,
std::min(CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL, tagItr->second.throttleDuration()));
if (returnedBackoff == CLIENT_KNOBS->TAG_THROTTLE_RECHECK_INTERVAL) {
break;
}
}
}
}
}
returnedBackoff *= deterministicRandom()->random01();
// Set backoff for next time
if (errCode == error_code_commit_proxy_memory_limit_exceeded ||
errCode == error_code_grv_proxy_memory_limit_exceeded ||
errCode == error_code_transaction_throttled_hot_shard ||
errCode == error_code_transaction_rejected_range_locked) {
backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF);
} else {
backoff = std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, trState->options.maxBackoff);
}
return returnedBackoff;
}
TransactionOptions::TransactionOptions(Database const& cx) {
reset(cx);
if (buggify()) {
commitOnFirstProxy = true;
}
}
void TransactionOptions::clear() {
maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF;
getReadVersionFlags = 0;
sizeLimit = CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT;
maxTransactionLoggingFieldLength = 0;
checkWritesEnabled = false;
causalWriteRisky = false;
commitOnFirstProxy = false;
debugDump = false;
lockAware = false;
readOnly = false;
firstInBatch = false;
includePort = false;
reportConflictingKeys = false;
tags = TagSet{};
readTags = TagSet{};
priority = TransactionPriority::DEFAULT;
expensiveClearCostEstimation = false;
useGrvCache = false;
skipGrvCache = false;
rawAccess = false;
bypassStorageQuota = false;
enableReplicaConsistencyCheck = false;
maxGrvQueueDelayMS = Optional<int64_t>();
requiredReplicas = 0;
}
TransactionOptions::TransactionOptions() {
clear();
}
void TransactionOptions::reset(Database const& cx) {
clear();
lockAware = cx->lockAware;
if (cx->apiVersionAtLeast(630)) {
includePort = true;
}
}
void Transaction::resetImpl(bool generateNewSpan) {
flushTrLogsIfEnabled();
trState = trState->cloneAndReset(createTrLogInfoProbabilistically(trState->cx), generateNewSpan);
tr = CommitTransactionRequest(trState->spanContext);
extraConflictRanges.clear();
commitResult = Promise<Void>();
committing = Future<Void>();
cancelWatches();
}
TagSet const& Transaction::getTags() const {
return trState->options.tags;
}
void Transaction::reset() {
resetImpl(false);
}
void Transaction::fullReset() {
resetImpl(true);
span = Span(trState->spanContext, "Transaction"_loc);
backoff = CLIENT_KNOBS->DEFAULT_BACKOFF;
}
int Transaction::apiVersionAtLeast(int minVersion) const {
return trState->cx->apiVersionAtLeast(minVersion);
}
class MutationBlock {
public:
bool mutated;
bool cleared;
ValueRef setValue;
MutationBlock() : mutated(false) {}
explicit MutationBlock(bool _cleared) : mutated(true), cleared(_cleared) {}
explicit MutationBlock(ValueRef value) : mutated(true), cleared(false), setValue(value) {}
};
bool compareBegin(KeyRangeRef lhs, KeyRangeRef rhs) {
return lhs.begin < rhs.begin;
}
// If there is any intersection between the two given sets of ranges, returns a range that
// falls within the intersection
Optional<KeyRangeRef> intersects(VectorRef<KeyRangeRef> lhs, VectorRef<KeyRangeRef> rhs) {
if (lhs.size() && rhs.size()) {
std::sort(lhs.begin(), lhs.end(), compareBegin);
std::sort(rhs.begin(), rhs.end(), compareBegin);
int l = 0, r = 0;
while (l < lhs.size() && r < rhs.size()) {
if (lhs[l].end <= rhs[r].begin)
l++;
else if (rhs[r].end <= lhs[l].begin)
r++;
else
return lhs[l] & rhs[r];
}
}
return Optional<KeyRangeRef>();
}
Future<Void> checkWrites(Uncancellable,
Reference<TransactionState> trState,
Future<Void> committed,
Promise<Void> outCommitted,
CommitTransactionRequest req) {
Version version{ 0 };
try {
co_await committed;
// If the commit is successful, by definition the transaction still exists for now. Grab the version, and don't
// use it again.
version = trState->committedVersion;
outCommitted.send(Void());
} catch (Error& e) {
outCommitted.sendError(e);
co_return;
}
co_await delay(deterministicRandom()->random01()); // delay between 0 and 1 seconds
KeyRangeMap<MutationBlock> expectedValues;
auto& mutations = req.transaction.mutations;
const int mCount = mutations.size(); // debugging info for traceEvent
for (const auto& mutation : mutations) {
if (mutation.type == MutationRef::SetValue)
expectedValues.insert(singleKeyRange(mutation.param1), MutationBlock(mutation.param2));
else if (mutation.type == MutationRef::ClearRange)
expectedValues.insert(KeyRangeRef(mutation.param1, mutation.param2), MutationBlock(true));
}
try {
Transaction tr(trState->cx);
tr.setVersion(version);
int checkedRanges = 0;
auto ranges = expectedValues.ranges();
for (auto it = ranges.begin(); it != ranges.end(); ++it) {
MutationBlock m = it->value();
if (m.mutated) {
checkedRanges++;
if (m.cleared) {
RangeResult shouldBeEmpty = co_await tr.getRange(it->range(), 1);
if (shouldBeEmpty.size()) {
TraceEvent(SevError, "CheckWritesFailed")
.detail("Class", "Clear")
.detail("KeyBegin", it->range().begin)
.detail("KeyEnd", it->range().end);
co_return;
}
} else {
Optional<Value> val = co_await tr.get(it->range().begin);
if (!val.present() || val.get() != m.setValue) {
TraceEvent evt(SevError, "CheckWritesFailed");
evt.detail("Class", "Set").detail("Key", it->range().begin).detail("Expected", m.setValue);
if (!val.present())
evt.detail("Actual", "_Value Missing_");
else
evt.detail("Actual", val.get());
co_return;
}
}
}
}
TraceEvent("CheckWritesSuccess")
.detail("Version", version)
.detail("MutationCount", mCount)
.detail("CheckedRanges", checkedRanges);
} catch (Error& e) {
bool ok = e.code() == error_code_transaction_too_old || e.code() == error_code_future_version;
TraceEvent(ok ? SevWarn : SevError, "CheckWritesFailed").error(e);
throw;
}
}
static Future<Void> commitDummyTransaction(Reference<TransactionState> trState, KeyRange range) {
Transaction tr(trState->cx);
int retries = 0;
Span span("NAPI:dummyTransaction"_loc, trState->spanContext);
tr.span.setParent(span.context);
while (true) {
Error err;
try {
TraceEvent("CommitDummyTransaction").detail("Key", range.begin).detail("Retries", retries);
tr.trState->options = trState->options;
tr.trState->taskID = trState->taskID;
tr.trState->authToken = trState->authToken;
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
tr.setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
tr.addReadConflictRange(range);
tr.addWriteConflictRange(range);
co_await tr.commit();
co_return;
} catch (Error& e) {
err = e;
}
TraceEvent("CommitDummyTransactionError")
.errorUnsuppressed(err)
.detail("Key", range.begin)
.detail("Retries", retries);
co_await tr.onError(err);
++retries;
}
}
static Future<Optional<CommitResult>> determineCommitStatus(Reference<TransactionState> trState,
Version minPossibleCommitVersion,
Version maxPossibleCommitVersion,
IdempotencyIdRef idempotencyId) {
Transaction tr(trState->cx);
int retries = 0;
Version expiredVersion{ 0 };
Span span("NAPI:determineCommitStatus"_loc, trState->spanContext);
tr.span.setParent(span.context);
while (true) {
Error err;
try {
tr.trState->options = trState->options;
tr.trState->taskID = trState->taskID;
tr.trState->authToken = trState->authToken;
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
KeyBackedObjectProperty<IdempotencyIdsExpiredVersion, _Unversioned> expiredKey(idempotencyIdsExpiredVersion,
Unversioned());
IdempotencyIdsExpiredVersion expiredVal = co_await expiredKey.getD(&tr);
expiredVersion = expiredVal.expired;
if (expiredVersion >= minPossibleCommitVersion) {
throw commit_unknown_result_fatal();
}
Version rv = co_await tr.getReadVersion();
TraceEvent("DetermineCommitStatusAttempt")
.detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
.detail("Retries", retries)
.detail("ReadVersion", rv)
.detail("ExpiredVersion", expiredVersion)
.detail("MinPossibleCommitVersion", minPossibleCommitVersion)
.detail("MaxPossibleCommitVersion", maxPossibleCommitVersion);
KeyRange possibleRange =
KeyRangeRef(BinaryWriter::toValue(bigEndian64(minPossibleCommitVersion), Unversioned())
.withPrefix(idempotencyIdKeys.begin),
BinaryWriter::toValue(bigEndian64(maxPossibleCommitVersion + 1), Unversioned())
.withPrefix(idempotencyIdKeys.begin));
RangeResult range = co_await tr.getRange(possibleRange, CLIENT_KNOBS->TOO_MANY);
ASSERT(!range.more);
for (const auto& kv : range) {
auto commitResult = kvContainsIdempotencyId(kv, idempotencyId);
if (commitResult.present()) {
TraceEvent("DetermineCommitStatus")
.detail("Committed", 1)
.detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
.detail("Retries", retries);
co_return commitResult;
}
}
TraceEvent("DetermineCommitStatus")
.detail("Committed", 0)
.detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
.detail("Retries", retries);
co_return Optional<CommitResult>();
} catch (Error& e) {
err = e;
}
TraceEvent("DetermineCommitStatusError")
.errorUnsuppressed(err)
.detail("IdempotencyId", idempotencyId.asStringRefUnsafe())
.detail("Retries", retries);
co_await tr.onError(err);
++retries;
}
}
void Transaction::cancelWatches(Error const& e) {
for (auto& watch : watches)
if (!watch->onChangeTrigger.isSet())
watch->onChangeTrigger.sendError(e);
watches.clear();
}
void Transaction::setupWatches() {
try {
Future<Version> watchVersion = getCommittedVersion() > 0 ? getCommittedVersion() : getReadVersion();
for (auto& watch : watches)
watch->setWatch(
watchValueMap(watchVersion,
watch->key,
watch->value,
trState->cx,
trState->options.readTags,
trState->spanContext,
trState->taskID,
trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>(),
trState->useProvisionalProxies));
watches.clear();
} catch (Error&) {
ASSERT(false); // The above code must NOT throw because commit has already occurred.
throw internal_error();
}
}
ACTOR Future<Optional<ClientTrCommitCostEstimation>> estimateCommitCosts(Reference<TransactionState> trState,
CommitTransactionRef const* transaction) {
state ClientTrCommitCostEstimation trCommitCosts;
state KeyRangeRef keyRange;
state int i = 0;
for (; i < transaction->mutations.size(); ++i) {
auto const& mutation = transaction->mutations[i];
if (mutation.type == MutationRef::Type::SetValue || mutation.isAtomicOp()) {
trCommitCosts.opsCount++;
trCommitCosts.writeCosts += getWriteOperationCost(mutation.expectedSize());
} else if (mutation.type == MutationRef::Type::ClearRange) {
trCommitCosts.opsCount++;
keyRange = KeyRangeRef(mutation.param1, mutation.param2);
if (trState->options.expensiveClearCostEstimation) {
StorageMetrics m = wait(trState->cx->getStorageMetrics(keyRange, CLIENT_KNOBS->TOO_MANY, trState));
trCommitCosts.clearIdxCosts.emplace_back(i, getWriteOperationCost(m.bytes));
trCommitCosts.writeCosts += getWriteOperationCost(m.bytes);
++trCommitCosts.expensiveCostEstCount;
++trState->cx->transactionsExpensiveClearCostEstCount;
} else {
std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(
trState, keyRange, CLIENT_KNOBS->TOO_MANY, Reverse::False, &StorageServerInterface::getShardState));
if (locations.empty()) {
continue;
}
uint64_t bytes = 0;
if (locations.size() == 1) {
bytes = CLIENT_KNOBS->INCOMPLETE_SHARD_PLUS;
} else { // small clear on the boundary will hit two shards but be much smaller than the shard size
bytes = CLIENT_KNOBS->INCOMPLETE_SHARD_PLUS * 2 +
(locations.size() - 2) * (int64_t)trState->cx->smoothMidShardSize.smoothTotal();
}
trCommitCosts.clearIdxCosts.emplace_back(i, getWriteOperationCost(bytes));
trCommitCosts.writeCosts += getWriteOperationCost(bytes);
}
}
}
// sample on written bytes
if (!trState->cx->sampleOnCost(trCommitCosts.writeCosts))
return Optional<ClientTrCommitCostEstimation>();
// sample clear op: the expectation of #sampledOp is every COMMIT_SAMPLE_COST sample once
// we also scale the cost of mutations whose cost is less than COMMIT_SAMPLE_COST as scaledCost =
// min(COMMIT_SAMPLE_COST, cost) If we have 4 transactions: A - 100 1-cost mutations: E[sampled ops] = 1, E[sampled
// cost] = 100 B - 1 100-cost mutation: E[sampled ops] = 1, E[sampled cost] = 100 C - 50 2-cost mutations: E[sampled
// ops] = 1, E[sampled cost] = 100 D - 1 150-cost mutation and 150 1-cost mutations: E[sampled ops] = 3, E[sampled
// cost] = 150cost * 1 + 150 * 100cost * 0.01 = 300
ASSERT(trCommitCosts.writeCosts > 0);
std::deque<std::pair<int, uint64_t>> newClearIdxCosts;
for (const auto& [idx, cost] : trCommitCosts.clearIdxCosts) {
if (trCommitCosts.writeCosts >= CLIENT_KNOBS->COMMIT_SAMPLE_COST) {
double mul = trCommitCosts.writeCosts / std::max(1.0, (double)CLIENT_KNOBS->COMMIT_SAMPLE_COST);
if (deterministicRandom()->random01() < cost * mul / trCommitCosts.writeCosts) {
newClearIdxCosts.emplace_back(
idx, cost < CLIENT_KNOBS->COMMIT_SAMPLE_COST ? CLIENT_KNOBS->COMMIT_SAMPLE_COST : cost);
}
} else if (deterministicRandom()->random01() < (double)cost / trCommitCosts.writeCosts) {
newClearIdxCosts.emplace_back(
idx, cost < CLIENT_KNOBS->COMMIT_SAMPLE_COST ? CLIENT_KNOBS->COMMIT_SAMPLE_COST : cost);
}
}
trCommitCosts.clearIdxCosts.swap(newClearIdxCosts);
return trCommitCosts;
}
ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState, CommitTransactionRequest req) {
state TraceInterval interval("TransactionCommit");
state double startTime = now();
state Span span("NAPI:tryCommit"_loc, trState->spanContext);
state Optional<UID> debugID = trState->readOptions.present() ? trState->readOptions.get().debugID : Optional<UID>();
if (debugID.present()) {
TraceEvent(interval.begin()).detail("Parent", debugID.get());
}
// If the read version hasn't already been fetched, then we had no reads and don't need (expensive) full causal
// consistency.
state Future<Void> startFuture = trState->startTransaction(GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY);
try {
if (CLIENT_BUGGIFY) {
throw deterministicRandom()->randomChoice(std::vector<Error>{ not_committed(), transaction_too_old() });
}
if (req.tagSet.present() && trState->options.priority < TransactionPriority::IMMEDIATE) {
state Future<Optional<ClientTrCommitCostEstimation>> commitCostFuture =
estimateCommitCosts(trState, &req.transaction);
wait(startFuture);
wait(store(req.commitCostEstimation, commitCostFuture));
} else {
wait(startFuture);
}
req.transaction.read_snapshot = trState->readVersion();
if (CLIENT_BUGGIFY) {
throw commit_proxy_memory_limit_exceeded();
}
startTime = now();
state Optional<UID> commitID = Optional<UID>();
if (debugID.present()) {
commitID = nondeterministicRandom()->randomUniqueID();
g_traceBatch.addAttach("CommitAttachID", debugID.get().first(), commitID.get().first());
g_traceBatch.addEvent("CommitDebug", commitID.get().first(), "NativeAPI.commit.Before");
}
req.debugID = commitID;
state Future<CommitID> reply;
// Only gets filled in in the happy path where we don't have to commit on the first proxy or use provisional
// proxies
state int alternativeChosen = -1;
// Only valid if alternativeChosen >= 0
state Reference<CommitProxyInfo> proxiesUsed;
if (trState->options.commitOnFirstProxy) {
if (trState->cx->clientInfo->get().firstCommitProxy.present()) {
reply = throwErrorOr(brokenPromiseToMaybeDelivered(
trState->cx->clientInfo->get().firstCommitProxy.get().commit.tryGetReply(req)));
} else {
const std::vector<CommitProxyInterface>& proxies = trState->cx->clientInfo->get().commitProxies;
reply = proxies.size() ? throwErrorOr(brokenPromiseToMaybeDelivered(proxies[0].commit.tryGetReply(req)))
: Never();
}
} else {
proxiesUsed = trState->cx->getCommitProxies(trState->useProvisionalProxies);
reply = basicLoadBalance(proxiesUsed,
&CommitProxyInterface::commit,
req,
TaskPriority::DefaultPromiseEndpoint,
AtMostOnce::True,
&alternativeChosen);
}
state double grvTime = now();
choose {
when(wait(trState->cx->onProxiesChanged())) {
reply.cancel();
throw request_maybe_delivered();
}
when(CommitID ci = wait(reply)) {
Version v = ci.version;
if (v != invalidVersion) {
if (CLIENT_BUGGIFY) {
throw commit_unknown_result();
}
trState->cx->updateCachedReadVersion(grvTime, v);
if (debugID.present())
TraceEvent(interval.end()).detail("CommittedVersion", v);
trState->committedVersion = v;
if (v > trState->cx->metadataVersionCache[trState->cx->mvCacheInsertLocation].first) {
trState->cx->mvCacheInsertLocation =
(trState->cx->mvCacheInsertLocation + 1) % trState->cx->metadataVersionCache.size();
trState->cx->metadataVersionCache[trState->cx->mvCacheInsertLocation] =
std::make_pair(v, ci.metadataVersion);
}
Standalone<StringRef> ret = makeString(10);
placeVersionstamp(mutateString(ret), v, ci.txnBatchId);
trState->versionstampPromise.send(ret);
trState->numErrors = 0;
++trState->cx->transactionsCommitCompleted;
trState->cx->transactionCommittedMutations += req.transaction.mutations.size();
trState->cx->transactionCommittedMutationBytes += req.transaction.mutations.expectedSize();
if (commitID.present())
g_traceBatch.addEvent("CommitDebug", commitID.get().first(), "NativeAPI.commit.After");
double latency = now() - startTime;
trState->cx->commitLatencies.addSample(latency);
trState->cx->latencies.addSample(now() - trState->startTime);
if (trState->trLogInfo)
trState->trLogInfo->addLog(
FdbClientLogEvents::EventCommit_V2(startTime,
trState->cx->clientLocality.dcId(),
latency,
req.transaction.mutations.size(),
req.transaction.mutations.expectedSize(),
ci.version,
req));
if (trState->automaticIdempotency && alternativeChosen >= 0) {
// Automatic idempotency means we're responsible for best effort idempotency id clean up
proxiesUsed->getInterface(alternativeChosen)
.expireIdempotencyId.send(
ExpireIdempotencyIdRequest{ ci.version, uint8_t(ci.txnBatchId >> 8) });
}
return Void();
} else {
// clear the RYW transaction which contains previous conflicting keys
trState->conflictingKeys.reset();
if (ci.conflictingKRIndices.present()) {
trState->conflictingKeys =
std::make_shared<CoalescedKeyRangeMap<Value>>(conflictingKeysFalse, specialKeys.end);
state Standalone<VectorRef<int>> conflictingKRIndices = ci.conflictingKRIndices.get();
// drop duplicate indices and merge overlapped ranges
// Note: addReadConflictRange in native transaction object does not merge overlapped ranges
state std::unordered_set<int> mergedIds(conflictingKRIndices.begin(),
conflictingKRIndices.end());
for (auto const& rCRIndex : mergedIds) {
const KeyRangeRef kr = req.transaction.read_conflict_ranges[rCRIndex];
const KeyRange krWithPrefix = KeyRangeRef(kr.begin.withPrefix(conflictingKeysRange.begin),
kr.end.withPrefix(conflictingKeysRange.begin));
trState->conflictingKeys->insert(krWithPrefix, conflictingKeysTrue);
}
}
if (debugID.present())
TraceEvent(interval.end()).detail("Conflict", 1);
if (commitID.present())
g_traceBatch.addEvent("CommitDebug", commitID.get().first(), "NativeAPI.commit.After");
throw not_committed();
}
}
}
} catch (Error& e) {
if (e.code() == error_code_request_maybe_delivered || e.code() == error_code_commit_unknown_result ||
e.code() == error_code_never_reply) {
// We don't know if the commit happened, and it might even still be in flight.
if (!trState->options.causalWriteRisky || req.idempotencyId.valid()) {
// Make sure it's not still in flight, either by ensuring the master we submitted to is dead, or the
// version we submitted with is dead, or by committing a conflicting transaction successfully
// if ( cx->getCommitProxies()->masterGeneration <= originalMasterGeneration )
// To ensure the original request is not in flight, we need a key range which intersects its read
// conflict ranges We pick a key range which also intersects its write conflict ranges, since that
// avoids potentially creating conflicts where there otherwise would be none We make the range as small
// as possible (a single key range) to minimize conflicts The intersection will never be empty, because
// if it were (since !causalWriteRisky) makeSelfConflicting would have been applied automatically to req
KeyRangeRef selfConflictingRange =
intersects(req.transaction.write_conflict_ranges, req.transaction.read_conflict_ranges).get();
CODE_PROBE(true, "Waiting for dummy transaction to report commit_unknown_result");
wait(commitDummyTransaction(trState, singleKeyRange(selfConflictingRange.begin)));
if (req.idempotencyId.valid()) {
Optional<CommitResult> commitResult = wait(determineCommitStatus(
trState,
req.transaction.read_snapshot,
req.transaction.read_snapshot + CLIENT_KNOBS->MAX_WRITE_TRANSACTION_LIFE_VERSIONS,
req.idempotencyId));
if (commitResult.present()) {
trState->committedVersion = commitResult.get().commitVersion;
Standalone<StringRef> ret = makeString(10);
placeVersionstamp(
mutateString(ret), commitResult.get().commitVersion, commitResult.get().batchIndex);
trState->versionstampPromise.send(ret);
CODE_PROBE(true, "AutomaticIdempotencyCommitted");
return Void();
} else {
CODE_PROBE(true, "AutomaticIdempotencyNotCommitted");
throw transaction_too_old();
}
}
}
// The user needs to be informed that we aren't sure whether the commit happened. Standard retry loops
// retry it anyway (relying on transaction idempotence) but a client might do something else.
throw commit_unknown_result();
} else {
if (e.code() != error_code_transaction_too_old && e.code() != error_code_not_committed &&
e.code() != error_code_database_locked && e.code() != error_code_commit_proxy_memory_limit_exceeded &&
e.code() != error_code_grv_proxy_memory_limit_exceeded &&
e.code() != error_code_batch_transaction_throttled && e.code() != error_code_tag_throttled &&
e.code() != error_code_process_behind && e.code() != error_code_future_version &&
e.code() != error_code_transaction_throttled_hot_shard &&
e.code() != error_code_transaction_rejected_range_locked) {
TraceEvent(SevError, "TryCommitError").error(e);
}
if (trState->trLogInfo)
trState->trLogInfo->addLog(FdbClientLogEvents::EventCommitError(
startTime, trState->cx->clientLocality.dcId(), static_cast<int>(e.code()), req));
throw;
}
}
}
Future<Void> Transaction::commitMutations() {
try {
// if this is a read-only transaction return immediately
if (!tr.transaction.write_conflict_ranges.size() && !tr.transaction.mutations.size()) {
trState->numErrors = 0;
trState->committedVersion = invalidVersion;
trState->versionstampPromise.sendError(no_commit_version());
return Void();
}
++trState->cx->transactionsCommitStarted;
if (trState->options.readOnly)
return transaction_read_only();
trState->cx->mutationsPerCommit.addSample(tr.transaction.mutations.size());
trState->cx->bytesPerCommit.addSample(tr.transaction.mutations.expectedSize());
if (trState->options.tags.size())
tr.tagSet = trState->options.tags;
size_t transactionSize = getSize();
if (transactionSize > (uint64_t)FLOW_KNOBS->PACKET_WARNING) {
TraceEvent(SevWarn, "LargeTransaction")
.suppressFor(1.0)
.detail("Size", transactionSize)
.detail("NumMutations", tr.transaction.mutations.size())
.detail("ReadConflictSize", tr.transaction.read_conflict_ranges.expectedSize())
.detail("WriteConflictSize", tr.transaction.write_conflict_ranges.expectedSize())
.detail("DebugIdentifier", trState->trLogInfo ? trState->trLogInfo->identifier : "");
}
if (!apiVersionAtLeast(300)) {
transactionSize =
tr.transaction.mutations.expectedSize(); // Old API versions didn't account for conflict ranges when
// determining whether to throw transaction_too_large
}
if (transactionSize > trState->options.sizeLimit) {
return transaction_too_large();
}
bool isCheckingWrites = trState->options.checkWritesEnabled && deterministicRandom()->random01() < 0.01;
for (const auto& extraConflictRange : extraConflictRanges)
if (extraConflictRange.isReady() && extraConflictRange.get().first < extraConflictRange.get().second)
tr.transaction.read_conflict_ranges.emplace_back(
tr.arena, extraConflictRange.get().first, extraConflictRange.get().second);
if (tr.idempotencyId.valid()) {
// We need to be able confirm that this transaction is no longer in
// flight, and if the idempotency id is in the read and write
// conflict range we can use that.
BinaryWriter wr(Unversioned());
wr.serializeBytes("\xFF/SC/"_sr);
wr.serializeBytes(tr.idempotencyId.asStringRefUnsafe());
auto r = singleKeyRange(wr.toValue(), tr.arena);
tr.transaction.read_conflict_ranges.push_back(tr.arena, r);
tr.transaction.write_conflict_ranges.push_back(tr.arena, r);
}
if (!trState->options.causalWriteRisky &&
!intersects(tr.transaction.write_conflict_ranges, tr.transaction.read_conflict_ranges).present())
makeSelfConflicting();
if (isCheckingWrites) {
// add all writes into the read conflict range...
tr.transaction.read_conflict_ranges.append(
tr.arena, tr.transaction.write_conflict_ranges.begin(), tr.transaction.write_conflict_ranges.size());
}
if (trState->options.debugDump) {
UID u = nondeterministicRandom()->randomUniqueID();
TraceEvent("TransactionDump", u).log();
for (auto i = tr.transaction.mutations.begin(); i != tr.transaction.mutations.end(); ++i)
TraceEvent("TransactionMutation", u)
.detail("T", i->type)
.detail("P1", i->param1)
.detail("P2", i->param2);
}
if (trState->options.lockAware) {
tr.flags = tr.flags | CommitTransactionRequest::FLAG_IS_LOCK_AWARE;
}
if (trState->options.firstInBatch) {
tr.flags = tr.flags | CommitTransactionRequest::FLAG_FIRST_IN_BATCH;
}
if (trState->options.bypassStorageQuota) {
tr.flags = tr.flags | CommitTransactionRequest::FLAG_BYPASS_STORAGE_QUOTA;
}
if (trState->options.reportConflictingKeys) {
tr.transaction.report_conflicting_keys = true;
}
Future<Void> commitResult = tryCommit(trState, tr);
if (isCheckingWrites) {
Promise<Void> committed;
checkWrites(Uncancellable(), trState, commitResult, committed, tr);
return committed.getFuture();
}
return commitResult;
} catch (Error& e) {
if (e.code() == error_code_transaction_throttled_hot_shard ||
e.code() == error_code_transaction_rejected_range_locked) {
TraceEvent("TransactionThrottledHotShard").error(e);
return onError(e);
}
TraceEvent("ClientCommitError").error(e);
return Future<Void>(e);
} catch (...) {
Error e(error_code_unknown_error);
TraceEvent("ClientCommitError").error(e);
return Future<Void>(e);
}
}
ACTOR Future<Void> commitAndWatch(Transaction* self) {
try {
wait(self->commitMutations());
self->getDatabase()->transactionTracingSample =
(self->getCommittedVersion() % 60000000) < (60000000 * FLOW_KNOBS->TRACING_SAMPLE_RATE);
if (!self->watches.empty()) {
self->setupWatches();
}
if (!self->apiVersionAtLeast(700)) {
self->reset();
}
return Void();
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled) {
if (!self->watches.empty()) {
self->cancelWatches(e);
}
self->trState->versionstampPromise.sendError(transaction_invalid_version());
if (!self->apiVersionAtLeast(700)) {
self->reset();
}
}
throw;
}
}
Future<Void> Transaction::commit() {
ASSERT(!committing.isValid());
committing = commitAndWatch(this);
return committing;
}
// Returns a thread-local mt19937_64 seeded once with 32 bytes of OS entropy.
// Used for AUTOMATIC_IDEMPOTENCY ID generation in non-simulation runs.
static std::mt19937_64& getIdempotencyRng() {
static thread_local std::mt19937_64 rng = []() {
uint32_t seed_data[8];
platform::getRandomBytes(seed_data, sizeof(seed_data));
std::seed_seq seq(seed_data, seed_data + 8);
return std::mt19937_64(seq);
}();
return rng;
}
void Transaction::setOption(FDBTransactionOptions::Option option, Optional<StringRef> value) {
switch (option) {
case FDBTransactionOptions::INITIALIZE_NEW_DATABASE:
validateOptionValueNotPresent(value);
if (trState->readVersionFuture.isValid())
throw read_version_already_set();
trState->readVersionFuture = Version(0);
trState->options.causalWriteRisky = true;
break;
case FDBTransactionOptions::CAUSAL_READ_RISKY:
validateOptionValueNotPresent(value);
trState->options.getReadVersionFlags |= GetReadVersionRequest::FLAG_CAUSAL_READ_RISKY;
break;
case FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE:
validateOptionValueNotPresent(value);
trState->options.priority = TransactionPriority::IMMEDIATE;
break;
case FDBTransactionOptions::PRIORITY_BATCH:
validateOptionValueNotPresent(value);
trState->options.priority = TransactionPriority::BATCH;
break;
case FDBTransactionOptions::CAUSAL_WRITE_RISKY:
validateOptionValueNotPresent(value);
trState->options.causalWriteRisky = true;
break;
case FDBTransactionOptions::COMMIT_ON_FIRST_PROXY:
validateOptionValueNotPresent(value);
trState->options.commitOnFirstProxy = true;
break;
case FDBTransactionOptions::CHECK_WRITES_ENABLE:
validateOptionValueNotPresent(value);
trState->options.checkWritesEnabled = true;
break;
case FDBTransactionOptions::DEBUG_DUMP:
validateOptionValueNotPresent(value);
trState->options.debugDump = true;
break;
case FDBTransactionOptions::TRANSACTION_LOGGING_ENABLE:
setOption(FDBTransactionOptions::DEBUG_TRANSACTION_IDENTIFIER, value);
setOption(FDBTransactionOptions::LOG_TRANSACTION);
break;
case FDBTransactionOptions::DEBUG_TRANSACTION_IDENTIFIER:
validateOptionValuePresent(value);
if (value.get().size() > 100 || value.get().size() == 0) {
throw invalid_option_value();
}
if (trState->trLogInfo) {
if (trState->trLogInfo->identifier.empty()) {
trState->trLogInfo->identifier = value.get().printable();
} else if (trState->trLogInfo->identifier != value.get().printable()) {
TraceEvent(SevWarn, "CannotChangeDebugTransactionIdentifier")
.detail("PreviousIdentifier", trState->trLogInfo->identifier)
.detail("NewIdentifier", value.get());
throw client_invalid_operation();
}
} else {
trState->trLogInfo =
makeReference<TransactionLogInfo>(value.get().printable(), TransactionLogInfo::DONT_LOG);
trState->trLogInfo->maxFieldLength = trState->options.maxTransactionLoggingFieldLength;
}
if (trState->readOptions.present() && trState->readOptions.get().debugID.present()) {
TraceEvent(SevInfo, "TransactionBeingTraced")
.detail("DebugTransactionID", trState->trLogInfo->identifier)
.detail("ServerTraceID", trState->readOptions.get().debugID.get());
}
break;
case FDBTransactionOptions::LOG_TRANSACTION:
validateOptionValueNotPresent(value);
if (trState->trLogInfo && !trState->trLogInfo->identifier.empty()) {
trState->trLogInfo->logTo(TransactionLogInfo::TRACE_LOG);
} else {
TraceEvent(SevWarn, "DebugTransactionIdentifierNotSet")
.detail("Error", "Debug Transaction Identifier option must be set before logging the transaction");
throw client_invalid_operation();
}
break;
case FDBTransactionOptions::TRANSACTION_LOGGING_MAX_FIELD_LENGTH:
validateOptionValuePresent(value);
{
int maxFieldLength = extractIntOption(value, -1, std::numeric_limits<int32_t>::max());
if (maxFieldLength == 0) {
throw invalid_option_value();
}
trState->options.maxTransactionLoggingFieldLength = maxFieldLength;
}
if (trState->trLogInfo) {
trState->trLogInfo->maxFieldLength = trState->options.maxTransactionLoggingFieldLength;
}
break;
case FDBTransactionOptions::SERVER_REQUEST_TRACING:
validateOptionValueNotPresent(value);
debugTransaction(deterministicRandom()->randomUniqueID());
if (trState->trLogInfo && !trState->trLogInfo->identifier.empty() && trState->readOptions.present() &&
trState->readOptions.get().debugID.present()) {
TraceEvent(SevInfo, "TransactionBeingTraced")
.detail("DebugTransactionID", trState->trLogInfo->identifier)
.detail("ServerTraceID", trState->readOptions.get().debugID.get());
}
break;
case FDBTransactionOptions::MAX_RETRY_DELAY:
validateOptionValuePresent(value);
trState->options.maxBackoff = extractIntOption(value, 0, std::numeric_limits<int32_t>::max()) / 1000.0;
break;
case FDBTransactionOptions::MAX_GRV_QUEUE_DELAY:
validateOptionValuePresent(value);
trState->options.maxGrvQueueDelayMS = extractIntOption(value, 0, std::numeric_limits<int32_t>::max());
break;
case FDBTransactionOptions::SIZE_LIMIT:
validateOptionValuePresent(value);
trState->options.sizeLimit = extractIntOption(value, 32, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT);
break;
case FDBTransactionOptions::LOCK_AWARE:
validateOptionValueNotPresent(value);
if (!trState->readOptions.present()) {
trState->readOptions = ReadOptions();
}
trState->readOptions.get().lockAware = true;
trState->options.lockAware = true;
trState->options.readOnly = false;
break;
case FDBTransactionOptions::READ_LOCK_AWARE:
validateOptionValueNotPresent(value);
if (!trState->readOptions.present()) {
trState->readOptions = ReadOptions();
}
trState->readOptions.get().lockAware = true;
if (!trState->options.lockAware) {
trState->options.lockAware = true;
trState->options.readOnly = true;
}
break;
case FDBTransactionOptions::FIRST_IN_BATCH:
validateOptionValueNotPresent(value);
trState->options.firstInBatch = true;
break;
case FDBTransactionOptions::USE_PROVISIONAL_PROXIES:
validateOptionValueNotPresent(value);
trState->options.getReadVersionFlags |= GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES;
trState->useProvisionalProxies = UseProvisionalProxies::True;
break;
case FDBTransactionOptions::INCLUDE_PORT_IN_ADDRESS:
validateOptionValueNotPresent(value);
trState->options.includePort = true;
break;
case FDBTransactionOptions::TAG:
validateOptionValuePresent(value);
trState->options.tags.addTag(value.get());
break;
case FDBTransactionOptions::AUTO_THROTTLE_TAG:
validateOptionValuePresent(value);
trState->options.tags.addTag(value.get());
trState->options.readTags.addTag(value.get());
break;
case FDBTransactionOptions::SPAN_PARENT:
validateOptionValuePresent(value);
if (value.get().size() != 33) {
throw invalid_option_value();
}
CODE_PROBE(true, "Adding link in FDBTransactionOptions::SPAN_PARENT");
span.setParent(BinaryReader::fromStringRef<SpanContext>(value.get(), IncludeVersion()));
break;
case FDBTransactionOptions::REPORT_CONFLICTING_KEYS:
validateOptionValueNotPresent(value);
trState->options.reportConflictingKeys = true;
break;
case FDBTransactionOptions::EXPENSIVE_CLEAR_COST_ESTIMATION_ENABLE:
validateOptionValueNotPresent(value);
trState->options.expensiveClearCostEstimation = true;
break;
case FDBTransactionOptions::USE_GRV_CACHE:
validateOptionValueNotPresent(value);
if (apiVersionAtLeast(ApiVersion::withGrvCache().version()) && !trState->cx->sharedStatePtr) {
throw invalid_option();
}
if (trState->numErrors == 0) {
trState->options.useGrvCache = true;
}
break;
case FDBTransactionOptions::SKIP_GRV_CACHE:
validateOptionValueNotPresent(value);
trState->options.skipGrvCache = true;
break;
case FDBTransactionOptions::READ_SYSTEM_KEYS:
case FDBTransactionOptions::ACCESS_SYSTEM_KEYS:
case FDBTransactionOptions::RAW_ACCESS:
// System key access implies raw access. Native API handles the raw access,
// system key access is handled in RYW.
validateOptionValueNotPresent(value);
trState->options.rawAccess = true;
break;
case FDBTransactionOptions::BYPASS_STORAGE_QUOTA:
trState->options.bypassStorageQuota = true;
break;
case FDBTransactionOptions::AUTHORIZATION_TOKEN:
if (value.present())
trState->authToken = WipedString(value.get());
else
trState->authToken.reset();
break;
case FDBTransactionOptions::IDEMPOTENCY_ID:
validateOptionValuePresent(value);
if (!(value.get().size() >= 16 && value.get().size() < 256)) {
Error e = invalid_option();
TraceEvent(SevWarn, "IdempotencyIdInvalidSize")
.error(e)
.detail("IdempotencyId", value.get().printable())
.detail("Recommendation", "Use an idempotency id that's at least 16 bytes and less than 256 bytes");
throw e;
}
tr.idempotencyId = IdempotencyIdRef(tr.arena, IdempotencyIdRef(value.get()));
trState->automaticIdempotency = false;
break;
case FDBTransactionOptions::AUTOMATIC_IDEMPOTENCY:
validateOptionValueNotPresent(value);
if (!tr.idempotencyId.valid()) {
StringRef id = makeString(16, tr.arena);
if (g_network->isSimulated()) {
deterministicRandom()->randomBytes(mutateString(id), 16);
} else {
auto& rng = getIdempotencyRng();
uint64_t buf[2] = { rng(), rng() };
memcpy(mutateString(id), buf, 16);
}
tr.idempotencyId = IdempotencyIdRef(id);
}
trState->automaticIdempotency = true;
break;
case FDBTransactionOptions::READ_SERVER_SIDE_CACHE_ENABLE:
trState->readOptions.withDefault(ReadOptions()).cacheResult = CacheResult::True;
break;
case FDBTransactionOptions::READ_SERVER_SIDE_CACHE_DISABLE:
trState->readOptions.withDefault(ReadOptions()).cacheResult = CacheResult::False;
break;
case FDBTransactionOptions::READ_PRIORITY_LOW:
trState->readOptions.withDefault(ReadOptions()).type = ReadType::LOW;
break;
case FDBTransactionOptions::READ_PRIORITY_NORMAL:
trState->readOptions.withDefault(ReadOptions()).type = ReadType::NORMAL;
break;
case FDBTransactionOptions::READ_PRIORITY_HIGH:
trState->readOptions.withDefault(ReadOptions()).type = ReadType::HIGH;
break;
case FDBTransactionOptions::ENABLE_REPLICA_CONSISTENCY_CHECK:
validateOptionValueNotPresent(value);
trState->options.enableReplicaConsistencyCheck = true;
break;
case FDBTransactionOptions::CONSISTENCY_CHECK_REQUIRED_REPLICAS:
validateOptionValuePresent(value);
trState->options.requiredReplicas = extractIntOption(value, -2, std::numeric_limits<int64_t>::max());
default:
break;
}
}
ACTOR Future<GetReadVersionReply> getConsistentReadVersion(SpanContext parentSpan,
DatabaseContext* cx,
uint32_t transactionCount,
TransactionPriority priority,
uint32_t flags,
TransactionTagMap<uint32_t> tags,
Optional<UID> debugID,
Optional<int64_t> maxGrvQueueDelayMS) {
state Span span("NAPI:getConsistentReadVersion"_loc, parentSpan);
++cx->transactionReadVersionBatches;
if (debugID.present())
g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getConsistentReadVersion.Before");
loop {
try {
state GetReadVersionRequest req(span.context,
transactionCount,
priority,
cx->ssVersionVectorCache.getMaxVersion(),
flags,
tags,
debugID,
maxGrvQueueDelayMS);
state Future<Void> onProxiesChanged = cx->onProxiesChanged();
choose {
when(wait(onProxiesChanged)) {
onProxiesChanged = cx->onProxiesChanged();
}
when(GetReadVersionReply v =
wait(basicLoadBalance(cx->getGrvProxies(UseProvisionalProxies(
flags & GetReadVersionRequest::FLAG_USE_PROVISIONAL_PROXIES)),
&GrvProxyInterface::getConsistentReadVersion,
req,
cx->taskID))) {
if (tags.size() != 0) {
auto& priorityThrottledTags = cx->throttledTags[priority];
for (auto& tag : tags) {
auto itr = v.tagThrottleInfo.find(tag.first);
if (itr == v.tagThrottleInfo.end()) {
CODE_PROBE(true, "Removing client throttle");
priorityThrottledTags.erase(tag.first);
} else {
CODE_PROBE(true, "Setting client throttle");
auto result = priorityThrottledTags.try_emplace(tag.first, itr->second);
if (!result.second) {
result.first->second.update(itr->second);
}
}
}
}
if (debugID.present())
g_traceBatch.addEvent(
"TransactionDebug", debugID.get().first(), "NativeAPI.getConsistentReadVersion.After");
ASSERT(v.version > 0);
cx->minAcceptableReadVersion = std::min(cx->minAcceptableReadVersion, v.version);
if (cx->versionVectorCacheActive(v.ssVersionVectorDelta)) {
if (cx->isCurrentGrvProxy(v.proxyId)) {
cx->ssVersionVectorCache.applyDelta(v.ssVersionVectorDelta);
} else {
continue; // stale GRV reply, retry
}
}
return v;
}
}
} catch (Error& e) {
if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled &&
e.code() != error_code_grv_proxy_memory_limit_exceeded &&
e.code() != error_code_transaction_grv_queue_rejected)
TraceEvent(SevError, "GetConsistentReadVersionError").error(e);
throw;
}
}
}
ACTOR Future<Void> readVersionBatcher(DatabaseContext* cx,
FutureStream<DatabaseContext::VersionRequest> versionStream,
TransactionPriority priority,
uint32_t flags,
Optional<int64_t> maxGrvQueueDelayMS) {
state std::vector<Promise<GetReadVersionReply>> requests;
state PromiseStream<Future<Void>> addActor;
state Future<Void> collection = actorCollection(addActor.getFuture());
state Future<Void> timeout;
state Optional<UID> debugID;
state bool send_batch;
state Reference<Histogram> batchSizeDist = Histogram::getHistogram(
"GrvBatcher"_sr, "ClientGrvBatchSize"_sr, Histogram::Unit::countLinear, 0, CLIENT_KNOBS->MAX_BATCH_SIZE * 2);
state Reference<Histogram> batchIntervalDist =
Histogram::getHistogram("GrvBatcher"_sr,
"ClientGrvBatchInterval"_sr,
Histogram::Unit::milliseconds,
0,
CLIENT_KNOBS->GRV_BATCH_TIMEOUT * 1000000 * 2);
state Reference<Histogram> grvReplyLatencyDist =
Histogram::getHistogram("GrvBatcher"_sr, "ClientGrvReplyLatency"_sr, Histogram::Unit::milliseconds);
state double lastRequestTime = now();
state TransactionTagMap<uint32_t> tags;
// dynamic batching
state PromiseStream<double> replyTimes;
state double batchTime = 0;
state Span span("NAPI:readVersionBatcher"_loc);
loop {
send_batch = false;
choose {
when(DatabaseContext::VersionRequest req = waitNext(versionStream)) {
if (req.debugID.present()) {
if (!debugID.present()) {
debugID = nondeterministicRandom()->randomUniqueID();
}
g_traceBatch.addAttach("TransactionAttachID", req.debugID.get().first(), debugID.get().first());
}
span.addLink(req.spanContext);
requests.push_back(req.reply);
for (auto tag : req.tags) {
++tags[tag];
}
if (requests.size() == CLIENT_KNOBS->MAX_BATCH_SIZE) {
send_batch = true;
++cx->transactionGrvFullBatches;
} else if (!timeout.isValid()) {
timeout = delay(batchTime, TaskPriority::GetConsistentReadVersion);
}
}
when(wait(timeout.isValid() ? timeout : Never())) {
send_batch = true;
++cx->transactionGrvTimedOutBatches;
}
// dynamic batching monitors reply latencies
when(double reply_latency = waitNext(replyTimes.getFuture())) {
double target_latency = reply_latency * 0.5;
batchTime = std::min(0.1 * target_latency + 0.9 * batchTime, CLIENT_KNOBS->GRV_BATCH_TIMEOUT);
grvReplyLatencyDist->sampleSeconds(reply_latency);
}
when(wait(collection)) {} // for errors
}
if (send_batch) {
int count = requests.size();
ASSERT(count);
batchSizeDist->sampleRecordCounter(count);
auto requestTime = now();
batchIntervalDist->sampleSeconds(requestTime - lastRequestTime);
lastRequestTime = requestTime;
// dynamic batching
Promise<GetReadVersionReply> GRVReply;
requests.push_back(GRVReply);
addActor.send(ready(timeReply(GRVReply.getFuture(), replyTimes)));
Future<Void> batch = incrementalBroadcastWithError(
getConsistentReadVersion(
span.context, cx, count, priority, flags, std::move(tags), std::move(debugID), maxGrvQueueDelayMS),
std::move(requests),
CLIENT_KNOBS->BROADCAST_BATCH_SIZE);
span = Span("NAPI:readVersionBatcher"_loc);
tags.clear();
debugID = Optional<UID>();
requests.clear();
addActor.send(batch);
timeout = Future<Void>();
}
}
}
ACTOR Future<Version> extractReadVersion(Reference<TransactionState> trState,
Location location,
SpanContext spanContext,
Future<GetReadVersionReply> f,
Promise<Optional<Value>> metadataVersion) {
state Span span(spanContext, location, trState->spanContext);
GetReadVersionReply rep = wait(f);
if (CLIENT_BUGGIFY) {
throw grv_proxy_memory_limit_exceeded();
}
double replyTime = now();
double latency = replyTime - trState->startTime;
trState->cx->lastProxyRequestTime = trState->startTime;
trState->cx->updateCachedReadVersion(trState->startTime, rep.version);
if (rep.rkBatchThrottled) {
trState->cx->lastRkBatchThrottleTime = replyTime;
}
if (rep.rkDefaultThrottled) {
trState->cx->lastRkDefaultThrottleTime = replyTime;
}
trState->cx->GRVLatencies.addSample(latency);
if (trState->trLogInfo)
trState->trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V3(
trState->startTime, trState->cx->clientLocality.dcId(), latency, trState->options.priority, rep.version));
if (rep.locked && !trState->options.lockAware)
throw database_locked();
++trState->cx->transactionReadVersionsCompleted;
switch (trState->options.priority) {
case TransactionPriority::IMMEDIATE:
++trState->cx->transactionImmediateReadVersionsCompleted;
break;
case TransactionPriority::DEFAULT:
++trState->cx->transactionDefaultReadVersionsCompleted;
break;
case TransactionPriority::BATCH:
++trState->cx->transactionBatchReadVersionsCompleted;
break;
default:
ASSERT(false);
}
if (trState->options.tags.size() != 0) {
auto& priorityThrottledTags = trState->cx->throttledTags[trState->options.priority];
for (auto& tag : trState->options.tags) {
auto itr = priorityThrottledTags.find(tag);
if (itr != priorityThrottledTags.end()) {
if (itr->second.expired()) {
priorityThrottledTags.erase(itr);
} else if (itr->second.throttleDuration() > 0) {
CODE_PROBE(true, "throttling transaction after getting read version");
++trState->cx->transactionReadVersionsThrottled;
throw tag_throttled();
}
}
}
for (auto& tag : trState->options.tags) {
auto itr = priorityThrottledTags.find(tag);
if (itr != priorityThrottledTags.end()) {
itr->second.addReleased(1);
}
}
}
if (rep.version > trState->cx->metadataVersionCache[trState->cx->mvCacheInsertLocation].first) {
trState->cx->mvCacheInsertLocation =
(trState->cx->mvCacheInsertLocation + 1) % trState->cx->metadataVersionCache.size();
trState->cx->metadataVersionCache[trState->cx->mvCacheInsertLocation] =
std::make_pair(rep.version, rep.metadataVersion);
}
metadataVersion.send(rep.metadataVersion);
if (trState->cx->versionVectorCacheActive(rep.ssVersionVectorDelta)) {
if (trState->cx->isCurrentGrvProxy(rep.proxyId)) {
trState->cx->ssVersionVectorCache.applyDelta(rep.ssVersionVectorDelta);
} else {
trState->cx->ssVersionVectorCache.clear();
}
}
return rep.version;
}
bool rkThrottlingCooledDown(DatabaseContext* cx, TransactionPriority priority) {
if (priority == TransactionPriority::IMMEDIATE) {
return true;
} else if (priority == TransactionPriority::BATCH) {
if (cx->lastRkBatchThrottleTime == 0.0) {
return true;
}
return (now() - cx->lastRkBatchThrottleTime > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
} else if (priority == TransactionPriority::DEFAULT) {
if (cx->lastRkDefaultThrottleTime == 0.0) {
return true;
}
return (now() - cx->lastRkDefaultThrottleTime > CLIENT_KNOBS->GRV_CACHE_RK_COOLDOWN);
}
return false;
}
ACTOR static Future<Void> backgroundGrvUpdater(DatabaseContext* cx) {
state Transaction tr;
state double grvDelay = 0.001;
state Backoff backoff;
try {
loop {
if (CLIENT_KNOBS->FORCE_GRV_CACHE_OFF)
return Void();
wait(refreshTransaction(cx, &tr));
state double curTime = now();
state double lastTime = cx->getLastGrvTime();
state double lastProxyTime = cx->lastProxyRequestTime;
TraceEvent(SevDebug, "BackgroundGrvUpdaterBefore")
.detail("CurTime", curTime)
.detail("LastTime", lastTime)
.detail("GrvDelay", grvDelay)
.detail("CachedReadVersion", cx->getCachedReadVersion())
.detail("CachedTime", cx->getLastGrvTime())
.detail("Gap", curTime - lastTime)
.detail("Bound", CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay);
if (curTime - lastTime >= (CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) ||
curTime - lastProxyTime > CLIENT_KNOBS->MAX_PROXY_CONTACT_LAG) {
try {
tr.setOption(FDBTransactionOptions::SKIP_GRV_CACHE);
wait(success(tr.getReadVersion()));
cx->lastProxyRequestTime = curTime;
grvDelay = (grvDelay + (now() - curTime)) / 2.0;
TraceEvent(SevDebug, "BackgroundGrvUpdaterSuccess")
.detail("GrvDelay", grvDelay)
.detail("CachedReadVersion", cx->getCachedReadVersion())
.detail("CachedTime", cx->getLastGrvTime());
backoff = Backoff();
} catch (Error& e) {
TraceEvent(SevInfo, "BackgroundGrvUpdaterTxnError").errorUnsuppressed(e);
wait(tr.onError(e));
wait(backoff.onError());
}
} else {
wait(
delay(std::max(0.001,
std::min(CLIENT_KNOBS->MAX_PROXY_CONTACT_LAG - (curTime - lastProxyTime),
(CLIENT_KNOBS->MAX_VERSION_CACHE_LAG - grvDelay) - (curTime - lastTime)))));
}
}
} catch (Error& e) {
TraceEvent(SevInfo, "BackgroundGrvUpdaterFailed").errorUnsuppressed(e);
throw;
}
}
Future<Version> TransactionState::getReadVersion(uint32_t flags) {
ASSERT(!readVersionFuture.isValid());
if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !options.skipGrvCache &&
(deterministicRandom()->random01() <= CLIENT_KNOBS->DEBUG_USE_GRV_CACHE_CHANCE || options.useGrvCache) &&
rkThrottlingCooledDown(cx.getPtr(), options.priority)) {
// Upon our first request to use cached RVs, start the background updater
if (!cx->grvUpdateHandler.isValid()) {
cx->grvUpdateHandler = backgroundGrvUpdater(cx.getPtr());
}
Version rv = cx->getCachedReadVersion();
double lastTime = cx->getLastGrvTime();
double requestTime = now();
if (requestTime - lastTime <= CLIENT_KNOBS->MAX_VERSION_CACHE_LAG && rv != Version(0)) {
ASSERT(!debug_checkVersionTime(rv, requestTime, "CheckStaleness"));
return rv;
} // else go through regular GRV path
}
++cx->transactionReadVersions;
flags |= options.getReadVersionFlags;
switch (options.priority) {
case TransactionPriority::IMMEDIATE:
flags |= GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE;
++cx->transactionImmediateReadVersions;
break;
case TransactionPriority::DEFAULT:
flags |= GetReadVersionRequest::PRIORITY_DEFAULT;
++cx->transactionDefaultReadVersions;
break;
case TransactionPriority::BATCH:
flags |= GetReadVersionRequest::PRIORITY_BATCH;
++cx->transactionBatchReadVersions;
break;
default:
ASSERT(false);
}
if (options.tags.size() != 0) {
double maxThrottleDelay = 0.0;
bool canRecheck = false;
auto& priorityThrottledTags = cx->throttledTags[options.priority];
for (auto& tag : options.tags) {
auto itr = priorityThrottledTags.find(tag);
if (itr != priorityThrottledTags.end()) {
if (!itr->second.expired()) {
maxThrottleDelay = std::max(maxThrottleDelay, itr->second.throttleDuration());
canRecheck = itr->second.canRecheck();
} else {
priorityThrottledTags.erase(itr);
}
}
}
if (maxThrottleDelay > 0.0 && !canRecheck) { // TODO: allow delaying?
CODE_PROBE(true, "Throttling tag before GRV request");
++cx->transactionReadVersionsThrottled;
return tag_throttled();
} else {
CODE_PROBE(maxThrottleDelay > 0.0, "Rechecking throttle");
}
for (auto& tag : options.tags) {
auto itr = priorityThrottledTags.find(tag);
if (itr != priorityThrottledTags.end()) {
itr->second.updateChecked();
}
}
}
Location location = "NAPI:getReadVersion"_loc;
SpanContext derivedSpanContext = generateSpanID(cx->transactionTracingSample, spanContext);
Optional<UID> versionDebugID = readOptions.present() ? readOptions.get().debugID : Optional<UID>();
// Include the max GRV queue delay in the batcher key so coalesced requests
// share the same proxy-side admission threshold.
auto& batcher = cx->versionBatcher[DatabaseContext::VersionBatcherKey(flags, options.maxGrvQueueDelayMS)];
if (!batcher.actor.isValid()) {
batcher.actor = readVersionBatcher(
cx.getPtr(), batcher.stream.getFuture(), options.priority, flags, options.maxGrvQueueDelayMS);
}
auto const req = DatabaseContext::VersionRequest(derivedSpanContext, options.tags, versionDebugID);
batcher.stream.send(req);
startTime = now();
return extractReadVersion(
Reference<TransactionState>::addRef(this), location, spanContext, req.reply.getFuture(), metadataVersion);
}
Optional<Version> Transaction::getCachedReadVersion() const {
if (trState->readVersionFuture.canGet()) {
return trState->readVersion();
} else {
return Optional<Version>();
}
}
double Transaction::getTagThrottledDuration() const {
return 0.0;
}
Future<Standalone<StringRef>> Transaction::getVersionstamp() {
if (committing.isValid()) {
return transaction_invalid_version();
}
return trState->versionstampPromise.getFuture();
}
// Gets the protocol version reported by a coordinator via the protocol info interface
Future<ProtocolVersion> getCoordinatorProtocol(NetworkAddress coordinatorAddress) {
RequestStream<ProtocolInfoRequest> requestStream(
Endpoint::wellKnown({ coordinatorAddress }, WLTOKEN_PROTOCOL_INFO));
ProtocolInfoReply reply = co_await retryBrokenPromise(requestStream, ProtocolInfoRequest{});
co_return reply.version;
}
// Gets the protocol version reported by a coordinator in its connect packet
// If we are unable to get a version from the connect packet (e.g. because we lost connection with the peer), then this
// function will return with an unset result.
// If an expected version is given, this future won't return if the actual protocol version matches the expected version
Future<Optional<ProtocolVersion>> getCoordinatorProtocolFromConnectPacket(NetworkAddress coordinatorAddress,
Optional<ProtocolVersion> expectedVersion) {
Optional<Reference<AsyncVar<Optional<ProtocolVersion>> const>> protocolVersion =
FlowTransport::transport().getPeerProtocolAsyncVar(coordinatorAddress);
if (!protocolVersion.present()) {
TraceEvent(SevWarnAlways, "GetCoordinatorProtocolPeerMissing").detail("Address", coordinatorAddress);
co_await delay(FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT);
co_return Optional<ProtocolVersion>();
}
while (true) {
if (protocolVersion.get()->get().present() && protocolVersion.get()->get() != expectedVersion) {
co_return protocolVersion.get()->get();
}
Future<Void> change = protocolVersion.get()->onChange();
if (!protocolVersion.get()->get().present()) {
// If we still don't have any connection info after a timeout, retry sending the protocol version request
change = timeout(change, FLOW_KNOBS->CONNECTION_MONITOR_TIMEOUT, Void());
}
co_await change;
if (!protocolVersion.get()->get().present()) {
co_return protocolVersion.get()->get();
}
}
}
// Returns the protocol version reported by the given coordinator
// If an expected version is given, the future won't return until the protocol version is different than expected
ACTOR Future<ProtocolVersion> getClusterProtocolImpl(
Reference<AsyncVar<Optional<ClientLeaderRegInterface>> const> coordinator,
Optional<ProtocolVersion> expectedVersion) {
state bool needToConnect = true;
state Future<ProtocolVersion> protocolVersion = Never();
loop {
if (!coordinator->get().present()) {
wait(coordinator->onChange());
} else {
state NetworkAddress coordinatorAddress;
if (coordinator->get().get().hostname.present()) {
state Hostname h = coordinator->get().get().hostname.get();
wait(store(coordinatorAddress, h.resolveWithRetry()));
} else {
coordinatorAddress = coordinator->get().get().getLeader.getEndpoint().getPrimaryAddress();
}
if (needToConnect) {
// Even though we typically rely on the connect packet to get the protocol version, we need to send some
// request in order to start a connection. This protocol version request serves that purpose.
protocolVersion = getCoordinatorProtocol(coordinatorAddress);
needToConnect = false;
}
choose {
when(wait(coordinator->onChange())) {
needToConnect = true;
}
when(ProtocolVersion pv = wait(protocolVersion)) {
if (!expectedVersion.present() || expectedVersion.get() != pv) {
return pv;
}
protocolVersion = Never();
}
// Older versions of FDB don't have an endpoint to return the protocol version, so we get this info from
// the connect packet
when(Optional<ProtocolVersion> pv =
wait(getCoordinatorProtocolFromConnectPacket(coordinatorAddress, expectedVersion))) {
if (pv.present()) {
return pv.get();
} else {
needToConnect = true;
}
}
}
}
}
}
// Returns the protocol version reported by the coordinator this client is currently connected to
// If an expected version is given, the future won't return until the protocol version is different than expected
// Note: this will never return if the server is running a protocol from FDB 5.0 or older
Future<ProtocolVersion> DatabaseContext::getClusterProtocol(Optional<ProtocolVersion> expectedVersion) {
return getClusterProtocolImpl(coordinator, expectedVersion);
}
double ClientTagThrottleData::throttleDuration() const {
if (expiration <= now()) {
return 0.0;
}
double capacity =
(smoothRate.smoothTotal() - smoothReleased.smoothRate()) * CLIENT_KNOBS->TAG_THROTTLE_SMOOTHING_WINDOW;
if (capacity >= 1) {
return 0.0;
}
if (tpsRate == 0) {
return std::max(0.0, expiration - now());
}
return std::min(expiration - now(), capacity / tpsRate);
}
uint32_t Transaction::getSize() {
auto s = tr.transaction.mutations.expectedSize() + tr.transaction.read_conflict_ranges.expectedSize() +
tr.transaction.write_conflict_ranges.expectedSize();
return s;
}
Future<Void> Transaction::onError(Error const& e) {
if (g_network->isSimulated() && ++trState->numErrors % 10 == 0) {
TraceEvent(SevWarnAlways, "TransactionTooManyRetries")
.errorUnsuppressed(e)
.detail("NumRetries", trState->numErrors);
}
if (e.code() == error_code_success) {
return client_invalid_operation();
}
if (e.code() == error_code_not_committed || e.code() == error_code_commit_unknown_result ||
e.code() == error_code_database_locked || e.code() == error_code_commit_proxy_memory_limit_exceeded ||
e.code() == error_code_grv_proxy_memory_limit_exceeded || e.code() == error_code_process_behind ||
e.code() == error_code_batch_transaction_throttled || e.code() == error_code_tag_throttled ||
e.code() == error_code_transaction_throttled_hot_shard ||
(e.code() == error_code_transaction_rejected_range_locked &&
CLIENT_KNOBS->TRANSACTION_LOCK_REJECTION_RETRIABLE)) {
if (e.code() == error_code_not_committed)
++trState->cx->transactionsNotCommitted;
else if (e.code() == error_code_commit_unknown_result)
++trState->cx->transactionsMaybeCommitted;
else if (e.code() == error_code_commit_proxy_memory_limit_exceeded ||
e.code() == error_code_grv_proxy_memory_limit_exceeded)
++trState->cx->transactionsResourceConstrained;
else if (e.code() == error_code_process_behind)
++trState->cx->transactionsProcessBehind;
else if (e.code() == error_code_batch_transaction_throttled || e.code() == error_code_tag_throttled ||
e.code() == error_code_transaction_throttled_hot_shard) {
++trState->cx->transactionsThrottled;
} else if (e.code() == error_code_transaction_rejected_range_locked) {
++trState->cx->transactionsLockRejected;
}
double backoff = getBackoff(e.code());
reset();
return delay(backoff, trState->taskID);
} else if (e.code() == error_code_transaction_rejected_range_locked) {
ASSERT(!CLIENT_KNOBS->TRANSACTION_LOCK_REJECTION_RETRIABLE);
++trState->cx->transactionsLockRejected; // throw error
}
if (e.code() == error_code_transaction_too_old || e.code() == error_code_future_version) {
if (e.code() == error_code_transaction_too_old)
++trState->cx->transactionsTooOld;
else if (e.code() == error_code_future_version)
++trState->cx->transactionsFutureVersions;
double maxBackoff = trState->options.maxBackoff;
reset();
return delay(std::min(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, maxBackoff), trState->taskID);
}
return e;
}
Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx,
KeyRange keys,
Optional<Reference<TransactionState>> trState);
Future<StorageMetrics> doGetStorageMetrics(Database cx,
Version version,
KeyRange keys,
Reference<LocationInfo> locationInfo,
Optional<Reference<TransactionState>> trState) {
Error err;
try {
WaitMetricsRequest req(version, keys, StorageMetrics(), StorageMetrics());
req.min.bytes = 0;
req.max.bytes = -1;
StorageMetrics m = co_await loadBalance(
locationInfo->locations(), &StorageServerInterface::waitMetrics, req, TaskPriority::DataDistribution);
co_return m;
} catch (Error& e) {
err = e;
}
if (err.code() == error_code_wrong_shard_server || err.code() == error_code_all_alternatives_failed) {
cx->invalidateCache(keys);
co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
} else if (err.code() == error_code_future_version) {
co_await delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution);
} else {
TraceEvent(SevError, "DoGetStorageMetricsError").error(err);
throw err;
}
co_return co_await getStorageMetricsLargeKeyRange(cx, keys, trState);
}
Future<StorageMetrics> getStorageMetricsLargeKeyRange(Database cx,
KeyRange keys,
Optional<Reference<TransactionState>> trState) {
Span span("NAPI:GetStorageMetricsLargeKeyRange"_loc);
if (trState.present()) {
co_await trState.get()->startTransaction();
}
Version version = trState.present() ? trState.get()->readVersion() : latestVersion;
std::vector<KeyRangeLocationInfo> locations = co_await getKeyRangeLocations(cx,
keys,
std::numeric_limits<int>::max(),
Reverse::False,
&StorageServerInterface::waitMetrics,
span.context,
Optional<UID>(),
UseProvisionalProxies::False,
version);
int nLocs = locations.size();
std::vector<Future<StorageMetrics>> fx(nLocs);
StorageMetrics total;
KeyRef partBegin, partEnd;
for (int i = 0; i < nLocs; i++) {
partBegin = (i == 0) ? keys.begin : locations[i].range.begin;
partEnd = (i == nLocs - 1) ? keys.end : locations[i].range.end;
fx[i] = doGetStorageMetrics(cx, version, KeyRangeRef(partBegin, partEnd), locations[i].locations, trState);
}
co_await waitForAll(fx);
for (int i = 0; i < nLocs; i++) {
total += fx[i].get();
}
co_return total;
}
Future<Void> trackBoundedStorageMetrics(Version version,
KeyRange keys,
Reference<LocationInfo> location,
StorageMetrics x,
StorageMetrics halfError,
PromiseStream<StorageMetrics> deltaStream) {
try {
while (true) {
WaitMetricsRequest req(version, keys, x - halfError, x + halfError);
StorageMetrics nextX =
co_await loadBalance(location->locations(), &StorageServerInterface::waitMetrics, req);
deltaStream.send(nextX - x);
x = nextX;
}
} catch (Error& e) {
deltaStream.sendError(e);
throw e;
}
}
Future<StorageMetrics> waitStorageMetricsMultipleLocations(Version version,
std::vector<KeyRangeLocationInfo> locations,
StorageMetrics min,
StorageMetrics max,
StorageMetrics permittedError) {
int nLocs = locations.size();
std::vector<Future<StorageMetrics>> fx(nLocs);
StorageMetrics total;
PromiseStream<StorageMetrics> deltas;
std::vector<Future<Void>> wx(fx.size());
StorageMetrics halfErrorPerMachine = permittedError * (0.5 / nLocs);
StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs - 1);
StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs - 1);
for (int i = 0; i < nLocs; i++) {
WaitMetricsRequest req(version, locations[i].range, StorageMetrics(), StorageMetrics());
req.min.bytes = 0;
req.max.bytes = -1;
fx[i] = loadBalance(locations[i].locations->locations(),
&StorageServerInterface::waitMetrics,
req,
TaskPriority::DataDistribution);
}
co_await waitForAll(fx);
// invariant: true total is between (total-permittedError/2, total+permittedError/2)
for (int i = 0; i < nLocs; i++)
total += fx[i].get();
if (!total.allLessOrEqual(maxPlus))
co_return total;
if (!minMinus.allLessOrEqual(total))
co_return total;
for (int i = 0; i < nLocs; i++)
wx[i] = trackBoundedStorageMetrics(
version, locations[i].range, locations[i].locations, fx[i].get(), halfErrorPerMachine, deltas);
while (true) {
StorageMetrics delta = co_await deltas.getFuture();
total += delta;
if (!total.allLessOrEqual(maxPlus))
co_return total;
if (!minMinus.allLessOrEqual(total))
co_return total;
}
}
Future<StorageMetrics> extractMetrics(Future<std::pair<Optional<StorageMetrics>, int>> fMetrics) {
std::pair<Optional<StorageMetrics>, int> x = co_await fMetrics;
co_return x.first.get();
}
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getReadHotRanges(Database cx, KeyRange keys) {
Span span("NAPI:GetReadHotRanges"_loc);
while (true) {
int64_t shardLimit = 100; // Shard limit here does not really matter since this function is currently only used
// to find the read-hot sub ranges within a read-hot shard.
std::vector<KeyRangeLocationInfo> locations =
co_await getKeyRangeLocations(cx,
keys,
shardLimit,
Reverse::False,
&StorageServerInterface::getReadHotRanges,
span.context,
Optional<UID>(),
UseProvisionalProxies::False,
latestVersion);
Error err;
try {
// TODO: how to handle this?
// This function is called whenever a shard becomes read-hot. But somehow the shard was split across more
// than one storage server after becoming read-hot and before this function is called, i.e. a race
// condition. Should we abort and wait for the newly split shards to be hot again?
int nLocs = locations.size();
// if (nLocs > 1) {
// TraceEvent("RHDDebug")
// .detail("NumSSIs", nLocs)
// .detail("KeysBegin", keys.begin.printable().c_str())
// .detail("KeysEnd", keys.end.printable().c_str());
// }
std::vector<Future<ReadHotSubRangeReply>> fReplies(nLocs);
KeyRef partBegin, partEnd;
for (int i = 0; i < nLocs; i++) {
partBegin = (i == 0) ? keys.begin : locations[i].range.begin;
partEnd = (i == nLocs - 1) ? keys.end : locations[i].range.end;
ReadHotSubRangeRequest req(KeyRangeRef(partBegin, partEnd));
fReplies[i] = loadBalance(locations[i].locations->locations(),
&StorageServerInterface::getReadHotRanges,
req,
TaskPriority::DataDistribution);
}
co_await waitForAll(fReplies);
if (nLocs == 1) {
CODE_PROBE(true, "Single-shard read hot range request");
co_return fReplies[0].get().readHotRanges;
} else {
CODE_PROBE(true, "Multi-shard read hot range request");
Standalone<VectorRef<ReadHotRangeWithMetrics>> results;
for (int i = 0; i < nLocs; i++) {
results.append(results.arena(),
fReplies[i].get().readHotRanges.begin(),
fReplies[i].get().readHotRanges.size());
results.arena().dependsOn(fReplies[i].get().readHotRanges.arena());
}
co_return results;
}
} catch (Error& e) {
err = e;
}
if (err.code() != error_code_wrong_shard_server && err.code() != error_code_all_alternatives_failed) {
TraceEvent(SevError, "GetReadHotSubRangesError").error(err);
throw err;
}
cx->invalidateCache(keys);
co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
}
}
Future<Optional<StorageMetrics>> waitStorageMetricsWithLocation(Version version,
KeyRange keys,
std::vector<KeyRangeLocationInfo> locations,
StorageMetrics min,
StorageMetrics max,
StorageMetrics permittedError) {
Future<StorageMetrics> fx;
if (locations.size() > 1) {
fx = waitStorageMetricsMultipleLocations(version, locations, min, max, permittedError);
} else {
WaitMetricsRequest req(version, keys, min, max);
fx = loadBalance(locations[0].locations->locations(),
&StorageServerInterface::waitMetrics,
req,
TaskPriority::DataDistribution);
}
StorageMetrics x = co_await fx;
co_return x;
}
Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(Database cx,
KeyRange keys,
StorageMetrics min,
StorageMetrics max,
StorageMetrics permittedError,
int shardLimit,
int expectedShardCount,
Optional<Reference<TransactionState>> trState) {
Span span("NAPI:WaitStorageMetrics"_loc, generateSpanID(cx->transactionTracingSample));
double startTime = now();
int retryCount = 0;
while (true) {
if (trState.present()) {
co_await trState.get()->startTransaction();
}
Version version = trState.present() ? trState.get()->readVersion() : latestVersion;
std::vector<KeyRangeLocationInfo> locations =
co_await getKeyRangeLocations(cx,
keys,
shardLimit,
Reverse::False,
&StorageServerInterface::waitMetrics,
span.context,
Optional<UID>(),
UseProvisionalProxies::False,
version);
if (expectedShardCount >= 0 && locations.size() != expectedShardCount) {
// NOTE(xwang): This happens only when a split shard haven't been moved to another location. We may need to
// change this if we allow split shard stay the same location.
co_return std::make_pair(Optional<StorageMetrics>(), locations.size());
}
// SOMEDAY: Right now, if there are too many shards we delay and check again later. There may be a better
// solution to this. How could this happen?
if (locations.size() >= shardLimit) {
TraceEvent(SevWarn, "WaitStorageMetricsPenalty")
.detail("Keys", keys)
.detail("Limit", shardLimit)
.detail("LocationSize", locations.size())
.detail("JitteredSecondsOfPenitence", CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY);
co_await delayJittered(CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, TaskPriority::DataDistribution);
// make sure that the next getKeyRangeLocations() call will actually re-fetch the range
cx->invalidateCache(keys);
continue;
}
Error err;
try {
Optional<StorageMetrics> res =
co_await waitStorageMetricsWithLocation(version, keys, locations, min, max, permittedError);
if (res.present()) {
co_return std::make_pair(res, -1);
}
continue;
} catch (Error& e) {
err = e;
}
retryCount++;
// Stays at SevDebug. The previous SevDebug→SevWarn upgrade after 60s
// elapsed didn't actually filter for stuck shards: the SS-side
// waitMetrics is a long-poll with a STORAGE_METRIC_TIMEOUT of 600s,
// and on timeout the SS deliberately returns wrong_shard_server with
// WAIT_METRICS_WRONG_SHARD_CHANCE = 0.1 to force clients to refresh
// their location cache. So most calls that ever hit this catch are
// already past 60s elapsed by design, and the SevWarn was firing on
// normal cluster operation. DD-init stall visibility lives on the
// DDInit* events instead (PR #12913).
TraceEvent(SevDebug, "WaitStorageMetricsHandleError")
.error(err)
.detail("Keys", keys)
.detail("Elapsed", now() - startTime)
.detail("Retries", retryCount);
if (err.code() == error_code_wrong_shard_server || err.code() == error_code_all_alternatives_failed) {
cx->invalidateCache(keys);
co_await delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution);
} else if (err.code() == error_code_future_version) {
co_await delay(CLIENT_KNOBS->FUTURE_VERSION_RETRY_DELAY, TaskPriority::DataDistribution);
} else {
TraceEvent(SevError, "WaitStorageMetricsError").error(err);
throw err;
}
}
}
Future<std::pair<Optional<StorageMetrics>, int>> DatabaseContext::waitStorageMetrics(
KeyRange const& keys,
StorageMetrics const& min,
StorageMetrics const& max,
StorageMetrics const& permittedError,
int shardLimit,
int expectedShardCount,
Optional<Reference<TransactionState>> trState) {
return ::waitStorageMetrics(Database(Reference<DatabaseContext>::addRef(this)),
keys,
min,
max,
permittedError,
shardLimit,
expectedShardCount,
trState);
}
Future<StorageMetrics> DatabaseContext::getStorageMetrics(KeyRange const& keys,
int shardLimit,
Optional<Reference<TransactionState>> trState) {
if (shardLimit > 0) {
StorageMetrics m;
m.bytes = -1;
return extractMetrics(::waitStorageMetrics(Database(Reference<DatabaseContext>::addRef(this)),
keys,
StorageMetrics(),
m,
StorageMetrics(),
shardLimit,
-1,
trState));
} else {
return ::getStorageMetricsLargeKeyRange(Database(Reference<DatabaseContext>::addRef(this)), keys, trState);
}
}
Future<Standalone<VectorRef<DDMetricsRef>>> waitDataDistributionMetricsList(Database cx,
KeyRange keys,
int shardLimit) {
GetDDMetricsReply rep = co_await commitProxyLoadBalance(
cx, makeReqBuilder<GetDDMetricsRequest>(keys, shardLimit), &CommitProxyInterface::getDDMetrics);
co_return rep.storageMetricsList;
}
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> DatabaseContext::getReadHotRanges(KeyRange const& keys) {
return ::getReadHotRanges(Database(Reference<DatabaseContext>::addRef(this)), keys);
}
static int getRangeSplitPointsLocationLimit(int splitPointLimit, int maxLocations, int avoidLocationLimit) {
int locationLimit = splitPointLimit >= 0 && splitPointLimit < maxLocations ? splitPointLimit + 1 : maxLocations;
if (locationLimit == avoidLocationLimit) {
ASSERT(maxLocations > 1);
locationLimit += locationLimit < maxLocations ? 1 : -1;
}
return locationLimit;
}
class RangeSplitPointsBuilder {
Standalone<VectorRef<KeyRef>> results;
int remaining;
public:
RangeSplitPointsBuilder(KeyRef begin, int limit) : remaining(limit) {
results.push_back_deep(results.arena(), begin);
}
int getRemaining() const { return remaining; }
void appendShardBoundary(KeyRef boundary) {
if (results.back() == boundary || remaining == 0) {
return;
}
results.push_back_deep(results.arena(), boundary);
if (remaining > 0) {
--remaining;
}
}
void appendSplitPoints(Standalone<VectorRef<KeyRef>> const& splitPoints) {
int splitPointCount = splitPoints.size();
if (remaining >= 0) {
splitPointCount = std::min(splitPointCount, remaining);
}
if (splitPointCount == 0) {
return;
}
results.append(results.arena(), splitPoints.begin(), splitPointCount);
results.arena().dependsOn(splitPoints.arena());
if (remaining > 0) {
remaining -= splitPointCount;
}
}
Standalone<VectorRef<KeyRef>> finish(KeyRef end) {
if (results.back() != end) {
results.push_back_deep(results.arena(), end);
}
return results;
}
};
ACTOR Future<Standalone<VectorRef<KeyRef>>> getRangeSplitPoints(Reference<TransactionState> trState,
KeyRange keys,
int64_t chunkSize,
int limit) {
state Span span("NAPI:GetRangeSplitPoints"_loc, trState->spanContext);
state Key beginKey = keys.begin;
state RangeSplitPointsBuilder results(keys.begin, limit);
if (limit == 0) {
return results.finish(keys.end);
}
loop {
state std::vector<KeyRangeLocationInfo> locations = wait(getKeyRangeLocations(
trState,
KeyRangeRef(beginKey, keys.end),
getRangeSplitPointsLocationLimit(
results.getRemaining(), CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT),
Reverse::False,
&StorageServerInterface::getRangeSplitPoints));
try {
state int nLocs = locations.size();
if (limit >= 0) {
state int i = 0;
for (; i < nLocs; i++) {
if (i > 0 || beginKey != keys.begin) {
results.appendShardBoundary(locations[i].range.begin);
}
if (results.getRemaining() == 0) {
break;
}
KeyRef partBegin = (i == 0) ? beginKey : locations[i].range.begin;
KeyRef partEnd = std::min(keys.end, locations[i].range.end);
SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize, results.getRemaining());
SplitRangeReply reply = wait(loadBalance(locations[i].locations->locations(),
&StorageServerInterface::getRangeSplitPoints,
req,
TaskPriority::DataDistribution));
results.appendSplitPoints(reply.splitPoints);
}
} else {
state std::vector<Future<SplitRangeReply>> fReplies(nLocs);
for (int i = 0; i < nLocs; i++) {
KeyRef partBegin = (i == 0) ? beginKey : locations[i].range.begin;
KeyRef partEnd = std::min(keys.end, locations[i].range.end);
SplitRangeRequest req(KeyRangeRef(partBegin, partEnd), chunkSize, limit);
fReplies[i] = loadBalance(locations[i].locations->locations(),
&StorageServerInterface::getRangeSplitPoints,
req,
TaskPriority::DataDistribution);
}
wait(waitForAll(fReplies));
for (int i = 0; i < nLocs; i++) {
if (i > 0 || beginKey != keys.begin) {
results.appendShardBoundary(locations[i].range.begin);
}
results.appendSplitPoints(fReplies[i].get().splitPoints);
}
}
if (results.getRemaining() == 0 || keys.end <= locations.back().range.end) {
return results.finish(keys.end);
}
beginKey = locations.back().range.end;
} catch (Error& e) {
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) {
trState->cx->invalidateCache(keys);
beginKey = keys.begin;
results = RangeSplitPointsBuilder(keys.begin, limit);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
} else {
TraceEvent(SevError, "GetRangeSplitPoints").error(e);
throw;
}
}
}
}
Future<Standalone<VectorRef<KeyRef>>> Transaction::getRangeSplitPoints(KeyRange const& keys,
int64_t chunkSize,
int limit) {
return ::getRangeSplitPoints(trState, keys, chunkSize, limit);
}
TEST_CASE("/fdbclient/NativeAPI/rangeSplitPoints/locationLimit") {
constexpr int maxLocations = 1000;
constexpr int dataDistributionLocationLimit = 100;
ASSERT(getRangeSplitPointsLocationLimit(-1, maxLocations, dataDistributionLocationLimit) == maxLocations);
ASSERT(getRangeSplitPointsLocationLimit(0, maxLocations, dataDistributionLocationLimit) == 1);
ASSERT(getRangeSplitPointsLocationLimit(16, maxLocations, dataDistributionLocationLimit) == 17);
ASSERT(getRangeSplitPointsLocationLimit(99, maxLocations, dataDistributionLocationLimit) == 101);
ASSERT(getRangeSplitPointsLocationLimit(9, maxLocations, 10) == 11);
ASSERT(getRangeSplitPointsLocationLimit(maxLocations - 1, maxLocations, dataDistributionLocationLimit) ==
maxLocations);
ASSERT(getRangeSplitPointsLocationLimit(maxLocations, maxLocations, dataDistributionLocationLimit) == maxLocations);
ASSERT(getRangeSplitPointsLocationLimit(
std::numeric_limits<int>::max(), maxLocations, dataDistributionLocationLimit) == maxLocations);
ASSERT(getRangeSplitPointsLocationLimit(maxLocations, maxLocations, maxLocations) == maxLocations - 1);
return Void();
}
TEST_CASE("/fdbclient/NativeAPI/rangeSplitPoints/multipleShards") {
Standalone<VectorRef<KeyRef>> firstShard;
firstShard.push_back_deep(firstShard.arena(), "A1"_sr);
firstShard.push_back_deep(firstShard.arena(), "A2"_sr);
Standalone<VectorRef<KeyRef>> secondShard;
secondShard.push_back_deep(secondShard.arena(), "B1"_sr);
secondShard.push_back_deep(secondShard.arena(), "B2"_sr);
Standalone<VectorRef<KeyRef>> firstShardEndingAtBoundary;
firstShardEndingAtBoundary.push_back_deep(firstShardEndingAtBoundary.arena(), "B"_sr);
RangeSplitPointsBuilder zero("A"_sr, 0);
zero.appendSplitPoints(firstShard);
zero.appendShardBoundary("B"_sr);
Standalone<VectorRef<KeyRef>> zeroResults = zero.finish("Z"_sr);
ASSERT(zeroResults.size() == 2 && zeroResults[0] == "A"_sr && zeroResults[1] == "Z"_sr);
RangeSplitPointsBuilder one("A"_sr, 1);
one.appendSplitPoints(firstShard);
ASSERT(one.getRemaining() == 0);
one.appendShardBoundary("B"_sr);
Standalone<VectorRef<KeyRef>> oneResults = one.finish("Z"_sr);
ASSERT(oneResults.size() == 3 && oneResults[1] == "A1"_sr && oneResults[2] == "Z"_sr);
RangeSplitPointsBuilder two("A"_sr, 2);
two.appendShardBoundary("B"_sr);
ASSERT(two.getRemaining() == 1);
two.appendSplitPoints(secondShard);
ASSERT(two.getRemaining() == 0);
two.appendShardBoundary("C"_sr);
Standalone<VectorRef<KeyRef>> twoResults = two.finish("Z"_sr);
ASSERT(twoResults.size() == 4 && twoResults[1] == "B"_sr && twoResults[2] == "B1"_sr && twoResults[3] == "Z"_sr);
RangeSplitPointsBuilder four("A"_sr, 4);
four.appendSplitPoints(firstShard);
ASSERT(four.getRemaining() == 2);
four.appendShardBoundary("B"_sr);
ASSERT(four.getRemaining() == 1);
four.appendSplitPoints(secondShard);
ASSERT(four.getRemaining() == 0);
Standalone<VectorRef<KeyRef>> fourResults = four.finish("Z"_sr);
ASSERT(fourResults.size() == 6 && fourResults[1] == "A1"_sr && fourResults[2] == "A2"_sr &&
fourResults[3] == "B"_sr && fourResults[4] == "B1"_sr && fourResults[5] == "Z"_sr);
RangeSplitPointsBuilder duplicateBoundary("A"_sr, 2);
duplicateBoundary.appendSplitPoints(firstShardEndingAtBoundary);
ASSERT(duplicateBoundary.getRemaining() == 1);
duplicateBoundary.appendShardBoundary("B"_sr);
ASSERT(duplicateBoundary.getRemaining() == 1);
duplicateBoundary.appendSplitPoints(secondShard);
Standalone<VectorRef<KeyRef>> duplicateBoundaryResults = duplicateBoundary.finish("Z"_sr);
ASSERT(duplicateBoundaryResults.size() == 4 && duplicateBoundaryResults[1] == "B"_sr &&
duplicateBoundaryResults[2] == "B1"_sr && duplicateBoundaryResults[3] == "Z"_sr);
RangeSplitPointsBuilder unlimited("A"_sr, -1);
unlimited.appendSplitPoints(firstShard);
unlimited.appendShardBoundary("B"_sr);
unlimited.appendSplitPoints(secondShard);
unlimited.appendShardBoundary("C"_sr);
Standalone<VectorRef<KeyRef>> unlimitedResults = unlimited.finish("Z"_sr);
ASSERT(unlimitedResults.size() == 8 && unlimitedResults[1] == "A1"_sr && unlimitedResults[2] == "A2"_sr &&
unlimitedResults[3] == "B"_sr && unlimitedResults[4] == "B1"_sr && unlimitedResults[5] == "B2"_sr &&
unlimitedResults[6] == "C"_sr && unlimitedResults[7] == "Z"_sr);
return Void();
}
Future<Version> setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) {
ReadYourWritesTransaction tr(cx);
while (true) {
Error err;
try {
tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
if (lockAware) {
tr.setOption(FDBTransactionOptions::LOCK_AWARE);
}
tr.set(perpetualStorageWiggleKey, enable ? "1"_sr : "0"_sr);
co_await tr.commit();
co_return tr.getCommittedVersion();
} catch (Error& e) {
err = e;
}
co_await tr.onError(err);
}
}
Future<std::vector<std::pair<UID, StorageWiggleValue>>> readStorageWiggleValues(Database cx,
bool primary,
bool use_system_priority) {
StorageWiggleData wiggleState;
auto metadataMap = wiggleState.wigglingStorageServer(PrimaryRegion(primary));
auto tr = makeReference<ReadYourWritesTransaction>(cx);
while (true) {
Error err;
try {
KeyBackedRangeResult<std::pair<UID, StorageWiggleValue>> res;
tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
if (use_system_priority) {
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
}
res = co_await metadataMap.getRange(tr, UID(0, 0), Optional<UID>(), CLIENT_KNOBS->TOO_MANY);
co_await tr->commit();
co_return res.results;
} catch (Error& e) {
err = e;
}
co_await tr->onError(err);
}
}
ACTOR Future<Void> splitStorageMetricsStream(PromiseStream<Key> resultStream,
Database cx,
KeyRange keys,
StorageMetrics limit,
StorageMetrics estimated,
Optional<int> minSplitBytes) {
state Span span("NAPI:SplitStorageMetricsStream"_loc);
state Key beginKey = keys.begin;
state Key globalLastKey = beginKey;
resultStream.send(beginKey);
// track used across loops
state StorageMetrics globalUsed;
loop {
state std::vector<KeyRangeLocationInfo> locations =
wait(getKeyRangeLocations(cx,
KeyRangeRef(beginKey, keys.end),
CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
Reverse::False,
&StorageServerInterface::splitMetrics,
span.context,
Optional<UID>(),
UseProvisionalProxies::False,
latestVersion));
try {
//TraceEvent("SplitStorageMetrics").detail("Locations", locations.size());
state StorageMetrics localUsed = globalUsed;
state Key localLastKey = globalLastKey;
state Standalone<VectorRef<KeyRef>> results;
state int i = 0;
for (; i < locations.size(); i++) {
SplitMetricsRequest req(locations[i].range,
limit,
localUsed,
estimated,
i == locations.size() - 1 && keys.end <= locations.back().range.end,
minSplitBytes);
SplitMetricsReply res = wait(loadBalance(locations[i].locations->locations(),
&StorageServerInterface::splitMetrics,
req,
TaskPriority::DataDistribution));
if (res.splits.size() &&
res.splits[0] <= localLastKey) { // split points are out of order, possibly because
// of moving data, throw error to retry
ASSERT_WE_THINK(false); // FIXME: This seems impossible and doesn't seem to be covered by testing
throw all_alternatives_failed();
}
if (res.splits.size()) {
results.append(results.arena(), res.splits.begin(), res.splits.size());
results.arena().dependsOn(res.splits.arena());
localLastKey = res.splits.back();
}
localUsed = res.used;
//TraceEvent("SplitStorageMetricsResult").detail("Used", used.bytes).detail("Location", i).detail("Size", res.splits.size());
}
globalUsed = localUsed;
// only truncate split at end
if (keys.end <= locations.back().range.end &&
globalUsed.allLessOrEqual(limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT) &&
results.size() > 1) {
results.resize(results.arena(), results.size() - 1);
localLastKey = results.back();
}
globalLastKey = localLastKey;
for (auto& splitKey : results) {
resultStream.send(splitKey);
}
if (keys.end <= locations.back().range.end) {
resultStream.send(keys.end);
resultStream.sendError(end_of_stream());
break;
} else {
beginKey = locations.back().range.end;
}
} catch (Error& e) {
if (e.code() == error_code_operation_cancelled) {
throw e;
}
if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) {
TraceEvent(SevError, "SplitStorageMetricsStreamError").error(e);
resultStream.sendError(e);
throw;
}
cx->invalidateCache(keys);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
}
}
return Void();
}
Future<Void> DatabaseContext::splitStorageMetricsStream(const PromiseStream<Key>& resultStream,
KeyRange const& keys,
StorageMetrics const& limit,
StorageMetrics const& estimated,
Optional<int> const& minSplitBytes) {
return ::splitStorageMetricsStream(
resultStream, Database(Reference<DatabaseContext>::addRef(this)), keys, limit, estimated, minSplitBytes);
}
ACTOR Future<Optional<Standalone<VectorRef<KeyRef>>>> splitStorageMetricsWithLocations(
std::vector<KeyRangeLocationInfo> locations,
KeyRange keys,
StorageMetrics limit,
StorageMetrics estimated,
Optional<int> minSplitBytes) {
state StorageMetrics used;
state Standalone<VectorRef<KeyRef>> results;
results.push_back_deep(results.arena(), keys.begin);
//TraceEvent("SplitStorageMetrics").detail("Locations", locations.size());
try {
state int i = 0;
for (; i < locations.size(); i++) {
state Key beginKey = locations[i].range.begin;
loop {
KeyRangeRef range(beginKey, locations[i].range.end);
SplitMetricsRequest req(range, limit, used, estimated, i == locations.size() - 1, minSplitBytes);
SplitMetricsReply res = wait(loadBalance(locations[i].locations->locations(),
&StorageServerInterface::splitMetrics,
req,
TaskPriority::DataDistribution));
if (res.splits.size() &&
res.splits[0] <= results.back()) { // split points are out of order, possibly
// because of moving data, throw error to retry
ASSERT_WE_THINK(false); // FIXME: This seems impossible and doesn't seem to be covered by testing
throw all_alternatives_failed();
}
if (res.splits.size()) {
results.append(results.arena(), res.splits.begin(), res.splits.size());
results.arena().dependsOn(res.splits.arena());
}
used = res.used;
if (res.more && res.splits.size()) {
// Next request will return split points after this one
beginKey = KeyRef(beginKey.arena(), res.splits.back());
} else {
break;
}
//TraceEvent("SplitStorageMetricsResult").detail("Used", used.bytes).detail("Location", i).detail("Size", res.splits.size());
}
}
if (used.allLessOrEqual(limit * CLIENT_KNOBS->STORAGE_METRICS_UNFAIR_SPLIT_LIMIT) && results.size() > 1) {
results.resize(results.arena(), results.size() - 1);
}
if (keys.end <= locations.back().range.end) {
results.push_back_deep(results.arena(), keys.end);
}
return results;
} catch (Error& e) {
if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) {
TraceEvent(SevError, "SplitStorageMetricsError").error(e);
throw;
}
}
return Optional<Standalone<VectorRef<KeyRef>>>();
}
ACTOR Future<Standalone<VectorRef<KeyRef>>> splitStorageMetrics(Database cx,
KeyRange keys,
StorageMetrics limit,
StorageMetrics estimated,
Optional<int> minSplitBytes) {
state Span span("NAPI:SplitStorageMetrics"_loc);
loop {
state std::vector<KeyRangeLocationInfo> locations =
wait(getKeyRangeLocations(cx,
keys,
CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT,
Reverse::False,
&StorageServerInterface::splitMetrics,
span.context,
Optional<UID>(),
UseProvisionalProxies::False,
latestVersion));
// SOMEDAY: Right now, if there are too many shards we delay and check again later. There may be a better
// solution to this.
if (locations.size() == CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT) {
wait(delay(CLIENT_KNOBS->STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, TaskPriority::DataDistribution));
cx->invalidateCache(keys);
continue;
}
Optional<Standalone<VectorRef<KeyRef>>> results =
wait(splitStorageMetricsWithLocations(locations, keys, limit, estimated, minSplitBytes));
if (results.present()) {
return results.get();
}
cx->invalidateCache(keys);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution));
}
}
Future<Standalone<VectorRef<KeyRef>>> DatabaseContext::splitStorageMetrics(KeyRange const& keys,
StorageMetrics const& limit,
StorageMetrics const& estimated,
Optional<int> const& minSplitBytes) {
return ::splitStorageMetrics(
Database(Reference<DatabaseContext>::addRef(this)), keys, limit, estimated, minSplitBytes);
}
void Transaction::checkDeferredError() const {
trState->cx->checkDeferredError();
}
Reference<TransactionLogInfo> Transaction::createTrLogInfoProbabilistically(const Database& cx) {
if (!cx->isError() && cx->globalConfig) {
// Note: For internal (fdbserver) databases, globalConfig->init() may not have been called,
// so we need to handle the case where globalConfig exists but isn't initialized yet.
// In that case, get() will return an empty Reference and we'll use the default value.
double sampleRate =
cx->globalConfig->get<double>(fdbClientInfoTxnSampleRate, std::numeric_limits<double>::infinity());
double clientSamplingProbability = std::isinf(sampleRate) ? CLIENT_KNOBS->CSI_SAMPLING_PROBABILITY : sampleRate;
if (((networkOptions.logClientInfo.present() && networkOptions.logClientInfo.get()) || buggify()) &&
deterministicRandom()->random01() < clientSamplingProbability &&
(!g_network->isSimulated() || !g_simulator->speedUpSimulation)) {
return makeReference<TransactionLogInfo>(TransactionLogInfo::DATABASE);
}
}
return Reference<TransactionLogInfo>();
}
void Transaction::setTransactionID(UID id) {
ASSERT(getSize() == 0);
trState->spanContext = SpanContext(id, trState->spanContext.spanID, trState->spanContext.m_Flags);
tr.spanContext = trState->spanContext;
span.context = trState->spanContext;
}
void Transaction::setToken(uint64_t token) {
ASSERT(getSize() == 0);
trState->spanContext = SpanContext(trState->spanContext.traceID, token);
}
void enableClientInfoLogging() {
ASSERT(networkOptions.logClientInfo.present() == false);
networkOptions.logClientInfo = true;
TraceEvent(SevInfo, "ClientInfoLoggingEnabled").log();
}
Future<Void> snapCreate(Database cx, Standalone<StringRef> snapCmd, UID snapUID) {
TraceEvent("SnapCreateEnter").detail("SnapCmd", snapCmd).detail("UID", snapUID);
try {
co_await commitProxyLoadBalance(cx,
makeReqBuilder<ProxySnapRequest>(snapCmd, snapUID, snapUID),
&CommitProxyInterface::proxySnapReq,
AtMostOnce::True);
TraceEvent("SnapCreateExit").detail("SnapCmd", snapCmd).detail("UID", snapUID);
} catch (Error& e) {
TraceEvent("SnapCreateError").error(e).detail("SnapCmd", snapCmd.toString()).detail("UID", snapUID);
throw;
}
}
ACTOR template <class T>
static Future<Void> createCheckpointImpl(T tr,
std::vector<KeyRange> ranges,
CheckpointFormat format,
Optional<UID> actionId) {
ASSERT(!ranges.empty());
ASSERT(actionId.present());
TraceEvent(SevDebug, "CreateCheckpointTransactionBegin").detail("Ranges", describe(ranges));
state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY));
ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY);
state std::unordered_map<UID, std::vector<KeyRange>> rangeMap;
state std::unordered_map<UID, std::vector<UID>> srcMap;
for (const auto& range : ranges) {
RangeResult keyServers = wait(krmGetRanges(tr, keyServersPrefix, range));
ASSERT(!keyServers.more);
for (int i = 0; i < keyServers.size() - 1; ++i) {
const KeyRangeRef currentRange(keyServers[i].key, keyServers[i + 1].key);
std::vector<UID> src;
std::vector<UID> dest;
UID srcId;
UID destId;
decodeKeyServersValue(UIDtoTagMap, keyServers[i].value, src, dest, srcId, destId);
rangeMap[srcId].push_back(currentRange);
srcMap.emplace(srcId, src);
}
}
if (format == DataMoveRocksCF) {
for (const auto& [srcId, ranges] : rangeMap) {
// The checkpoint request is sent to all replicas, in case any of them is unhealthy.
// An alternative is to choose a healthy replica.
const UID checkpointID = UID(deterministicRandom()->randomUInt64(), srcId.first());
CheckpointMetaData checkpoint(ranges, format, srcMap[srcId], checkpointID, actionId.get());
checkpoint.setState(CheckpointMetaData::Pending);
tr->set(checkpointKeyFor(checkpointID), checkpointValue(checkpoint));
TraceEvent(SevDebug, "CreateCheckpointTransactionShard")
.detail("CheckpointKey", checkpointKeyFor(checkpointID))
.detail("CheckpointMetaData", checkpoint.toString());
}
} else {
throw not_implemented();
}
return Void();
}
Future<Void> createCheckpoint(Reference<ReadYourWritesTransaction> tr,
const std::vector<KeyRange>& ranges,
CheckpointFormat format,
Optional<UID> actionId) {
return holdWhile(tr, createCheckpointImpl(tr, ranges, format, actionId));
}
Future<Void> createCheckpoint(Transaction* tr,
const std::vector<KeyRange>& ranges,
CheckpointFormat format,
Optional<UID> actionId) {
return createCheckpointImpl(tr, ranges, format, actionId);
}
// Gets CheckpointMetaData of the specific keyrange, version and format from one of the storage servers, if none of the
// servers have the checkpoint, a checkpoint_not_found error is returned.
static Future<CheckpointMetaData> getCheckpointMetaDataInternal(KeyRange range,
Version version,
CheckpointFormat format,
Optional<UID> actionId,
Reference<LocationInfo> alternatives,
double timeout) {
TraceEvent(SevDebug, "GetCheckpointMetaDataInternalBegin")
.detail("Range", range)
.detail("Version", version)
.detail("Format", static_cast<int>(format))
.detail("Locations", alternatives->description());
std::vector<Future<ErrorOr<CheckpointMetaData>>> futures;
int index = 0;
for (index = 0; index < alternatives->size(); ++index) {
// For each shard, all storage servers are checked, only one is required.
futures.push_back(errorOr(timeoutError(alternatives->getInterface(index).checkpoint.getReply(
GetCheckpointRequest({ range }, version, format, actionId)),
timeout)));
}
Optional<Error> error;
co_await waitForAll(futures);
TraceEvent(SevDebug, "GetCheckpointMetaDataInternalWaitEnd").detail("Range", range).detail("Version", version);
for (index = 0; index < futures.size(); ++index) {
if (!futures[index].isReady()) {
error = timed_out();
TraceEvent(SevDebug, "GetCheckpointMetaDataInternalSSTimeout")
.detail("Range", range)
.detail("Version", version)
.detail("StorageServer", alternatives->getInterface(index).uniqueID);
continue;
}
if (futures[index].get().isError()) {
const Error& e = futures[index].get().getError();
TraceEvent(SevWarn, "GetCheckpointMetaDataInternalError")
.errorUnsuppressed(e)
.detail("Range", range)
.detail("Version", version)
.detail("StorageServer", alternatives->getInterface(index).uniqueID);
if (e.code() != error_code_checkpoint_not_found || !error.present()) {
error = e;
}
} else {
co_return futures[index].get().get();
}
}
ASSERT(error.present());
throw error.get();
}
ACTOR static Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpointMetaDataForRange(
Database cx,
KeyRange range,
Version version,
CheckpointFormat format,
Optional<UID> actionId,
double timeout) {
state Span span("NAPI:GetCheckpointMetaDataForRange"_loc);
state int index = 0;
state std::vector<Future<CheckpointMetaData>> futures;
state std::vector<KeyRangeLocationInfo> locations;
loop {
locations.clear();
TraceEvent(SevDebug, "GetCheckpointMetaDataForRangeBegin")
.detail("Range", range.toString())
.detail("Version", version)
.detail("Format", static_cast<int>(format));
futures.clear();
try {
wait(store(locations,
getKeyRangeLocations(cx,
range,
CLIENT_KNOBS->TOO_MANY,
Reverse::False,
&StorageServerInterface::checkpoint,
span.context,
Optional<UID>(),
UseProvisionalProxies::False,
latestVersion)));
for (index = 0; index < locations.size(); ++index) {
futures.push_back(getCheckpointMetaDataInternal(
locations[index].range, version, format, actionId, locations[index].locations, timeout));
TraceEvent(SevDebug, "GetCheckpointShardBegin")
.detail("Range", locations[index].range)
.detail("Version", version)
.detail("StorageServers", locations[index].locations->description());
}
choose {
when(wait(cx->connectionFileChanged())) {
cx->invalidateCache(range);
}
when(wait(waitForAll(futures))) {
break;
}
when(wait(delay(timeout))) {
TraceEvent(SevWarn, "GetCheckpointTimeout").detail("Range", range).detail("Version", version);
}
}
} catch (Error& e) {
TraceEvent(SevWarn, "GetCheckpointError").errorUnsuppressed(e).detail("Range", range);
if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed ||
e.code() == error_code_connection_failed || e.code() == error_code_broken_promise) {
cx->invalidateCache(range);
wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY));
} else {
throw;
}
}
}
std::vector<std::pair<KeyRange, CheckpointMetaData>> res;
for (index = 0; index < futures.size(); ++index) {
TraceEvent(SevDebug, "GetCheckpointShardEnd")
.detail("Range", locations[index].range)
.detail("Checkpoint", futures[index].get().toString());
res.emplace_back(locations[index].range, futures[index].get());
}
return res;
}
Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>> getCheckpointMetaData(Database cx,
std::vector<KeyRange> ranges,
Version version,
CheckpointFormat format,
Optional<UID> actionId,
double timeout) {
std::vector<Future<std::vector<std::pair<KeyRange, CheckpointMetaData>>>> futures;
// TODO(heliu): Avoid send requests to the same shard.
for (const auto& range : ranges) {
futures.push_back(getCheckpointMetaDataForRange(cx, range, version, format, actionId, timeout));
}
std::vector<std::vector<std::pair<KeyRange, CheckpointMetaData>>> results = co_await getAll(futures);
std::vector<std::pair<KeyRange, CheckpointMetaData>> res;
for (const auto& r : results) {
ASSERT(!r.empty());
res.insert(res.end(), r.begin(), r.end());
}
co_return res;
}
Future<bool> checkSafeExclusions(Database cx, std::vector<AddressExclusion> exclusions) {
TraceEvent("ExclusionSafetyCheckBegin")
.detail("NumExclusion", exclusions.size())
.detail("Exclusions", describe(exclusions));
bool ddCheck{ false };
try {
ExclusionSafetyCheckReply _ddCheck =
co_await commitProxyLoadBalance(cx,
makeReqBuilder<ExclusionSafetyCheckRequest>(exclusions),
&CommitProxyInterface::exclusionSafetyCheckReq);
ddCheck = _ddCheck.safe;
} catch (Error& e) {
if (e.code() != error_code_actor_cancelled) {
TraceEvent("ExclusionSafetyCheckError")
.error(e)
.detail("NumExclusion", exclusions.size())
.detail("Exclusions", describe(exclusions));
}
throw;
}
TraceEvent("ExclusionSafetyCheckCoordinators").log();
ClientCoordinators coordinatorList(cx->getConnectionRecord());
std::vector<Future<Optional<LeaderInfo>>> leaderServers;
leaderServers.reserve(coordinatorList.clientLeaderServers.size());
for (const auto& clientLeaderServer : coordinatorList.clientLeaderServers) {
if (clientLeaderServer.hostname.present()) {
leaderServers.push_back(retryGetReplyFromHostname(GetLeaderRequest(coordinatorList.clusterKey, UID()),
clientLeaderServer.hostname.get(),
WLTOKEN_CLIENTLEADERREG_GETLEADER,
TaskPriority::CoordinationReply));
} else {
leaderServers.push_back(retryBrokenPromise(clientLeaderServer.getLeader,
GetLeaderRequest(coordinatorList.clusterKey, UID()),
TaskPriority::CoordinationReply));
}
}
// Wait for quorum so we don't dismiss live coordinators as unreachable by acting too fast
auto res = co_await race(smartQuorum(leaderServers, leaderServers.size() / 2 + 1, 1.0), delay(3.0));
if (res.index() == 1) {
TraceEvent("ExclusionSafetyCheckNoCoordinatorQuorum").log();
co_return false;
}
int attemptCoordinatorExclude = 0;
int coordinatorsUnavailable = 0;
for (int i = 0; i < leaderServers.size(); ++i) {
NetworkAddress leaderAddress =
coordinatorList.clientLeaderServers[i].getLeader.getEndpoint().getPrimaryAddress();
if (leaderServers[i].isReady()) {
if ((std::count(
exclusions.begin(), exclusions.end(), AddressExclusion(leaderAddress.ip, leaderAddress.port)) ||
std::count(exclusions.begin(), exclusions.end(), AddressExclusion(leaderAddress.ip)))) {
attemptCoordinatorExclude++;
}
} else {
coordinatorsUnavailable++;
}
}
int faultTolerance = (leaderServers.size() - 1) / 2 - coordinatorsUnavailable;
bool coordinatorCheck = (attemptCoordinatorExclude <= faultTolerance);
TraceEvent("ExclusionSafetyCheckFinish")
.detail("CoordinatorListSize", leaderServers.size())
.detail("NumExclusions", exclusions.size())
.detail("FaultTolerance", faultTolerance)
.detail("AttemptCoordinatorExclude", attemptCoordinatorExclude)
.detail("CoordinatorCheck", coordinatorCheck)
.detail("DataDistributorCheck", ddCheck);
co_return ddCheck&& coordinatorCheck;
}
// returns true if we can connect to the given worker interface
ACTOR Future<bool> verifyInterfaceActor(Reference<FlowLock> connectLock, ClientWorkerInterface workerInterf) {
wait(connectLock->take());
state FlowLock::Releaser releaser(*connectLock);
state ClientLeaderRegInterface leaderInterf(workerInterf.address());
choose {
when(Optional<LeaderInfo> rep =
wait(brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())))) {
return true;
}
when(wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT))) {
// NOTE : change timeout time here if necessary
return false;
}
}
}
static Future<int64_t> rebootWorkerActor(DatabaseContext* cx, ValueRef addr, bool check, int duration) {
// ignore negative value
if (duration < 0)
duration = 0;
if (!cx->getConnectionRecord())
co_return 0;
// fetch all workers' addresses and interfaces from CC
RangeResult kvs = co_await getWorkerInterfaces(cx->getConnectionRecord());
ASSERT(!kvs.more);
// map worker network address to its interface
std::map<Key, ClientWorkerInterface> workerInterfaces;
for (const auto& it : kvs) {
auto workerInterf = BinaryReader::fromStringRef<ClientWorkerInterface>(it.value, IncludeVersion());
Key primaryAddress = it.key.endsWith(":tls"_sr) ? it.key.removeSuffix(":tls"_sr) : it.key;
workerInterfaces[primaryAddress] = workerInterf;
// Also add mapping from a worker's second address(if present) to its interface
if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) {
Key secondAddress =
StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString());
secondAddress = secondAddress.endsWith(":tls"_sr) ? secondAddress.removeSuffix(":tls"_sr) : secondAddress;
workerInterfaces[secondAddress] = workerInterf;
}
}
// split and get all the requested addresses to send reboot requests
std::vector<std::string> addressesVec;
boost::algorithm::split(addressesVec, addr.toString(), boost::is_any_of(","));
// Note: reuse this knob from fdbcli, change it if necessary
Reference<FlowLock> connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM));
std::vector<Future<bool>> verifyInterfs;
for (const auto& requestedAddress : addressesVec) {
// step 1: check that the requested address is in the worker list provided by CC
if (!workerInterfaces.count(Key(requestedAddress)))
co_return 0;
// step 2: try to establish connections to the requested worker
verifyInterfs.push_back(verifyInterfaceActor(connectLock, workerInterfaces[Key(requestedAddress)]));
}
// step 3: check if we can establish connections to all requested workers, return if not
co_await waitForAll(verifyInterfs);
for (const auto& f : verifyInterfs) {
if (!f.get())
co_return 0;
}
// step 4: After verifying we can connect to all requested workers, send reboot requests together
for (const auto& address : addressesVec) {
// Note: We want to make sure these requests are sent in parallel
workerInterfaces[Key(address)].reboot.send(RebootRequest(false, check, duration));
}
co_return 1;
}
Future<int64_t> DatabaseContext::rebootWorker(StringRef addr, bool check, int duration) {
return rebootWorkerActor(this, addr, check, duration);
}
Future<Void> DatabaseContext::forceRecoveryWithDataLoss(StringRef dcId) {
return forceRecovery(getConnectionRecord(), dcId);
}
static Future<Void> createSnapshotActor(DatabaseContext* cx, UID snapUID, StringRef snapCmd) {
co_await mgmtSnapCreate(cx->clone(), snapCmd, snapUID);
}
Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_command) {
std::string uid_str = uid.toString();
if (!std::all_of(uid_str.begin(), uid_str.end(), [](unsigned char c) { return std::isxdigit(c); }) ||
uid_str.size() != 32) {
// only 32-length hex string is considered as a valid UID
throw snap_invalid_uid_string();
}
return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
}
void sharedStateDelRef(DatabaseSharedState* ssPtr) {
if (--ssPtr->refCount == 0) {
delete ssPtr;
}
}
Future<DatabaseSharedState*> DatabaseContext::initSharedState() {
ASSERT(!sharedStatePtr); // Don't re-initialize shared state if a pointer already exists
auto* newState = new DatabaseSharedState();
// Increment refcount by 1 on creation to account for the one held in MultiVersionApi map
// Therefore, on initialization, refCount should be 2 (after also going to setSharedState)
newState->refCount++;
newState->delRef = &sharedStateDelRef;
setSharedState(newState);
return newState;
}
void DatabaseContext::setSharedState(DatabaseSharedState* p) {
ASSERT(p->protocolVersion == currentProtocolVersion());
sharedStatePtr = p;
sharedStatePtr->refCount++;
}
Reference<DatabaseContext::TransactionT> DatabaseContext::createTransaction() {
return makeReference<ReadYourWritesTransaction>(Database(Reference<DatabaseContext>::addRef(this)));
}
static Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> getHotRangeMetricsActor(Reference<DatabaseContext> db,
StorageServerInterface ssi,
ReadHotSubRangeRequest req) {
ErrorOr<ReadHotSubRangeReply> fs = co_await ssi.getReadHotRanges.tryGetReply(req);
if (fs.isError()) {
fmt::print("Error({}): cannot get read hot metrics from storage server {}.\n",
fs.getError().what(),
ssi.address().toString());
co_return Standalone<VectorRef<ReadHotRangeWithMetrics>>();
} else {
co_return fs.get().readHotRanges;
}
}
Future<Standalone<VectorRef<ReadHotRangeWithMetrics>>> DatabaseContext::getHotRangeMetrics(
StorageServerInterface ssi,
const KeyRange& keys,
ReadHotSubRangeRequest::SplitType type,
int splitCount) {
return getHotRangeMetricsActor(
Reference<DatabaseContext>::addRef(this), ssi, ReadHotSubRangeRequest(keys, type, splitCount));
}
int64_t getMaxKeySize(KeyRef const& key) {
return getMaxWriteKeySize(key, true);
}
int64_t getMaxReadKeySize(KeyRef const& key) {
return getMaxKeySize(key);
}
int64_t getMaxWriteKeySize(KeyRef const& key, bool hasRawAccess) {
return key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT;
}
int64_t getMaxClearKeySize(KeyRef const& key) {
return getMaxKeySize(key);
}
namespace NativeAPI {
Future<std::vector<std::pair<StorageServerInterface, ProcessClass>>> getServerListAndProcessClasses(Transaction* tr) {
Future<std::vector<ProcessData>> workers = getWorkers(tr);
Future<RangeResult> serverList = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
co_await (success(workers) && success(serverList));
ASSERT(!serverList.get().more && serverList.get().size() < CLIENT_KNOBS->TOO_MANY);
std::map<Optional<Standalone<StringRef>>, ProcessData> id_data;
for (const auto& worker : workers.get())
id_data[worker.locality.processId()] = worker;
std::vector<std::pair<StorageServerInterface, ProcessClass>> results;
for (const auto& server : serverList.get()) {
auto ssi = decodeServerListValue(server.value);
results.emplace_back(ssi, id_data[ssi.locality.processId()].processClass);
}
co_return results;
}
} // namespace NativeAPI
Reference in New Issue View Git Blame Copy Permalink