wbtiger
/
foundationdb
mirror of https://github.com/apple/foundationdb.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into readaware

This commit is contained in:
Xiaoxi Wang 2022-04-08 11:50:01 -07:00
parent c858b0eaa9 4eb0adc51c
commit 5113277e9a
74 changed files with 2721 additions and 790 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/c/CMakeLists.txt
View File

@ -1,6 +1,8 @@
set(FDB_C_SRCS
fdb_c.cpp
foundationdb/fdb_c.h)
foundationdb/fdb_c.h
foundationdb/fdb_c_internal.h
foundationdb/fdb_c_types.h)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/foundationdb)

17
bindings/c/fdb_c.cpp
View File

@ -19,6 +19,7 @@
*/
#include "fdbclient/FDBTypes.h"
#include "flow/ProtocolVersion.h"
#include <cstdint>
#define FDB_API_VERSION 710
#define FDB_INCLUDE_LEGACY_TYPES
@ -26,6 +27,7 @@
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/MultiVersionAssignmentVars.h"
#include "foundationdb/fdb_c.h"
#include "foundationdb/fdb_c_internal.h"
int g_api_version = 0;
@ -293,6 +295,10 @@ extern "C" DLLEXPORT fdb_error_t fdb_future_get_mappedkeyvalue_array(FDBFuture*
*out_more = rrr.more;);
}
extern "C" DLLEXPORT fdb_error_t fdb_future_get_shared_state(FDBFuture* f, DatabaseSharedState** outPtr) {
CATCH_AND_RETURN(*outPtr = (DatabaseSharedState*)((TSAV(DatabaseSharedState*, f)->get())););
}
extern "C" DLLEXPORT fdb_error_t fdb_future_get_string_array(FDBFuture* f, const char*** out_strings, int* out_count) {
CATCH_AND_RETURN(Standalone<VectorRef<const char*>> na = TSAV(Standalone<VectorRef<const char*>>, f)->get();
*out_strings = (const char**)na.begin();
@ -426,6 +432,17 @@ extern "C" DLLEXPORT FDBFuture* fdb_database_create_snapshot(FDBDatabase* db,
.extractPtr());
}
extern "C" DLLEXPORT FDBFuture* fdb_database_create_shared_state(FDBDatabase* db) {
return (FDBFuture*)(DB(db)->createSharedState().extractPtr());
}
extern "C" DLLEXPORT void fdb_database_set_shared_state(FDBDatabase* db, DatabaseSharedState* p) {
try {
DB(db)->setSharedState(p);
} catch (...) {
}
}
// Get network thread busyness (updated every 1s)
// A value of 0 indicates that the client is more or less idle
// A value of 1 (or more) indicates that the client is saturated

11
bindings/c/foundationdb/fdb_c.h
View File

@ -58,21 +58,12 @@
#include <stdint.h>
#include "fdb_c_options.g.h"
#include "fdb_c_types.h"
#ifdef __cplusplus
extern "C" {
#endif
/* Pointers to these opaque types represent objects in the FDB API */
typedef struct FDB_future FDBFuture;
typedef struct FDB_result FDBResult;
typedef struct FDB_database FDBDatabase;
typedef struct FDB_tenant FDBTenant;
typedef struct FDB_transaction FDBTransaction;
typedef int fdb_error_t;
typedef int fdb_bool_t;
DLLEXPORT const char* fdb_get_error(fdb_error_t code);
DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_error_t code);

52
bindings/c/foundationdb/fdb_c_internal.h Normal file
View File

@ -0,0 +1,52 @@
/*
* fdb_c_internal.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDB_C_INTERNAL_H
#define FDB_C_INTERNAL_H
#include "flow/ProtocolVersion.h"
#pragma once
#ifndef DLLEXPORT
#define DLLEXPORT
#endif
#ifndef WARN_UNUSED_RESULT
#define WARN_UNUSED_RESULT
#endif
#include "fdb_c_types.h"
#ifdef __cplusplus
extern "C" {
#endif
// forward declaration and typedef
typedef struct DatabaseSharedState DatabaseSharedState;
DLLEXPORT FDBFuture* fdb_database_create_shared_state(FDBDatabase* db);
DLLEXPORT void fdb_database_set_shared_state(FDBDatabase* db, DatabaseSharedState* p);
DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_future_get_shared_state(FDBFuture* f, DatabaseSharedState** outPtr);
#ifdef __cplusplus
}
#endif
#endif

47
bindings/c/foundationdb/fdb_c_types.h Normal file
View File

@ -0,0 +1,47 @@
/*
* fdb_c_types.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDB_C_TYPES_H
#define FDB_C_TYPES_H
#pragma once
#ifndef DLLEXPORT
#define DLLEXPORT
#endif
#ifdef __cplusplus
extern "C" {
#endif
/* Pointers to these opaque types represent objects in the FDB API */
typedef struct FDB_future FDBFuture;
typedef struct FDB_result FDBResult;
typedef struct FDB_cluster FDBCluster;
typedef struct FDB_database FDBDatabase;
typedef struct FDB_tenant FDBTenant;
typedef struct FDB_transaction FDBTransaction;
typedef int fdb_error_t;
typedef int fdb_bool_t;
#ifdef __cplusplus
}
#endif
#endif

3
bindings/c/test/apitester/TesterCorrectnessWorkload.cpp
View File

@ -83,8 +83,7 @@ private:
auto results = std::make_shared<std::vector<std::optional<std::string>>>();
execTransaction(
[kvPairs, results](auto ctx) {
// TODO: Enable after merging with GRV caching
// ctx->tx()->setOption(FDB_TR_OPTION_USE_GRV_CACHE);
ctx->tx()->setOption(FDB_TR_OPTION_USE_GRV_CACHE);
auto futures = std::make_shared<std::vector<Future>>();
for (const auto& kv : *kvPairs) {
futures->push_back(ctx->tx()->get(kv.key, false));

4
documentation/sphinx/source/configuration.rst
View File

@ -271,6 +271,7 @@ Using the default parameters, a process will restart immediately if it fails and
# maxlogssize = 100MiB
# class =
# memory = 8GiB
# memory-vsize =
# storage-memory = 1GiB
# cache-memory = 2GiB
# locality-machineid =
@ -292,7 +293,8 @@ Contains default parameters for all fdbserver processes on this machine. These s
* ``logsize``: Roll over to a new log file after the current log file reaches the specified size. The default value is 10MiB.
* ``maxlogssize``: Delete the oldest log file when the total size of all log files exceeds the specified size. If set to 0B, old log files will not be deleted. The default value is 100MiB.
* ``class``: Process class specifying the roles that will be taken in the cluster. Recommended options are ``storage``, ``transaction``, ``stateless``. See :ref:`guidelines-process-class-config` for process class config recommendations.
* ``memory``: Maximum memory used by the process. The default value is 8GiB. When specified without a unit, MiB is assumed. This parameter does not change the memory allocation of the program. Rather, it sets a hard limit beyond which the process will kill itself and be restarted. The default value of 8GiB is double the intended memory usage in the default configuration (providing an emergency buffer to deal with memory leaks or similar problems). It is *not* recommended to decrease the value of this parameter below its default value. It may be *increased* if you wish to allocate a very large amount of storage engine memory or cache. In particular, when the ``storage-memory`` or ``cache-memory`` parameters are increased, the ``memory`` parameter should be increased by an equal amount.
* ``memory``: Maximum resident memory used by the process. The default value is 8GiB. When specified without a unit, MiB is assumed. Setting to 0 means unlimited. This parameter does not change the memory allocation of the program. Rather, it sets a hard limit beyond which the process will kill itself and be restarted. The default value of 8GiB is double the intended memory usage in the default configuration (providing an emergency buffer to deal with memory leaks or similar problems). It is *not* recommended to decrease the value of this parameter below its default value. It may be *increased* if you wish to allocate a very large amount of storage engine memory or cache. In particular, when the ``storage-memory`` or ``cache-memory`` parameters are increased, the ``memory`` parameter should be increased by an equal amount.
* ``memory-vsize``: Maximum virtual memory used by the process. The default value is 0, which means unlimited. When specified without a unit, MiB is assumed. Same as ``memory``, this parameter does not change the memory allocation of the program. Rather, it sets a hard limit beyond which the process will kill itself and be restarted.
* ``storage-memory``: Maximum memory used for data storage. This parameter is used *only* with memory storage engine, not the ssd storage engine. The default value is 1GiB. When specified without a unit, MB is assumed. Clusters will be restricted to using this amount of memory per process for purposes of data storage. Memory overhead associated with storing the data is counted against this total. If you increase the ``storage-memory`` parameter, you should also increase the ``memory`` parameter by the same amount.
* ``cache-memory``: Maximum memory used for caching pages from disk. The default value is 2GiB. When specified without a unit, MiB is assumed. If you increase the ``cache-memory`` parameter, you should also increase the ``memory`` parameter by the same amount.
* ``locality-machineid``: Machine identifier key. All processes on a machine should share a unique id. By default, processes on a machine determine a unique id to share. This does not generally need to be set.

2
documentation/sphinx/source/release-notes/release-notes-700.rst
View File

@ -1,5 +1,3 @@
.. _release-notes:
#############
Release Notes
#############

58
documentation/sphinx/source/release-notes/release-notes-710.rst Normal file
View File

@ -0,0 +1,58 @@
.. _release-notes:
#############
Release Notes
#############
7.1.0
=====
Features
--------
* Added ``USE_GRV_CACHE`` transaction option to allow read versions to be locally cached on the client side for latency optimizations. `(PR #5725) <https://github.com/apple/foundationdb/pull/5725>`_ `(PR #6664) <https://github.com/apple/foundationdb/pull/6664>`_
Performance
-----------
Reliability
-----------
Fixes
-----
Status
------
Bindings
--------
Other Changes
-------------
* OpenTracing support is now deprecated in favor of OpenTelemetry tracing, which will be enabled in a future release. `(PR #6478) <https://github.com/apple/foundationdb/pull/6478/files>`_
* Changed ``memory`` option to limit resident memory instead of virtual memory. Added a new ``memory_vsize`` option if limiting virtual memory is desired. `(PR #6719) <https://github.com/apple/foundationdb/pull/6719>`_
Earlier release notes
---------------------
* :doc:`7.0 (API Version 700) </release-notes/release-notes-700>`
* :doc:`6.3 (API Version 630) </release-notes/release-notes-630>`
* :doc:`6.2 (API Version 620) </release-notes/release-notes-620>`
* :doc:`6.1 (API Version 610) </release-notes/release-notes-610>`
* :doc:`6.0 (API Version 600) </release-notes/release-notes-600>`
* :doc:`5.2 (API Version 520) </release-notes/release-notes-520>`
* :doc:`5.1 (API Version 510) </release-notes/release-notes-510>`
* :doc:`5.0 (API Version 500) </release-notes/release-notes-500>`
* :doc:`4.6 (API Version 460) </release-notes/release-notes-460>`
* :doc:`4.5 (API Version 450) </release-notes/release-notes-450>`
* :doc:`4.4 (API Version 440) </release-notes/release-notes-440>`
* :doc:`4.3 (API Version 430) </release-notes/release-notes-430>`
* :doc:`4.2 (API Version 420) </release-notes/release-notes-420>`
* :doc:`4.1 (API Version 410) </release-notes/release-notes-410>`
* :doc:`4.0 (API Version 400) </release-notes/release-notes-400>`
* :doc:`3.0 (API Version 300) </release-notes/release-notes-300>`
* :doc:`2.0 (API Version 200) </release-notes/release-notes-200>`
* :doc:`1.0 (API Version 100) </release-notes/release-notes-100>`
* :doc:`Beta 3 (API Version 23) </release-notes/release-notes-023>`
* :doc:`Beta 2 (API Version 22) </release-notes/release-notes-022>`
* :doc:`Beta 1 (API Version 21) </release-notes/release-notes-021>`
* :doc:`Alpha 6 (API Version 16) </release-notes/release-notes-016>`
* :doc:`Alpha 5 (API Version 14) </release-notes/release-notes-014>`

3
fdbbackup/FileConverter.h
View File

@ -47,6 +47,7 @@ enum {
OPT_BEGIN_VERSION_FILTER,
OPT_END_VERSION_FILTER,
OPT_KNOB,
OPT_SAVE_FILE,
OPT_HELP
};
@ -74,6 +75,8 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP },
{ OPT_BEGIN_VERSION_FILTER, "--begin-version-filter", SO_REQ_SEP },
{ OPT_END_VERSION_FILTER, "--end-version-filter", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_SAVE_FILE, "-s", SO_NONE },
{ OPT_SAVE_FILE, "--save", SO_NONE },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },

135
fdbbackup/FileDecoder.actor.cpp
View File

@ -22,8 +22,13 @@
#include <cstdlib>
#include <iostream>
#include <limits>
#include <memory>
#include <string>
#include <vector>
#include <fcntl.h>
#ifdef _WIN32
#include <io.h>
#endif
#include "fdbbackup/BackupTLSConfig.h"
#include "fdbclient/BuildFlags.h"
@ -37,6 +42,7 @@
#include "fdbclient/MutationList.h"
#include "flow/ArgParseUtil.h"
#include "flow/IRandom.h"
#include "flow/Platform.h"
#include "flow/Trace.h"
#include "flow/flow.h"
#include "flow/serialize.h"
@ -83,7 +89,8 @@ void printDecodeUsage() {
" --end-version-filter END_VERSION\n"
" The version range's end version (exclusive) for filtering.\n"
" --knob-KNOBNAME KNOBVALUE\n"
" Changes a knob value. KNOBNAME should be lowercase."
" Changes a knob value. KNOBNAME should be lowercase.\n"
" -s, --save Save a copy of downloaded files (default: not saving).\n"
"\n";
return;
}
@ -100,6 +107,7 @@ struct DecodeParams {
std::string log_dir, trace_format, trace_log_group;
BackupTLSConfig tlsConfig;
bool list_only = false;
bool save_file_locally = false;
std::string prefix; // Key prefix for filtering
Version beginVersionFilter = 0;
Version endVersionFilter = std::numeric_limits<Version>::max();
@ -146,35 +154,15 @@ struct DecodeParams {
for (const auto& [knob, value] : knobs) {
s.append(", KNOB-").append(knob).append(" = ").append(value);
}
s.append(", SaveFile: ").append(save_file_locally ? "true" : "false");
return s;
}
void updateKnobs() {
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
for (const auto& [knobName, knobValueString] : knobs) {
try {
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName
<< "'\n";
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n";
TraceEvent(SevError, "FailedToSetKnob")
.errorUnsuppressed(e)
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
throw;
}
}
}
IKnobCollection::setupKnobs(knobs);
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
g_knobs.initialize(Randomize::True, IsSimulated::False);
IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
}
};
@ -310,6 +298,10 @@ int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) {
break;
}
case OPT_SAVE_FILE:
param->save_file_locally = true;
break;
#ifndef TLS_DISABLED
case TLSConfig::OPT_TLS_PLUGIN:
args->OptionArg();
@ -375,16 +367,17 @@ struct VersionedMutations {
*
* DecodeProgress progress(logfile);
* wait(progress->openFile(container));
* while (!progress->finished()) {
* VersionedMutations m = wait(progress->getNextBatch());
* ...
* while (1) {
* Optional<VersionedMutations> batch = wait(progress->getNextBatch());
* if (!batch.present()) break;
* ... // process the batch mutations
* }
*
* Internally, the decoding process is done block by block -- each block is
* decoded into a list of key/value pairs, which are then decoded into batches
* of mutations. Because a version's mutations can be split into many key/value
* pairs, the decoding of mutation batch needs to look ahead one more pair. So
* at any time this object might have two blocks of data in memory.
* pairs, the decoding of mutation needs to look ahead to find all batches that
* belong to the same version.
*/
class DecodeProgress {
std::vector<Standalone<VectorRef<KeyValueRef>>> blocks;
@ -392,31 +385,30 @@ class DecodeProgress {
public:
DecodeProgress() = default;
DecodeProgress(const LogFile& file) : file(file) {}
DecodeProgress(const LogFile& file, bool save) : file(file), save(save) {}
// If there are no more mutations to pull from the file.
bool finished() const { return done; }
~DecodeProgress() {
if (lfd != -1) {
close(lfd);
}
}
// Open and loads file into memory
Future<Void> openFile(Reference<IBackupContainer> container) { return openFileImpl(this, container); }
// The following are private APIs:
// PRECONDITION: finished() must return false before calling this function.
// Returns the next batch of mutations along with the arena backing it.
// Note the returned batch can be empty when the file has unfinished
// version batch data that are in the next file.
VersionedMutations getNextBatch() {
ASSERT(!finished());
VersionedMutations vms;
Optional<VersionedMutations> getNextBatch() {
for (auto& [version, m] : mutationBlocksByVersion) {
if (m.isComplete()) {
VersionedMutations vms;
vms.version = version;
std::vector<MutationRef> mutations = fileBackup::decodeMutationLogValue(m.serializedMutations);
TraceEvent("Decode").detail("Version", vms.version).detail("N", mutations.size());
vms.mutations.insert(vms.mutations.end(), mutations.begin(), mutations.end());
vms.serializedMutations = m.serializedMutations;
vms.mutations = fileBackup::decodeMutationLogValue(vms.serializedMutations);
TraceEvent("Decode").detail("Version", vms.version).detail("N", vms.mutations.size());
mutationBlocksByVersion.erase(version);
return vms;
}
@ -426,13 +418,27 @@ public:
if (!mutationBlocksByVersion.empty()) {
TraceEvent(SevWarn, "UnfishedBlocks").detail("NumberOfVersions", mutationBlocksByVersion.size());
}
done = true;
return vms;
return Optional<VersionedMutations>();
}
ACTOR static Future<Void> openFileImpl(DecodeProgress* self, Reference<IBackupContainer> container) {
Reference<IAsyncFile> fd = wait(container->readFile(self->file.fileName));
self->fd = fd;
if (self->save) {
std::string dir = self->file.fileName;
std::size_t found = self->file.fileName.find_last_of('/');
if (found != std::string::npos) {
std::string path = self->file.fileName.substr(0, found);
if (!directoryExists(path)) {
platform::createDirectory(path);
}
}
self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC);
if (self->lfd == -1) {
TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName);
throw platform_error();
}
}
while (!self->eof) {
wait(readAndDecodeFile(self));
}
@ -457,10 +463,34 @@ public:
}
// Decode a file block into log_key and log_value chunks
Standalone<VectorRef<KeyValueRef>> chunks =
state Standalone<VectorRef<KeyValueRef>> chunks =
wait(fileBackup::decodeMutationLogFileBlock(self->fd, self->offset, len));
self->blocks.push_back(chunks);
if (self->save) {
ASSERT(self->lfd != -1);
// Read the chunck one more time
state Standalone<StringRef> buf = makeString(len);
int rLen = wait(self->fd->read(mutateString(buf), len, self->offset));
if (rLen != len)
throw restore_bad_read();
int wlen = write(self->lfd, buf.begin(), len);
if (wlen != len) {
TraceEvent(SevError, "WriteLocalFileFailed")
.detail("File", self->file.fileName)
.detail("Offset", self->offset)
.detail("Len", len)
.detail("Wrote", wlen);
throw platform_error();
}
TraceEvent("WriteLocalFile")
.detail("Name", self->file.fileName)
.detail("Len", len)
.detail("Offset", self->offset);
}
TraceEvent("ReadFile")
.detail("Name", self->file.fileName)
.detail("Len", len)
@ -483,7 +513,8 @@ public:
Reference<IAsyncFile> fd;
int64_t offset = 0;
bool eof = false;
bool done = false;
bool save = false;
int lfd = -1; // local file descriptor
};
ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile file, UID uid, DecodeParams params) {
@ -492,10 +523,14 @@ ACTOR Future<Void> process_file(Reference<IBackupContainer> container, LogFile f
return Void();
}
state DecodeProgress progress(file);
state DecodeProgress progress(file, params.save_file_locally);
wait(progress.openFile(container));
while (!progress.finished()) {
VersionedMutations vms = progress.getNextBatch();
while (true) {
auto batch = progress.getNextBatch();
if (!batch.present())
break;
const VersionedMutations& vms = batch.get();
if (vms.version < params.beginVersionFilter || vms.version >= params.endVersionFilter) {
TraceEvent("SkipVersion").detail("Version", vms.version);
continue;
@ -570,10 +605,10 @@ ACTOR Future<Void> decode_logs(DecodeParams params) {
int main(int argc, char** argv) {
try {
CSimpleOpt* args =
new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE);
std::unique_ptr<CSimpleOpt> args(
new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE));
file_converter::DecodeParams param;
int status = file_converter::parseDecodeCommandLine(&param, args);
int status = file_converter::parseDecodeCommandLine(&param, args.get());
std::cout << "Params: " << param.toString() << "\n";
if (status != FDB_EXIT_SUCCESS) {
file_converter::printDecodeUsage();

64
fdbbackup/backup.actor.cpp
View File

@ -24,6 +24,7 @@
#include "flow/Arena.h"
#include "flow/ArgParseUtil.h"
#include "flow/Error.h"
#include "flow/SystemMonitor.h"
#include "flow/Trace.h"
#define BOOST_DATE_TIME_NO_LIB
#include <boost/interprocess/managed_shared_memory.hpp>
@ -171,6 +172,7 @@ enum {
OPT_KNOB,
OPT_TRACE_LOG_GROUP,
OPT_MEMLIMIT,
OPT_VMEMLIMIT,
OPT_LOCALITY,
// DB constants
@ -212,6 +214,7 @@ CSimpleOpt::SOption g_rgAgentOptions[] = {
{ OPT_LOCALITY, "--locality-", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -257,6 +260,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -283,6 +287,7 @@ CSimpleOpt::SOption g_rgBackupModifyOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -323,6 +328,7 @@ CSimpleOpt::SOption g_rgBackupStatusOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -352,6 +358,7 @@ CSimpleOpt::SOption g_rgBackupAbortOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -378,6 +385,7 @@ CSimpleOpt::SOption g_rgBackupCleanupOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -410,6 +418,7 @@ CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -440,6 +449,7 @@ CSimpleOpt::SOption g_rgBackupWaitOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -466,6 +476,7 @@ CSimpleOpt::SOption g_rgBackupPauseOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -495,6 +506,7 @@ CSimpleOpt::SOption g_rgBackupExpireOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -531,6 +543,7 @@ CSimpleOpt::SOption g_rgBackupDeleteOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -561,6 +574,7 @@ CSimpleOpt::SOption g_rgBackupDescribeOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -593,6 +607,7 @@ CSimpleOpt::SOption g_rgBackupDumpOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -640,6 +655,7 @@ CSimpleOpt::SOption g_rgBackupListOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -675,6 +691,7 @@ CSimpleOpt::SOption g_rgBackupQueryOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -720,6 +737,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -757,6 +775,7 @@ CSimpleOpt::SOption g_rgDBAgentOptions[] = {
{ OPT_LOCALITY, "--locality-", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -788,6 +807,7 @@ CSimpleOpt::SOption g_rgDBStartOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -820,6 +840,7 @@ CSimpleOpt::SOption g_rgDBStatusOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -851,6 +872,7 @@ CSimpleOpt::SOption g_rgDBSwitchOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -883,6 +905,7 @@ CSimpleOpt::SOption g_rgDBAbortOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -911,6 +934,7 @@ CSimpleOpt::SOption g_rgDBPauseOptions[] = {
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
@ -3411,6 +3435,7 @@ int main(int argc, char* argv[]) {
DstOnly dstOnly{ false };
LocalityData localities;
uint64_t memLimit = 8LL << 30;
uint64_t virtualMemLimit = 0; // unlimited
Optional<uint64_t> ti;
BackupTLSConfig tlsConfig;
Version dumpBegin = 0;
@ -3756,6 +3781,15 @@ int main(int argc, char* argv[]) {
}
memLimit = ti.get();
break;
case OPT_VMEMLIMIT:
ti = parse_with_suffix(args->OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse virtual memory limit from `%s'\n", args->OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
virtualMemLimit = ti.get();
break;
case OPT_BLOB_CREDENTIALS:
tlsConfig.blobCredentials.push_back(args->OptionArg());
break;
@ -3885,7 +3919,7 @@ int main(int argc, char* argv[]) {
Error::init();
std::set_new_handler(&platform::outOfMemory);
setMemoryQuota(memLimit);
setMemoryQuota(virtualMemLimit);
Database db;
Database sourceDb;
@ -3902,33 +3936,11 @@ int main(int argc, char* argv[]) {
return FDB_EXIT_ERROR;
}
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
for (const auto& [knobName, knobValueString] : knobs) {
try {
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
fprintf(stderr,
"WARNING: Invalid value '%s' for knob option '%s'\n",
knobValueString.c_str(),
knobName.c_str());
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
TraceEvent(SevError, "FailedToSetKnob")
.error(e)
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
throw;
}
}
}
Future<Void> memoryUsageMonitor = startMemoryUsageMonitor(memLimit);
IKnobCollection::setupKnobs(knobs);
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
g_knobs.initialize(Randomize::False, IsSimulated::False);
IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
TraceEvent("ProgramStart")
.setMaxEventLength(12000)

41
fdbcli/fdbcli.actor.cpp
View File

@ -42,10 +42,12 @@
#include "fdbclient/Tuple.h"
#include "fdbclient/ThreadSafeTransaction.h"
#include "flow/flow.h"
#include "flow/ArgParseUtil.h"
#include "flow/DeterministicRandom.h"
#include "flow/FastRef.h"
#include "flow/Platform.h"
#include "flow/SystemMonitor.h"
#include "flow/TLSConfig.actor.h"
#include "flow/ThreadHelper.actor.h"
@ -98,6 +100,7 @@ enum {
OPT_KNOB,
OPT_DEBUG_TLS,
OPT_API_VERSION,
OPT_MEMORY,
};
CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
@ -121,6 +124,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP },
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_DEBUG_TLS, "--debug-tls", SO_NONE },
{ OPT_API_VERSION, "--api-version", SO_REQ_SEP },
{ OPT_MEMORY, "--memory", SO_REQ_SEP },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
@ -453,6 +457,7 @@ static void printProgramUsage(const char* name) {
" --debug-tls Prints the TLS configuration and certificate chain, then exits.\n"
" Useful in reporting and diagnosing TLS issues.\n"
" --build-flags Print build information and exit.\n"
" --memory Resident memory limit of the CLI (defaults to 8GiB).\n"
" -v, --version Print FoundationDB CLI version information and exit.\n"
" -h, --help Display this help and exit.\n");
}
@ -990,6 +995,7 @@ struct CLIOptions {
std::string tlsVerifyPeers;
std::string tlsCAPath;
std::string tlsPassword;
uint64_t memLimit = 8uLL << 30;
std::vector<std::pair<std::string, std::string>> knobs;
@ -1021,33 +1027,10 @@ struct CLIOptions {
}
void setupKnobs() {
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
for (const auto& [knobName, knobValueString] : knobs) {
try {
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
fprintf(stderr,
"WARNING: Invalid value '%s' for knob option '%s'\n",
knobValueString.c_str(),
knobName.c_str());
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
TraceEvent(SevError, "FailedToSetKnob")
.error(e)
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
exit_code = FDB_EXIT_ERROR;
}
}
}
IKnobCollection::setupKnobs(knobs);
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
g_knobs.initialize(Randomize::False, IsSimulated::False);
IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False);
}
int processArg(CSimpleOpt& args) {
@ -1076,6 +1059,11 @@ struct CLIOptions {
}
break;
}
case OPT_MEMORY: {
std::string memoryArg(args.OptionArg());
memLimit = parse_with_suffix(memoryArg, "MiB").orDefault(8uLL << 30);
break;
}
case OPT_TRACE:
trace = true;
break;
@ -2140,8 +2128,6 @@ int main(int argc, char** argv) {
platformInit();
Error::init();
std::set_new_handler(&platform::outOfMemory);
uint64_t memLimit = 8LL << 30;
setMemoryQuota(memLimit);
registerCrashHandler();
@ -2253,6 +2239,7 @@ int main(int argc, char** argv) {
if (opt.exit_code != -1) {
return opt.exit_code;
}
Future<Void> memoryUsageMonitor = startMemoryUsageMonitor(opt.memLimit);
Future<int> cliFuture = runCli(opt);
Future<Void> timeoutFuture = opt.exit_timeout ? timeExit(opt.exit_timeout) : Never();
auto f = stopNetworkAfter(success(cliFuture) || timeoutFuture);

4
fdbclient/BlobWorkerCommon.h
View File

@ -44,6 +44,7 @@ struct BlobWorkerStats {
int numRangesAssigned;
int mutationBytesBuffered;
int activeReadRequests;
int granulesPendingSplitCheck;
Future<Void> logger;
@ -62,10 +63,11 @@ struct BlobWorkerStats {
readReqDeltaBytesReturned("ReadReqDeltaBytesReturned", cc), commitVersionChecks("CommitVersionChecks", cc),
granuleUpdateErrors("GranuleUpdateErrors", cc), granuleRequestTimeouts("GranuleRequestTimeouts", cc),
readRequestsWithBegin("ReadRequestsWithBegin", cc), readRequestsCollapsed("ReadRequestsCollapsed", cc),
numRangesAssigned(0), mutationBytesBuffered(0), activeReadRequests(0) {
numRangesAssigned(0), mutationBytesBuffered(0), activeReadRequests(0), granulesPendingSplitCheck(0) {
specialCounter(cc, "NumRangesAssigned", [this]() { return this->numRangesAssigned; });
specialCounter(cc, "MutationBytesBuffered", [this]() { return this->mutationBytesBuffered; });
specialCounter(cc, "ActiveReadRequests", [this]() { return this->activeReadRequests; });
specialCounter(cc, "GranulesPendingSplitCheck", [this]() { return this->granulesPendingSplitCheck; });
logger = traceCounters("BlobWorkerMetrics", id, interval, &cc, "BlobWorkerMetrics");
}

1
fdbclient/ClientKnobs.cpp
View File

@ -50,7 +50,6 @@ void ClientKnobs::initialize(Randomize randomize) {
init( MAX_GENERATIONS_OVERRIDE, 0 );
init( MAX_GENERATIONS_SIM, 50 ); //Disable network connections after this many generations in simulation, should be less than RECOVERY_DELAY_START_GENERATION
init( COORDINATOR_HOSTNAME_RESOLVE_DELAY, 0.05 );
init( COORDINATOR_RECONNECTION_DELAY, 1.0 );
init( CLIENT_EXAMPLE_AMOUNT, 20 );
init( MAX_CLIENT_STATUS_AGE, 1.0 );

1
fdbclient/ClientKnobs.h
View File

@ -49,7 +49,6 @@ public:
double MAX_GENERATIONS_OVERRIDE;
double MAX_GENERATIONS_SIM;
double COORDINATOR_HOSTNAME_RESOLVE_DELAY;
double COORDINATOR_RECONNECTION_DELAY;
int CLIENT_EXAMPLE_AMOUNT;
double MAX_CLIENT_STATUS_AGE;

3
fdbclient/CoordinationInterface.h
View File

@ -28,6 +28,7 @@
#include "fdbclient/CommitProxyInterface.h"
#include "fdbclient/ClusterInterface.h"
#include "fdbclient/WellKnownEndpoints.h"
#include "flow/Hostname.h"
const int MAX_CLUSTER_FILE_BYTES = 60000;
@ -35,10 +36,12 @@ struct ClientLeaderRegInterface {
RequestStream<struct GetLeaderRequest> getLeader;
RequestStream<struct OpenDatabaseCoordRequest> openDatabase;
RequestStream<struct CheckDescriptorMutableRequest> checkDescriptorMutable;
Optional<Hostname> hostname;
ClientLeaderRegInterface() {}
ClientLeaderRegInterface(NetworkAddress remote);
ClientLeaderRegInterface(INetwork* local);
ClientLeaderRegInterface(Hostname hostname) : hostname(hostname) {}
bool operator==(const ClientLeaderRegInterface& rhs) const {
return getLeader == rhs.getLeader && openDatabase == rhs.openDatabase;

5
fdbclient/DatabaseConfiguration.cpp
View File

@ -660,6 +660,11 @@ void DatabaseConfiguration::applyMutation(MutationRef m) {
}
}
bool DatabaseConfiguration::involveMutation(MutationRef m) {
return (m.type == MutationRef::SetValue && m.param1.startsWith(configKeysPrefix)) ||
(m.type == MutationRef::ClearRange && KeyRangeRef(m.param1, m.param2).intersects(configKeys));
}
bool DatabaseConfiguration::set(KeyRef key, ValueRef value) {
makeConfigurationMutable();
mutableConfiguration.get()[key.toString()] = value.toString();

2
fdbclient/DatabaseConfiguration.h
View File

@ -104,6 +104,8 @@ struct DatabaseConfiguration {
DatabaseConfiguration();
void applyMutation(MutationRef mutation);
// return true if mutation will cause configuration changes
bool involveMutation(MutationRef mutation);
bool set(KeyRef key,
ValueRef value); // Returns true if a configuration option that requires recovery to take effect is changed
bool clear(KeyRangeRef keys);

6
fdbclient/DatabaseContext.h
View File

@ -29,6 +29,7 @@
#include <unordered_map>
#pragma once
#include "fdbclient/FDBTypes.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/KeyRangeMap.h"
#include "fdbclient/CommitProxyInterface.h"
@ -519,6 +520,11 @@ public:
int outstandingWatches;
int maxOutstandingWatches;
// Manage any shared state that may be used by MVC
DatabaseSharedState* sharedStatePtr;
Future<DatabaseSharedState*> initSharedState();
void setSharedState(DatabaseSharedState* p);
// GRV Cache
// Database-level read version cache storing the most recent successful GRV as well as the time it was requested.
double lastGrvTime;

44
fdbclient/FDBTypes.h
View File

@ -28,8 +28,29 @@
#include <unordered_set>
#include "flow/Arena.h"
#include "flow/FastRef.h"
#include "flow/ProtocolVersion.h"
#include "flow/flow.h"
enum class TraceFlags : uint8_t { unsampled = 0b00000000, sampled = 0b00000001 };
inline TraceFlags operator&(TraceFlags lhs, TraceFlags rhs) {
return static_cast<TraceFlags>(static_cast<std::underlying_type_t<TraceFlags>>(lhs) &
static_cast<std::underlying_type_t<TraceFlags>>(rhs));
}
struct SpanContext {
UID traceID;
uint64_t spanID;
TraceFlags m_Flags;
SpanContext() : traceID(UID()), spanID(0), m_Flags(TraceFlags::unsampled) {}
SpanContext(UID traceID, uint64_t spanID, TraceFlags flags) : traceID(traceID), spanID(spanID), m_Flags(flags) {}
SpanContext(UID traceID, uint64_t spanID) : traceID(traceID), spanID(spanID), m_Flags(TraceFlags::unsampled) {}
SpanContext(Arena arena, const SpanContext& span)
: traceID(span.traceID), spanID(span.spanID), m_Flags(span.m_Flags) {}
bool isSampled() const { return (m_Flags & TraceFlags::sampled) == TraceFlags::sampled; }
};
typedef int64_t Version;
typedef uint64_t LogEpoch;
typedef uint64_t Sequence;
@ -1331,6 +1352,29 @@ struct TenantMode {
uint32_t mode;
};
struct GRVCacheSpace {
Version cachedReadVersion;
double lastGrvTime;
GRVCacheSpace() : cachedReadVersion(Version(0)), lastGrvTime(0.0) {}
};
// This structure can be extended in the future to include additional features that required a shared state
struct DatabaseSharedState {
// These two members should always be listed first, in this order.
// This is to preserve compatibility with future updates of this shared state
// and ensures the MVC does not attempt to access methods incorrectly
// due to newly introduced offsets in the structure.
const ProtocolVersion protocolVersion;
void (*delRef)(DatabaseSharedState*);
Mutex mutexLock;
GRVCacheSpace grvCacheSpace;
std::atomic<int> refCount;
DatabaseSharedState()
: protocolVersion(currentProtocolVersion), mutexLock(Mutex()), grvCacheSpace(GRVCacheSpace()), refCount(0) {}
};
inline bool isValidPerpetualStorageWiggleLocality(std::string locality) {
int pos = locality.find(':');

44
fdbclient/GenericManagementAPI.actor.h
View File

@ -125,6 +125,21 @@ bool isCompleteConfiguration(std::map<std::string, std::string> const& options);
ConfigureAutoResult parseConfig(StatusObject const& status);
template <typename Transaction, class T>
struct transaction_future_type {
using type = typename Transaction::template FutureT<T>;
};
template <typename Transaction, class T>
struct transaction_future_type<Transaction*, T> {
using type = typename transaction_future_type<Transaction, T>::type;
};
template <typename Transaction, class T>
struct transaction_future_type<Reference<Transaction>, T> {
using type = typename transaction_future_type<Transaction, T>::type;
};
// Management API written in template code to support both IClientAPI and NativeAPI
namespace ManagementAPI {
@ -636,7 +651,8 @@ Future<Optional<TenantMapEntry>> tryGetTenantTransaction(Transaction tr, TenantN
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
Optional<Value> val = wait(safeThreadFutureToFuture(tr->get(tenantMapKey)));
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantFuture = tr->get(tenantMapKey);
Optional<Value> val = wait(safeThreadFutureToFuture(tenantFuture));
return val.map<TenantMapEntry>([](Optional<Value> v) { return decodeTenantEntry(v.get()); });
}
@ -688,10 +704,13 @@ Future<Optional<TenantMapEntry>> createTenantTransaction(Transaction tr, TenantN
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
state Future<Optional<TenantMapEntry>> tenantEntryFuture = tryGetTenantTransaction(tr, name);
state Future<Optional<Value>> tenantDataPrefixFuture = safeThreadFutureToFuture(tr->get(tenantDataPrefixKey));
state Future<Optional<Value>> lastIdFuture = safeThreadFutureToFuture(tr->get(tenantLastIdKey));
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantDataPrefixFuture =
tr->get(tenantDataPrefixKey);
state typename transaction_future_type<Transaction, Optional<Value>>::type lastIdFuture = tr->get(tenantLastIdKey);
state typename transaction_future_type<Transaction, Optional<Value>>::type tenantModeFuture =
tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr));
Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tr->get(configKeysPrefix.withSuffix("tenant_mode"_sr))));
Optional<Value> tenantMode = wait(safeThreadFutureToFuture(tenantModeFuture));
if (!tenantMode.present() || tenantMode.get() == StringRef(format("%d", TenantMode::DISABLED))) {
throw tenants_disabled();
@ -702,13 +721,15 @@ Future<Optional<TenantMapEntry>> createTenantTransaction(Transaction tr, TenantN
return Optional<TenantMapEntry>();
}
state Optional<Value> lastIdVal = wait(lastIdFuture);
Optional<Value> tenantDataPrefix = wait(tenantDataPrefixFuture);
state Optional<Value> lastIdVal = wait(safeThreadFutureToFuture(lastIdFuture));
Optional<Value> tenantDataPrefix = wait(safeThreadFutureToFuture(tenantDataPrefixFuture));
state TenantMapEntry newTenant(lastIdVal.present() ? TenantMapEntry::prefixToId(lastIdVal.get()) + 1 : 0,
tenantDataPrefix.present() ? (KeyRef)tenantDataPrefix.get() : ""_sr);
RangeResult contents = wait(safeThreadFutureToFuture(tr->getRange(prefixRange(newTenant.prefix), 1)));
state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
tr->getRange(prefixRange(newTenant.prefix), 1);
RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
if (!contents.empty()) {
throw tenant_prefix_allocator_conflict();
}
@ -774,7 +795,9 @@ Future<Void> deleteTenantTransaction(Transaction tr, TenantNameRef name) {
return Void();
}
RangeResult contents = wait(safeThreadFutureToFuture(tr->getRange(prefixRange(tenantEntry.get().prefix), 1)));
state typename transaction_future_type<Transaction, RangeResult>::type prefixRangeFuture =
tr->getRange(prefixRange(tenantEntry.get().prefix), 1);
RangeResult contents = wait(safeThreadFutureToFuture(prefixRangeFuture));
if (!contents.empty()) {
throw tenant_not_empty();
}
@ -832,8 +855,9 @@ Future<std::map<TenantName, TenantMapEntry>> listTenantsTransaction(Transaction
tr->setOption(FDBTransactionOptions::RAW_ACCESS);
tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE);
RangeResult results = wait(safeThreadFutureToFuture(
tr->getRange(firstGreaterOrEqual(range.begin), firstGreaterOrEqual(range.end), limit)));
state typename transaction_future_type<Transaction, RangeResult>::type listFuture =
tr->getRange(firstGreaterOrEqual(range.begin), firstGreaterOrEqual(range.end), limit);
RangeResult results = wait(safeThreadFutureToFuture(listFuture));
std::map<TenantName, TenantMapEntry> tenants;
for (auto kv : results) {

24
fdbclient/HighContentionPrefixAllocator.actor.h
View File

@ -65,8 +65,9 @@ private:
state int64_t window = 0;
loop {
RangeResult range =
wait(safeThreadFutureToFuture(tr->getRange(self->counters.range(), 1, Snapshot::True, Reverse::True)));
state typename TransactionT::template FutureT<RangeResult> rangeFuture =
tr->getRange(self->counters.range(), 1, Snapshot::True, Reverse::True);
RangeResult range = wait(safeThreadFutureToFuture(rangeFuture));
if (range.size() > 0) {
start = self->counters.unpack(range[0].key).getInt(0);
@ -83,11 +84,12 @@ private:
int64_t inc = 1;
tr->atomicOp(self->counters.get(start).key(), StringRef((uint8_t*)&inc, 8), MutationRef::AddValue);
Future<Optional<Value>> countFuture =
safeThreadFutureToFuture(tr->get(self->counters.get(start).key(), Snapshot::True));
state typename TransactionT::template FutureT<Optional<Value>> countFuture =
tr->get(self->counters.get(start).key(), Snapshot::True);
// }
Optional<Value> countValue = wait(countFuture);
Optional<Value> countValue = wait(safeThreadFutureToFuture(countFuture));
int64_t count = 0;
if (countValue.present()) {
@ -110,15 +112,17 @@ private:
state int64_t candidate = deterministicRandom()->randomInt(start, start + window);
// if thread safety is needed, this should be locked {
state Future<RangeResult> latestCounterFuture =
safeThreadFutureToFuture(tr->getRange(self->counters.range(), 1, Snapshot::True, Reverse::True));
state Future<Optional<Value>> candidateValueFuture =
safeThreadFutureToFuture(tr->get(self->recent.get(candidate).key()));
state typename TransactionT::template FutureT<RangeResult> latestCounterFuture =
tr->getRange(self->counters.range(), 1, Snapshot::True, Reverse::True);
state typename TransactionT::template FutureT<Optional<Value>> candidateValueFuture =
tr->get(self->recent.get(candidate).key());
tr->setOption(FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE);
tr->set(self->recent.get(candidate).key(), ValueRef());
// }
wait(success(latestCounterFuture) && success(candidateValueFuture));
wait(success(safeThreadFutureToFuture(latestCounterFuture)) &&
success(safeThreadFutureToFuture(candidateValueFuture)));
int64_t currentWindowStart = 0;
if (latestCounterFuture.get().size() > 0) {
currentWindowStart = self->counters.unpack(latestCounterFuture.get()[0].key).getInt(0);

5
fdbclient/IClientApi.h
View File

@ -20,6 +20,7 @@
#ifndef FDBCLIENT_ICLIENTAPI_H
#define FDBCLIENT_ICLIENTAPI_H
#include "flow/ProtocolVersion.h"
#pragma once
#include "fdbclient/FDBOptions.g.h"
@ -151,6 +152,10 @@ public:
// Management API, create snapshot
virtual ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) = 0;
// Interface to manage shared state across multiple connections to the same Database
virtual ThreadFuture<DatabaseSharedState*> createSharedState() = 0;
virtual void setSharedState(DatabaseSharedState* p) = 0;
// used in template functions as the Transaction type that can be created through createTransaction()
using TransactionT = ITransaction;
};

25
fdbclient/IKnobCollection.cpp
View File

@ -95,6 +95,31 @@ IKnobCollection& IKnobCollection::getMutableGlobalKnobCollection() {
return *globalKnobCollection();
}
void IKnobCollection::setupKnobs(const std::vector<std::pair<std::string, std::string>>& knobs) {
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
for (const auto& [knobName, knobValueString] : knobs) {
try {
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
std::cerr << "WARNING: Invalid value '" << knobValueString << "' for knob option '" << knobName
<< "'\n";
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
std::cerr << "ERROR: Failed to set knob option '" << knobName << "': " << e.what() << "\n";
TraceEvent(SevError, "FailedToSetKnob")
.errorUnsuppressed(e)
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
throw e;
}
}
}
}
ConfigMutationRef IKnobCollection::createSetMutation(Arena arena, KeyRef key, ValueRef value) {
ConfigKey configKey = ConfigKeyRef::decodeKey(key);
auto knobValue =

5
fdbclient/IKnobCollection.h
View File

@ -69,6 +69,11 @@ public:
static void setGlobalKnobCollection(Type, Randomize, IsSimulated);
static IKnobCollection const& getGlobalKnobCollection();
static IKnobCollection& getMutableGlobalKnobCollection();
// Sets up a list of <knob, value> pairs. If encounter a failure,
// immediately throws the error.
static void setupKnobs(const std::vector<std::pair<std::string, std::string>>& knobs);
static ConfigMutationRef createSetMutation(Arena, KeyRef, ValueRef);
static ConfigMutationRef createClearMutation(Arena, KeyRef);
};

2
fdbclient/MonitorLeader.actor.cpp
View File

@ -559,7 +559,7 @@ ACTOR Future<Void> monitorNominee(Key key,
.detail("OldAddr", coord.getLeader.getEndpoint().getPrimaryAddress().toString());
if (rep.getError().code() == error_code_request_maybe_delivered) {
// Delay to prevent tight resolving loop due to outdated DNS cache
wait(delay(CLIENT_KNOBS->COORDINATOR_HOSTNAME_RESOLVE_DELAY));
wait(delay(FLOW_KNOBS->HOSTNAME_RESOLVE_DELAY));
throw coordinators_changed();
} else {
throw rep.getError();

80
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -26,6 +26,7 @@
#include "fdbclient/ClientVersion.h"
#include "fdbclient/LocalClientAPI.h"
#include "flow/ThreadPrimitives.h"
#include "flow/network.h"
#include "flow/Platform.h"
#include "flow/ProtocolVersion.h"
@ -469,6 +470,26 @@ ThreadFuture<Void> DLDatabase::createSnapshot(const StringRef& uid, const String
return toThreadFuture<Void>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) { return Void(); });
}
ThreadFuture<DatabaseSharedState*> DLDatabase::createSharedState() {
if (!api->databaseCreateSharedState) {
return unsupported_operation();
}
FdbCApi::FDBFuture* f = api->databaseCreateSharedState(db);
return toThreadFuture<DatabaseSharedState*>(api, f, [](FdbCApi::FDBFuture* f, FdbCApi* api) {
DatabaseSharedState* res;
FdbCApi::fdb_error_t error = api->futureGetSharedState(f, &res);
ASSERT(!error);
return res;
});
}
void DLDatabase::setSharedState(DatabaseSharedState* p) {
if (!api->databaseSetSharedState) {
throw unsupported_operation();
}
api->databaseSetSharedState(db, p);
}
// Get network thread busyness
double DLDatabase::getMainThreadBusyness() {
if (api->databaseGetMainThreadBusyness != nullptr) {
@ -545,6 +566,10 @@ void DLApi::init() {
loadClientFunction(&api->createDatabase, lib, fdbCPath, "fdb_create_database", headerVersion >= 610);
loadClientFunction(&api->databaseOpenTenant, lib, fdbCPath, "fdb_database_open_tenant", headerVersion >= 710);
loadClientFunction(
&api->databaseCreateSharedState, lib, fdbCPath, "fdb_database_create_shared_state", headerVersion >= 710);
loadClientFunction(
&api->databaseSetSharedState, lib, fdbCPath, "fdb_database_set_shared_state", headerVersion >= 710);
loadClientFunction(
&api->databaseCreateTransaction, lib, fdbCPath, "fdb_database_create_transaction", headerVersion >= 0);
loadClientFunction(&api->databaseSetOption, lib, fdbCPath, "fdb_database_set_option", headerVersion >= 0);
@ -643,6 +668,7 @@ void DLApi::init() {
&api->futureGetKeyValueArray, lib, fdbCPath, "fdb_future_get_keyvalue_array", headerVersion >= 0);
loadClientFunction(
&api->futureGetMappedKeyValueArray, lib, fdbCPath, "fdb_future_get_mappedkeyvalue_array", headerVersion >= 700);
loadClientFunction(&api->futureGetSharedState, lib, fdbCPath, "fdb_future_get_shared_state", headerVersion >= 710);
loadClientFunction(&api->futureSetCallback, lib, fdbCPath, "fdb_future_set_callback", headerVersion >= 0);
loadClientFunction(&api->futureCancel, lib, fdbCPath, "fdb_future_cancel", headerVersion >= 0);
loadClientFunction(&api->futureDestroy, lib, fdbCPath, "fdb_future_destroy", headerVersion >= 0);
@ -1276,7 +1302,6 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api,
: dbState(new DatabaseState(clusterFilePath, versionMonitorDb)) {
dbState->db = db;
dbState->dbVar->set(db);
if (openConnectors) {
if (!api->localClientDisabled) {
dbState->addClient(api->getLocalClient());
@ -1391,6 +1416,18 @@ ThreadFuture<Void> MultiVersionDatabase::createSnapshot(const StringRef& uid, co
return abortableFuture(f, dbState->dbVar->get().onChange);
}
ThreadFuture<DatabaseSharedState*> MultiVersionDatabase::createSharedState() {
auto dbVar = dbState->dbVar->get();
auto f = dbVar.value ? dbVar.value->createSharedState() : ThreadFuture<DatabaseSharedState*>(Never());
return abortableFuture(f, dbVar.onChange);
}
void MultiVersionDatabase::setSharedState(DatabaseSharedState* p) {
if (dbState->db) {
dbState->db->setSharedState(p);
}
}
// Get network thread busyness
// Return the busyness for the main thread. When using external clients, take the larger of the local client
// and the external client's busyness.
@ -1497,6 +1534,11 @@ void MultiVersionDatabase::DatabaseState::protocolVersionChanged(ProtocolVersion
TraceEvent("ProtocolVersionChanged")
.detail("NewProtocolVersion", protocolVersion)
.detail("OldProtocolVersion", dbProtocolVersion);
// When the protocol version changes, clear the corresponding entry in the shared state map
// so it can be re-initialized. Only do so if there was a valid previous protocol version.
if (dbProtocolVersion.present()) {
MultiVersionApi::api->clearClusterSharedStateMapEntry(clusterFilePath);
}
dbProtocolVersion = protocolVersion;
@ -1607,8 +1649,15 @@ void MultiVersionDatabase::DatabaseState::updateDatabase(Reference<IDatabase> ne
.detail("ClusterFilePath", clusterFilePath);
}
}
dbVar->set(db);
if (db.isValid() && dbProtocolVersion.present() && MultiVersionApi::apiVersionAtLeast(710)) {
auto updateResult = MultiVersionApi::api->updateClusterSharedStateMap(clusterFilePath, db);
auto handler = mapThreadFuture<Void, Void>(updateResult, [this](ErrorOr<Void> result) {
dbVar->set(db);
return ErrorOr<Void>(Void());
});
} else {
dbVar->set(db);
}
ASSERT(protocolVersionMonitor.isValid());
protocolVersionMonitor.cancel();
@ -2264,6 +2313,31 @@ void MultiVersionApi::updateSupportedVersions() {
}
}
ThreadFuture<Void> MultiVersionApi::updateClusterSharedStateMap(std::string clusterFilePath, Reference<IDatabase> db) {
MutexHolder holder(lock);
if (clusterSharedStateMap.find(clusterFilePath) == clusterSharedStateMap.end()) {
clusterSharedStateMap[clusterFilePath] = db->createSharedState();
} else {
ThreadFuture<DatabaseSharedState*> entry = clusterSharedStateMap[clusterFilePath];
return mapThreadFuture<DatabaseSharedState*, Void>(entry, [db](ErrorOr<DatabaseSharedState*> result) {
if (result.isError()) {
return ErrorOr<Void>(result.getError());
}
auto ssPtr = result.get();
db->setSharedState(ssPtr);
return ErrorOr<Void>(Void());
});
}
return Void();
}
void MultiVersionApi::clearClusterSharedStateMapEntry(std::string clusterFilePath) {
MutexHolder holder(lock);
auto ssPtr = clusterSharedStateMap[clusterFilePath].get();
ssPtr->delRef(ssPtr);
clusterSharedStateMap.erase(clusterFilePath);
}
std::vector<std::string> parseOptionValues(std::string valueStr) {
std::string specialCharacters = "\\";
specialCharacters += ENV_VAR_PATH_SEPARATOR;

16
fdbclient/MultiVersionTransaction.h
View File

@ -20,6 +20,7 @@
#ifndef FDBCLIENT_MULTIVERSIONTRANSACTION_H
#define FDBCLIENT_MULTIVERSIONTRANSACTION_H
#include "flow/ProtocolVersion.h"
#pragma once
#include "bindings/c/foundationdb/fdb_c_options.g.h"
@ -149,6 +150,9 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
int uidLength,
uint8_t const* snapshotCommmand,
int snapshotCommandLength);
FDBFuture* (*databaseCreateSharedState)(FDBDatabase* database);
void (*databaseSetSharedState)(FDBDatabase* database, DatabaseSharedState* p);
double (*databaseGetMainThreadBusyness)(FDBDatabase* database);
FDBFuture* (*databaseGetServerProtocol)(FDBDatabase* database, uint64_t expectedVersion);
@ -285,6 +289,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted<FdbCApi> {
FDBMappedKeyValue const** outKVM,
int* outCount,
fdb_bool_t* outMore);
fdb_error_t (*futureGetSharedState)(FDBFuture* f, DatabaseSharedState** outPtr);
fdb_error_t (*futureSetCallback)(FDBFuture* f, FDBCallback callback, void* callback_parameter);
void (*futureCancel)(FDBFuture* f);
void (*futureDestroy)(FDBFuture* f);
@ -433,6 +438,9 @@ public:
ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override;
private:
const Reference<FdbCApi> api;
FdbCApi::FDBDatabase*
@ -708,6 +716,9 @@ public:
ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override;
// private:
struct LegacyVersionMonitor;
@ -830,6 +841,8 @@ public:
bool callbackOnMainThread;
bool localClientDisabled;
ThreadFuture<Void> updateClusterSharedStateMap(std::string clusterFilePath, Reference<IDatabase> db);
void clearClusterSharedStateMapEntry(std::string clusterFilePath);
static bool apiVersionAtLeast(int minVersion);
@ -853,6 +866,9 @@ private:
Reference<ClientInfo> localClient;
std::map<std::string, ClientDesc> externalClientDescriptions;
std::map<std::string, std::vector<Reference<ClientInfo>>> externalClients;
// Map of clusterFilePath -> DatabaseSharedState pointer Future
// Upon cluster version upgrade, clear the map entry for that cluster
std::map<std::string, ThreadFuture<DatabaseSharedState*>> clusterSharedStateMap;
bool networkStartSetup;
volatile bool networkSetup;

64
fdbclient/NativeAPI.actor.cpp
View File

@ -71,6 +71,7 @@
#include "flow/Error.h"
#include "flow/FastRef.h"
#include "flow/IRandom.h"
#include "flow/ProtocolVersion.h"
#include "flow/flow.h"
#include "flow/genericactors.actor.h"
#include "flow/Knobs.h"
@ -214,7 +215,25 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) {
}
}
void updateCachedReadVersionShared(double t, Version v, DatabaseSharedState* p) {
MutexHolder mutex(p->mutexLock);
if (v >= p->grvCacheSpace.cachedReadVersion) {
TraceEvent(SevDebug, "CacheReadVersionUpdate")
.detail("Version", v)
.detail("CurTime", t)
.detail("LastVersion", p->grvCacheSpace.cachedReadVersion)
.detail("LastTime", p->grvCacheSpace.lastGrvTime);
p->grvCacheSpace.cachedReadVersion = v;
if (t > p->grvCacheSpace.lastGrvTime) {
p->grvCacheSpace.lastGrvTime = t;
}
}
}
void DatabaseContext::updateCachedReadVersion(double t, Version v) {
if (sharedStatePtr) {
return updateCachedReadVersionShared(t, v, sharedStatePtr);
}
if (v >= cachedReadVersion) {
TraceEvent(SevDebug, "CachedReadVersionUpdate")
.detail("Version", v)
@ -233,10 +252,18 @@ void DatabaseContext::updateCachedReadVersion(double t, Version v) {
}
Version DatabaseContext::getCachedReadVersion() {
if (sharedStatePtr) {
MutexHolder mutex(sharedStatePtr->mutexLock);
return sharedStatePtr->grvCacheSpace.cachedReadVersion;
}
return cachedReadVersion;
}
double DatabaseContext::getLastGrvTime() {
if (sharedStatePtr) {
MutexHolder mutex(sharedStatePtr->mutexLock);
return sharedStatePtr->grvCacheSpace.lastGrvTime;
}
return lastGrvTime;
}
@ -1363,11 +1390,12 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
bytesPerCommit(1000), bgLatencies(1000), bgGranulesPerRequest(1000), outstandingWatches(0), lastGrvTime(0.0),
cachedReadVersion(0), lastRkBatchThrottleTime(0.0), lastRkDefaultThrottleTime(0.0), lastProxyRequestTime(0.0),
transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor),
coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0),
detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
bytesPerCommit(1000), bgLatencies(1000), bgGranulesPerRequest(1000), outstandingWatches(0), sharedStatePtr(nullptr),
lastGrvTime(0.0), cachedReadVersion(0), lastRkBatchThrottleTime(0.0), lastRkDefaultThrottleTime(0.0),
lastProxyRequestTime(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)),
connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {
dbId = deterministicRandom()->randomUniqueID();
@ -1664,6 +1692,9 @@ DatabaseContext::~DatabaseContext() {
if (grvUpdateHandler.isValid()) {
grvUpdateHandler.cancel();
}
if (sharedStatePtr) {
sharedStatePtr->delRef(sharedStatePtr);
}
for (auto it = server_interf.begin(); it != server_interf.end(); it = server_interf.erase(it))
it->second->notifyContextDestroyed();
ASSERT_ABORT(server_interf.empty());
@ -8143,6 +8174,29 @@ Future<Void> DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c
return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command);
}
void sharedStateDelRef(DatabaseSharedState* ssPtr) {
if (--ssPtr->refCount == 0) {
delete ssPtr;
}
}
Future<DatabaseSharedState*> DatabaseContext::initSharedState() {
ASSERT(!sharedStatePtr); // Don't re-initialize shared state if a pointer already exists
DatabaseSharedState* newState = new DatabaseSharedState();
// Increment refcount by 1 on creation to account for the one held in MultiVersionApi map
// Therefore, on initialization, refCount should be 2 (after also going to setSharedState)
newState->refCount++;
newState->delRef = &sharedStateDelRef;
setSharedState(newState);
return newState;
}
void DatabaseContext::setSharedState(DatabaseSharedState* p) {
ASSERT(p->protocolVersion == currentProtocolVersion);
sharedStatePtr = p;
sharedStatePtr->refCount++;
}
ACTOR Future<Void> storageFeedVersionUpdater(StorageServerInterface interf, ChangeFeedStorageData* self) {
state Promise<Void> destroyed = self->destroyed;
loop {

4
fdbclient/NativeAPI.actor.h
View File

@ -459,6 +459,10 @@ public:
std::vector<Reference<Watch>> watches;
Span span;
// used in template functions as returned Future type
template <typename Type>
using FutureT = Future<Type>;
private:
Future<Version> getReadVersion(uint32_t flags);

1
fdbclient/S3BlobStore.actor.cpp
View File

@ -32,6 +32,7 @@
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string.hpp>
#include "fdbrpc/IAsyncFile.h"
#include "flow/Hostname.h"
#include "flow/UnitTest.h"
#include "fdbclient/rapidxml/rapidxml.hpp"
#include "fdbclient/FDBAWSCredentialsProvider.h"

5
fdbclient/ServerKnobs.cpp
View File

@ -720,6 +720,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( PEER_LATENCY_DEGRADATION_PERCENTILE, 0.90 );
init( PEER_LATENCY_DEGRADATION_THRESHOLD, 0.05 );
init( PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD, 0.1 );
init( PEER_DEGRADATION_CONNECTION_FAILURE_COUNT, 1 );
init( WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER, true );
// Test harness
init( WORKER_POLL_DELAY, 1.0 );
@ -859,6 +861,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX, 5.0 );
init( BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT, 1.5 );
init( BGCC_TIMEOUT, isSimulated ? 10.0 : 120.0 );
init( BGCC_MIN_INTERVAL, isSimulated ? 1.0 : 10.0 );
// clang-format on
if (clientKnobs) {

7
fdbclient/ServerKnobs.h
View File

@ -662,6 +662,11 @@ public:
double PEER_LATENCY_DEGRADATION_PERCENTILE; // The percentile latency used to check peer health.
double PEER_LATENCY_DEGRADATION_THRESHOLD; // The latency threshold to consider a peer degraded.
double PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD; // The percentage of timeout to consider a peer degraded.
int PEER_DEGRADATION_CONNECTION_FAILURE_COUNT; // The number of connection failures experienced during measurement
// period to consider a peer degraded.
bool WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER; // When enabled, the worker's health monitor also report any recent
// destroyed peers who are part of the transaction system to
// cluster controller.
// Test harness
double WORKER_POLL_DELAY;
@ -812,6 +817,8 @@ public:
double BLOB_MANAGER_STATUS_EXP_BACKOFF_MIN;
double BLOB_MANAGER_STATUS_EXP_BACKOFF_MAX;
double BLOB_MANAGER_STATUS_EXP_BACKOFF_EXPONENT;
double BGCC_TIMEOUT;
double BGCC_MIN_INTERVAL;
ServerKnobs(Randomize, ClientKnobs*, IsSimulated);
void initialize(Randomize, ClientKnobs*, IsSimulated);

11
fdbclient/ThreadSafeTransaction.cpp
View File

@ -25,6 +25,7 @@
#include "fdbclient/versions.h"
#include "fdbclient/GenericManagementAPI.actor.h"
#include "fdbclient/NativeAPI.actor.h"
#include "flow/ProtocolVersion.h"
// Users of ThreadSafeTransaction might share Reference<ThreadSafe...> between different threads as long as they don't
// call addRef (e.g. C API follows this). Therefore, it is unsafe to call (explicitly or implicitly) this->addRef in any
@ -101,6 +102,16 @@ ThreadFuture<Void> ThreadSafeDatabase::createSnapshot(const StringRef& uid, cons
return onMainThread([db, snapUID, cmd]() -> Future<Void> { return db->createSnapshot(snapUID, cmd); });
}
ThreadFuture<DatabaseSharedState*> ThreadSafeDatabase::createSharedState() {
DatabaseContext* db = this->db;
return onMainThread([db]() -> Future<DatabaseSharedState*> { return db->initSharedState(); });
}
void ThreadSafeDatabase::setSharedState(DatabaseSharedState* p) {
DatabaseContext* db = this->db;
onMainThreadVoid([db, p]() { db->setSharedState(p); }, nullptr);
}
// Return the main network thread busyness
double ThreadSafeDatabase::getMainThreadBusyness() {
ASSERT(g_network);

4
fdbclient/ThreadSafeTransaction.h
View File

@ -20,6 +20,7 @@
#ifndef FDBCLIENT_THREADSAFETRANSACTION_H
#define FDBCLIENT_THREADSAFETRANSACTION_H
#include "flow/ProtocolVersion.h"
#pragma once
#include "fdbclient/ReadYourWrites.h"
@ -58,6 +59,9 @@ public:
ThreadFuture<Void> forceRecoveryWithDataLoss(const StringRef& dcid) override;
ThreadFuture<Void> createSnapshot(const StringRef& uid, const StringRef& snapshot_command) override;
ThreadFuture<DatabaseSharedState*> createSharedState() override;
void setSharedState(DatabaseSharedState* p) override;
private:
friend class ThreadSafeTenant;
friend class ThreadSafeTransaction;

159
fdbmonitor/fdbmonitor.cpp
View File

@ -80,6 +80,9 @@
#include "fdbclient/SimpleIni.h"
#include "fdbclient/versions.h"
constexpr uint64_t DEFAULT_MEMORY_LIMIT = 8LL << 30;
constexpr double MEMORY_CHECK_INTERVAL = 2.0; // seconds
#ifdef __linux__
typedef fd_set* fdb_fd_set;
#elif defined(__APPLE__) || defined(__FreeBSD__)
@ -397,6 +400,47 @@ int mkdir(std::string const& directory) {
return 0;
}
// Parse size value with same format as parse_with_suffix in flow.h
uint64_t parseWithSuffix(const char* to_parse, const char* default_unit = nullptr) {
char* end_ptr = nullptr;
uint64_t ret = strtoull(to_parse, &end_ptr, 10);
if (end_ptr == to_parse) {
// failed to parse
return 0;
}
const char* unit = default_unit;
if (*end_ptr != 0) {
unit = end_ptr;
}
if (unit == nullptr) {
// no unit found
return 0;
}
if (strcmp(end_ptr, "B") == 0) {
// do nothing
} else if (strcmp(unit, "KB") == 0) {
ret *= static_cast<uint64_t>(1e3);
} else if (strcmp(unit, "KiB") == 0) {
ret *= 1ull << 10;
} else if (strcmp(unit, "MB") == 0) {
ret *= static_cast<uint64_t>(1e6);
} else if (strcmp(unit, "MiB") == 0) {
ret *= 1ull << 20;
} else if (strcmp(unit, "GB") == 0) {
ret *= static_cast<uint64_t>(1e9);
} else if (strcmp(unit, "GiB") == 0) {
ret *= 1ull << 30;
} else if (strcmp(unit, "TB") == 0) {
ret *= static_cast<uint64_t>(1e12);
} else if (strcmp(unit, "TiB") == 0) {
ret *= 1ull << 40;
} else {
// unrecognized unit
ret = 0;
}
return ret;
}
struct Command {
private:
std::vector<std::string> commands;
@ -416,6 +460,7 @@ public:
const char* delete_envvars;
bool deconfigured;
bool kill_on_configuration_change;
uint64_t memory_rss;
// one pair for each of stdout and stderr
int pipes[2][2];
@ -423,7 +468,7 @@ public:
Command() : argv(nullptr) {}
Command(const CSimpleIni& ini, std::string _section, ProcessID id, fdb_fd_set fds, int* maxfd)
: fds(fds), argv(nullptr), section(_section), fork_retry_time(-1), quiet(false), delete_envvars(nullptr),
deconfigured(false), kill_on_configuration_change(true) {
deconfigured(false), kill_on_configuration_change(true), memory_rss(0) {
char _ssection[strlen(section.c_str()) + 22];
snprintf(_ssection, strlen(section.c_str()) + 22, "%s", id.c_str());
ssection = _ssection;
@ -529,6 +574,22 @@ public:
log_msg(SevError, "Unable to resolve command for %s\n", ssection.c_str());
return;
}
const char* mem_rss = get_value_multi(ini, "memory", ssection.c_str(), section.c_str(), "general", nullptr);
#ifdef __linux__
if (mem_rss) {
memory_rss = parseWithSuffix(mem_rss, "MiB");
} else {
memory_rss = DEFAULT_MEMORY_LIMIT;
}
#else
if (mem_rss) {
// While the memory check is not currently implemented on non-Linux by fdbmonitor, the "memory" option is
// still pass to fdbserver, which will crash itself if the limit is exceeded.
log_msg(SevWarn, "Memory monitoring by fdbmonitor is not supported by current system\n");
}
#endif
std::stringstream ss(binary);
std::copy(std::istream_iterator<std::string>(ss),
std::istream_iterator<std::string>(),
@ -537,6 +598,7 @@ public:
const char* id_s = ssection.c_str() + strlen(section.c_str()) + 1;
for (auto i : keys) {
// For "memory" option, despite they are handled by fdbmonitor, we still pass it to fdbserver.
if (isParameterNameEqual(i.pItem, "command") || isParameterNameEqual(i.pItem, "restart-delay") ||
isParameterNameEqual(i.pItem, "initial-restart-delay") ||
isParameterNameEqual(i.pItem, "restart-backoff") ||
@ -642,6 +704,31 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONFFILE, "--conffile", SO_REQ_SEP }
{ OPT_HELP, "--help", SO_NONE },
SO_END_OF_OPTIONS };
// Return resident memory in bytes for the given process, or 0 if error.
uint64_t getRss(ProcessID id) {
#ifndef __linux__
// TODO: implement for non-linux
return 0;
#else
pid_t pid = id_pid[id];
char stat_path[100];
snprintf(stat_path, sizeof(stat_path), "/proc/%d/statm", pid);
FILE* stat_file = fopen(stat_path, "r");
if (stat_file == nullptr) {
log_msg(SevWarn, "Unable to open stat file for %s\n", id.c_str());
return 0;
}
long rss = 0;
int ret = fscanf(stat_file, "%*s%ld", &rss);
if (ret == 0) {
log_msg(SevWarn, "Unable to parse rss size for %s\n", id.c_str());
return 0;
}
fclose(stat_file);
return static_cast<uint64_t>(rss) * sysconf(_SC_PAGESIZE);
#endif
}
void start_process(Command* cmd, ProcessID id, uid_t uid, gid_t gid, int delay, sigset_t* mask) {
if (!cmd->argv)
return;
@ -797,7 +884,7 @@ bool argv_equal(const char** a1, const char** a2) {
return true;
}
void kill_process(ProcessID id, bool wait = true) {
void kill_process(ProcessID id, bool wait = true, bool cleanup = true) {
pid_t pid = id_pid[id];
log_msg(SevInfo, "Killing process %d\n", pid);
@ -807,8 +894,10 @@ void kill_process(ProcessID id, bool wait = true) {
waitpid(pid, nullptr, 0);
}
pid_id.erase(pid);
id_pid.erase(id);
if (cleanup) {
pid_id.erase(pid);
id_pid.erase(id);
}
}
void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb_fd_set rfds, int* maxfd) {
@ -1477,6 +1566,7 @@ int main(int argc, char** argv) {
#endif
bool reload = true;
double last_rss_check = timer();
while (1) {
if (reload) {
reload = false;
@ -1534,21 +1624,37 @@ int main(int argc, char** argv) {
}
double end_time = std::numeric_limits<double>::max();
double now = timer();
// True if any process has a resident memory limit
bool need_rss_check = false;
for (auto& i : id_command) {
if (i.second->fork_retry_time >= 0) {
end_time = std::min(i.second->fork_retry_time, end_time);
}
// If process has a resident memory limit and is currently running
if (i.second->memory_rss > 0 && id_pid.count(i.first) > 0) {
need_rss_check = true;
}
}
bool timeout_for_rss_check = false;
if (need_rss_check && end_time > last_rss_check + MEMORY_CHECK_INTERVAL) {
end_time = last_rss_check + MEMORY_CHECK_INTERVAL;
timeout_for_rss_check = true;
}
struct timespec tv;
double timeout = -1;
if (end_time < std::numeric_limits<double>::max()) {
timeout = std::max(0.0, end_time - timer());
timeout = std::max(0.0, end_time - now);
if (timeout > 0) {
tv.tv_sec = timeout;
tv.tv_nsec = 1e9 * (timeout - tv.tv_sec);
}
}
bool is_timeout = false;
#ifdef __linux__
/* Block until something interesting happens (while atomically
unblocking signals) */
@ -1561,7 +1667,10 @@ int main(int argc, char** argv) {
}
if (nfds == 0) {
reload = true;
is_timeout = true;
if (!timeout_for_rss_check) {
reload = true;
}
}
#elif defined(__APPLE__) || defined(__FreeBSD__)
int nev = 0;
@ -1572,7 +1681,10 @@ int main(int argc, char** argv) {
}
if (nev == 0) {
reload = true;
is_timeout = true;
if (!timeout_for_rss_check) {
reload = true;
}
}
if (nev > 0) {
@ -1621,6 +1733,39 @@ int main(int argc, char** argv) {
}
#endif
if (is_timeout && timeout_for_rss_check) {
last_rss_check = timer();
std::vector<ProcessID> oom_ids;
for (auto& i : id_command) {
if (id_pid.count(i.first) == 0) {
// process is not running
continue;
}
uint64_t rss_limit = i.second->memory_rss;
if (rss_limit == 0) {
continue;
}
uint64_t current_rss = getRss(i.first);
if (current_rss > rss_limit) {
log_process_msg(SevWarn,
i.second->ssection.c_str(),
"Process %d being killed for exceeding resident memory limit, current %" PRIu64
" , limit %" PRIu64 "\n",
id_pid[i.first],
current_rss,
i.second->memory_rss);
oom_ids.push_back(i.first);
}
}
// kill process without waiting, and rely on the SIGCHLD handling logic below to restart the process.
for (auto& id : oom_ids) {
kill_process(id, false /*wait*/, false /*cleanup*/);
}
if (oom_ids.size() > 0) {
child_exited = true;
}
}
/* select() could have returned because received an exit signal */
if (exit_signal > 0) {
switch (exit_signal) {

21
fdbrpc/HealthMonitor.actor.cpp
View File

@ -36,6 +36,10 @@ void HealthMonitor::purgeOutdatedHistory() {
--count;
ASSERT(count >= 0);
peerClosedHistory.pop_front();
if (count == 0) {
peerClosedNum.erase(p.second);
}
} else {
break;
}
@ -44,10 +48,27 @@ void HealthMonitor::purgeOutdatedHistory() {
bool HealthMonitor::tooManyConnectionsClosed(const NetworkAddress& peerAddress) {
purgeOutdatedHistory();
if (peerClosedNum.find(peerAddress) == peerClosedNum.end()) {
return false;
}
return peerClosedNum[peerAddress] > FLOW_KNOBS->HEALTH_MONITOR_CONNECTION_MAX_CLOSED;
}
int HealthMonitor::closedConnectionsCount(const NetworkAddress& peerAddress) {
purgeOutdatedHistory();
if (peerClosedNum.find(peerAddress) == peerClosedNum.end()) {
return 0;
}
return peerClosedNum[peerAddress];
}
std::unordered_set<NetworkAddress> HealthMonitor::getRecentClosedPeers() {
purgeOutdatedHistory();
std::unordered_set<NetworkAddress> closedPeers;
for (const auto& [peerAddr, count] : peerClosedNum) {
if (count > 0) {
closedPeers.insert(peerAddr);
}
}
return closedPeers;
}

1
fdbrpc/HealthMonitor.h
View File

@ -31,6 +31,7 @@ public:
void reportPeerClosed(const NetworkAddress& peerAddress);
bool tooManyConnectionsClosed(const NetworkAddress& peerAddress);
int closedConnectionsCount(const NetworkAddress& peerAddress);
std::unordered_set<NetworkAddress> getRecentClosedPeers();
private:
void purgeOutdatedHistory();

213
fdbrpc/SimExternalConnection.actor.cpp
View File

@ -67,97 +67,6 @@ public:
}
};
bool MockDNS::findMockTCPEndpoint(const std::string& host, const std::string& service) {
std::string hostname = host + ":" + service;
return hostnameToAddresses.find(hostname) != hostnameToAddresses.end();
}
void MockDNS::addMockTCPEndpoint(const std::string& host,
const std::string& service,
const std::vector<NetworkAddress>& addresses) {
if (findMockTCPEndpoint(host, service)) {
throw operation_failed();
}
hostnameToAddresses[host + ":" + service] = addresses;
}
void MockDNS::updateMockTCPEndpoint(const std::string& host,
const std::string& service,
const std::vector<NetworkAddress>& addresses) {
if (!findMockTCPEndpoint(host, service)) {
throw operation_failed();
}
hostnameToAddresses[host + ":" + service] = addresses;
}
void MockDNS::removeMockTCPEndpoint(const std::string& host, const std::string& service) {
if (!findMockTCPEndpoint(host, service)) {
throw operation_failed();
}
hostnameToAddresses.erase(host + ":" + service);
}
std::vector<NetworkAddress> MockDNS::getTCPEndpoint(const std::string& host, const std::string& service) {
if (!findMockTCPEndpoint(host, service)) {
throw operation_failed();
}
return hostnameToAddresses[host + ":" + service];
}
void MockDNS::clearMockTCPEndpoints() {
hostnameToAddresses.clear();
}
std::string MockDNS::toString() {
std::string ret;
for (auto it = hostnameToAddresses.begin(); it != hostnameToAddresses.end(); ++it) {
if (it != hostnameToAddresses.begin()) {
ret += ';';
}
ret += it->first + ',';
const std::vector<NetworkAddress>& addresses = it->second;
for (int i = 0; i < addresses.size(); ++i) {
ret += addresses[i].toString();
if (i != addresses.size() - 1) {
ret += ',';
}
}
}
return ret;
}
MockDNS MockDNS::parseFromString(const std::string& s) {
std::map<std::string, std::vector<NetworkAddress>> mockDNS;
for (int p = 0; p < s.length();) {
int pSemiColumn = s.find_first_of(';', p);
if (pSemiColumn == s.npos) {
pSemiColumn = s.length();
}
std::string oneMapping = s.substr(p, pSemiColumn - p);
std::string hostname;
std::vector<NetworkAddress> addresses;
for (int i = 0; i < oneMapping.length();) {
int pComma = oneMapping.find_first_of(',', i);
if (pComma == oneMapping.npos) {
pComma = oneMapping.length();
}
if (!i) {
// The first part is hostname
hostname = oneMapping.substr(i, pComma - i);
} else {
addresses.push_back(NetworkAddress::parse(oneMapping.substr(i, pComma - i)));
}
i = pComma + 1;
}
mockDNS[hostname] = addresses;
p = pSemiColumn + 1;
}
return MockDNS(mockDNS);
}
void SimExternalConnection::close() {
socket.close();
}
@ -222,33 +131,45 @@ UID SimExternalConnection::getDebugID() const {
}
std::vector<NetworkAddress> SimExternalConnection::resolveTCPEndpointBlocking(const std::string& host,
const std::string& service) {
const std::string& service,
DNSCache* dnsCache) {
ip::tcp::resolver resolver(ios);
ip::tcp::resolver::query query(host, service);
auto iter = resolver.resolve(query);
decltype(iter) end;
std::vector<NetworkAddress> addrs;
while (iter != end) {
auto endpoint = iter->endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
try {
auto iter = resolver.resolve(host, service);
decltype(iter) end;
std::vector<NetworkAddress> addrs;
while (iter != end) {
auto endpoint = iter->endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
}
++iter;
}
++iter;
if (addrs.empty()) {
throw lookup_failed();
}
dnsCache->add(host, service, addrs);
return addrs;
} catch (...) {
dnsCache->remove(host, service);
throw lookup_failed();
}
return addrs;
}
ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host, std::string service) {
ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpointImpl(std::string host,
std::string service,
DNSCache* dnsCache) {
wait(delayJittered(0.1));
return SimExternalConnection::resolveTCPEndpointBlocking(host, service);
return SimExternalConnection::resolveTCPEndpointBlocking(host, service, dnsCache);
}
Future<std::vector<NetworkAddress>> SimExternalConnection::resolveTCPEndpoint(const std::string& host,
const std::string& service) {
return resolveTCPEndpointImpl(host, service);
const std::string& service,
DNSCache* dnsCache) {
return resolveTCPEndpointImpl(host, service, dnsCache);
}
Future<Reference<IConnection>> SimExternalConnection::connect(NetworkAddress toAddr) {
@ -309,51 +230,6 @@ TEST_CASE("fdbrpc/SimExternalClient") {
}
TEST_CASE("fdbrpc/MockDNS") {
state MockDNS mockDNS;
state std::vector<NetworkAddress> networkAddresses;
state NetworkAddress address1(IPAddress(0x13131313), 1);
state NetworkAddress address2(IPAddress(0x14141414), 2);
networkAddresses.push_back(address1);
networkAddresses.push_back(address2);
mockDNS.addMockTCPEndpoint("testhost1", "port1", networkAddresses);
ASSERT(mockDNS.findMockTCPEndpoint("testhost1", "port1"));
ASSERT(!mockDNS.findMockTCPEndpoint("testhost1", "port2"));
std::vector<NetworkAddress> resolvedNetworkAddresses = mockDNS.getTCPEndpoint("testhost1", "port1");
ASSERT(resolvedNetworkAddresses.size() == 2);
ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address1) !=
resolvedNetworkAddresses.end());
ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address2) !=
resolvedNetworkAddresses.end());
// Adding a hostname twice should fail.
try {
mockDNS.addMockTCPEndpoint("testhost1", "port1", networkAddresses);
} catch (Error& e) {
ASSERT(e.code() == error_code_operation_failed);
}
// Updating an unexisted hostname should fail.
try {
mockDNS.updateMockTCPEndpoint("testhost2", "port2", networkAddresses);
} catch (Error& e) {
ASSERT(e.code() == error_code_operation_failed);
}
// Removing an unexisted hostname should fail.
try {
mockDNS.removeMockTCPEndpoint("testhost2", "port2");
} catch (Error& e) {
ASSERT(e.code() == error_code_operation_failed);
}
mockDNS.clearMockTCPEndpoints();
// Updating any hostname right after clearing endpoints should fail.
try {
mockDNS.updateMockTCPEndpoint("testhost1", "port1", networkAddresses);
} catch (Error& e) {
ASSERT(e.code() == error_code_operation_failed);
}
return Void();
}
TEST_CASE("fdbrpc/MockTCPEndpoints") {
state std::vector<NetworkAddress> networkAddresses;
state NetworkAddress address1(IPAddress(0x13131313), 1);
networkAddresses.push_back(address1);
@ -363,18 +239,6 @@ TEST_CASE("fdbrpc/MockTCPEndpoints") {
ASSERT(resolvedNetworkAddresses.size() == 1);
ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address1) !=
resolvedNetworkAddresses.end());
// Adding a hostname twice should fail.
try {
INetworkConnections::net()->addMockTCPEndpoint("testhost1", "port1", networkAddresses);
} catch (Error& e) {
ASSERT(e.code() == error_code_operation_failed);
}
// Removing an unexisted hostname should fail.
try {
INetworkConnections::net()->removeMockTCPEndpoint("testhost2", "port2");
} catch (Error& e) {
ASSERT(e.code() == error_code_operation_failed);
}
INetworkConnections::net()->removeMockTCPEndpoint("testhost1", "port1");
state NetworkAddress address2(IPAddress(0x14141414), 2);
networkAddresses.push_back(address2);
@ -387,21 +251,4 @@ TEST_CASE("fdbrpc/MockTCPEndpoints") {
return Void();
}
TEST_CASE("fdbrpc/MockDNSParsing") {
std::string mockDNSString;
INetworkConnections::net()->parseMockDNSFromString(mockDNSString);
ASSERT(INetworkConnections::net()->convertMockDNSToString() == mockDNSString);
mockDNSString = "testhost1:port1,[::1]:4800:tls(fromHostname)";
INetworkConnections::net()->parseMockDNSFromString(mockDNSString);
ASSERT(INetworkConnections::net()->convertMockDNSToString() == mockDNSString);
mockDNSString = "testhost1:port1,[::1]:4800,[2001:db8:85a3::8a2e:370:7334]:4800;testhost2:port2,[2001:"
"db8:85a3::8a2e:370:7334]:4800:tls(fromHostname),8.8.8.8:12";
INetworkConnections::net()->parseMockDNSFromString(mockDNSString);
ASSERT(INetworkConnections::net()->convertMockDNSToString() == mockDNSString);
return Void();
}
void forceLinkSimExternalConnectionTests() {}

36
fdbrpc/SimExternalConnection.h
View File

@ -28,34 +28,6 @@
#include <boost/asio.hpp>
// MockDNS is a class maintaining a <hostname, vector<NetworkAddress>> mapping, mocking a DNS in simulation.
class MockDNS {
public:
MockDNS() {}
explicit MockDNS(const std::map<std::string, std::vector<NetworkAddress>>& mockDNS)
: hostnameToAddresses(mockDNS) {}
bool findMockTCPEndpoint(const std::string& host, const std::string& service);
void addMockTCPEndpoint(const std::string& host,
const std::string& service,
const std::vector<NetworkAddress>& addresses);
void updateMockTCPEndpoint(const std::string& host,
const std::string& service,
const std::vector<NetworkAddress>& addresses);
void removeMockTCPEndpoint(const std::string& host, const std::string& service);
void clearMockTCPEndpoints();
std::vector<NetworkAddress> getTCPEndpoint(const std::string& host, const std::string& service);
void operator=(MockDNS const& rhs) { hostnameToAddresses = rhs.hostnameToAddresses; }
// Convert hostnameToAddresses to string. The format is:
// hostname1,host1Address1,host1Address2;hostname2,host2Address1,host2Address2...
std::string toString();
static MockDNS parseFromString(const std::string& s);
private:
std::map<std::string, std::vector<NetworkAddress>> hostnameToAddresses;
};
class SimExternalConnection final : public IConnection, public ReferenceCounted<SimExternalConnection> {
boost::asio::ip::tcp::socket socket;
SimExternalConnection(boost::asio::ip::tcp::socket&& socket);
@ -76,8 +48,12 @@ public:
int write(SendBuffer const* buffer, int limit) override;
NetworkAddress getPeerAddress() const override;
UID getDebugID() const override;
static Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host, const std::string& service);
static std::vector<NetworkAddress> resolveTCPEndpointBlocking(const std::string& host, const std::string& service);
static Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host,
const std::string& service,
DNSCache* dnsCache);
static std::vector<NetworkAddress> resolveTCPEndpointBlocking(const std::string& host,
const std::string& service,
DNSCache* dnsCache);
static Future<Reference<IConnection>> connect(NetworkAddress toAddr);
};

96
fdbrpc/genericactors.actor.h
View File

@ -28,6 +28,8 @@
#include "flow/genericactors.actor.h"
#include "fdbrpc/fdbrpc.h"
#include "fdbclient/WellKnownEndpoints.h"
#include "flow/Hostname.h"
#include "flow/actorcompiler.h" // This must be the last #include.
ACTOR template <class Req>
@ -70,6 +72,100 @@ Future<REPLY_TYPE(Req)> retryBrokenPromise(RequestStream<Req> to, Req request, T
}
}
ACTOR template <class Req>
Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
Req request,
Hostname hostname,
WellKnownEndpoints token) {
// A wrapper of tryGetReply(request), except that the request is sent to an address resolved from a hostname.
// If resolving fails, return lookup_failed().
// Otherwise, return tryGetReply(request).
try {
wait(hostname.resolve());
} catch (...) {
return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
}
Optional<NetworkAddress> address = hostname.resolvedAddress;
*to = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
return to->tryGetReply(request);
}
ACTOR template <class Req>
Future<ErrorOr<REPLY_TYPE(Req)>> tryGetReplyFromHostname(RequestStream<Req>* to,
Req request,
Hostname hostname,
WellKnownEndpoints token,
TaskPriority taskID) {
// A wrapper of tryGetReply(request), except that the request is sent to an address resolved from a hostname.
// If resolving fails, return lookup_failed().
// Otherwise, return tryGetReply(request).
try {
wait(hostname.resolve());
} catch (...) {
return ErrorOr<REPLY_TYPE(Req)>(lookup_failed());
}
Optional<NetworkAddress> address = hostname.resolvedAddress;
*to = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
return to->tryGetReply(request, taskID);
}
ACTOR template <class Req>
Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(RequestStream<Req>* to,
Req request,
Hostname hostname,
WellKnownEndpoints token) {
// Like tryGetReplyFromHostname, except that request_maybe_delivered results in re-resolving the hostname.
// Suitable for use with hostname, where RequestStream is NOT initialized yet.
// Not normally useful for endpoints initialized with NetworkAddress.
loop {
wait(hostname.resolveWithRetry());
state Optional<NetworkAddress> address = hostname.resolvedAddress;
*to = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request));
if (reply.isError()) {
resetReply(request);
if (reply.getError().code() == error_code_request_maybe_delivered) {
// Connection failure.
hostname.resetToUnresolved();
INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
} else {
throw reply.getError();
}
} else {
return reply.get();
}
}
}
ACTOR template <class Req>
Future<REPLY_TYPE(Req)> retryGetReplyFromHostname(RequestStream<Req>* to,
Req request,
Hostname hostname,
WellKnownEndpoints token,
TaskPriority taskID) {
// Like tryGetReplyFromHostname, except that request_maybe_delivered results in re-resolving the hostname.
// Suitable for use with hostname, where RequestStream is NOT initialized yet.
// Not normally useful for endpoints initialized with NetworkAddress.
loop {
wait(hostname.resolveWithRetry());
state Optional<NetworkAddress> address = hostname.resolvedAddress;
*to = RequestStream<Req>(Endpoint::wellKnown({ address.get() }, token));
ErrorOr<REPLY_TYPE(Req)> reply = wait(to->tryGetReply(request, taskID));
if (reply.isError()) {
resetReply(request);
if (reply.getError().code() == error_code_request_maybe_delivered) {
// Connection failure.
hostname.resetToUnresolved();
INetworkConnections::net()->removeCachedDNS(hostname.host, hostname.service);
} else {
throw reply.getError();
}
} else {
return reply.get();
}
}
}
ACTOR template <class T>
Future<T> timeoutWarning(Future<T> what, double time, PromiseStream<Void> output) {
state Future<Void> end = delay(time);

48
fdbrpc/sim2.actor.cpp
View File

@ -965,30 +965,58 @@ public:
void addMockTCPEndpoint(const std::string& host,
const std::string& service,
const std::vector<NetworkAddress>& addresses) override {
mockDNS.addMockTCPEndpoint(host, service, addresses);
mockDNS.add(host, service, addresses);
}
void removeMockTCPEndpoint(const std::string& host, const std::string& service) override {
mockDNS.removeMockTCPEndpoint(host, service);
mockDNS.remove(host, service);
}
// Convert hostnameToAddresses from/to string. The format is:
// hostname1,host1Address1,host1Address2;hostname2,host2Address1,host2Address2...
void parseMockDNSFromString(const std::string& s) override { mockDNS = MockDNS::parseFromString(s); }
void parseMockDNSFromString(const std::string& s) override { mockDNS = DNSCache::parseFromString(s); }
std::string convertMockDNSToString() override { return mockDNS.toString(); }
Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host,
const std::string& service) override {
// If a <hostname, vector<NetworkAddress>> pair was injected to mock DNS, use it.
if (mockDNS.findMockTCPEndpoint(host, service)) {
return mockDNS.getTCPEndpoint(host, service);
Optional<std::vector<NetworkAddress>> mock = mockDNS.find(host, service);
if (mock.present()) {
return mock.get();
}
return SimExternalConnection::resolveTCPEndpoint(host, service);
return SimExternalConnection::resolveTCPEndpoint(host, service, &dnsCache);
}
Future<std::vector<NetworkAddress>> resolveTCPEndpointWithDNSCache(const std::string& host,
const std::string& service) override {
// If a <hostname, vector<NetworkAddress>> pair was injected to mock DNS, use it.
Optional<std::vector<NetworkAddress>> mock = mockDNS.find(host, service);
if (mock.present()) {
return mock.get();
}
Optional<std::vector<NetworkAddress>> cache = dnsCache.find(host, service);
if (cache.present()) {
return cache.get();
}
return SimExternalConnection::resolveTCPEndpoint(host, service, &dnsCache);
}
std::vector<NetworkAddress> resolveTCPEndpointBlocking(const std::string& host,
const std::string& service) override {
// If a <hostname, vector<NetworkAddress>> pair was injected to mock DNS, use it.
if (mockDNS.findMockTCPEndpoint(host, service)) {
return mockDNS.getTCPEndpoint(host, service);
Optional<std::vector<NetworkAddress>> mock = mockDNS.find(host, service);
if (mock.present()) {
return mock.get();
}
return SimExternalConnection::resolveTCPEndpointBlocking(host, service);
return SimExternalConnection::resolveTCPEndpointBlocking(host, service, &dnsCache);
}
std::vector<NetworkAddress> resolveTCPEndpointBlockingWithDNSCache(const std::string& host,
const std::string& service) override {
// If a <hostname, vector<NetworkAddress>> pair was injected to mock DNS, use it.
Optional<std::vector<NetworkAddress>> mock = mockDNS.find(host, service);
if (mock.present()) {
return mock.get();
}
Optional<std::vector<NetworkAddress>> cache = dnsCache.find(host, service);
if (cache.present()) {
return cache.get();
}
return SimExternalConnection::resolveTCPEndpointBlocking(host, service, &dnsCache);
}
ACTOR static Future<Reference<IConnection>> onConnect(Future<Void> ready, Reference<Sim2Conn> conn) {
wait(ready);
@ -2193,7 +2221,7 @@ public:
bool printSimTime;
private:
MockDNS mockDNS;
DNSCache mockDNS;
#ifdef ENABLE_SAMPLING
ActorLineageSet actorLineageSet;

61
fdbserver/BlobManager.actor.cpp
View File

@ -224,13 +224,16 @@ struct BlobManagerStats {
Counter ccRowsChecked;
Counter ccBytesChecked;
Counter ccMismatches;
Counter ccTimeouts;
Counter ccErrors;
Future<Void> logger;
// Current stats maintained for a given blob worker process
explicit BlobManagerStats(UID id, double interval, std::unordered_map<UID, BlobWorkerInterface>* workers)
: cc("BlobManagerStats", id.toString()), granuleSplits("GranuleSplits", cc),
granuleWriteHotSplits("GranuleWriteHotSplits", cc), ccGranulesChecked("CCGranulesChecked", cc),
ccRowsChecked("CCRowsChecked", cc), ccBytesChecked("CCBytesChecked", cc), ccMismatches("CCMismatches", cc) {
ccRowsChecked("CCRowsChecked", cc), ccBytesChecked("CCBytesChecked", cc), ccMismatches("CCMismatches", cc),
ccTimeouts("CCTimeouts", cc), ccErrors("CCErrors", cc) {
specialCounter(cc, "WorkerCount", [workers]() { return workers->size(); });
logger = traceCounters("BlobManagerMetrics", id, interval, &cc, "BlobManagerMetrics");
}
@ -2743,6 +2746,25 @@ static void blobManagerExclusionSafetyCheck(Reference<BlobManagerData> self,
req.reply.send(reply);
}
ACTOR Future<int64_t> bgccCheckGranule(Reference<BlobManagerData> bmData, KeyRange range) {
state std::pair<RangeResult, Version> fdbResult = wait(readFromFDB(bmData->db, range));
std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> blobResult =
wait(readFromBlob(bmData->db, bmData->bstore, range, 0, fdbResult.second));
if (!compareFDBAndBlob(fdbResult.first, blobResult, range, fdbResult.second, BM_DEBUG)) {
++bmData->stats.ccMismatches;
}
int64_t bytesRead = fdbResult.first.expectedSize();
++bmData->stats.ccGranulesChecked;
bmData->stats.ccRowsChecked += fdbResult.first.size();
bmData->stats.ccBytesChecked += bytesRead;
return bytesRead;
}
// FIXME: could eventually make this more thorough by storing some state in the DB or something
// FIXME: simpler solution could be to shuffle ranges
ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
@ -2775,32 +2797,31 @@ ACTOR Future<Void> bgConsistencyCheck(Reference<BlobManagerData> bmData) {
tries--;
}
state int64_t allowanceBytes = SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES;
if (tries == 0) {
if (BM_DEBUG) {
printf("BGCC couldn't find random range to check, skipping\n");
}
wait(rateLimiter->getAllowance(SERVER_KNOBS->BG_SNAPSHOT_FILE_TARGET_BYTES));
} else {
state std::pair<RangeResult, Version> fdbResult = wait(readFromFDB(bmData->db, range));
std::pair<RangeResult, Standalone<VectorRef<BlobGranuleChunkRef>>> blobResult =
wait(readFromBlob(bmData->db, bmData->bstore, range, 0, fdbResult.second));
if (!compareFDBAndBlob(fdbResult.first, blobResult, range, fdbResult.second, BM_DEBUG)) {
++bmData->stats.ccMismatches;
try {
Optional<int64_t> bytesRead =
wait(timeout(bgccCheckGranule(bmData, range), SERVER_KNOBS->BGCC_TIMEOUT));
if (bytesRead.present()) {
allowanceBytes = bytesRead.get();
} else {
++bmData->stats.ccTimeouts;
}
} catch (Error& e) {
if (e.code() == error_code_operation_cancelled) {
throw e;
}
TraceEvent(SevWarn, "BGCCError", bmData->id).error(e).detail("Epoch", bmData->epoch);
++bmData->stats.ccErrors;
}
int64_t bytesRead = fdbResult.first.expectedSize();
++bmData->stats.ccGranulesChecked;
bmData->stats.ccRowsChecked += fdbResult.first.size();
bmData->stats.ccBytesChecked += bytesRead;
// clear fdb result to release memory since it is a state variable
fdbResult = std::pair(RangeResult(), 0);
wait(rateLimiter->getAllowance(bytesRead));
}
// wait at least some interval if snapshot is small and to not overwhelm the system with reads (for example,
// empty database with one empty granule)
wait(rateLimiter->getAllowance(allowanceBytes) && delay(SERVER_KNOBS->BGCC_MIN_INTERVAL));
} else {
if (BM_DEBUG) {
fmt::print("BGCC found no workers, skipping\n", bmData->workerAssignments.size());

30
fdbserver/BlobWorker.actor.cpp
View File

@ -400,8 +400,7 @@ ACTOR Future<Void> updateGranuleSplitState(Transaction* tr,
fmt::print("{0} destroying old granule {1}\n", currentGranuleID.toString(), parentGranuleID.toString());
}
// FIXME: appears change feed destroy isn't working! ADD BACK
// wait(updateChangeFeed(tr, granuleIDToCFKey(parentGranuleID), ChangeFeedStatus::CHANGE_FEED_DESTROY));
wait(updateChangeFeed(tr, granuleIDToCFKey(parentGranuleID), ChangeFeedStatus::CHANGE_FEED_DESTROY));
Key oldGranuleLockKey = blobGranuleLockKeyFor(parentGranuleRange);
// FIXME: deleting granule lock can cause races where another granule with the same range starts way later
@ -863,6 +862,23 @@ ACTOR Future<BlobFileIndex> compactFromBlob(Reference<BlobWorkerData> bwData,
}
}
struct CounterHolder {
int* counter;
bool completed;
CounterHolder() : counter(nullptr), completed(true) {}
CounterHolder(int* counter) : counter(counter), completed(false) { (*counter)++; }
void complete() {
if (!completed) {
completed = true;
(*counter)--;
}
}
~CounterHolder() { complete(); }
};
ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bwData,
Reference<GranuleMetadata> metadata,
UID granuleID,
@ -878,6 +894,8 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
wait(delay(0, TaskPriority::BlobWorkerUpdateFDB));
state CounterHolder pendingCounter(&bwData->stats.granulesPendingSplitCheck);
if (BW_DEBUG) {
fmt::print("Granule [{0} - {1}) checking with BM for re-snapshot after {2} bytes\n",
metadata->keyRange.begin.printable(),
@ -956,6 +974,8 @@ ACTOR Future<BlobFileIndex> checkSplitAndReSnapshot(Reference<BlobWorkerData> bw
}
}
pendingCounter.complete();
if (BW_DEBUG) {
fmt::print("Granule [{0} - {1}) re-snapshotting after {2} bytes\n",
metadata->keyRange.begin.printable(),
@ -1844,6 +1864,12 @@ ACTOR Future<Void> blobGranuleUpdateFiles(Reference<BlobWorkerData> bwData,
.detail("GranuleID", startState.granuleID);
return Void();
}
if (e.code() == error_code_change_feed_not_registered) {
TraceEvent(SevInfo, "GranuleDestroyed", bwData->id)
.detail("Granule", metadata->keyRange)
.detail("GranuleID", startState.granuleID);
return Void();
}
++bwData->stats.granuleUpdateErrors;
if (BW_DEBUG) {
fmt::print("Granule file updater for [{0} - {1}) got error {2}, exiting\n",

3
fdbserver/ClusterRecovery.actor.cpp
View File

@ -869,7 +869,8 @@ ACTOR Future<Standalone<CommitTransactionRef>> provisionalMaster(Reference<Clust
.detail("MType", m->type)
.detail("Param1", m->param1)
.detail("Param2", m->param2);
if (isMetadataMutation(*m)) {
// emergency transaction only mean to do configuration change
if (parent->configuration.involveMutation(*m)) {
// We keep the mutations and write conflict ranges from this transaction, but not its read
// conflict ranges
Standalone<CommitTransactionRef> out;

11
fdbserver/CommitProxyServer.actor.cpp
View File

@ -2182,10 +2182,13 @@ ACTOR Future<Void> commitProxyServerCore(CommitProxyInterface proxy,
// ((SERVER_MEM_LIMIT * COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) / COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR) is
// only a approximate formula for limiting the memory used. COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR is an
// estimate based on experiments and not an accurate one.
state int64_t commitBatchesMemoryLimit = std::min(
SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT,
static_cast<int64_t>((SERVER_KNOBS->SERVER_MEM_LIMIT * SERVER_KNOBS->COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) /
SERVER_KNOBS->COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR));
state int64_t commitBatchesMemoryLimit = SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT;
if (SERVER_KNOBS->SERVER_MEM_LIMIT > 0) {
commitBatchesMemoryLimit = std::min(
commitBatchesMemoryLimit,
static_cast<int64_t>((SERVER_KNOBS->SERVER_MEM_LIMIT * SERVER_KNOBS->COMMIT_BATCHES_MEM_FRACTION_OF_TOTAL) /
SERVER_KNOBS->COMMIT_BATCHES_MEM_TO_TOTAL_MEM_SCALE_FACTOR));
}
TraceEvent(SevInfo, "CommitBatchesMemoryLimit").detail("BytesLimit", commitBatchesMemoryLimit);
addActor.send(monitorRemoteCommitted(&commitData));

5
fdbserver/ConfigFollowerInterface.h
View File

@ -22,6 +22,7 @@
#include "fdbclient/CommitTransaction.h"
#include "fdbclient/ConfigKnobs.h"
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBTypes.h"
#include "fdbrpc/fdbrpc.h"
@ -215,10 +216,12 @@ public:
RequestStream<ConfigFollowerCompactRequest> compact;
RequestStream<ConfigFollowerRollforwardRequest> rollforward;
RequestStream<ConfigFollowerGetCommittedVersionRequest> getCommittedVersion;
Optional<Hostname> hostname;
ConfigFollowerInterface();
void setupWellKnownEndpoints();
ConfigFollowerInterface(NetworkAddress const& remote);
ConfigFollowerInterface(Hostname hostname) : hostname(hostname) {}
bool operator==(ConfigFollowerInterface const& rhs) const;
bool operator!=(ConfigFollowerInterface const& rhs) const;
UID id() const { return _id; }
@ -226,6 +229,6 @@ public:
template <class Ar>
void serialize(Ar& ar) {
serializer(ar, _id, getSnapshotAndChanges, getChanges, compact, rollforward, getCommittedVersion);
serializer(ar, _id, getSnapshotAndChanges, getChanges, compact, rollforward, getCommittedVersion, hostname);
}
};

3
fdbserver/CoordinationInterface.h
View File

@ -31,6 +31,7 @@ struct GenerationRegInterface {
constexpr static FileIdentifier file_identifier = 16726744;
RequestStream<struct GenerationRegReadRequest> read;
RequestStream<struct GenerationRegWriteRequest> write;
Optional<Hostname> hostname;
// read(key,gen2) returns (value,gen,rgen).
// If there was no prior write(_,_,0) or a data loss fault,
@ -54,6 +55,7 @@ struct GenerationRegInterface {
GenerationRegInterface() {}
GenerationRegInterface(NetworkAddress remote);
GenerationRegInterface(INetwork* local);
GenerationRegInterface(Hostname hostname) : hostname(hostname){};
};
struct UniqueGeneration {
@ -128,6 +130,7 @@ struct LeaderElectionRegInterface : ClientLeaderRegInterface {
LeaderElectionRegInterface() {}
LeaderElectionRegInterface(NetworkAddress remote);
LeaderElectionRegInterface(INetwork* local);
LeaderElectionRegInterface(Hostname hostname) : ClientLeaderRegInterface(hostname) {}
};
struct CandidacyRequest {

227
fdbserver/KeyValueStoreRocksDB.actor.cpp
View File

@ -855,14 +855,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
UID id;
std::shared_ptr<rocksdb::RateLimiter> rateLimiter;
Reference<Histogram> commitLatencyHistogram;
Reference<Histogram> commitActionHistogram;
Reference<Histogram> commitQueueWaitHistogram;
Reference<Histogram> writeHistogram;
Reference<Histogram> deleteCompactRangeHistogram;
std::shared_ptr<ReadIteratorPool> readIterPool;
std::shared_ptr<PerfContextMetrics> perfContextMetrics;
int threadIndex;
ThreadReturnPromiseStream<std::tuple<int, std::string, double>> metricPromiseStream;
explicit Writer(DB& db,
CF& cf,
@ -879,22 +875,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
10, // fairness
rocksdb::RateLimiter::Mode::kWritesOnly,
SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE)
: nullptr),
commitLatencyHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_COMMIT_LATENCY_HISTOGRAM,
Histogram::Unit::microseconds)),
commitActionHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_COMMIT_ACTION_HISTOGRAM,
Histogram::Unit::microseconds)),
commitQueueWaitHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM,
Histogram::Unit::microseconds)),
writeHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_WRITE_HISTOGRAM,
Histogram::Unit::microseconds)),
deleteCompactRangeHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM,
Histogram::Unit::microseconds)) {
: nullptr) {
if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE) {
// Enable perf context on the same thread with the db thread
rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
@ -1055,7 +1036,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
double commitBeginTime;
if (a.getHistograms) {
commitBeginTime = timer_monotonic();
commitQueueWaitHistogram->sampleSeconds(commitBeginTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM.toString(), commitBeginTime - a.startTime));
}
Standalone<VectorRef<KeyRangeRef>> deletes;
DeleteVisitor dv(deletes, deletes.arena());
@ -1079,7 +1061,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
s = db->Write(options, a.batchToCommit.get());
readIterPool->update();
if (a.getHistograms) {
writeHistogram->sampleSeconds(timer_monotonic() - writeBeginTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_WRITE_HISTOGRAM.toString(), timer_monotonic() - writeBeginTime));
}
if (!s.ok()) {
@ -1095,13 +1078,17 @@ struct RocksDBKeyValueStore : IKeyValueStore {
ASSERT(db->SuggestCompactRange(cf, &begin, &end).ok());
}
if (a.getHistograms) {
deleteCompactRangeHistogram->sampleSeconds(timer_monotonic() - compactRangeBeginTime);
metricPromiseStream.send(std::make_tuple(threadIndex,
ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString(),
timer_monotonic() - compactRangeBeginTime));
}
}
if (a.getHistograms) {
double currTime = timer_monotonic();
commitActionHistogram->sampleSeconds(currTime - commitBeginTime);
commitLatencyHistogram->sampleSeconds(currTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString(), currTime - commitBeginTime));
metricPromiseStream.send(
std::make_tuple(threadIndex, ROCKSDB_COMMIT_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
}
if (doPerfContextMetrics) {
perfContextMetrics->set(threadIndex);
@ -1272,21 +1259,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
double readValueTimeout;
double readValuePrefixTimeout;
double readRangeTimeout;
Reference<Histogram> readRangeLatencyHistogram;
Reference<Histogram> readValueLatencyHistogram;
Reference<Histogram> readPrefixLatencyHistogram;
Reference<Histogram> readRangeActionHistogram;
Reference<Histogram> readValueActionHistogram;
Reference<Histogram> readPrefixActionHistogram;
Reference<Histogram> readRangeQueueWaitHistogram;
Reference<Histogram> readValueQueueWaitHistogram;
Reference<Histogram> readPrefixQueueWaitHistogram;
Reference<Histogram> readRangeNewIteratorHistogram;
Reference<Histogram> readValueGetHistogram;
Reference<Histogram> readPrefixGetHistogram;
std::shared_ptr<ReadIteratorPool> readIterPool;
std::shared_ptr<PerfContextMetrics> perfContextMetrics;
int threadIndex;
ThreadReturnPromiseStream<std::tuple<int, std::string, double>> metricPromiseStream;
explicit Reader(DB& db,
CF& cf,
@ -1294,43 +1270,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
std::shared_ptr<PerfContextMetrics> perfContextMetrics,
int threadIndex)
: db(db), cf(cf), readIterPool(readIterPool), perfContextMetrics(perfContextMetrics),
threadIndex(threadIndex),
readRangeLatencyHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READRANGE_LATENCY_HISTOGRAM,
Histogram::Unit::microseconds)),
readValueLatencyHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READVALUE_LATENCY_HISTOGRAM,
Histogram::Unit::microseconds)),
readPrefixLatencyHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READPREFIX_LATENCY_HISTOGRAM,
Histogram::Unit::microseconds)),
readRangeActionHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READRANGE_ACTION_HISTOGRAM,
Histogram::Unit::microseconds)),
readValueActionHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READVALUE_ACTION_HISTOGRAM,
Histogram::Unit::microseconds)),
readPrefixActionHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READPREFIX_ACTION_HISTOGRAM,
Histogram::Unit::microseconds)),
readRangeQueueWaitHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM,
Histogram::Unit::microseconds)),
readValueQueueWaitHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM,
Histogram::Unit::microseconds)),
readPrefixQueueWaitHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM,
Histogram::Unit::microseconds)),
readRangeNewIteratorHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM,
Histogram::Unit::microseconds)),
readValueGetHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READVALUE_GET_HISTOGRAM,
Histogram::Unit::microseconds)),
readPrefixGetHistogram(Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP,
ROCKSDB_READPREFIX_GET_HISTOGRAM,
Histogram::Unit::microseconds)) {
threadIndex(threadIndex) {
if (g_network->isSimulated()) {
// In simulation, increasing the read operation timeouts to 5 minutes, as some of the tests have
// very high load and single read thread cannot process all the load within the timeouts.
@ -1374,7 +1314,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
}
double readBeginTime = timer_monotonic();
if (a.getHistograms) {
readValueQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
}
Optional<TraceBatch> traceBatch;
if (a.debugID.present()) {
@ -1406,7 +1347,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
}
if (a.getHistograms) {
readValueGetHistogram->sampleSeconds(timer_monotonic() - dbGetBeginTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READVALUE_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime));
}
if (a.debugID.present()) {
@ -1424,8 +1366,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
if (a.getHistograms) {
double currTime = timer_monotonic();
readValueActionHistogram->sampleSeconds(currTime - readBeginTime);
readValueLatencyHistogram->sampleSeconds(currTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READVALUE_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READVALUE_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
}
if (doPerfContextMetrics) {
perfContextMetrics->set(threadIndex);
@ -1455,7 +1399,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
}
double readBeginTime = timer_monotonic();
if (a.getHistograms) {
readPrefixQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
}
Optional<TraceBatch> traceBatch;
if (a.debugID.present()) {
@ -1483,7 +1428,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0;
auto s = db->Get(options, cf, toSlice(a.key), &value);
if (a.getHistograms) {
readPrefixGetHistogram->sampleSeconds(timer_monotonic() - dbGetBeginTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READPREFIX_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime));
}
if (a.debugID.present()) {
@ -1503,8 +1449,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
}
if (a.getHistograms) {
double currTime = timer_monotonic();
readPrefixActionHistogram->sampleSeconds(currTime - readBeginTime);
readPrefixLatencyHistogram->sampleSeconds(currTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READPREFIX_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READPREFIX_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
}
if (doPerfContextMetrics) {
perfContextMetrics->set(threadIndex);
@ -1533,7 +1481,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
}
double readBeginTime = timer_monotonic();
if (a.getHistograms) {
readRangeQueueWaitHistogram->sampleSeconds(readBeginTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM.toString(), readBeginTime - a.startTime));
}
if (readBeginTime - a.startTime > readRangeTimeout) {
TraceEvent(SevWarn, "KVSTimeout")
@ -1554,7 +1503,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
double iterCreationBeginTime = a.getHistograms ? timer_monotonic() : 0;
ReadIterator readIter = readIterPool->getIterator();
if (a.getHistograms) {
readRangeNewIteratorHistogram->sampleSeconds(timer_monotonic() - iterCreationBeginTime);
metricPromiseStream.send(std::make_tuple(threadIndex,
ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
timer_monotonic() - iterCreationBeginTime));
}
auto cursor = readIter.iter;
cursor->Seek(toSlice(a.keys.begin));
@ -1582,7 +1533,9 @@ struct RocksDBKeyValueStore : IKeyValueStore {
double iterCreationBeginTime = a.getHistograms ? timer_monotonic() : 0;
ReadIterator readIter = readIterPool->getIterator();
if (a.getHistograms) {
readRangeNewIteratorHistogram->sampleSeconds(timer_monotonic() - iterCreationBeginTime);
metricPromiseStream.send(std::make_tuple(threadIndex,
ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString(),
timer_monotonic() - iterCreationBeginTime));
}
auto cursor = readIter.iter;
cursor->SeekForPrev(toSlice(a.keys.end));
@ -1624,8 +1577,10 @@ struct RocksDBKeyValueStore : IKeyValueStore {
a.result.send(result);
if (a.getHistograms) {
double currTime = timer_monotonic();
readRangeActionHistogram->sampleSeconds(currTime - readBeginTime);
readRangeLatencyHistogram->sampleSeconds(currTime - a.startTime);
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READRANGE_ACTION_HISTOGRAM.toString(), currTime - readBeginTime));
metricPromiseStream.send(std::make_tuple(
threadIndex, ROCKSDB_READRANGE_LATENCY_HISTOGRAM.toString(), currTime - a.startTime));
}
if (doPerfContextMetrics) {
perfContextMetrics->set(threadIndex);
@ -1651,6 +1606,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
FlowLock fetchSemaphore;
int numFetchWaiters;
std::shared_ptr<ReadIteratorPool> readIterPool;
std::vector<Future<Void>> actors;
struct Counters {
CounterCollection cc;
@ -1688,12 +1644,99 @@ struct RocksDBKeyValueStore : IKeyValueStore {
writeThread = createGenericThreadPool();
readThreads = createGenericThreadPool();
}
writeThread->addThread(
new Writer(db, defaultFdbCF, id, readIterPool, perfContextMetrics, SERVER_KNOBS->ROCKSDB_READ_PARALLELISM),
"fdb-rocksdb-wr");
struct Writer* writer =
new Writer(db, defaultFdbCF, id, readIterPool, perfContextMetrics, SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
if (SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0) {
actors.push_back(updateHistogram(writer->metricPromiseStream.getFuture()));
}
writeThread->addThread(writer, "fdb-rocksdb-wr");
TraceEvent("RocksDBReadThreads").detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM);
for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
readThreads->addThread(new Reader(db, defaultFdbCF, readIterPool, perfContextMetrics, i), "fdb-rocksdb-re");
struct Reader* reader = new Reader(db, defaultFdbCF, readIterPool, perfContextMetrics, i);
if (SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE > 0) {
actors.push_back(updateHistogram(reader->metricPromiseStream.getFuture()));
}
readThreads->addThread(reader, "fdb-rocksdb-re");
}
}
ACTOR Future<Void> updateHistogram(FutureStream<std::tuple<int, std::string, double>> metricFutureStream) {
state Reference<Histogram> commitLatencyHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_LATENCY_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> commitActionHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_ACTION_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> commitQueueWaitHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> writeHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_WRITE_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> deleteCompactRangeHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readRangeLatencyHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_LATENCY_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readValueLatencyHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_LATENCY_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readPrefixLatencyHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_LATENCY_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readRangeActionHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_ACTION_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readValueActionHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_ACTION_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readPrefixActionHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_ACTION_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readRangeQueueWaitHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readValueQueueWaitHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readPrefixQueueWaitHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readRangeNewIteratorHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readValueGetHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READVALUE_GET_HISTOGRAM, Histogram::Unit::microseconds);
state Reference<Histogram> readPrefixGetHistogram = Histogram::getHistogram(
ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_READPREFIX_GET_HISTOGRAM, Histogram::Unit::microseconds);
loop {
choose {
when(std::tuple<int, std::string, double> measure = waitNext(metricFutureStream)) {
std::string metricName = std::get<1>(measure);
double latency = std::get<2>(measure);
if (metricName == ROCKSDB_COMMIT_LATENCY_HISTOGRAM.toString()) {
commitLatencyHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString()) {
commitActionHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM.toString()) {
commitQueueWaitHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_WRITE_HISTOGRAM.toString()) {
writeHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM.toString()) {
deleteCompactRangeHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READRANGE_LATENCY_HISTOGRAM.toString()) {
readRangeLatencyHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READVALUE_LATENCY_HISTOGRAM.toString()) {
readValueLatencyHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READPREFIX_LATENCY_HISTOGRAM.toString()) {
readPrefixLatencyHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READRANGE_ACTION_HISTOGRAM.toString()) {
readRangeActionHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READVALUE_ACTION_HISTOGRAM.toString()) {
readValueActionHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READPREFIX_ACTION_HISTOGRAM.toString()) {
readPrefixActionHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READRANGE_QUEUEWAIT_HISTOGRAM.toString()) {
readRangeQueueWaitHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READVALUE_QUEUEWAIT_HISTOGRAM.toString()) {
readValueQueueWaitHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM.toString()) {
readPrefixQueueWaitHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM.toString()) {
readRangeNewIteratorHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READVALUE_GET_HISTOGRAM.toString()) {
readValueGetHistogram->sampleSeconds(latency);
} else if (metricName == ROCKSDB_READPREFIX_GET_HISTOGRAM.toString()) {
readPrefixGetHistogram->sampleSeconds(latency);
}
}
}
}
}

2
fdbserver/LeaderElection.actor.cpp
View File

@ -51,7 +51,7 @@ ACTOR Future<Void> submitCandidacy(Key key,
.detail("OldAddr", coord.candidacy.getEndpoint().getPrimaryAddress().toString());
if (rep.getError().code() == error_code_request_maybe_delivered) {
// Delay to prevent tight resolving loop due to outdated DNS cache
wait(delay(CLIENT_KNOBS->COORDINATOR_HOSTNAME_RESOLVE_DELAY));
wait(delay(FLOW_KNOBS->HOSTNAME_RESOLVE_DELAY));
throw coordinators_changed();
} else {
throw rep.getError();

5
fdbserver/RemoteIKeyValueStore.actor.h
View File

@ -454,6 +454,7 @@ struct RemoteIKeyValueStore : public IKeyValueStore {
}
state Future<Void> connectionCheckingDelay = delay(FLOW_KNOBS->FAILURE_DETECTION_DELAY);
state Future<ErrorOr<Void>> storeError = errorOr(self->interf.getError.getReply(IKVSGetErrorRequest{}));
state NetworkAddress childAddr = self->interf.getError.getEndpoint().getPrimaryAddress();
loop choose {
when(ErrorOr<Void> e = wait(storeError)) {
TraceEvent(SevDebug, "RemoteIKVSGetError")
@ -474,9 +475,7 @@ struct RemoteIKeyValueStore : public IKeyValueStore {
when(wait(connectionCheckingDelay)) {
// for the corner case where the child process stuck and waitpid also does not give update on it
// In this scenario, we need to manually reboot the storage engine process
if (IFailureMonitor::failureMonitor()
.getState(self->interf.getError.getEndpoint().getPrimaryAddress())
.isFailed()) {
if (IFailureMonitor::failureMonitor().getState(childAddr).isFailed()) {
TraceEvent(SevError, "RemoteKVStoreConnectionStuck").log();
throw please_reboot_remote_kv_store(); // this will reboot the worker
}

15
fdbserver/TLogServer.actor.cpp
View File

@ -1142,8 +1142,19 @@ ACTOR Future<Void> tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen
int8_t tagLocality = inputTag.locality;
if (isPseudoLocality(tagLocality)) {
if (logData->logSystem->get().isValid()) {
upTo = logData->logSystem->get()->popPseudoLocalityTag(inputTag, to);
tagLocality = tagLocalityLogRouter;
// if the configuration change from multi-region to single region mode, the delayed pop created during
// multi-region stage should be skipped. Same thing applies to the backup worker
if (isPseudoLocality(inputTag.locality) &&
logData->logSystem->get()->hasPseudoLocality(inputTag.locality)) {
upTo = logData->logSystem->get()->popPseudoLocalityTag(inputTag, to);
tagLocality = tagLocalityLogRouter;
} else {
ASSERT_WE_THINK(tagLocality == tagLocalityLogRouterMapped);
TraceEvent(SevWarn, "TLogPopNoPseudoLocality", self->dbgid)
.detail("Locality", tagLocality)
.detail("Version", upTo);
return Void();
}
} else {
TraceEvent(SevWarn, "TLogPopNoLogSystem", self->dbgid)
.detail("Locality", tagLocality)

73
fdbserver/fdbserver.actor.cpp
View File

@ -103,7 +103,7 @@ using namespace std::literals;
// clang-format off
enum {
OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_VMEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
@ -153,6 +153,7 @@ CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_KEY, "--key", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP },
{ OPT_STORAGEMEMLIMIT, "-M", SO_REQ_SEP },
{ OPT_STORAGEMEMLIMIT, "--storage-memory", SO_REQ_SEP },
{ OPT_CACHEMEMLIMIT, "--cache-memory", SO_REQ_SEP },
@ -634,7 +635,10 @@ static void printUsage(const char* name, bool devhelp) {
" Define a locality key. LOCALITYKEY is case-insensitive though"
" LOCALITYVALUE is not.");
printOptionUsage("-m SIZE, --memory SIZE",
" Memory limit. The default value is 8GiB. When specified"
" Resident memory limit. The default value is 8GiB. When specified"
" without a unit, MiB is assumed.");
printOptionUsage("--memory-vsize SIZE",
" Virtual memory limit. The default value is unlimited. When specified"
" without a unit, MiB is assumed.");
printOptionUsage("-M SIZE, --storage-memory SIZE",
" Maximum amount of memory used for storage. The default"
@ -1002,9 +1006,10 @@ struct CLIOptions {
NetworkAddressList publicAddresses, listenAddresses;
const char* targetKey = nullptr;
int64_t memLimit =
uint64_t memLimit =
8LL << 30; // Nice to maintain the same default value for memLimit and SERVER_KNOBS->SERVER_MEM_LIMIT and
// SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT
uint64_t virtualMemLimit = 0; // unlimited
uint64_t storageMemLimit = 1LL << 30;
bool buggifyEnabled = false, faultInjectionEnabled = true, restarting = false;
Optional<Standalone<StringRef>> zoneId;
@ -1434,6 +1439,15 @@ private:
}
memLimit = ti.get();
break;
case OPT_VMEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse virtual memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
virtualMemLimit = ti.get();
break;
case OPT_STORAGEMEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MB");
if (!ti.present()) {
@ -1780,47 +1794,28 @@ int main(int argc, char* argv[]) {
Randomize::True,
role == ServerRole::Simulation ? IsSimulated::True
: IsSimulated::False);
IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder));
IKnobCollection::getMutableGlobalKnobCollection().setKnob("conn_file", KnobValue::create(opts.connFile));
if (role != ServerRole::Simulation) {
IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit",
KnobValue::create(int64_t{ opts.memLimit }));
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
g_knobs.setKnob("log_directory", KnobValue::create(opts.logFolder));
g_knobs.setKnob("conn_file", KnobValue::create(opts.connFile));
if (role != ServerRole::Simulation && opts.memLimit > 0) {
g_knobs.setKnob("commit_batches_mem_bytes_hard_limit",
KnobValue::create(static_cast<int64_t>(opts.memLimit)));
}
for (const auto& [knobName, knobValueString] : opts.knobs) {
try {
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
fprintf(stderr,
"WARNING: Invalid value '%s' for knob option '%s'\n",
knobName.c_str(),
knobValueString.c_str());
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
TraceEvent(SevError, "FailedToSetKnob")
.error(e)
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
throw;
}
}
}
IKnobCollection::getMutableGlobalKnobCollection().setKnob("server_mem_limit",
KnobValue::create(int64_t{ opts.memLimit }));
IKnobCollection::setupKnobs(opts.knobs);
g_knobs.setKnob("server_mem_limit", KnobValue::create(static_cast<int64_t>(opts.memLimit)));
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
IKnobCollection::getMutableGlobalKnobCollection().initialize(
Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
g_knobs.initialize(Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
if (opts.memLimit <= FLOW_KNOBS->PAGE_CACHE_4K) {
if (opts.memLimit > 0 && opts.virtualMemLimit > 0 && opts.memLimit > opts.virtualMemLimit) {
fprintf(stderr, "ERROR : --memory-vsize has to be no less than --memory");
flushAndExit(FDB_EXIT_ERROR);
}
if (opts.memLimit > 0 && opts.memLimit <= FLOW_KNOBS->PAGE_CACHE_4K) {
fprintf(stderr, "ERROR: --memory has to be larger than --cache-memory\n");
flushAndExit(FDB_EXIT_ERROR);
}
@ -1937,11 +1932,13 @@ int main(int argc, char* argv[]) {
.detail("BuggifyEnabled", opts.buggifyEnabled)
.detail("FaultInjectionEnabled", opts.faultInjectionEnabled)
.detail("MemoryLimit", opts.memLimit)
.detail("VirtualMemoryLimit", opts.virtualMemLimit)
.trackLatest("ProgramStart");
Error::init();
std::set_new_handler(&platform::outOfMemory);
setMemoryQuota(opts.memLimit);
Future<Void> memoryUsageMonitor = startMemoryUsageMonitor(opts.memLimit);
setMemoryQuota(opts.virtualMemLimit);
Future<Optional<Void>> f;

101
fdbserver/storageserver.actor.cpp
View File

@ -158,7 +158,6 @@ struct AddingShard : NonCopyable {
Future<Void> fetchClient; // holds FetchKeys() actor
Promise<Void> fetchComplete;
Promise<Void> readWrite;
PromiseStream<Key> changeFeedRemovals;
// During the Fetching phase, it saves newer mutations whose version is greater or equal to fetchClient's
// fetchVersion, while the shard is still busy catching up with fetchClient. It applies these updates after fetching
@ -1887,20 +1886,22 @@ ACTOR Future<Void> overlappingChangeFeedsQ(StorageServer* data, OverlappingChang
std::map<Key, std::tuple<KeyRange, Version, Version>> rangeIds;
for (auto r : ranges) {
for (auto& it : r.value()) {
// Can't tell other SS about a change feed create or stopVersion that may get rolled back, and we only need
// to tell it about the metadata if req.minVersion > metadataVersion, since it will get the information from
// its own private mutations if it hasn't processed up that version yet
metadataVersion = std::max(metadataVersion, it->metadataCreateVersion);
if (!it->removing) {
// Can't tell other SS about a change feed create or stopVersion that may get rolled back, and we only
// need to tell it about the metadata if req.minVersion > metadataVersion, since it will get the
// information from its own private mutations if it hasn't processed up that version yet
metadataVersion = std::max(metadataVersion, it->metadataCreateVersion);
Version stopVersion;
if (it->stopVersion != MAX_VERSION && req.minVersion > it->stopVersion) {
stopVersion = it->stopVersion;
metadataVersion = std::max(metadataVersion, stopVersion);
} else {
stopVersion = MAX_VERSION;
Version stopVersion;
if (it->stopVersion != MAX_VERSION && req.minVersion > it->stopVersion) {
stopVersion = it->stopVersion;
metadataVersion = std::max(metadataVersion, stopVersion);
} else {
stopVersion = MAX_VERSION;
}
rangeIds[it->id] = std::tuple(it->range, it->emptyVersion, stopVersion);
}
rangeIds[it->id] = std::tuple(it->range, it->emptyVersion, stopVersion);
}
}
state OverlappingChangeFeedsReply reply;
@ -2550,6 +2551,7 @@ ACTOR Future<Void> changeFeedStreamQ(StorageServer* data, ChangeFeedStreamReques
req.reply.send(feedReply);
if (req.begin == req.end) {
data->activeFeedQueries--;
req.reply.sendError(end_of_stream());
return Void();
}
@ -5059,11 +5061,23 @@ ACTOR Future<Version> fetchChangeFeed(StorageServer* data,
}
}
ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data, KeyRange keys, Version fetchVersion) {
ACTOR Future<std::vector<Key>> fetchChangeFeedMetadata(StorageServer* data,
KeyRange keys,
Version fetchVersion,
PromiseStream<Key> removals) {
TraceEvent(SevDebug, "FetchChangeFeedMetadata", data->thisServerID)
.detail("Range", keys.toString())
.detail("FetchVersion", fetchVersion);
std::vector<OverlappingChangeFeedEntry> feeds = wait(data->cx->getOverlappingChangeFeeds(keys, fetchVersion + 1));
state std::vector<OverlappingChangeFeedEntry> feeds =
wait(data->cx->getOverlappingChangeFeeds(keys, fetchVersion + 1));
while (removals.getFuture().isReady()) {
Key remove = waitNext(removals.getFuture());
for (int i = 0; i < feeds.size(); i++) {
if (feeds[i].rangeId == remove) {
swapAndPop(&feeds, i--);
}
}
}
std::vector<Key> feedIds;
feedIds.reserve(feeds.size());
// create change feed metadata if it does not exist
@ -5159,19 +5173,19 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
KeyRange keys,
Version beginVersion,
Version endVersion,
std::vector<Key> feedIds,
PromiseStream<Key> removals,
std::vector<Key>* feedIds,
std::unordered_set<Key> newFeedIds) {
state std::unordered_map<Key, Version> feedMaxFetched;
if (feedIds.empty() && newFeedIds.empty()) {
if (feedIds->empty() && newFeedIds.empty()) {
return feedMaxFetched;
}
// find overlapping range feeds
state std::map<Key, Future<Version>> feedFetches;
state PromiseStream<Key> removals;
data->changeFeedRemovals[fetchKeysID] = removals;
try {
for (auto& feedId : feedIds) {
for (auto& feedId : *feedIds) {
auto feedIt = data->uidChangeFeed.find(feedId);
// feed may have been moved away or deleted after move was scheduled, do nothing in that case
if (feedIt != data->uidChangeFeed.end() && !feedIt->second->removing) {
@ -5209,7 +5223,15 @@ ACTOR Future<std::unordered_map<Key, Version>> dispatchChangeFeeds(StorageServer
}
}
choose {
when(Key remove = waitNext(removals.getFuture())) { feedFetches.erase(remove); }
when(state Key remove = waitNext(removals.getFuture())) {
wait(delay(0));
feedFetches.erase(remove);
for (int i = 0; i < feedIds->size(); i++) {
if ((*feedIds)[i] == remove) {
swapAndPop(feedIds, i--);
}
}
}
when(wait(success(nextFeed))) {}
}
}
@ -5256,9 +5278,13 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
TraceEvent(SevDebug, interval.begin(), data->thisServerID)
.detail("KeyBegin", shard->keys.begin)
.detail("KeyEnd", shard->keys.end)
.detail("Version", data->version.get());
.detail("Version", data->version.get())
.detail("FKID", fetchKeysID);
state Future<std::vector<Key>> fetchCFMetadata = fetchChangeFeedMetadata(data, keys, data->version.get());
state PromiseStream<Key> removals;
data->changeFeedRemovals[fetchKeysID] = removals;
state Future<std::vector<Key>> fetchCFMetadata =
fetchChangeFeedMetadata(data, keys, data->version.get(), removals);
validate(data);
@ -5486,7 +5512,7 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
// we have written)
state Future<std::unordered_map<Key, Version>> feedFetchMain = dispatchChangeFeeds(
data, fetchKeysID, keys, 0, fetchVersion + 1, changeFeedsToFetch, std::unordered_set<Key>());
data, fetchKeysID, keys, 0, fetchVersion + 1, removals, &changeFeedsToFetch, std::unordered_set<Key>());
state Future<Void> fetchDurable = data->durableVersion.whenAtLeast(data->storageVersion() + 1);
state Future<Void> dataArrive = data->version.whenAtLeast(fetchVersion);
@ -5532,7 +5558,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
for (auto& r : ranges) {
for (auto& cfInfo : r.value()) {
TEST(true); // SS fetching new change feed that didn't exist when fetch started
newChangeFeeds.insert(cfInfo->id);
if (!cfInfo->removing) {
newChangeFeeds.insert(cfInfo->id);
}
}
}
for (auto& cfId : changeFeedsToFetch) {
@ -5541,8 +5569,15 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
// This is split into two fetches to reduce tail. Fetch [0 - fetchVersion+1)
// once fetchVersion is finalized, and [fetchVersion+1, transferredVersion) here once transferredVersion is
// finalized. Also fetch new change feeds alongside it
state Future<std::unordered_map<Key, Version>> feedFetchTransferred = dispatchChangeFeeds(
data, fetchKeysID, keys, fetchVersion + 1, shard->transferredVersion, changeFeedsToFetch, newChangeFeeds);
state Future<std::unordered_map<Key, Version>> feedFetchTransferred =
dispatchChangeFeeds(data,
fetchKeysID,
keys,
fetchVersion + 1,
shard->transferredVersion,
removals,
&changeFeedsToFetch,
newChangeFeeds);
TraceEvent(SevDebug, "FetchKeysHaveData", data->thisServerID)
.detail("FKID", interval.pairID)
@ -5632,7 +5667,9 @@ ACTOR Future<Void> fetchKeys(StorageServer* data, AddingShard* shard) {
TraceEvent(SevDebug, interval.end(), data->thisServerID)
.errorUnsuppressed(e)
.detail("Version", data->version.get());
if (!data->shuttingDown) {
data->changeFeedRemovals.erase(fetchKeysID);
}
if (e.code() == error_code_actor_cancelled && !data->shuttingDown && shard->phase >= AddingShard::Fetching) {
if (shard->phase < AddingShard::FetchingCF) {
data->storage.clearRange(keys);
@ -6224,7 +6261,7 @@ private:
feed->second->stopVersion = currentVersion;
addMutationToLog = true;
}
if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY && !createdFeed) {
if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY && !createdFeed && feed != data->uidChangeFeed.end()) {
TraceEvent(SevDebug, "DestroyingChangeFeed", data->thisServerID)
.detail("RangeID", changeFeedId.printable())
.detail("Range", changeFeedRange.toString())
@ -6250,6 +6287,12 @@ private:
data->changeFeedCleanupDurable[feed->first] = cleanupVersion;
}
if (status == ChangeFeedStatus::CHANGE_FEED_DESTROY) {
for (auto& it : data->changeFeedRemovals) {
it.second.send(changeFeedId);
}
}
if (addMutationToLog) {
auto& mLV = data->addVersionToMutationLog(data->data().getLatestVersion());
data->addMutationToMutationLog(

91
fdbserver/tester.actor.cpp
View File

@ -1320,17 +1320,58 @@ std::string toml_to_string(const T& value) {
}
}
std::vector<TestSpec> readTOMLTests_(std::string fileName) {
TestSpec spec;
struct TestSet {
KnobKeyValuePairs overrideKnobs;
std::vector<TestSpec> testSpecs;
};
namespace {
// In the current TOML scope, look for "knobs" field. If exists, translate all
// key value pairs into KnobKeyValuePairs
KnobKeyValuePairs getOverriddenKnobKeyValues(const toml::value& context) {
KnobKeyValuePairs result;
try {
const toml::array& overrideKnobs = toml::find(context, "knobs").as_array();
for (const toml::value& knob : overrideKnobs) {
for (const auto& [key, value_] : knob.as_table()) {
const std::string& value = toml_to_string(value_);
ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value);
if (std::get_if<NoKnobFound>(&parsedValue)) {
parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
}
if (std::get_if<NoKnobFound>(&parsedValue)) {
TraceEvent(SevError, "TestSpecUnrecognizedKnob")
.detail("KnobName", key)
.detail("OverrideValue", value);
continue;
}
result.set(key, parsedValue);
}
}
} catch (const std::out_of_range&) {
// No knobs field in this scope, this is not an error
}
return result;
}
} // namespace
TestSet readTOMLTests_(std::string fileName) {
Standalone<VectorRef<KeyValueRef>> workloadOptions;
std::vector<TestSpec> result;
TestSet result;
const toml::value& conf = toml::parse(fileName);
// Parse the global knob changes
result.overrideKnobs = getOverriddenKnobKeyValues(conf);
// Then parse each test
const toml::array& tests = toml::find(conf, "test").as_array();
for (const toml::value& test : tests) {
spec = TestSpec();
TestSpec spec;
// First handle all test-level settings
for (const auto& [k, v] : test.as_table()) {
@ -1361,29 +1402,9 @@ std::vector<TestSpec> readTOMLTests_(std::string fileName) {
}
// And then copy the knob attributes to spec.overrideKnobs
try {
const toml::array& overrideKnobs = toml::find(test, "knobs").as_array();
for (const toml::value& knob : overrideKnobs) {
for (const auto& [key, value_] : knob.as_table()) {
const std::string& value = toml_to_string(value_);
ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value);
if (std::get_if<NoKnobFound>(&parsedValue)) {
parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
}
if (std::get_if<NoKnobFound>(&parsedValue)) {
TraceEvent(SevError, "TestSpecUnrecognizedKnob")
.detail("KnobName", key)
.detail("OverrideValue", value);
continue;
}
spec.overrideKnobs.set(key, parsedValue);
}
}
} catch (const std::out_of_range&) {
// no knob overridden
}
spec.overrideKnobs = getOverriddenKnobKeyValues(test);
result.push_back(spec);
result.testSpecs.push_back(spec);
}
return result;
@ -1391,7 +1412,7 @@ std::vector<TestSpec> readTOMLTests_(std::string fileName) {
// A hack to catch and log std::exception, because TOML11 has very useful
// error messages, but the actor framework can't handle std::exception.
std::vector<TestSpec> readTOMLTests(std::string fileName) {
TestSet readTOMLTests(std::string fileName) {
try {
return readTOMLTests_(fileName);
} catch (std::exception& e) {
@ -1694,7 +1715,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
LocalityData locality,
UnitTestParameters testOptions,
Optional<TenantName> defaultTenant) {
state std::vector<TestSpec> testSpecs;
state TestSet testSet;
state std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup(nullptr);
auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
auto ci = makeReference<AsyncVar<Optional<ClusterInterface>>>();
std::vector<Future<Void>> actors;
@ -1725,7 +1747,7 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
options.push_back_deep(options.arena(),
KeyValueRef(LiteralStringRef("shuffleShards"), LiteralStringRef("true")));
spec.options.push_back_deep(spec.options.arena(), options);
testSpecs.push_back(spec);
testSet.testSpecs.push_back(spec);
} else if (whatToRun == TEST_TYPE_UNIT_TESTS) {
TestSpec spec;
Standalone<VectorRef<KeyValueRef>> options;
@ -1741,7 +1763,7 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
options.push_back_deep(options.arena(), KeyValueRef(kv.first, kv.second));
}
spec.options.push_back_deep(spec.options.arena(), options);
testSpecs.push_back(spec);
testSet.testSpecs.push_back(spec);
} else {
std::ifstream ifs;
ifs.open(fileName.c_str(), std::ifstream::in);
@ -1754,11 +1776,11 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
}
enableClientInfoLogging(); // Enable Client Info logging by default for tester
if (boost::algorithm::ends_with(fileName, ".txt")) {
testSpecs = readTests(ifs);
testSet.testSpecs = readTests(ifs);
} else if (boost::algorithm::ends_with(fileName, ".toml")) {
// TOML is weird about opening the file as binary on windows, so we
// just let TOML re-open the file instead of using ifs.
testSpecs = readTOMLTests(fileName);
testSet = readTOMLTests(fileName);
} else {
TraceEvent(SevError, "TestHarnessFail")
.detail("Reason", "unknown tests specification extension")
@ -1768,6 +1790,7 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
ifs.close();
}
knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(testSet.overrideKnobs);
Future<Void> tests;
if (at == TEST_HERE) {
auto db = makeReference<AsyncVar<ServerDBInfo>>();
@ -1775,10 +1798,10 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
actors.push_back(
reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
actors.push_back(reportErrors(testerServerCore(iTesters[0], connRecord, db, locality), "TesterServerCore"));
tests = runTests(cc, ci, iTesters, testSpecs, startingConfiguration, locality, defaultTenant);
tests = runTests(cc, ci, iTesters, testSet.testSpecs, startingConfiguration, locality, defaultTenant);
} else {
tests = reportErrors(
runTests(cc, ci, testSpecs, at, minTestersExpected, startingConfiguration, locality, defaultTenant),
runTests(cc, ci, testSet.testSpecs, at, minTestersExpected, startingConfiguration, locality, defaultTenant),
"RunTests");
}

53
fdbserver/worker.actor.cpp
View File

@ -210,18 +210,10 @@ ACTOR Future<Void> handleIOErrors(Future<Void> actor, IClosable* store, UID id,
state Future<ErrorOr<Void>> storeError = actor.isReady() ? Never() : errorOr(store->getError());
choose {
when(state ErrorOr<Void> e = wait(errorOr(actor))) {
TraceEvent(SevDebug, "HandleIOErrorsActorIsReady")
.detail("Error", e.isError() ? e.getError().code() : -1)
.detail("UID", id);
if (e.isError() && e.getError().code() == error_code_please_reboot) {
// no need to wait.
} else {
TraceEvent(SevDebug, "HandleIOErrorsActorBeforeOnClosed").detail("IsClosed", onClosed.isReady());
wait(onClosed);
TraceEvent(SevDebug, "HandleIOErrorsActorOnClosedFinished")
.detail("StoreError",
storeError.isReady() ? (storeError.get().isError() ? storeError.get().getError().code() : 0)
: -1);
}
if (e.isError() && e.getError().code() == error_code_broken_promise && !storeError.isReady()) {
wait(delay(0.00001 + FLOW_KNOBS->MAX_BUGGIFIED_DELAY));
@ -892,7 +884,8 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
if (workerInDb) {
for (const auto& [address, peer] : allPeers) {
if (peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
if (peer->connectFailedCount == 0 &&
peer->pingLatencies.getPopulationSize() < SERVER_KNOBS->PEER_LATENCY_CHECK_MIN_POPULATION) {
// Ignore peers that don't have enough samples.
// TODO(zhewu): Currently, FlowTransport latency monitor clears ping latency samples on a
// regular
@ -909,7 +902,8 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
// Note that currently we are not monitor storage servers, since lagging in storage servers
// today already can trigger server exclusion by data distributor.
if (peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) >
if (peer->connectFailedCount >= SERVER_KNOBS->PEER_DEGRADATION_CONNECTION_FAILURE_COUNT ||
peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE) >
SERVER_KNOBS->PEER_LATENCY_DEGRADATION_THRESHOLD ||
peer->timeoutCount / (double)(peer->pingLatencies.getPopulationSize()) >
SERVER_KNOBS->PEER_TIMEOUT_PERCENTAGE_DEGRADATION_THRESHOLD) {
@ -926,13 +920,34 @@ ACTOR Future<Void> healthMonitor(Reference<AsyncVar<Optional<ClusterControllerFu
.detail(
"CheckedPercentileLatency",
peer->pingLatencies.percentile(SERVER_KNOBS->PEER_LATENCY_DEGRADATION_PERCENTILE))
.detail("Count", peer->pingLatencies.getPopulationSize())
.detail("TimeoutCount", peer->timeoutCount);
.detail("PingCount", peer->pingLatencies.getPopulationSize())
.detail("PingTimeoutCount", peer->timeoutCount)
.detail("ConnectionFailureCount", peer->connectFailedCount);
req.degradedPeers.push_back(address);
}
}
}
if (SERVER_KNOBS->WORKER_HEALTH_REPORT_RECENT_DESTROYED_PEER) {
// When the worker cannot connect to a remote peer, the peer maybe erased from the list returned
// from getAllPeers(). Therefore, we also look through all the recent closed peers in the flow
// transport's health monitor. Note that all the closed peers stored here are caused by connection
// failure, but not normal connection close. Therefore, we report all such peers if they are also
// part of the transaction sub system.
for (const auto& address : FlowTransport::transport().healthMonitor()->getRecentClosedPeers()) {
if (allPeers.find(address) != allPeers.end()) {
// We have checked this peer in the above for loop.
continue;
}
if ((workerInPrimary && addressInDbAndPrimaryDc(address, dbInfo)) ||
(!workerInPrimary && addressInDbAndRemoteDc(address, dbInfo))) {
TraceEvent("HealthMonitorDetectRecentClosedPeer").suppressFor(30).detail("Peer", address);
req.degradedPeers.push_back(address);
}
}
}
}
if (!req.degradedPeers.empty()) {
@ -1551,8 +1566,11 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
memoryLimit,
false,
validateDataFiles,
SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled*/
(g_network->isSimulated() ? deterministicRandom()->coinflip() : true));
SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled */
(g_network->isSimulated()
? (/* Disable for RocksDB */ s.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
deterministicRandom()->coinflip())
: true));
Future<Void> kvClosed = kv->onClosed();
filesClosed.add(kvClosed);
@ -2124,8 +2142,11 @@ ACTOR Future<Void> workerServer(Reference<IClusterConnectionRecord> connRecord,
memoryLimit,
false,
false,
SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled*/
(g_network->isSimulated() ? deterministicRandom()->coinflip() : true));
SERVER_KNOBS->REMOTE_KV_STORE && /* testing mixed mode in simulation if remote kvs enabled */
(g_network->isSimulated()
? (/* Disable for RocksDB */ req.storeType != KeyValueStoreType::SSD_ROCKSDB_V1 &&
deterministicRandom()->coinflip())
: true));
Future<Void> kvClosed = data->onClosed();
filesClosed.add(kvClosed);

13
flow/Arena.cpp
View File

@ -304,11 +304,9 @@ void* ArenaBlock::allocate(Reference<ArenaBlock>& self, int bytes) {
ArenaBlock* ArenaBlock::create(int dataSize, Reference<ArenaBlock>& next) {
ArenaBlock* b;
if (dataSize <= SMALL - TINY_HEADER && !next) {
if (dataSize <= 16 - TINY_HEADER) {
b = (ArenaBlock*)FastAllocator<16>::allocate();
b->tinySize = 16;
INSTRUMENT_ALLOCATE("Arena16");
} else if (dataSize <= 32 - TINY_HEADER) {
static_assert(sizeof(ArenaBlock) <= 32); // Need to allocate at least sizeof(ArenaBlock) for an ArenaBlock*. See
// https://github.com/apple/foundationdb/issues/6753
if (dataSize <= 32 - TINY_HEADER) {
b = (ArenaBlock*)FastAllocator<32>::allocate();
b->tinySize = 32;
INSTRUMENT_ALLOCATE("Arena32");
@ -442,10 +440,7 @@ void ArenaBlock::destroy() {
void ArenaBlock::destroyLeaf() {
if (isTiny()) {
if (tinySize <= 16) {
FastAllocator<16>::release(this);
INSTRUMENT_RELEASE("Arena16");
} else if (tinySize <= 32) {
if (tinySize <= 32) {
FastAllocator<32>::release(this);
INSTRUMENT_RELEASE("Arena32");
} else {

2
flow/CMakeLists.txt
View File

@ -30,6 +30,8 @@ set(FLOW_SRCS
Hash3.h
Histogram.cpp
Histogram.h
Hostname.actor.cpp
Hostname.h
IDispatched.h
IRandom.h
IThreadPoolTest.actor.cpp

6
flow/FastAlloc.h
View File

@ -178,7 +178,7 @@ void releaseAllThreadMagazines();
int64_t getTotalUnusedAllocatedMemory();
inline constexpr int nextFastAllocatedSize(int x) {
assert(x > 0 && x <= 8192);
assert(x > 0 && x <= 16384);
if (x <= 16)
return 16;
else if (x <= 32)
@ -199,8 +199,10 @@ inline constexpr int nextFastAllocatedSize(int x) {
return 2048;
else if (x <= 4096)
return 4096;
else
else if (x <= 8192)
return 8192;
else
return 16384;
}
template <class Object>

230
flow/Hostname.actor.cpp Normal file
View File

@ -0,0 +1,230 @@
/*
* Hostname.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "flow/Hostname.h"
#include "flow/UnitTest.h"
#include "flow/actorcompiler.h" // has to be last include
Hostname Hostname::parse(const std::string& s) {
if (s.empty() || !Hostname::isHostname(s)) {
throw connection_string_invalid();
}
bool isTLS = false;
std::string f;
if (s.size() > 4 && strcmp(s.c_str() + s.size() - 4, ":tls") == 0) {
isTLS = true;
f = s.substr(0, s.size() - 4);
} else {
f = s;
}
auto colonPos = f.find_first_of(":");
return Hostname(f.substr(0, colonPos), f.substr(colonPos + 1), isTLS);
}
void Hostname::resetToUnresolved() {
if (status == Hostname::RESOLVED) {
status = UNRESOLVED;
resolvedAddress = Optional<NetworkAddress>();
}
}
ACTOR Future<Void> resolveImpl(Hostname* self) {
loop {
if (self->status == Hostname::UNRESOLVED) {
self->status = Hostname::RESOLVING;
try {
std::vector<NetworkAddress> addresses =
wait(INetworkConnections::net()->resolveTCPEndpointWithDNSCache(self->host, self->service));
NetworkAddress address = addresses[deterministicRandom()->randomInt(0, addresses.size())];
address.flags = 0; // Reset the parsed address to public
address.fromHostname = NetworkAddressFromHostname::True;
if (self->isTLS) {
address.flags |= NetworkAddress::FLAG_TLS;
}
self->resolvedAddress = address;
self->status = Hostname::RESOLVED;
break;
} catch (...) {
self->status = Hostname::UNRESOLVED;
self->resolveFinish.trigger();
self->resolvedAddress = Optional<NetworkAddress>();
throw lookup_failed();
}
} else if (self->status == Hostname::RESOLVING) {
wait(self->resolveFinish.onTrigger());
if (self->status == Hostname::RESOLVED) {
break;
}
// Otherwise, this means other threads failed on resolve, so here we go back to the loop and try to resolve
// again.
} else {
// status is RESOLVED, nothing to do.
break;
}
}
return Void();
}
ACTOR Future<Void> resolveWithRetryImpl(Hostname* self) {
loop {
try {
wait(resolveImpl(self));
return Void();
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled) {
throw;
}
wait(delay(FLOW_KNOBS->HOSTNAME_RESOLVE_DELAY));
}
}
}
Future<Void> Hostname::resolve() {
return resolveImpl(this);
}
Future<Void> Hostname::resolveWithRetry() {
return resolveWithRetryImpl(this);
}
void Hostname::resolveBlocking() {
if (status != RESOLVED) {
try {
std::vector<NetworkAddress> addresses =
INetworkConnections::net()->resolveTCPEndpointBlockingWithDNSCache(host, service);
NetworkAddress address = addresses[deterministicRandom()->randomInt(0, addresses.size())];
address.flags = 0; // Reset the parsed address to public
address.fromHostname = NetworkAddressFromHostname::True;
if (isTLS) {
address.flags |= NetworkAddress::FLAG_TLS;
}
resolvedAddress = address;
status = RESOLVED;
} catch (...) {
status = UNRESOLVED;
resolvedAddress = Optional<NetworkAddress>();
throw lookup_failed();
}
}
}
TEST_CASE("/flow/Hostname/hostname") {
std::string hn1s = "localhost:1234";
std::string hn2s = "host-name:1234";
std::string hn3s = "host.name:1234";
std::string hn4s = "host-name_part1.host-name_part2:1234:tls";
std::string hn5s = "127.0.0.1:1234";
std::string hn6s = "127.0.0.1:1234:tls";
std::string hn7s = "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:4800";
std::string hn8s = "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:4800:tls";
std::string hn9s = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
std::string hn10s = "2001:0db8:85a3:0000:0000:8a2e:0370:7334:tls";
std::string hn11s = "[::1]:4800";
std::string hn12s = "[::1]:4800:tls";
std::string hn13s = "1234";
auto hn1 = Hostname::parse(hn1s);
ASSERT(hn1.toString() == hn1s);
ASSERT(hn1.host == "localhost");
ASSERT(hn1.service == "1234");
ASSERT(!hn1.isTLS);
state Hostname hn2 = Hostname::parse(hn2s);
ASSERT(hn2.toString() == hn2s);
ASSERT(hn2.host == "host-name");
ASSERT(hn2.service == "1234");
ASSERT(!hn2.isTLS);
auto hn3 = Hostname::parse(hn3s);
ASSERT(hn3.toString() == hn3s);
ASSERT(hn3.host == "host.name");
ASSERT(hn3.service == "1234");
ASSERT(!hn3.isTLS);
auto hn4 = Hostname::parse(hn4s);
ASSERT(hn4.toString() == hn4s);
ASSERT(hn4.host == "host-name_part1.host-name_part2");
ASSERT(hn4.service == "1234");
ASSERT(hn4.isTLS);
ASSERT(!Hostname::isHostname(hn5s));
ASSERT(!Hostname::isHostname(hn6s));
ASSERT(!Hostname::isHostname(hn7s));
ASSERT(!Hostname::isHostname(hn8s));
ASSERT(!Hostname::isHostname(hn9s));
ASSERT(!Hostname::isHostname(hn10s));
ASSERT(!Hostname::isHostname(hn11s));
ASSERT(!Hostname::isHostname(hn12s));
ASSERT(!Hostname::isHostname(hn13s));
ASSERT(hn1.status == Hostname::UNRESOLVED && !hn1.resolvedAddress.present());
ASSERT(hn2.status == Hostname::UNRESOLVED && !hn2.resolvedAddress.present());
ASSERT(hn3.status == Hostname::UNRESOLVED && !hn3.resolvedAddress.present());
ASSERT(hn4.status == Hostname::UNRESOLVED && !hn4.resolvedAddress.present());
try {
wait(hn2.resolve());
} catch (Error& e) {
ASSERT(e.code() == error_code_lookup_failed);
}
ASSERT(hn2.status == Hostname::UNRESOLVED && !hn2.resolvedAddress.present());
try {
wait(timeoutError(hn2.resolveWithRetry(), 1));
} catch (Error& e) {
ASSERT(e.code() == error_code_timed_out);
}
ASSERT(hn2.status == Hostname::UNRESOLVED && !hn2.resolvedAddress.present());
try {
hn2.resolveBlocking();
} catch (Error& e) {
ASSERT(e.code() == error_code_lookup_failed);
}
ASSERT(hn2.status == Hostname::UNRESOLVED && !hn2.resolvedAddress.present());
state NetworkAddress address = NetworkAddress::parse("127.0.0.0:1234");
INetworkConnections::net()->addMockTCPEndpoint("host-name", "1234", { address });
// Test resolve.
wait(hn2.resolve());
ASSERT(hn2.status == Hostname::RESOLVED);
ASSERT(hn2.resolvedAddress.present() && hn2.resolvedAddress.get() == address);
// Test resolveWithRetry.
hn2.resetToUnresolved();
ASSERT(hn2.status == Hostname::UNRESOLVED && !hn2.resolvedAddress.present());
wait(hn2.resolveWithRetry());
ASSERT(hn2.status == Hostname::RESOLVED);
ASSERT(hn2.resolvedAddress.present() && hn2.resolvedAddress.get() == address);
// Test resolveBlocking.
hn2.resetToUnresolved();
ASSERT(hn2.status == Hostname::UNRESOLVED && !hn2.resolvedAddress.present());
hn2.resolveBlocking();
ASSERT(hn2.status == Hostname::RESOLVED);
ASSERT(hn2.resolvedAddress.present() && hn2.resolvedAddress.get() == address);
return Void();
}

86
flow/Hostname.h Normal file
View File

@ -0,0 +1,86 @@
/*
* Hostname.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FLOW_HOSTNAME_H
#define FLOW_HOSTNAME_H
#pragma once
#include "flow/network.h"
#include "flow/genericactors.actor.h"
struct Hostname {
std::string host;
std::string service; // decimal port number
bool isTLS;
Hostname(const std::string& host, const std::string& service, bool isTLS)
: host(host), service(service), isTLS(isTLS) {}
Hostname() : host(""), service(""), isTLS(false) {}
Hostname(const Hostname& rhs) { operator=(rhs); }
Hostname& operator=(const Hostname& rhs) {
// Copy everything except AsyncTrigger resolveFinish.
host = rhs.host;
service = rhs.service;
isTLS = rhs.isTLS;
resolvedAddress = rhs.resolvedAddress;
status = rhs.status;
return *this;
}
bool operator==(const Hostname& r) const { return host == r.host && service == r.service && isTLS == r.isTLS; }
bool operator!=(const Hostname& r) const { return !(*this == r); }
bool operator<(const Hostname& r) const {
if (isTLS != r.isTLS)
return isTLS < r.isTLS;
else if (host != r.host)
return host < r.host;
return service < r.service;
}
bool operator>(const Hostname& r) const { return r < *this; }
bool operator<=(const Hostname& r) const { return !(*this > r); }
bool operator>=(const Hostname& r) const { return !(*this < r); }
// Allow hostnames in forms like following:
// hostname:1234
// host.name:1234
// host-name:1234
// host-name_part1.host-name_part2:1234:tls
static bool isHostname(const std::string& s) {
std::regex validation("^([\\w\\-]+\\.?)+:([\\d]+){1,}(:tls)?$");
std::regex ipv4Validation("^([\\d]{1,3}\\.?){4,}:([\\d]+){1,}(:tls)?$");
return !std::regex_match(s, ipv4Validation) && std::regex_match(s, validation);
}
static Hostname parse(const std::string& s);
std::string toString() const { return host + ":" + service + (isTLS ? ":tls" : ""); }
Optional<NetworkAddress> resolvedAddress;
enum HostnameStatus { UNRESOLVED, RESOLVING, RESOLVED };
Future<Void> resolve();
Future<Void> resolveWithRetry();
void resolveBlocking(); // This one should only be used when resolving asynchronously is impossible.
// For all other cases, resolve() should be preferred.
void resetToUnresolved();
HostnameStatus status = UNRESOLVED;
AsyncTrigger resolveFinish;
};
#endif

3
flow/Knobs.cpp
View File

@ -40,6 +40,7 @@ FlowKnobs const* FLOW_KNOBS = &bootstrapGlobalFlowKnobs;
void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
init( AUTOMATIC_TRACE_DUMP, 1 );
init( PREVENT_FAST_SPIN_DELAY, .01 );
init( HOSTNAME_RESOLVE_DELAY, .05 );
init( CACHE_REFRESH_INTERVAL_WHEN_ALL_ALTERNATIVES_FAILED, 1.0 );
init( DELAY_JITTER_OFFSET, 0.9 );
@ -67,6 +68,8 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
init( HUGE_ARENA_LOGGING_BYTES, 100e6 );
init( HUGE_ARENA_LOGGING_INTERVAL, 5.0 );
init( MEMORY_USAGE_CHECK_INTERVAL, 1.0 );
// Chaos testing - enabled for simulation by default
init( ENABLE_CHAOS_FEATURES, isSimulated );
init( CHAOS_LOGGING_INTERVAL, 5.0 );

3
flow/Knobs.h
View File

@ -113,6 +113,7 @@ class FlowKnobs : public KnobsImpl<FlowKnobs> {
public:
int AUTOMATIC_TRACE_DUMP;
double PREVENT_FAST_SPIN_DELAY;
double HOSTNAME_RESOLVE_DELAY;
double CACHE_REFRESH_INTERVAL_WHEN_ALL_ALTERNATIVES_FAILED;
double DELAY_JITTER_OFFSET;
@ -129,6 +130,8 @@ public:
double HUGE_ARENA_LOGGING_BYTES;
double HUGE_ARENA_LOGGING_INTERVAL;
double MEMORY_USAGE_CHECK_INTERVAL;
// Chaos testing
bool ENABLE_CHAOS_FEATURES;
double CHAOS_LOGGING_INTERVAL;

77
flow/Net2.actor.cpp
View File

@ -166,8 +166,12 @@ public:
Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host,
const std::string& service) override;
Future<std::vector<NetworkAddress>> resolveTCPEndpointWithDNSCache(const std::string& host,
const std::string& service) override;
std::vector<NetworkAddress> resolveTCPEndpointBlocking(const std::string& host,
const std::string& service) override;
std::vector<NetworkAddress> resolveTCPEndpointBlockingWithDNSCache(const std::string& host,
const std::string& service) override;
Reference<IListener> listen(NetworkAddress localAddr) override;
// INetwork interface
@ -1837,6 +1841,14 @@ Future<Reference<IConnection>> Net2::connectExternal(NetworkAddress toAddr, cons
return connect(toAddr, host);
}
Future<Reference<IUDPSocket>> Net2::createUDPSocket(NetworkAddress toAddr) {
return UDPSocket::connect(&reactor.ios, toAddr, toAddr.ip.isV6());
}
Future<Reference<IUDPSocket>> Net2::createUDPSocket(bool isV6) {
return UDPSocket::connect(&reactor.ios, Optional<NetworkAddress>(), isV6);
}
ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* self,
std::string host,
std::string service) {
@ -1847,6 +1859,7 @@ ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* s
tcpResolver.async_resolve(tcp::resolver::query(host, service),
[=](const boost::system::error_code& ec, tcp::resolver::iterator iter) {
if (ec) {
self->dnsCache.remove(host, service);
promise.sendError(lookup_failed());
return;
}
@ -1866,6 +1879,7 @@ ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* s
}
if (addrs.empty()) {
self->dnsCache.remove(host, service);
promise.sendError(lookup_failed());
} else {
promise.send(addrs);
@ -1874,39 +1888,58 @@ ACTOR static Future<std::vector<NetworkAddress>> resolveTCPEndpoint_impl(Net2* s
wait(ready(result));
tcpResolver.cancel();
std::vector<NetworkAddress> ret = result.get();
self->dnsCache.add(host, service, ret);
return result.get();
}
Future<Reference<IUDPSocket>> Net2::createUDPSocket(NetworkAddress toAddr) {
return UDPSocket::connect(&reactor.ios, toAddr, toAddr.ip.isV6());
}
Future<Reference<IUDPSocket>> Net2::createUDPSocket(bool isV6) {
return UDPSocket::connect(&reactor.ios, Optional<NetworkAddress>(), isV6);
return ret;
}
Future<std::vector<NetworkAddress>> Net2::resolveTCPEndpoint(const std::string& host, const std::string& service) {
return resolveTCPEndpoint_impl(this, host, service);
}
Future<std::vector<NetworkAddress>> Net2::resolveTCPEndpointWithDNSCache(const std::string& host,
const std::string& service) {
Optional<std::vector<NetworkAddress>> cache = dnsCache.find(host, service);
if (cache.present()) {
return cache.get();
}
return resolveTCPEndpoint_impl(this, host, service);
}
std::vector<NetworkAddress> Net2::resolveTCPEndpointBlocking(const std::string& host, const std::string& service) {
tcp::resolver tcpResolver(reactor.ios);
tcp::resolver::query query(host, service);
auto iter = tcpResolver.resolve(query);
decltype(iter) end;
std::vector<NetworkAddress> addrs;
while (iter != end) {
auto endpoint = iter->endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
try {
auto iter = tcpResolver.resolve(host, service);
decltype(iter) end;
std::vector<NetworkAddress> addrs;
while (iter != end) {
auto endpoint = iter->endpoint();
auto addr = endpoint.address();
if (addr.is_v6()) {
addrs.emplace_back(IPAddress(addr.to_v6().to_bytes()), endpoint.port());
} else {
addrs.emplace_back(addr.to_v4().to_ulong(), endpoint.port());
}
++iter;
}
++iter;
if (addrs.empty()) {
throw lookup_failed();
}
return addrs;
} catch (...) {
dnsCache.remove(host, service);
throw lookup_failed();
}
return addrs;
}
std::vector<NetworkAddress> Net2::resolveTCPEndpointBlockingWithDNSCache(const std::string& host,
const std::string& service) {
Optional<std::vector<NetworkAddress>> cache = dnsCache.find(host, service);
if (cache.present()) {
return cache.get();
}
return resolveTCPEndpointBlocking(host, service);
}
bool Net2::isAddressOnThisHost(NetworkAddress const& addr) const {

3
flow/Platform.actor.cpp
View File

@ -1942,6 +1942,9 @@ std::string epochsToGMTString(double epochs) {
}
void setMemoryQuota(size_t limit) {
if (limit == 0) {
return;
}
#if defined(USE_SANITIZER)
// ASAN doesn't work with memory quotas: https://github.com/google/sanitizers/wiki/AddressSanitizer#ulimit--v
return;

12
flow/SystemMonitor.cpp
View File

@ -415,3 +415,15 @@ SystemStatistics customSystemMonitor(std::string const& eventName, StatisticsSta
statState->networkState = netData;
return currentStats;
}
Future<Void> startMemoryUsageMonitor(uint64_t memLimit) {
if (memLimit == 0) {
return Void();
}
auto checkMemoryUsage = [=]() {
if (getResidentMemoryUsage() > memLimit) {
platform::outOfMemory();
}
};
return recurring(checkMemoryUsage, FLOW_KNOBS->MEMORY_USAGE_CHECK_INTERVAL);
}

2
flow/SystemMonitor.h
View File

@ -156,4 +156,6 @@ SystemStatistics customSystemMonitor(std::string const& eventName,
bool machineMetrics = false);
SystemStatistics getSystemStatistics();
Future<Void> startMemoryUsageMonitor(uint64_t memLimit);
#endif /* FLOW_SYSTEM_MONITOR_H */

43
flow/ThreadHelper.actor.h
View File

@ -689,7 +689,7 @@ private:
// If instead, this actor is cancelled, we will also cancel the underlying "threadFuture"
// Note: we are required to have unique ownership of the "threadFuture"
ACTOR template <class T>
Future<T> safeThreadFutureToFuture(ThreadFuture<T> threadFuture) {
Future<T> safeThreadFutureToFutureImpl(ThreadFuture<T> threadFuture) {
Promise<Void> ready;
Future<Void> onReady = ready.getFuture();
UtilCallback<T>* callback = new UtilCallback<T>(threadFuture, ready.extractRawPointer());
@ -710,10 +710,45 @@ Future<T> safeThreadFutureToFuture(ThreadFuture<T> threadFuture) {
return threadFuture.get();
}
// do nothing, just for template functions' calls
// The allow anonymous_future type is used to prevent misuse of ThreadFutures.
// For Standalone types, the memory in some cases is actually stored in the ThreadFuture object,
// in which case we expect the caller to keep that ThreadFuture around until the result is no
// longer needed.
//
// We can provide some compile-time detection of this misuse by disallowing anonymous thread futures
// being passed in for certain types.
template <typename T>
struct allow_anonymous_future : std::true_type {};
template <typename T>
struct allow_anonymous_future<Standalone<T>> : std::false_type {};
template <typename T>
struct allow_anonymous_future<Optional<Standalone<T>>> : std::false_type {};
template <class T>
Future<T> safeThreadFutureToFuture(Future<T> future) {
// do nothing
typename std::enable_if<allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
const ThreadFuture<T>& threadFuture) {
return safeThreadFutureToFutureImpl(threadFuture);
}
template <class T>
typename std::enable_if<!allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
ThreadFuture<T>& threadFuture) {
return safeThreadFutureToFutureImpl(threadFuture);
}
template <class T>
typename std::enable_if<allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
const Future<T>& future) {
// Do nothing
return future;
}
template <class T>
typename std::enable_if<!allow_anonymous_future<T>::value, Future<T>>::type safeThreadFutureToFuture(
Future<T>& future) {
// Do nothing
return future;
}

511
flow/Tracing.actor.cpp
View File

@ -19,10 +19,9 @@
*/
#include "flow/Tracing.h"
#include "flow/UnitTest.h"
#include "flow/Knobs.h"
#include "flow/network.h"
#include <functional>
#include <iomanip>
#include <memory>
@ -43,6 +42,7 @@ constexpr float kQueueSizeLogInterval = 5.0;
struct NoopTracer : ITracer {
TracerType type() const override { return TracerType::DISABLED; }
void trace(Span const& span) override {}
void trace(OTELSpan const& span) override {}
};
struct LogfileTracer : ITracer {
@ -63,6 +63,35 @@ struct LogfileTracer : ITracer {
TraceEvent(SevInfo, "TracingSpanTag", span.context).detail("Key", key).detail("Value", value);
}
}
void trace(OTELSpan const& span) override {
TraceEvent te(SevInfo, "TracingSpan", span.context.traceID);
te.detail("SpanID", span.context.spanID)
.detail("Location", span.location.name)
.detail("Begin", format("%.6f", span.begin))
.detail("End", format("%.6f", span.end))
.detail("Kind", span.kind)
.detail("Status", span.status)
.detail("ParentSpanID", span.parentContext.spanID);
for (const auto& link : span.links) {
TraceEvent(SevInfo, "TracingSpanLink", span.context.traceID)
.detail("TraceID", link.traceID)
.detail("SpanID", link.spanID);
}
for (const auto& [key, value] : span.attributes) {
TraceEvent(SevInfo, "TracingSpanTag", span.context.traceID).detail("Key", key).detail("Value", value);
}
for (const auto& event : span.events) {
TraceEvent(SevInfo, "TracingSpanEvent", span.context.traceID)
.detail("Name", event.name)
.detail("Time", event.time);
for (const auto& [key, value] : event.attributes) {
TraceEvent(SevInfo, "TracingSpanEventAttribute", span.context.traceID)
.detail("Key", key)
.detail("Value", value);
}
}
}
};
struct TraceRequest {
@ -151,7 +180,6 @@ ACTOR Future<Void> traceLog(int* pendingMessages, bool* sendError) {
*/
struct UDPTracer : public ITracer {
protected:
// Serializes span fields as an array into the supplied TraceRequest
// buffer.
void serialize_span(const Span& span, TraceRequest& request) {
@ -179,6 +207,32 @@ protected:
serialize_vector(span.parents, request);
}
void serialize_span(const OTELSpan& span, TraceRequest& request) {
uint16_t size = 14;
request.write_byte(size | 0b10010000); // write as array
serialize_value(span.context.traceID.first(), request, 0xcf); // trace id
serialize_value(span.context.traceID.second(), request, 0xcf); // trace id
serialize_value(span.context.spanID, request, 0xcf); // spanid
// parent value
serialize_value(span.parentContext.traceID.first(), request, 0xcf); // trace id
serialize_value(span.parentContext.traceID.second(), request, 0xcf); // trace id
serialize_value(span.parentContext.spanID, request, 0xcf); // spanId
// Payload
serialize_string(span.location.name.toString(), request);
serialize_value(span.begin, request, 0xcb); // start time
serialize_value(span.end, request, 0xcb); // end
// Kind
serialize_value(span.kind, request, 0xcc);
// Status
serialize_value(span.status, request, 0xcc);
// Links
serialize_vector(span.links, request);
// Events
serialize_vector(span.events, request);
// Attributes
serialize_map(span.attributes, request);
}
private:
// Writes the given value in big-endian format to the request. Sets the
// first byte to msgpack_type.
@ -205,10 +259,12 @@ private:
request.write_byte(static_cast<uint8_t>(length));
} else if (length <= 65535) {
request.write_byte(0xda);
request.write_byte(static_cast<uint16_t>(length));
request.write_byte(reinterpret_cast<const uint8_t*>(&length)[1]);
request.write_byte(reinterpret_cast<const uint8_t*>(&length)[0]);
} else {
// TODO: Add support for longer strings if necessary.
ASSERT(false);
TraceEvent(SevWarn, "TracingSpanSerializeString")
.detail("Failed to MessagePack encode very large string", length);
ASSERT_WE_THINK(false);
}
request.write_bytes(c, length);
@ -225,7 +281,6 @@ private:
if (size == 0) {
return;
}
if (size <= 15) {
request.write_byte(static_cast<uint8_t>(size) | 0b10010000);
} else if (size <= 65535) {
@ -233,8 +288,9 @@ private:
request.write_byte(reinterpret_cast<const uint8_t*>(&size)[1]);
request.write_byte(reinterpret_cast<const uint8_t*>(&size)[0]);
} else {
// TODO: Add support for longer vectors if necessary.
ASSERT(false);
TraceEvent(SevWarn, "TracingSpanSerializeVector")
.detail("Failed to MessagePack encode very large vector", size);
ASSERT_WE_THINK(false);
}
for (const auto& parentContext : vec) {
@ -242,14 +298,76 @@ private:
}
}
inline void serialize_map(const std::unordered_map<StringRef, StringRef>& map, TraceRequest& request) {
// Writes the given vector of linked SpanContext's to the request. If the vector is
// empty, the request is not modified.
inline void serialize_vector(const SmallVectorRef<SpanContext>& vec, TraceRequest& request) {
int size = vec.size();
if (size <= 15) {
request.write_byte(static_cast<uint8_t>(size) | 0b10010000);
} else if (size <= 65535) {
request.write_byte(0xdc);
request.write_byte(reinterpret_cast<const uint8_t*>(&size)[1]);
request.write_byte(reinterpret_cast<const uint8_t*>(&size)[0]);
} else {
TraceEvent(SevWarn, "TracingSpanSerializeVector").detail("Failed to MessagePack encode large vector", size);
ASSERT_WE_THINK(false);
}
for (const auto& link : vec) {
serialize_value(link.traceID.first(), request, 0xcf); // trace id
serialize_value(link.traceID.second(), request, 0xcf); // trace id
serialize_value(link.spanID, request, 0xcf); // spanid
}
}
// Writes the given vector of linked SpanContext's to the request. If the vector is
// empty, the request is not modified.
inline void serialize_vector(const SmallVectorRef<OTELEventRef>& vec, TraceRequest& request) {
int size = vec.size();
if (size <= 15) {
request.write_byte(static_cast<uint8_t>(size) | 0b10010000);
} else if (size <= 65535) {
request.write_byte(0xdc);
request.write_byte(reinterpret_cast<const uint8_t*>(&size)[1]);
request.write_byte(reinterpret_cast<const uint8_t*>(&size)[0]);
} else {
TraceEvent(SevWarn, "TracingSpanSerializeVector").detail("Failed to MessagePack encode large vector", size);
ASSERT_WE_THINK(false);
}
for (const auto& event : vec) {
serialize_string(event.name.toString(), request); // event name
serialize_value(event.time, request, 0xcb); // event time
serialize_vector(event.attributes, request);
}
}
inline void serialize_vector(const SmallVectorRef<KeyValueRef>& vals, TraceRequest& request) {
int size = vals.size();
if (size <= 15) {
// N.B. We're actually writing this out as a fixmap here in messagepack format!
// fixmap 1000xxxx 0x80 - 0x8f
request.write_byte(static_cast<uint8_t>(size) | 0b10000000);
} else {
TraceEvent(SevWarn, "TracingSpanSerializeVector").detail("Failed to MessagePack encode large vector", size);
ASSERT_WE_THINK(false);
}
for (const auto& kv : vals) {
serialize_string(kv.key.toString(), request);
serialize_string(kv.value.toString(), request);
}
}
template <class Map>
inline void serialize_map(const Map& map, TraceRequest& request) {
int size = map.size();
if (size <= 15) {
request.write_byte(static_cast<uint8_t>(size) | 0b10000000);
} else {
// TODO: Add support for longer maps if necessary.
ASSERT(false);
TraceEvent(SevWarn, "TracingSpanSerializeMap").detail("Failed to MessagePack encode large map", size);
ASSERT_WE_THINK(false);
}
for (const auto& [key, value] : map) {
@ -291,7 +409,7 @@ struct FastUDPTracer : public UDPTracer {
TracerType type() const override { return TracerType::NETWORK_LOSSY; }
void trace(Span const& span) override {
void prepare(int size) {
static std::once_flag once;
std::call_once(once, [&]() {
log_actor_ = fastTraceLogger(&unready_socket_messages_, &failed_messages_, &total_messages_, &send_error_);
@ -307,7 +425,7 @@ struct FastUDPTracer : public UDPTracer {
socket_ = INetworkConnections::net()->createUDPSocket(destAddress);
});
if (span.location.name.size() == 0) {
if (size == 0) {
return;
}
@ -322,9 +440,9 @@ struct FastUDPTracer : public UDPTracer {
if (send_error_) {
return;
}
}
serialize_span(span, request_);
void write() {
int bytesSent = send(socket_fd_, request_.buffer.get(), request_.data_size, MSG_DONTWAIT);
if (bytesSent == -1) {
// Will forgo checking errno here, and assume all error messages
@ -335,6 +453,18 @@ struct FastUDPTracer : public UDPTracer {
request_.reset();
}
void trace(OTELSpan const& span) override {
prepare(span.location.name.size());
serialize_span(span, request_);
write();
}
void trace(Span const& span) override {
prepare(span.location.name.size());
serialize_span(span, request_);
write();
}
private:
TraceRequest request_;
@ -403,3 +533,352 @@ Span::~Span() {
g_tracer->trace(*this);
}
}
OTELSpan& OTELSpan::operator=(OTELSpan&& o) {
if (begin > 0.0 && o.context.isSampled() > 0) {
end = g_network->now();
g_tracer->trace(*this);
}
arena = std::move(o.arena);
context = o.context;
parentContext = o.parentContext;
begin = o.begin;
end = o.end;
location = o.location;
links = std::move(o.links);
events = std::move(o.events);
status = o.status;
kind = o.kind;
o.context = SpanContext();
o.parentContext = SpanContext();
o.kind = SpanKind::INTERNAL;
o.begin = 0.0;
o.end = 0.0;
o.status = SpanStatus::UNSET;
return *this;
}
OTELSpan::~OTELSpan() {
if (begin > 0.0 && context.isSampled()) {
end = g_network->now();
g_tracer->trace(*this);
}
}
TEST_CASE("/flow/Tracing/CreateOTELSpan") {
// Sampling disabled, no parent.
OTELSpan notSampled("foo"_loc);
ASSERT(!notSampled.context.isSampled());
// Force Sampling
OTELSpan sampled("foo"_loc, []() { return 1.0; });
ASSERT(sampled.context.isSampled());
// Ensure child traceID matches parent, when parent is sampled.
OTELSpan childTraceIDMatchesParent(
"foo"_loc, []() { return 1.0; }, SpanContext(UID(100, 101), 200, TraceFlags::sampled));
ASSERT(childTraceIDMatchesParent.context.traceID.first() ==
childTraceIDMatchesParent.parentContext.traceID.first());
ASSERT(childTraceIDMatchesParent.context.traceID.second() ==
childTraceIDMatchesParent.parentContext.traceID.second());
// When the parent isn't sampled AND it has legitimate values we should not sample a child,
// even if the child was randomly selected for sampling.
OTELSpan parentNotSampled(
"foo"_loc, []() { return 1.0; }, SpanContext(UID(1, 1), 1, TraceFlags::unsampled));
ASSERT(!parentNotSampled.context.isSampled());
// When the parent isn't sampled AND it has zero values for traceID and spanID this means
// we should defer to the child as the new root of the trace as there was no actual parent.
// If the child was sampled we should send the child trace with a null parent.
OTELSpan noParent(
"foo"_loc, []() { return 1.0; }, SpanContext(UID(0, 0), 0, TraceFlags::unsampled));
ASSERT(noParent.context.isSampled());
return Void();
};
TEST_CASE("/flow/Tracing/AddEvents") {
// Use helper method to add an OTELEventRef to an OTELSpan.
OTELSpan span1("span_with_event"_loc);
auto arena = span1.arena;
SmallVectorRef<KeyValueRef> attrs;
attrs.push_back(arena, KeyValueRef("foo"_sr, "bar"_sr));
span1.addEvent(LiteralStringRef("read_version"), 1.0, attrs);
ASSERT(span1.events[0].name.toString() == "read_version");
ASSERT(span1.events[0].time == 1.0);
ASSERT(span1.events[0].attributes.begin()->key.toString() == "foo");
ASSERT(span1.events[0].attributes.begin()->value.toString() == "bar");
// Use helper method to add an OTELEventRef with no attributes to an OTELSpan
OTELSpan span2("span_with_event"_loc);
span2.addEvent(StringRef(span2.arena, LiteralStringRef("commit_succeed")), 1234567.100);
ASSERT(span2.events[0].name.toString() == "commit_succeed");
ASSERT(span2.events[0].time == 1234567.100);
ASSERT(span2.events[0].attributes.size() == 0);
// Add fully constructed OTELEventRef to OTELSpan passed by value.
OTELSpan span3("span_with_event"_loc);
auto s3Arena = span3.arena;
SmallVectorRef<KeyValueRef> s3Attrs;
s3Attrs.push_back(s3Arena, KeyValueRef("xyz"_sr, "123"_sr));
span3.addEvent("commit_fail"_sr, 1234567.100, s3Attrs).addEvent("commit_succeed"_sr, 1111.001, s3Attrs);
ASSERT(span3.events[0].name.toString() == "commit_fail");
ASSERT(span3.events[0].time == 1234567.100);
ASSERT(span3.events[0].attributes.size() == 1);
ASSERT(span3.events[0].attributes.begin()->key.toString() == "xyz");
ASSERT(span3.events[0].attributes.begin()->value.toString() == "123");
ASSERT(span3.events[1].name.toString() == "commit_succeed");
ASSERT(span3.events[1].time == 1111.001);
ASSERT(span3.events[1].attributes.size() == 1);
ASSERT(span3.events[1].attributes.begin()->key.toString() == "xyz");
ASSERT(span3.events[1].attributes.begin()->value.toString() == "123");
return Void();
};
TEST_CASE("/flow/Tracing/AddAttributes") {
OTELSpan span1("span_with_attrs"_loc);
auto arena = span1.arena;
span1.addAttribute(StringRef(arena, LiteralStringRef("foo")), StringRef(arena, LiteralStringRef("bar")));
span1.addAttribute(StringRef(arena, LiteralStringRef("operation")), StringRef(arena, LiteralStringRef("grv")));
ASSERT_EQ(span1.attributes.size(), 3); // Includes default attribute of "address"
ASSERT(span1.attributes[1] == KeyValueRef("foo"_sr, "bar"_sr));
ASSERT(span1.attributes[2] == KeyValueRef("operation"_sr, "grv"_sr));
OTELSpan span3("span_with_attrs"_loc);
auto s3Arena = span3.arena;
span3.addAttribute(StringRef(s3Arena, LiteralStringRef("a")), StringRef(s3Arena, LiteralStringRef("1")))
.addAttribute(StringRef(s3Arena, LiteralStringRef("b")), LiteralStringRef("2"))
.addAttribute(StringRef(s3Arena, LiteralStringRef("c")), LiteralStringRef("3"));
ASSERT_EQ(span3.attributes.size(), 4); // Includes default attribute of "address"
ASSERT(span3.attributes[1] == KeyValueRef("a"_sr, "1"_sr));
ASSERT(span3.attributes[2] == KeyValueRef("b"_sr, "2"_sr));
ASSERT(span3.attributes[3] == KeyValueRef("c"_sr, "3"_sr));
return Void();
};
TEST_CASE("/flow/Tracing/AddLinks") {
OTELSpan span1("span_with_links"_loc);
span1.addLink(SpanContext(UID(100, 101), 200, TraceFlags::sampled));
span1.addLink(SpanContext(UID(200, 201), 300, TraceFlags::unsampled))
.addLink(SpanContext(UID(300, 301), 400, TraceFlags::sampled));
ASSERT(span1.links[0].traceID == UID(100, 101));
ASSERT(span1.links[0].spanID == 200);
ASSERT(span1.links[0].m_Flags == TraceFlags::sampled);
ASSERT(span1.links[1].traceID == UID(200, 201));
ASSERT(span1.links[1].spanID == 300);
ASSERT(span1.links[1].m_Flags == TraceFlags::unsampled);
ASSERT(span1.links[2].traceID == UID(300, 301));
ASSERT(span1.links[2].spanID == 400);
ASSERT(span1.links[2].m_Flags == TraceFlags::sampled);
OTELSpan span2("span_with_links"_loc);
auto link1 = SpanContext(UID(1, 1), 1, TraceFlags::sampled);
auto link2 = SpanContext(UID(2, 2), 2, TraceFlags::sampled);
auto link3 = SpanContext(UID(3, 3), 3, TraceFlags::sampled);
span2.addLinks({ link1, link2 }).addLinks({ link3 });
ASSERT(span2.links[0].traceID == UID(1, 1));
ASSERT(span2.links[0].spanID == 1);
ASSERT(span2.links[0].m_Flags == TraceFlags::sampled);
ASSERT(span2.links[1].traceID == UID(2, 2));
ASSERT(span2.links[1].spanID == 2);
ASSERT(span2.links[1].m_Flags == TraceFlags::sampled);
ASSERT(span2.links[2].traceID == UID(3, 3));
ASSERT(span2.links[2].spanID == 3);
ASSERT(span2.links[2].m_Flags == TraceFlags::sampled);
return Void();
};
uint64_t swapUint16BE(uint8_t* index) {
uint16_t value;
memcpy(&value, index, sizeof(value));
return fromBigEndian16(value);
}
uint64_t swapUint64BE(uint8_t* index) {
uint64_t value;
memcpy(&value, index, sizeof(value));
return fromBigEndian64(value);
}
double swapDoubleBE(uint8_t* index) {
double value;
memcpy(&value, index, sizeof(value));
char* const p = reinterpret_cast<char*>(&value);
for (size_t i = 0; i < sizeof(double) / 2; ++i)
std::swap(p[i], p[sizeof(double) - i - 1]);
return value;
}
std::string readMPString(uint8_t* index, int len) {
uint8_t data[len + 1];
std::copy(index, index + len, data);
data[len] = '\0';
return reinterpret_cast<char*>(data);
}
// Windows doesn't like lack of header and declaration of constructor for FastUDPTracer
#ifndef WIN32
TEST_CASE("/flow/Tracing/FastUDPMessagePackEncoding") {
OTELSpan span1("encoded_span"_loc);
auto request = TraceRequest{ .buffer = std::make_unique<uint8_t[]>(kTraceBufferSize),
.data_size = 0,
.buffer_size = kTraceBufferSize };
auto tracer = FastUDPTracer();
tracer.serialize_span(span1, request);
auto data = request.buffer.get();
ASSERT(data[0] == 0b10011110); // Default array size.
request.reset();
// Test - constructor OTELSpan(const Location& location, const SpanContext parent, const SpanContext& link)
// Will delegate to other constructors.
OTELSpan span2("encoded_span"_loc,
SpanContext(UID(100, 101), 1, TraceFlags::sampled),
SpanContext(UID(200, 201), 2, TraceFlags::sampled));
tracer.serialize_span(span2, request);
data = request.buffer.get();
ASSERT(data[0] == 0b10011110); // 14 element array.
// Verify the Parent Trace ID overwrites this spans Trace ID
ASSERT(data[1] == 0xcf);
ASSERT(swapUint64BE(&data[2]) == 100);
ASSERT(data[10] == 0xcf);
ASSERT(swapUint64BE(&data[11]) == 101);
ASSERT(data[19] == 0xcf);
// We don't care about the next 8 bytes, they are the ID for the span itself and will be random.
// Parent TraceID and Parent SpanID.
ASSERT(data[28] == 0xcf);
ASSERT(swapUint64BE(&data[29]) == 100);
ASSERT(data[37] == 0xcf);
ASSERT(swapUint64BE(&data[38]) == 101);
ASSERT(data[46] == 0xcf);
ASSERT(swapUint64BE(&data[47]) == 1);
// Read and verify span name
ASSERT(data[55] == (0b10100000 | strlen("encoded_span")));
ASSERT(strncmp(readMPString(&data[56], strlen("encoded_span")).c_str(), "encoded_span", strlen("encoded_span")) ==
0);
// Verify begin/end is encoded, we don't care about the values
ASSERT(data[68] == 0xcb);
ASSERT(data[77] == 0xcb);
// SpanKind
ASSERT(data[86] == 0xcc);
ASSERT(data[87] == static_cast<uint8_t>(SpanKind::SERVER));
// Status
ASSERT(data[88] == 0xcc);
ASSERT(data[89] == static_cast<uint8_t>(SpanStatus::OK));
// Linked SpanContext
ASSERT(data[90] == 0b10010001);
ASSERT(data[91] == 0xcf);
ASSERT(swapUint64BE(&data[92]) == 200);
ASSERT(data[100] == 0xcf);
ASSERT(swapUint64BE(&data[101]) == 201);
ASSERT(data[109] == 0xcf);
ASSERT(swapUint64BE(&data[110]) == 2);
// Events
ASSERT(data[118] == 0b10010000); // empty
// Attributes
ASSERT(data[119] == 0b10000001); // single k/v pair
ASSERT(data[120] == 0b10100111); // length of key string "address" == 7
request.reset();
// Exercise all fluent interfaces, include links, events, and attributes.
OTELSpan span3("encoded_span_3"_loc);
auto s3Arena = span3.arena;
SmallVectorRef<KeyValueRef> attrs;
attrs.push_back(s3Arena, KeyValueRef("foo"_sr, "bar"_sr));
span3.addAttribute("operation"_sr, "grv"_sr)
.addLink(SpanContext(UID(300, 301), 400, TraceFlags::sampled))
.addEvent(StringRef(s3Arena, LiteralStringRef("event1")), 100.101, attrs);
tracer.serialize_span(span3, request);
data = request.buffer.get();
ASSERT(data[0] == 0b10011110); // 14 element array.
// We don't care about the next 54 bytes as there is no parent and a randomly assigned Trace and SpanID
// Read and verify span name
ASSERT(data[55] == (0b10100000 | strlen("encoded_span_3")));
ASSERT(strncmp(readMPString(&data[56], strlen("encoded_span_3")).c_str(),
"encoded_span_3",
strlen("encoded_span_3")) == 0);
// Verify begin/end is encoded, we don't care about the values
ASSERT(data[70] == 0xcb);
ASSERT(data[79] == 0xcb);
// SpanKind
ASSERT(data[88] == 0xcc);
ASSERT(data[89] == static_cast<uint8_t>(SpanKind::SERVER));
// Status
ASSERT(data[90] == 0xcc);
ASSERT(data[91] == static_cast<uint8_t>(SpanStatus::OK));
// Linked SpanContext
ASSERT(data[92] == 0b10010001);
ASSERT(data[93] == 0xcf);
ASSERT(swapUint64BE(&data[94]) == 300);
ASSERT(data[102] == 0xcf);
ASSERT(swapUint64BE(&data[103]) == 301);
ASSERT(data[111] == 0xcf);
ASSERT(swapUint64BE(&data[112]) == 400);
// Events
ASSERT(data[120] == 0b10010001); // empty
ASSERT(data[121] == (0b10100000 | strlen("event1")));
ASSERT(strncmp(readMPString(&data[122], strlen("event1")).c_str(), "event1", strlen("event1")) == 0);
ASSERT(data[128] == 0xcb);
ASSERT(swapDoubleBE(&data[129]) == 100.101);
// Events Attributes
ASSERT(data[137] == 0b10000001); // single k/v pair
ASSERT(data[138] == 0b10100011); // length of key string "foo" == 3
ASSERT(strncmp(readMPString(&data[139], strlen("foo")).c_str(), "foo", strlen("foo")) == 0);
ASSERT(data[142] == 0b10100011); // length of key string "bar" == 3
ASSERT(strncmp(readMPString(&data[143], strlen("bar")).c_str(), "bar", strlen("bar")) == 0);
// Attributes
ASSERT(data[146] == 0b10000010); // two k/v pair
// Reconstruct map from MessagePack wire format data and verify.
std::unordered_map<std::string, std::string> attributes;
auto index = 147;
// We & out the bits here that contain the length the initial 4 higher order bits are
// to signify this is a string of len <= 31 chars.
auto firstKeyLength = static_cast<uint8_t>(data[index] & 0b00011111);
index++;
auto firstKey = readMPString(&data[index], firstKeyLength);
index += firstKeyLength;
auto firstValueLength = static_cast<uint8_t>(data[index] & 0b00011111);
index++;
auto firstValue = readMPString(&data[index], firstValueLength);
index += firstValueLength;
attributes[firstKey] = firstValue;
auto secondKeyLength = static_cast<uint8_t>(data[index] & 0b00011111);
index++;
auto secondKey = readMPString(&data[index], secondKeyLength);
index += secondKeyLength;
auto secondValueLength = static_cast<uint8_t>(data[index] & 0b00011111);
index++;
auto secondValue = readMPString(&data[index], secondValueLength);
attributes[secondKey] = secondValue;
// We don't know what the value for address will be, so just verify it is in the map.
ASSERT(attributes.find("address") != attributes.end());
ASSERT(strncmp(attributes["operation"].c_str(), "grv", strlen("grv")) == 0);
request.reset();
// Test message pack encoding for string >= 256 && <= 65535 chars
const char* longString = "yGUtj42gSKfdqib3f0Ri4OVhD7eWyTbKsH/g9+x4UWyXry7NIBFIapPV9f1qdTRl"
"2jXcZI8Ua/Gp8k9EBn7peaEN1uj4w9kf4FQ2Lalu0VrA4oquQoaKYr+wPsLBak9i"
"uyZDF9sX/HW4pVvQhPQdXQWME5E7m58XFMpZ3H8HNXuytWInEuh97SRLlI0RhrvG"
"ixNpYtYlvghsLCrEdZMMGnS2gXgGufIdg1xKJd30fUbZLHcYIC4DTnL5RBpkbQCR"
"SGKKUrpIb/7zePhBDi+gzUzyAcbQ2zUbFWI1KNi3zQk58uUG6wWJZkw+GCs7Cc3V"
"OUxOljwCJkC4QTgdsbbFhxUC+rtoHV5xAqoTQwR0FXnWigUjP7NtdL6huJUr3qRv"
"40c4yUI1a4+P5vJa";
auto span4 = OTELSpan();
auto location = Location();
location.name = StringRef(span4.arena, longString);
span4.location = location;
tracer.serialize_span(span4, request);
data = request.buffer.get();
ASSERT(data[0] == 0b10011110); // 14 element array.
// We don't care about the next 54 bytes as there is no parent and a randomly assigned Trace and SpanID
// Read and verify span name
ASSERT(data[55] == 0xda);
auto locationLength = swapUint16BE(&data[56]);
ASSERT(locationLength == strlen(longString));
ASSERT(strncmp(readMPString(&data[58], locationLength).c_str(), longString, strlen(longString)) == 0);
return Void();
};
#endif

204
flow/Tracing.h
View File

@ -106,6 +106,196 @@ struct Span {
std::unordered_map<StringRef, StringRef> tags;
};
// OTELSpan
//
// OTELSpan is a tracing implementation which, for the most part, complies with the W3C Trace Context specification
// https://www.w3.org/TR/trace-context/ and the OpenTelemetry API
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md.
//
// The major differences between OTELSpan and the current Span implementation, which is based off the OpenTracing.io
// specification https://opentracing.io/ are as follows.
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span
//
// OTELSpans have...
// 1. A SpanContext which consists of 3 attributes.
//
// TraceId - A valid trace identifier is a 16-byte array with at least one non-zero byte.
// SpanId - A valid span identifier is an 8-byte array with at least one non-zero byte.
// TraceFlags - 1 byte, bit field for flags.
//
// TraceState is not implemented, specifically we do not provide some of the following APIs
// https://www.w3.org/TR/trace-context/#mutating-the-tracestate-field In particular APIs to delete/update a specific,
// arbitrary key/value pair, as this complies with the OTEL specification where SpanContexts are immutable.
// 2. A begin/end and those values are serialized, unlike the Span implementation which has an end but serializes with a
// begin and calculated duration field.
// 3. A SpanKind
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind
// 4. A SpanStatus
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status
// 5. A singular parent SpanContext, which may optionally be null, as opposed to our Span implementation which allows
// for a list of parents.
// 6. An "attributes" rather than "tags", however the implementation is essentially the same, a set of key/value of
// strings, stored here as a SmallVectorRef<KeyValueRef> rather than map as a convenience.
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/common.md#attributes
// 7. An optional list of linked SpanContexts.
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#specifying-links
// 8. An optional list of timestamped Events.
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#add-events
enum class SpanKind : uint8_t { INTERNAL = 0, CLIENT = 1, SERVER = 2, PRODUCER = 3, CONSUMER = 4 };
enum class SpanStatus : uint8_t { UNSET = 0, OK = 1, ERR = 2 };
struct OTELEventRef {
OTELEventRef() {}
OTELEventRef(const StringRef& name,
const double& time,
const SmallVectorRef<KeyValueRef>& attributes = SmallVectorRef<KeyValueRef>())
: name(name), time(time), attributes(attributes) {}
OTELEventRef(Arena& arena, const OTELEventRef& other)
: name(arena, other.name), time(other.time), attributes(arena, other.attributes) {}
StringRef name;
double time = 0.0;
SmallVectorRef<KeyValueRef> attributes;
};
class OTELSpan {
public:
OTELSpan(const SpanContext& context,
const Location& location,
const SpanContext& parentContext,
const std::initializer_list<SpanContext>& links = {})
: context(context), location(location), parentContext(parentContext), links(arena, links.begin(), links.end()),
begin(g_network->now()) {
// We've simplified the logic here, essentially we're now always setting trace and span ids and relying on the
// TraceFlags to determine if we're sampling. Therefore if the parent is sampled, we simply overwrite this
// span's traceID with the parent trace id.
if (parentContext.isSampled()) {
this->context.traceID = UID(parentContext.traceID.first(), parentContext.traceID.second());
this->context.m_Flags = TraceFlags::sampled;
} else {
// However there are two other cases.
// 1. A legitamite parent span exists but it was not selected for tracing.
// 2. There is no actual parent, just a default arg parent provided by the constructor AND the "child" span
// was selected for sampling. For case 1. we handle below by marking the child as unsampled. For case 2 we
// needn't do anything, and can rely on the values in this OTELSpan
if (parentContext.traceID.first() != 0 && parentContext.traceID.second() != 0 &&
parentContext.spanID != 0) {
this->context.m_Flags = TraceFlags::unsampled;
}
}
this->kind = SpanKind::SERVER;
this->status = SpanStatus::OK;
this->attributes.push_back(
this->arena, KeyValueRef("address"_sr, StringRef(this->arena, g_network->getLocalAddress().toString())));
}
OTELSpan(const Location& location,
const SpanContext& parent = SpanContext(),
const std::initializer_list<SpanContext>& links = {})
: OTELSpan(
SpanContext(UID(deterministicRandom()->randomUInt64(), deterministicRandom()->randomUInt64()), // traceID
deterministicRandom()->randomUInt64(), // spanID
deterministicRandom()->random01() < FLOW_KNOBS->TRACING_SAMPLE_RATE // sampled or unsampled
? TraceFlags::sampled
: TraceFlags::unsampled),
location,
parent,
links) {}
OTELSpan(const Location& location, const SpanContext parent, const SpanContext& link)
: OTELSpan(location, parent, { link }) {}
// NOTE: This constructor is primarly for unit testing until we sort out how to enable/disable a Knob dynamically in
// a test.
OTELSpan(const Location& location,
const std::function<double()>& rateProvider,
const SpanContext& parent = SpanContext(),
const std::initializer_list<SpanContext>& links = {})
: OTELSpan(SpanContext(UID(deterministicRandom()->randomUInt64(), deterministicRandom()->randomUInt64()),
deterministicRandom()->randomUInt64(),
deterministicRandom()->random01() < rateProvider() ? TraceFlags::sampled
: TraceFlags::unsampled),
location,
parent,
links) {}
OTELSpan(const OTELSpan&) = delete;
OTELSpan(OTELSpan&& o) {
arena = std::move(o.arena);
context = o.context;
location = o.location;
parentContext = std::move(o.parentContext);
kind = o.kind;
begin = o.begin;
end = o.end;
links = std::move(o.links);
events = std::move(o.events);
status = o.status;
o.context = SpanContext();
o.parentContext = SpanContext();
o.kind = SpanKind::INTERNAL;
o.begin = 0.0;
o.end = 0.0;
o.status = SpanStatus::UNSET;
}
OTELSpan() {}
~OTELSpan();
OTELSpan& operator=(OTELSpan&& o);
OTELSpan& operator=(const OTELSpan&) = delete;
void swap(OTELSpan& other) {
std::swap(arena, other.arena);
std::swap(context, other.context);
std::swap(location, other.location);
std::swap(parentContext, other.parentContext);
std::swap(kind, other.kind);
std::swap(status, other.status);
std::swap(begin, other.begin);
std::swap(end, other.end);
std::swap(links, other.links);
std::swap(events, other.events);
}
OTELSpan& addLink(const SpanContext& linkContext) {
links.push_back(arena, linkContext);
return *this;
}
OTELSpan& addLinks(const std::initializer_list<SpanContext>& linkContexts = {}) {
for (auto const& sc : linkContexts) {
links.push_back(arena, sc);
}
return *this;
}
OTELSpan& addEvent(const OTELEventRef& event) {
events.push_back_deep(arena, event);
return *this;
}
OTELSpan& addEvent(const StringRef& name,
const double& time,
const SmallVectorRef<KeyValueRef>& attrs = SmallVectorRef<KeyValueRef>()) {
return addEvent(OTELEventRef(name, time, attrs));
}
OTELSpan& addAttribute(const StringRef& key, const StringRef& value) {
attributes.push_back_deep(arena, KeyValueRef(key, value));
return *this;
}
Arena arena;
SpanContext context;
Location location;
SpanContext parentContext;
SpanKind kind;
SmallVectorRef<SpanContext> links;
double begin = 0.0, end = 0.0;
SmallVectorRef<KeyValueRef> attributes; // not necessarily sorted
SmallVectorRef<OTELEventRef> events;
SpanStatus status;
};
// The user selects a tracer using a string passed to fdbserver on boot.
// Clients should not refer to TracerType directly, and mappings of names to
// values in this enum can change without notice.
@ -121,6 +311,7 @@ struct ITracer {
virtual TracerType type() const = 0;
// passed ownership to the tracer
virtual void trace(Span const& span) = 0;
virtual void trace(OTELSpan const& span) = 0;
};
void openTracer(TracerType type);
@ -137,3 +328,16 @@ struct SpannedDeque : Deque<T> {
span = std::move(other.span);
}
};
template <class T>
struct OTELSpannedDeque : Deque<T> {
OTELSpan span;
explicit OTELSpannedDeque(Location loc) : span(loc) {}
OTELSpannedDeque(OTELSpannedDeque&& other) : Deque<T>(std::move(other)), span(std::move(other.span)) {}
OTELSpannedDeque(OTELSpannedDeque const&) = delete;
OTELSpannedDeque& operator=(OTELSpannedDeque const&) = delete;
OTELSpannedDeque& operator=(OTELSpannedDeque&& other) {
*static_cast<Deque<T>*>(this) = std::move(other);
span = std::move(other.span);
}
};

187
flow/network.cpp
View File

@ -63,23 +63,6 @@ bool IPAddress::isValid() const {
return std::get<uint32_t>(addr) != 0;
}
Hostname Hostname::parse(const std::string& s) {
if (s.empty()) {
throw connection_string_invalid();
}
bool isTLS = false;
std::string f;
if (s.size() > 4 && strcmp(s.c_str() + s.size() - 4, ":tls") == 0) {
isTLS = true;
f = s.substr(0, s.size() - 4);
} else {
f = s;
}
auto colonPos = f.find_first_of(":");
return Hostname(f.substr(0, colonPos), f.substr(colonPos + 1), isTLS);
}
FDB_DEFINE_BOOLEAN_PARAM(NetworkAddressFromHostname);
NetworkAddress NetworkAddress::parse(std::string const& s) {
@ -178,6 +161,118 @@ std::string formatIpPort(const IPAddress& ip, uint16_t port) {
return format(patt, ip.toString().c_str(), port);
}
Optional<std::vector<NetworkAddress>> DNSCache::find(const std::string& host, const std::string& service) {
auto it = hostnameToAddresses.find(host + ":" + service);
if (it != hostnameToAddresses.end()) {
return it->second;
}
return {};
}
void DNSCache::add(const std::string& host, const std::string& service, const std::vector<NetworkAddress>& addresses) {
hostnameToAddresses[host + ":" + service] = addresses;
}
void DNSCache::remove(const std::string& host, const std::string& service) {
auto it = hostnameToAddresses.find(host + ":" + service);
if (it != hostnameToAddresses.end()) {
hostnameToAddresses.erase(it);
}
}
void DNSCache::clear() {
hostnameToAddresses.clear();
}
std::string DNSCache::toString() {
std::string ret;
for (auto it = hostnameToAddresses.begin(); it != hostnameToAddresses.end(); ++it) {
if (it != hostnameToAddresses.begin()) {
ret += ';';
}
ret += it->first + ',';
const std::vector<NetworkAddress>& addresses = it->second;
for (int i = 0; i < addresses.size(); ++i) {
ret += addresses[i].toString();
if (i != addresses.size() - 1) {
ret += ',';
}
}
}
return ret;
}
DNSCache DNSCache::parseFromString(const std::string& s) {
std::map<std::string, std::vector<NetworkAddress>> dnsCache;
for (int p = 0; p < s.length();) {
int pSemiColumn = s.find_first_of(';', p);
if (pSemiColumn == s.npos) {
pSemiColumn = s.length();
}
std::string oneMapping = s.substr(p, pSemiColumn - p);
std::string hostname;
std::vector<NetworkAddress> addresses;
for (int i = 0; i < oneMapping.length();) {
int pComma = oneMapping.find_first_of(',', i);
if (pComma == oneMapping.npos) {
pComma = oneMapping.length();
}
if (!i) {
// The first part is hostname
hostname = oneMapping.substr(i, pComma - i);
} else {
addresses.push_back(NetworkAddress::parse(oneMapping.substr(i, pComma - i)));
}
i = pComma + 1;
}
dnsCache[hostname] = addresses;
p = pSemiColumn + 1;
}
return DNSCache(dnsCache);
}
TEST_CASE("/flow/DNSCache") {
DNSCache dnsCache;
std::vector<NetworkAddress> networkAddresses;
NetworkAddress address1(IPAddress(0x13131313), 1), address2(IPAddress(0x14141414), 2);
networkAddresses.push_back(address1);
networkAddresses.push_back(address2);
dnsCache.add("testhost1", "port1", networkAddresses);
ASSERT(dnsCache.find("testhost1", "port1").present());
ASSERT(!dnsCache.find("testhost1", "port2").present());
std::vector<NetworkAddress> resolvedNetworkAddresses = dnsCache.find("testhost1", "port1").get();
ASSERT(resolvedNetworkAddresses.size() == 2);
ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address1) !=
resolvedNetworkAddresses.end());
ASSERT(std::find(resolvedNetworkAddresses.begin(), resolvedNetworkAddresses.end(), address2) !=
resolvedNetworkAddresses.end());
dnsCache.remove("testhost1", "port1");
ASSERT(!dnsCache.find("testhost1", "port1").present());
dnsCache.add("testhost1", "port2", networkAddresses);
ASSERT(dnsCache.find("testhost1", "port2").present());
dnsCache.clear();
ASSERT(!dnsCache.find("testhost1", "port2").present());
return Void();
}
TEST_CASE("/flow/DNSCacheParsing") {
std::string dnsCacheString;
ASSERT(DNSCache::parseFromString(dnsCacheString).toString() == dnsCacheString);
dnsCacheString = "testhost1:port1,[::1]:4800:tls(fromHostname)";
ASSERT(DNSCache::parseFromString(dnsCacheString).toString() == dnsCacheString);
dnsCacheString = "testhost1:port1,[::1]:4800,[2001:db8:85a3::8a2e:370:7334]:4800;testhost2:port2,[2001:db8:85a3::"
"8a2e:370:7334]:4800:tls(fromHostname),8.8.8.8:12";
ASSERT(DNSCache::parseFromString(dnsCacheString).toString() == dnsCacheString);
return Void();
}
Future<Reference<IConnection>> INetworkConnections::connect(const std::string& host,
const std::string& service,
bool isTLS) {
@ -252,62 +347,4 @@ TEST_CASE("/flow/network/ipaddress") {
return Void();
}
TEST_CASE("/flow/network/hostname") {
std::string hn1s = "localhost:1234";
std::string hn2s = "host-name:1234";
std::string hn3s = "host.name:1234";
std::string hn4s = "host-name_part1.host-name_part2:1234:tls";
std::string hn5s = "127.0.0.1:1234";
std::string hn6s = "127.0.0.1:1234:tls";
std::string hn7s = "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:4800";
std::string hn8s = "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:4800:tls";
std::string hn9s = "2001:0db8:85a3:0000:0000:8a2e:0370:7334";
std::string hn10s = "2001:0db8:85a3:0000:0000:8a2e:0370:7334:tls";
std::string hn11s = "[::1]:4800";
std::string hn12s = "[::1]:4800:tls";
std::string hn13s = "1234";
auto hn1 = Hostname::parse(hn1s);
ASSERT(hn1.toString() == hn1s);
ASSERT(hn1.host == "localhost");
ASSERT(hn1.service == "1234");
ASSERT(!hn1.isTLS);
auto hn2 = Hostname::parse(hn2s);
ASSERT(hn2.toString() == hn2s);
ASSERT(hn2.host == "host-name");
ASSERT(hn2.service == "1234");
ASSERT(!hn2.isTLS);
auto hn3 = Hostname::parse(hn3s);
ASSERT(hn3.toString() == hn3s);
ASSERT(hn3.host == "host.name");
ASSERT(hn3.service == "1234");
ASSERT(!hn3.isTLS);
auto hn4 = Hostname::parse(hn4s);
ASSERT(hn4.toString() == hn4s);
ASSERT(hn4.host == "host-name_part1.host-name_part2");
ASSERT(hn4.service == "1234");
ASSERT(hn4.isTLS);
ASSERT(Hostname::isHostname(hn1s));
ASSERT(Hostname::isHostname(hn2s));
ASSERT(Hostname::isHostname(hn3s));
ASSERT(Hostname::isHostname(hn4s));
ASSERT(!Hostname::isHostname(hn5s));
ASSERT(!Hostname::isHostname(hn6s));
ASSERT(!Hostname::isHostname(hn7s));
ASSERT(!Hostname::isHostname(hn8s));
ASSERT(!Hostname::isHostname(hn9s));
ASSERT(!Hostname::isHostname(hn10s));
ASSERT(!Hostname::isHostname(hn11s));
ASSERT(!Hostname::isHostname(hn12s));
ASSERT(!Hostname::isHostname(hn13s));
return Void();
}
NetworkInfo::NetworkInfo() : handshakeLock(new FlowLock(FLOW_KNOBS->TLS_HANDSHAKE_LIMIT)) {}

70
flow/network.h
View File

@ -135,43 +135,6 @@ inline TaskPriority incrementPriorityIfEven(TaskPriority p) {
class Void;
struct Hostname {
std::string host;
std::string service; // decimal port number
bool isTLS;
Hostname(std::string host, std::string service, bool isTLS) : host(host), service(service), isTLS(isTLS) {}
Hostname() : host(""), service(""), isTLS(false) {}
bool operator==(const Hostname& r) const { return host == r.host && service == r.service && isTLS == r.isTLS; }
bool operator!=(const Hostname& r) const { return !(*this == r); }
bool operator<(const Hostname& r) const {
if (isTLS != r.isTLS)
return isTLS < r.isTLS;
else if (host != r.host)
return host < r.host;
return service < r.service;
}
bool operator>(const Hostname& r) const { return r < *this; }
bool operator<=(const Hostname& r) const { return !(*this > r); }
bool operator>=(const Hostname& r) const { return !(*this < r); }
// Allow hostnames in forms like following:
// hostname:1234
// host.name:1234
// host-name:1234
// host-name_part1.host-name_part2:1234:tls
static bool isHostname(const std::string& s) {
std::regex validation("^([\\w\\-]+\\.?)+:([\\d]+){1,}(:tls)?$");
std::regex ipv4Validation("^([\\d]{1,3}\\.?){4,}:([\\d]+){1,}(:tls)?$");
return !std::regex_match(s, ipv4Validation) && std::regex_match(s, validation);
}
static Hostname parse(const std::string& s);
std::string toString() const { return host + ":" + service + (isTLS ? ":tls" : ""); }
};
struct IPAddress {
typedef boost::asio::ip::address_v6::bytes_type IPAddressStore;
static_assert(std::is_same<IPAddressStore, std::array<uint8_t, 16>>::value,
@ -697,6 +660,27 @@ public:
virtual boost::asio::ip::udp::socket::native_handle_type native_handle() = 0;
};
// DNSCache is a class maintaining a <hostname, vector<NetworkAddress>> mapping.
class DNSCache {
public:
DNSCache() = default;
explicit DNSCache(const std::map<std::string, std::vector<NetworkAddress>>& dnsCache)
: hostnameToAddresses(dnsCache) {}
Optional<std::vector<NetworkAddress>> find(const std::string& host, const std::string& service);
void add(const std::string& host, const std::string& service, const std::vector<NetworkAddress>& addresses);
void remove(const std::string& host, const std::string& service);
void clear();
// Convert hostnameToAddresses to string. The format is:
// hostname1,host1Address1,host1Address2;hostname2,host2Address1,host2Address2...
std::string toString();
static DNSCache parseFromString(const std::string& s);
private:
std::map<std::string, std::vector<NetworkAddress>> hostnameToAddresses;
};
class INetworkConnections {
public:
// Methods for making and accepting network connections. Logically this is part of the INetwork abstraction
@ -724,10 +708,17 @@ public:
// NetworkAddresses
virtual Future<std::vector<NetworkAddress>> resolveTCPEndpoint(const std::string& host,
const std::string& service) = 0;
// Similar to resolveTCPEndpoint(), except that this one uses DNS cache.
virtual Future<std::vector<NetworkAddress>> resolveTCPEndpointWithDNSCache(const std::string& host,
const std::string& service) = 0;
// Resolve host name and service name. This one should only be used when resolving asynchronously is impossible. For
// all other cases, resolveTCPEndpoint() should be preferred.
virtual std::vector<NetworkAddress> resolveTCPEndpointBlocking(const std::string& host,
const std::string& service) = 0;
// Resolve host name and service name with DNS cache. This one should only be used when resolving asynchronously is
// impossible. For all other cases, resolveTCPEndpointWithDNSCache() should be preferred.
virtual std::vector<NetworkAddress> resolveTCPEndpointBlockingWithDNSCache(const std::string& host,
const std::string& service) = 0;
// Convenience function to resolve host/service and connect to one of its NetworkAddresses randomly
// isTLS has to be a parameter here because it is passed to connect() as part of the toAddr object.
@ -741,6 +732,11 @@ public:
static INetworkConnections* net() {
return static_cast<INetworkConnections*>((void*)g_network->global(INetwork::enNetworkConnections));
}
void removeCachedDNS(const std::string& host, const std::string& service) { dnsCache.remove(host, service); }
DNSCache dnsCache;
// Returns the interface that should be used to make and accept socket connections
};

1
tests/fast/PhysicalShardMove.toml
View File

@ -4,7 +4,6 @@ storageEngineType = 4
processesPerMachine = 1
coordinators = 3
machineCount = 15
disableRemoteKVS = true
[[test]]
testTitle = 'PhysicalShardMove'