From 1615977695da9ecd7bda261bfac939ff3b7cef8b Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 22 Jan 2021 21:46:36 -0800 Subject: [PATCH 001/426] Added StreamCipher --- fdbserver/workloads/UnitTests.actor.cpp | 2 + flow/CMakeLists.txt | 2 + flow/StreamCipher.cpp | 138 ++++++++++++++++++++++++ flow/StreamCipher.h | 51 +++++++++ 4 files changed, 193 insertions(+) create mode 100644 flow/StreamCipher.cpp create mode 100644 flow/StreamCipher.h diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 10fa028354..6b633ad5ab 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -28,6 +28,7 @@ void forceLinkFlowTests(); void forceLinkVersionedMapTests(); void forceLinkMemcpyTests(); void forceLinkMemcpyPerfTests(); +void forceLinkStreamCipherTests(); struct UnitTestWorkload : TestWorkload { bool enabled; @@ -49,6 +50,7 @@ struct UnitTestWorkload : TestWorkload { forceLinkVersionedMapTests(); forceLinkMemcpyTests(); forceLinkMemcpyPerfTests(); + forceLinkStreamCipherTests(); } std::string description() const override { return "UnitTests"; } diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index f98e6ab67a..fc858a2caf 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -50,6 +50,8 @@ set(FLOW_SRCS SignalSafeUnwind.cpp SignalSafeUnwind.h SimpleOpt.h + StreamCipher.cpp + StreamCipher.h SystemMonitor.cpp SystemMonitor.h TDMetric.actor.h diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp new file mode 100644 index 0000000000..cd6dc4f832 --- /dev/null +++ b/flow/StreamCipher.cpp @@ -0,0 +1,138 @@ +/* + * StreamCipher.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flow/StreamCipher.h" +#include "flow/UnitTest.h" + +EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) + : ctx(EVP_CIPHER_CTX_new()) { + EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, 12, nullptr); + EVP_EncryptInit_ex(ctx, nullptr, nullptr, key, iv); +} + +EncryptionStreamCipher::~EncryptionStreamCipher() { + EVP_CIPHER_CTX_free(ctx); +} + +StringRef EncryptionStreamCipher::encrypt(unsigned char const* plaintext, int len, Arena& arena) { + auto ciphertext = new (arena) unsigned char[len + AES_BLOCK_SIZE]; + int bytes{ 0 }; + EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, len); + return StringRef(ciphertext, bytes); +} + +StringRef EncryptionStreamCipher::finish(Arena& arena) { + auto ciphertext = new (arena) unsigned char[AES_BLOCK_SIZE]; + int bytes{ 0 }; + EVP_EncryptFinal_ex(ctx, ciphertext, &bytes); + return StringRef(ciphertext, bytes); +} + +DecryptionStreamCipher::DecryptionStreamCipher(unsigned char const* key, unsigned char const* iv) + : ctx(EVP_CIPHER_CTX_new()) { + + EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, 12, nullptr); + EVP_DecryptInit_ex(ctx, nullptr, nullptr, key, iv); +} + +DecryptionStreamCipher::~DecryptionStreamCipher() { + EVP_CIPHER_CTX_free(ctx); +} + +StringRef DecryptionStreamCipher::decrypt(unsigned char const* ciphertext, int len, Arena& arena) { + auto plaintext = new (arena) unsigned char[len]; + int bytesDecrypted{ 0 }; + EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, len); + int finalBlockBytes{ 0 }; + EVP_DecryptFinal_ex(ctx, plaintext + bytesDecrypted, &finalBlockBytes); + return StringRef(plaintext, bytesDecrypted + finalBlockBytes); +} + +StringRef DecryptionStreamCipher::finish(Arena& arena) { + auto plaintext = new (arena) unsigned char[AES_BLOCK_SIZE]; + int finalBlockBytes{ 0 }; + EVP_DecryptFinal_ex(ctx, plaintext, &finalBlockBytes); + return StringRef(plaintext, finalBlockBytes); +} + +void forceLinkStreamCipherTests() {} + +TEST_CASE("flow/StreamCipher") { + std::array key; + generateRandomData(key.data(), key.size()); + + std::array iv; + generateRandomData(iv.data(), iv.size()); + + Arena arena; + std::vector plaintext(deterministicRandom()->randomInt(0, 10001)); + generateRandomData(&plaintext.front(), plaintext.size()); + std::vector ciphertext(plaintext.size() + AES_BLOCK_SIZE); + std::vector decryptedtext(plaintext.size() + AES_BLOCK_SIZE); + + TraceEvent("StreamCipherTestStart") + .detail("PlaintextSize", plaintext.size()) + .detail("AESBlockSize", AES_BLOCK_SIZE); + { + EncryptionStreamCipher encryptor(key.data(), iv.data()); + int index = 0; + int encryptedOffset = 0; + while (index < plaintext.size()) { + const auto chunkSize = std::min(deterministicRandom()->randomInt(1, 101), plaintext.size() - index); + const auto encrypted = encryptor.encrypt(&plaintext[index], chunkSize, arena); + TraceEvent("StreamCipherTestEcryptedChunk") + .detail("EncryptedSize", encrypted.size()) + .detail("EncryptedOffset", encryptedOffset) + .detail("Index", index); + std::copy(encrypted.begin(), encrypted.end(), &ciphertext[encryptedOffset]); + encryptedOffset += encrypted.size(); + index += chunkSize; + } + const auto encrypted = encryptor.finish(arena); + std::copy(encrypted.begin(), encrypted.end(), &ciphertext[encryptedOffset]); + ciphertext.resize(encryptedOffset + encrypted.size()); + } + + { + DecryptionStreamCipher decryptor(key.data(), iv.data()); + int index = 0; + int decryptedOffset = 0; + while (index < plaintext.size()) { + const auto chunkSize = std::min(deterministicRandom()->randomInt(1, 101), plaintext.size() - index); + const auto decrypted = decryptor.decrypt(&ciphertext[index], chunkSize, arena); + TraceEvent("StreamCipherTestDecryptedChunk") + .detail("DecryptedSize", decrypted.size()) + .detail("DecryptedOffset", decryptedOffset) + .detail("Index", index); + std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]); + decryptedOffset += decrypted.size(); + index += chunkSize; + } + const auto decrypted = decryptor.finish(arena); + std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]); + ASSERT(decryptedOffset + decrypted.size() == plaintext.size()); + decryptedtext.resize(decryptedOffset + decrypted.size()); + } + + ASSERT(plaintext == decryptedtext); + return Void(); +} diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h new file mode 100644 index 0000000000..2af0f9ac07 --- /dev/null +++ b/flow/StreamCipher.h @@ -0,0 +1,51 @@ +/* + * StreamCipher.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FLOW_STREAM_CIPHER_H__ +#define __FLOW_STREAM_CIPHER_H__ + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/flow.h" + +#include +#include +#include +#include + +class EncryptionStreamCipher final : NonCopyable, public ReferenceCounted { + EVP_CIPHER_CTX* ctx; +public: + EncryptionStreamCipher(unsigned char const* key, unsigned char const* salt); + ~EncryptionStreamCipher(); + StringRef encrypt(unsigned char const* plaintext, int len, Arena&); + StringRef finish(Arena&); +}; + +class DecryptionStreamCipher final : NonCopyable, public ReferenceCounted { + EVP_CIPHER_CTX* ctx; +public: + DecryptionStreamCipher(unsigned char const* key, unsigned char const* salt); + ~DecryptionStreamCipher(); + StringRef decrypt(unsigned char const* ciphertext, int len, Arena&); + StringRef finish(Arena&); +}; + +#endif From 88bc157bd0f7a75bbbd2121b7fd092687400a3b9 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 22 Jan 2021 21:47:36 -0800 Subject: [PATCH 002/426] Added AsyncFileEncrypted --- fdbrpc/AsyncFileEncrypted.actor.cpp | 254 ++++++++++++++++++++++++++++ fdbrpc/AsyncFileEncrypted.h | 72 ++++++++ fdbrpc/CMakeLists.txt | 2 + fdbrpc/IAsyncFile.h | 32 ++-- fdbrpc/Net2FileSystem.cpp | 7 +- fdbrpc/sim2.actor.cpp | 7 +- flow/CMakeLists.txt | 3 +- flow/Knobs.cpp | 4 + flow/Knobs.h | 4 + flow/StreamCipher.cpp | 18 +- flow/StreamCipher.h | 9 +- 11 files changed, 383 insertions(+), 29 deletions(-) create mode 100644 fdbrpc/AsyncFileEncrypted.actor.cpp create mode 100644 fdbrpc/AsyncFileEncrypted.h diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp new file mode 100644 index 0000000000..2fadd545af --- /dev/null +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -0,0 +1,254 @@ +/* + * AsyncFileEncrypted.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbrpc/AsyncFileEncrypted.h" +#include "flow/StreamCipher.h" +#include "flow/UnitTest.h" +#include "flow/xxhash.h" +#include "flow/actorcompiler.h" // must be last include + +class AsyncFileEncryptedImpl { +public: + static auto getFirstBlockIV(const std::string& filename) { + StreamCipher::IV iv; + auto hash = XXH3_128bits(filename.c_str(), filename.size()); + auto high = reinterpret_cast(&hash.high64); + auto low = reinterpret_cast(&hash.low64); + std::copy(high, high + 8, &iv[0]); + std::copy(low, low + 6, &iv[8]); + iv[14] = iv[15] = 0; // last 16 bits identify block + return iv; + } + + ACTOR static Future> readBlock(AsyncFileEncrypted* self, uint16_t block) { + state Arena arena; + state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE]; + int bytes = wait( + self->file->read(encrypted, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * block)); + DecryptionStreamCipher decryptor(AsyncFileEncrypted::getKey(), self->getIV(block)); + auto decrypted = decryptor.decrypt(encrypted, bytes, arena); + return Standalone(decrypted, arena); + } + + ACTOR static Future read(AsyncFileEncrypted* self, void* data, int length, int offset) { + state const uint16_t firstBlock = offset / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE; + state const uint16_t lastBlock = (offset + length - 1) / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE; + state uint16_t block; + state unsigned char* output = reinterpret_cast(data); + state int bytesRead = 0; + for (block = firstBlock; block <= lastBlock; ++block) { + state StringRef plaintext; + auto it = self->readBuffers.find(block); + if (it != self->readBuffers.end()) { + plaintext = it->second; + } else { + Standalone _plaintext = wait(readBlock(self, block)); + ASSERT(_plaintext.size() == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); + if (self->readBuffers.size() == FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { + // TODO: Improve eviction policy + self->readBuffers.erase(self->readBuffers.begin()); + } + self->readBuffers[block] = _plaintext; + plaintext = _plaintext; + } + ASSERT(plaintext.size() == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); + auto start = (block == firstBlock) ? plaintext.begin() + (offset % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) + : plaintext.begin(); + auto end = (block == lastBlock) + ? plaintext.begin() + ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) + : plaintext.end(); + if ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE == 0) { + end = plaintext.end(); + } + std::copy(start, end, output); + output += (end - start); + bytesRead += (end - start); + } + return bytesRead; + } + + ACTOR static Future write(AsyncFileEncrypted* self, void const* data, int length, int64_t offset) { + ASSERT(self->canWrite); + ASSERT(offset == self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock); + state unsigned char const* input = reinterpret_cast(data); + while (length > 0) { + const auto chunkSize = std::min(length, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE - self->offsetInBlock); + Arena arena; + auto encrypted = self->encryptor->encrypt(input, chunkSize, arena); + std::copy(encrypted.begin(), encrypted.end(), &self->writeBuffer[self->offsetInBlock]); + offset += encrypted.size(); + self->offsetInBlock += chunkSize; + length -= chunkSize; + input += chunkSize; + if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) { + wait(self->writeLastBlockToFile()); + self->offsetInBlock = 0; + ASSERT(self->currentBlock < std::numeric_limits::max()); + ++self->currentBlock; + self->encryptor = std::make_unique(AsyncFileEncrypted::getKey(), self->getIV(self->currentBlock)); + } + } + return Void(); + } + + ACTOR static Future sync(AsyncFileEncrypted* self) { + ASSERT(self->canWrite); + wait(self->writeLastBlockToFile()); + wait(self->file->sync()); + return Void(); + } + + ACTOR static Future initializeKey(Reference keyFile, int64_t offset) { + ASSERT(!AsyncFileEncrypted::key.present()); + AsyncFileEncrypted::key = StreamCipher::Key{}; + state int keySize = AsyncFileEncrypted::key.get().size(); + if (g_network->isSimulated()) { + generateRandomData(AsyncFileEncrypted::key.get().data(), keySize); + return Void(); + } else { + int bytesRead = wait(keyFile->read(AsyncFileEncrypted::key.get().data(), keySize, offset)); + ASSERT(bytesRead == keySize); + return Void(); + } + } + + ACTOR static Future zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) { + // TODO: Could optimize this + Arena arena; + auto zeroes = new (arena) unsigned char[length]; + memset(zeroes, 0, length); + wait(self->write(zeroes, length, offset)); + return Void(); + } +}; + +AsyncFileEncrypted::AsyncFileEncrypted(Reference file, bool canWrite) + : file(file), canWrite(canWrite), currentBlock(0) { + firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename()); + if (canWrite) { + encryptor = std::make_unique(AsyncFileEncrypted::getKey(), getIV(currentBlock)); + writeBuffer = std::vector(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0); + } +} + +void AsyncFileEncrypted::addref() { + ReferenceCounted::addref(); +} + +void AsyncFileEncrypted::delref() { + ReferenceCounted::delref(); +} + +Future AsyncFileEncrypted::read(void* data, int length, int64_t offset) { + return AsyncFileEncryptedImpl::read(this, data, length, offset); +} + +Future AsyncFileEncrypted::write(void const* data, int length, int64_t offset) { + return AsyncFileEncryptedImpl::write(this, data, length, offset); +} + +Future AsyncFileEncrypted::zeroRange(int64_t offset, int64_t length) { + return AsyncFileEncryptedImpl::zeroRange(this, offset, length); +} + +Future AsyncFileEncrypted::truncate(int64_t size) { + ASSERT(false); // TODO: Not yet implemented + return Void(); +} + +Future AsyncFileEncrypted::sync() { + return AsyncFileEncryptedImpl::sync(this); +} + +Future AsyncFileEncrypted::flush() { + return Void(); +} + +Future AsyncFileEncrypted::size() const { + return currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + offsetInBlock; +} + +std::string AsyncFileEncrypted::getFilename() const { + return file->getFilename(); +} + +Future AsyncFileEncrypted::readZeroCopy(void** data, int* length, int64_t offset) { + ASSERT(false); // Not implemented + return Void(); +} + +void AsyncFileEncrypted::releaseZeroCopy(void* data, int length, int64_t offset) { + ASSERT(false); // Not implemented +} + +int64_t AsyncFileEncrypted::debugFD() const { + return 0; +} + +StreamCipher::IV AsyncFileEncrypted::getIV(uint16_t block) const { + auto iv = firstBlockIV; + iv[14] = block / 256; + iv[15] = block % 256; + return iv; +} + +Future AsyncFileEncrypted::writeLastBlockToFile() { + return file->write(&writeBuffer[0], offsetInBlock, currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); +} + +Optional AsyncFileEncrypted::key; + +StreamCipher::Key AsyncFileEncrypted::getKey() { + return key.get(); +} + +Future AsyncFileEncrypted::initializeKey(const Reference& keyFile, int64_t offset) { + return AsyncFileEncryptedImpl::initializeKey(keyFile, offset); +} + +TEST_CASE("fdbrpc/AsyncFileEncrypted") { + state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000); + state std::vector writeBuffer(bytes, 0); + generateRandomData(&writeBuffer.front(), bytes); + state std::vector readBuffer(bytes, 0); + ASSERT(g_network->isSimulated()); + wait(AsyncFileEncrypted::initializeKey(Reference{})); + int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | + IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED | + IAsyncFile::OPEN_NO_AIO; + state Reference file = wait(IAsyncFileSystem::filesystem()->open("/tmp/test", flags, 0600)); + state int bytesWritten = 0; + while (bytesWritten < bytes) { + chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten); + wait(file->write(&writeBuffer[bytesWritten], chunkSize, bytesWritten)); + bytesWritten += chunkSize; + } + wait(file->sync()); + state int bytesRead = 0; + state int chunkSize; + while (bytesRead < bytes) { + chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead); + int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead)); + ASSERT(bytesReadInChunk == chunkSize); + bytesRead += bytesReadInChunk; + } + ASSERT(writeBuffer == readBuffer); + return Void(); +} diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h new file mode 100644 index 0000000000..d5807d0ccf --- /dev/null +++ b/fdbrpc/AsyncFileEncrypted.h @@ -0,0 +1,72 @@ +/* + * AsyncFileEncrypted.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FDBRPC_ASYNC_FILE_ENCRYPTED_H__ +#define __FDBRPC_ASYNC_FILE_ENCRYPTED_H__ + +#include "fdbrpc/IAsyncFile.h" +#include "flow/FastRef.h" +#include "flow/flow.h" +#include "flow/IRandom.h" +#include "flow/StreamCipher.h" + +#include + +/* + * Append-only file encrypted using AES-128-GCM. + * */ +class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted { + Reference file; + StreamCipher::IV firstBlockIV; + StreamCipher::IV getIV(uint16_t block) const; + bool canWrite; + Future writeLastBlockToFile(); + friend class AsyncFileEncryptedImpl; + static Optional key; + static StreamCipher::Key getKey(); + + // Reading: + std::map> readBuffers; + + // Writing (append only): + std::unique_ptr encryptor; + uint16_t currentBlock{ 0 }; + int offsetInBlock{ 0 }; + std::vector writeBuffer; + +public: + AsyncFileEncrypted(Reference, bool canWrite); + void addref() override; + void delref() override; + Future read(void* data, int length, int64_t offset) override; + Future write(void const* data, int length, int64_t offset) override; + Future zeroRange(int64_t offset, int64_t length) override; + Future truncate(int64_t size) override; + Future sync() override; + Future flush() override; + Future size() const override; + std::string getFilename() const override; + Future readZeroCopy(void** data, int* length, int64_t offset) override; + void releaseZeroCopy(void* data, int length, int64_t offset) override; + int64_t debugFD() const override; + static Future initializeKey(const Reference& keyFile, int64_t offset = 0); +}; + +#endif diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt index af84676be7..da8d4fe52a 100644 --- a/fdbrpc/CMakeLists.txt +++ b/fdbrpc/CMakeLists.txt @@ -8,6 +8,8 @@ set(FDBRPC_SRCS AsyncFileCached.actor.cpp AsyncFileNonDurable.actor.cpp AsyncFileWriteChecker.cpp + AsyncFileEncrypted.actor.cpp + AsyncFileEncrypted.h FailureMonitor.actor.cpp FlowTransport.actor.cpp genericactors.actor.h diff --git a/fdbrpc/IAsyncFile.h b/fdbrpc/IAsyncFile.h index 569a4f6ac6..6512c14b42 100644 --- a/fdbrpc/IAsyncFile.h +++ b/fdbrpc/IAsyncFile.h @@ -35,21 +35,25 @@ public: virtual ~IAsyncFile(); // Pass these to g_network->open to get an IAsyncFile enum { - // Implementation relies on the low bits being the same as the SQLite flags (this is validated by a static_assert there) - OPEN_READONLY = 0x1, - OPEN_READWRITE = 0x2, - OPEN_CREATE = 0x4, - OPEN_EXCLUSIVE = 0x10, - + // Implementation relies on the low bits being the same as the SQLite flags (this is validated by a + // static_assert there) + OPEN_READONLY = 0x1, + OPEN_READWRITE = 0x2, + OPEN_CREATE = 0x4, + OPEN_EXCLUSIVE = 0x10, + // Further flag values are arbitrary bits - OPEN_UNBUFFERED = 0x10000, - OPEN_UNCACHED = 0x20000, - OPEN_LOCK = 0x40000, - OPEN_ATOMIC_WRITE_AND_CREATE = 0x80000, // A temporary file is opened, and on the first call to sync() it is atomically renamed to the given filename - OPEN_LARGE_PAGES = 0x100000, - OPEN_NO_AIO = 0x200000, // Don't use AsyncFileKAIO or similar implementations that rely on filesystem support for AIO - OPEN_CACHED_READ_ONLY = 0x400000 // AsyncFileCached opens files read/write even if you specify read only - }; + OPEN_UNBUFFERED = 0x10000, + OPEN_UNCACHED = 0x20000, + OPEN_LOCK = 0x40000, + OPEN_ATOMIC_WRITE_AND_CREATE = 0x80000, // A temporary file is opened, and on the first call to sync() it is + // atomically renamed to the given filename + OPEN_LARGE_PAGES = 0x100000, + OPEN_NO_AIO = + 0x200000, // Don't use AsyncFileKAIO or similar implementations that rely on filesystem support for AIO + OPEN_CACHED_READ_ONLY = 0x400000, // AsyncFileCached opens files read/write even if you specify read only + OPEN_ENCRYPTED = 0x800000 // File is encrypted using AES-128-GCM (must be either read-only or write-only) + }; virtual void addref() = 0; virtual void delref() = 0; diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index 9fa61f29b3..db9c8fc0dd 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -33,6 +33,7 @@ #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbrpc/AsyncFileEIO.actor.h" +#include "fdbrpc/AsyncFileEncrypted.h" #include "fdbrpc/AsyncFileWinASIO.actor.h" #include "fdbrpc/AsyncFileKAIO.actor.h" #include "flow/AsioReactor.h" @@ -69,7 +70,11 @@ Future> Net2FileSystem::open(const std::string& file #endif f = Net2AsyncFile::open(filename, flags, mode, static_cast ((void*) g_network->global(INetwork::enASIOService))); if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) - f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); + f = map(f, [](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); + if (flags & IAsyncFile::OPEN_ENCRYPTED) + f = map(f, [flags](Reference r) { + return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); + }); return f; } diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index f97eb07b1b..0443fb5a48 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -31,6 +31,7 @@ #include "flow/Util.h" #include "fdbrpc/IAsyncFile.h" #include "fdbrpc/AsyncFileCached.actor.h" +#include "fdbrpc/AsyncFileEncrypted.h" #include "fdbrpc/AsyncFileNonDurable.actor.h" #include "flow/crc32c.h" #include "fdbrpc/TraceFileIO.h" @@ -2060,7 +2061,11 @@ Future> Sim2FileSystem::open(const std::string& file } Future> f = AsyncFileDetachable::open( machineCache[actualFilename] ); if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) - f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); + f = map(f, [](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); + if (flags & IAsyncFile::OPEN_ENCRYPTED) + f = map(f, [flags](Reference r) { + return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); + }); return f; } else diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index fc858a2caf..54d56209ca 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -132,8 +132,7 @@ target_link_libraries(flow PRIVATE ${FLOW_LIBS}) if(USE_VALGRIND) target_link_libraries(flow PUBLIC Valgrind) endif() -# TODO(atn34) Re-enable TLS for OPEN_FOR_IDE build once #2201 is resolved -if(NOT WITH_TLS OR OPEN_FOR_IDE) +if(NOT WITH_TLS) target_compile_definitions(flow PUBLIC TLS_DISABLED) else() target_link_libraries(flow PUBLIC OpenSSL::SSL) diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index a83100e9db..d71e3f0b9e 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -121,6 +121,10 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( EIO_MAX_PARALLELISM, 4 ); init( EIO_USE_ODIRECT, 0 ); + //AsyncFileEncrypted + init( ENCRYPTION_BLOCK_SIZE, 4096 ); + init( MAX_DECRYPTED_BLOCKS, 10 ); + //AsyncFileKAIO init( MAX_OUTSTANDING, 64 ); init( MIN_SUBMIT, 10 ); diff --git a/flow/Knobs.h b/flow/Knobs.h index 1450af992a..dbd5dae369 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -138,6 +138,10 @@ public: int EIO_MAX_PARALLELISM; int EIO_USE_ODIRECT; + // AsyncFileEncrypted + int ENCRYPTION_BLOCK_SIZE; + int MAX_DECRYPTED_BLOCKS; + //AsyncFileKAIO int MAX_OUTSTANDING; int MIN_SUBMIT; diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index cd6dc4f832..ef2c8cf0d6 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -24,8 +24,8 @@ EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) : ctx(EVP_CIPHER_CTX_new()) { EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, 12, nullptr); - EVP_EncryptInit_ex(ctx, nullptr, nullptr, key, iv); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); + EVP_EncryptInit_ex(ctx, nullptr, nullptr, key.data(), iv.data()); } EncryptionStreamCipher::~EncryptionStreamCipher() { @@ -46,12 +46,12 @@ StringRef EncryptionStreamCipher::finish(Arena& arena) { return StringRef(ciphertext, bytes); } -DecryptionStreamCipher::DecryptionStreamCipher(unsigned char const* key, unsigned char const* iv) +DecryptionStreamCipher::DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) : ctx(EVP_CIPHER_CTX_new()) { EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, 12, nullptr); - EVP_DecryptInit_ex(ctx, nullptr, nullptr, key, iv); + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); + EVP_DecryptInit_ex(ctx, nullptr, nullptr, key.data(), iv.data()); } DecryptionStreamCipher::~DecryptionStreamCipher() { @@ -77,10 +77,10 @@ StringRef DecryptionStreamCipher::finish(Arena& arena) { void forceLinkStreamCipherTests() {} TEST_CASE("flow/StreamCipher") { - std::array key; + StreamCipher::Key key; generateRandomData(key.data(), key.size()); - std::array iv; + StreamCipher::IV iv; generateRandomData(iv.data(), iv.size()); Arena arena; @@ -93,7 +93,7 @@ TEST_CASE("flow/StreamCipher") { .detail("PlaintextSize", plaintext.size()) .detail("AESBlockSize", AES_BLOCK_SIZE); { - EncryptionStreamCipher encryptor(key.data(), iv.data()); + EncryptionStreamCipher encryptor(key, iv); int index = 0; int encryptedOffset = 0; while (index < plaintext.size()) { @@ -113,7 +113,7 @@ TEST_CASE("flow/StreamCipher") { } { - DecryptionStreamCipher decryptor(key.data(), iv.data()); + DecryptionStreamCipher decryptor(key, iv); int index = 0; int decryptedOffset = 0; while (index < plaintext.size()) { diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index 2af0f9ac07..bc734854e9 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -30,10 +30,15 @@ #include #include +namespace StreamCipher { +using Key = std::array; +using IV = std::array; +}; // namespace StreamCipher + class EncryptionStreamCipher final : NonCopyable, public ReferenceCounted { EVP_CIPHER_CTX* ctx; public: - EncryptionStreamCipher(unsigned char const* key, unsigned char const* salt); + EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv); ~EncryptionStreamCipher(); StringRef encrypt(unsigned char const* plaintext, int len, Arena&); StringRef finish(Arena&); @@ -42,7 +47,7 @@ public: class DecryptionStreamCipher final : NonCopyable, public ReferenceCounted { EVP_CIPHER_CTX* ctx; public: - DecryptionStreamCipher(unsigned char const* key, unsigned char const* salt); + DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv); ~DecryptionStreamCipher(); StringRef decrypt(unsigned char const* ciphertext, int len, Arena&); StringRef finish(Arena&); From 037279c843bfe78323e3bb6887c6ef12186aaae5 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 25 Jan 2021 10:42:14 -0800 Subject: [PATCH 003/426] Disable StreamCipher if TLS is disabled --- fdbrpc/CMakeLists.txt | 9 +++++++-- fdbrpc/Net2FileSystem.cpp | 2 ++ fdbrpc/sim2.actor.cpp | 2 ++ fdbserver/workloads/UnitTests.actor.cpp | 4 ++++ flow/CMakeLists.txt | 9 +++++++-- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt index da8d4fe52a..bd2d83f312 100644 --- a/fdbrpc/CMakeLists.txt +++ b/fdbrpc/CMakeLists.txt @@ -8,8 +8,6 @@ set(FDBRPC_SRCS AsyncFileCached.actor.cpp AsyncFileNonDurable.actor.cpp AsyncFileWriteChecker.cpp - AsyncFileEncrypted.actor.cpp - AsyncFileEncrypted.h FailureMonitor.actor.cpp FlowTransport.actor.cpp genericactors.actor.h @@ -31,6 +29,13 @@ set(FDBRPC_SRCS TimedRequest.h TraceFileIO.cpp) +if(WITH_TLS) + set(FDBRPC_SRCS + ${FDBRPC_SRCS} + AsyncFileEncrypted.h + AsyncFileEncrypted.actor.cpp) +endif() + set(FDBRPC_THIRD_PARTY_SRCS libcoroutine/Common.c libcoroutine/Coro.c) diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index db9c8fc0dd..854b551c45 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -71,10 +71,12 @@ Future> Net2FileSystem::open(const std::string& file f = Net2AsyncFile::open(filename, flags, mode, static_cast ((void*) g_network->global(INetwork::enASIOService))); if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); +#ifndef TLS_DISABLED if (flags & IAsyncFile::OPEN_ENCRYPTED) f = map(f, [flags](Reference r) { return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); }); +#endif return f; } diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 0443fb5a48..cdf9ea1187 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -2062,10 +2062,12 @@ Future> Sim2FileSystem::open(const std::string& file Future> f = AsyncFileDetachable::open( machineCache[actualFilename] ); if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); +#ifndef TLS_DISABLED if (flags & IAsyncFile::OPEN_ENCRYPTED) f = map(f, [flags](Reference r) { return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); }); +#endif return f; } else diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 6b633ad5ab..fbc2598253 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -28,7 +28,9 @@ void forceLinkFlowTests(); void forceLinkVersionedMapTests(); void forceLinkMemcpyTests(); void forceLinkMemcpyPerfTests(); +#ifndef TLS_DISABLED void forceLinkStreamCipherTests(); +#endif struct UnitTestWorkload : TestWorkload { bool enabled; @@ -50,7 +52,9 @@ struct UnitTestWorkload : TestWorkload { forceLinkVersionedMapTests(); forceLinkMemcpyTests(); forceLinkMemcpyPerfTests(); +#ifndef TLS_DISABLED forceLinkStreamCipherTests(); +#endif } std::string description() const override { return "UnitTests"; } diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index 54d56209ca..54e379392d 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -50,8 +50,6 @@ set(FLOW_SRCS SignalSafeUnwind.cpp SignalSafeUnwind.h SimpleOpt.h - StreamCipher.cpp - StreamCipher.h SystemMonitor.cpp SystemMonitor.h TDMetric.actor.h @@ -95,6 +93,13 @@ set(FLOW_SRCS xxhash.c xxhash.h) +if(WITH_TLS) + set(FLOW_SRCS + ${FLOW_SRCS} + StreamCipher.cpp + StreamCipher.h) +endif() + add_library(stacktrace stacktrace.amalgamation.cpp stacktrace.h) if (USE_ASAN) target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER) From 1612c449885d4b34e4c2ea862850d6acfa14495e Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 25 Jan 2021 12:09:29 -0800 Subject: [PATCH 004/426] Disable StreamCipher for Windows --- fdbrpc/CMakeLists.txt | 2 +- fdbrpc/Net2FileSystem.cpp | 4 +++- fdbrpc/sim2.actor.cpp | 4 +++- fdbserver/workloads/UnitTests.actor.cpp | 4 ++-- flow/CMakeLists.txt | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt index bd2d83f312..c93456e019 100644 --- a/fdbrpc/CMakeLists.txt +++ b/fdbrpc/CMakeLists.txt @@ -29,7 +29,7 @@ set(FDBRPC_SRCS TimedRequest.h TraceFileIO.cpp) -if(WITH_TLS) +if(WITH_TLS AND NOT WIN32) set(FDBRPC_SRCS ${FDBRPC_SRCS} AsyncFileEncrypted.h diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index 854b551c45..0fc289eb7c 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -33,7 +33,9 @@ #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbrpc/AsyncFileEIO.actor.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) #include "fdbrpc/AsyncFileEncrypted.h" +#endif #include "fdbrpc/AsyncFileWinASIO.actor.h" #include "fdbrpc/AsyncFileKAIO.actor.h" #include "flow/AsioReactor.h" @@ -71,7 +73,7 @@ Future> Net2FileSystem::open(const std::string& file f = Net2AsyncFile::open(filename, flags, mode, static_cast ((void*) g_network->global(INetwork::enASIOService))); if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); -#ifndef TLS_DISABLED +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) if (flags & IAsyncFile::OPEN_ENCRYPTED) f = map(f, [flags](Reference r) { return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index cdf9ea1187..b0521ac8a4 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -31,7 +31,9 @@ #include "flow/Util.h" #include "fdbrpc/IAsyncFile.h" #include "fdbrpc/AsyncFileCached.actor.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) #include "fdbrpc/AsyncFileEncrypted.h" +#endif #include "fdbrpc/AsyncFileNonDurable.actor.h" #include "flow/crc32c.h" #include "fdbrpc/TraceFileIO.h" @@ -2062,7 +2064,7 @@ Future> Sim2FileSystem::open(const std::string& file Future> f = AsyncFileDetachable::open( machineCache[actualFilename] ); if(FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); -#ifndef TLS_DISABLED +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) if (flags & IAsyncFile::OPEN_ENCRYPTED) f = map(f, [flags](Reference r) { return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index fbc2598253..4597f1f72e 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -28,7 +28,7 @@ void forceLinkFlowTests(); void forceLinkVersionedMapTests(); void forceLinkMemcpyTests(); void forceLinkMemcpyPerfTests(); -#ifndef TLS_DISABLED +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) void forceLinkStreamCipherTests(); #endif @@ -52,7 +52,7 @@ struct UnitTestWorkload : TestWorkload { forceLinkVersionedMapTests(); forceLinkMemcpyTests(); forceLinkMemcpyPerfTests(); -#ifndef TLS_DISABLED +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) forceLinkStreamCipherTests(); #endif } diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index 54e379392d..83f2de73ea 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -93,7 +93,7 @@ set(FLOW_SRCS xxhash.c xxhash.h) -if(WITH_TLS) +if(WITH_TLS AND NOT WIN32) set(FLOW_SRCS ${FLOW_SRCS} StreamCipher.cpp From 06e495737cf86103d6d02e1085095bbfa1047945 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 25 Jan 2021 13:44:55 -0800 Subject: [PATCH 005/426] Added BenchEncrypt --- flowbench/BenchEncrypt.cpp | 81 ++++++++++++++++++++++++++++++++++++++ flowbench/CMakeLists.txt | 6 +++ 2 files changed, 87 insertions(+) create mode 100644 flowbench/BenchEncrypt.cpp diff --git a/flowbench/BenchEncrypt.cpp b/flowbench/BenchEncrypt.cpp new file mode 100644 index 0000000000..625ad800f6 --- /dev/null +++ b/flowbench/BenchEncrypt.cpp @@ -0,0 +1,81 @@ +/* + * BenchEncrypt.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark/benchmark.h" + +#include "flow/StreamCipher.h" +#include "flowbench/GlobalData.h" + +static StreamCipher::Key getRandomKey() { + StreamCipher::Key key; + generateRandomData(key.data(), key.size()); + return key; +} + +static StreamCipher::IV getRandomIV() { + StreamCipher::IV iv; + generateRandomData(iv.data(), iv.size()); + return iv; +} + +static inline Standalone encrypt(const StreamCipher::Key& key, const StreamCipher::IV& iv, + unsigned char const* data, size_t len) { + EncryptionStreamCipher encryptor(key, iv); + Arena arena; + auto encrypted = encryptor.encrypt(data, len, arena); + return Standalone(encrypted, arena); +} + +static void bench_encrypt(benchmark::State& state) { + auto bytes = state.range(0); + auto chunks = state.range(1); + auto chunkSize = bytes / chunks; + auto key = getRandomKey(); + auto iv = getRandomIV(); + auto data = getKey(bytes); + while (state.KeepRunning()) { + for (int chunk = 0; chunk < chunks; ++chunk) { + benchmark::DoNotOptimize(encrypt(key, iv, data.begin() + chunk * chunkSize, chunkSize)); + } + } + state.SetBytesProcessed(bytes * static_cast(state.iterations())); +} + +static void bench_decrypt(benchmark::State& state) { + auto bytes = state.range(0); + auto chunks = state.range(1); + auto chunkSize = bytes / chunks; + auto key = getRandomKey(); + auto iv = getRandomIV(); + auto data = getKey(bytes); + auto encrypted = encrypt(key, iv, data.begin(), data.size()); + while (state.KeepRunning()) { + Arena arena; + DecryptionStreamCipher decryptor(key, iv); + for (int chunk = 0; chunk < chunks; ++chunk) { + benchmark::DoNotOptimize( + Standalone(decryptor.decrypt(encrypted.begin() + chunk * chunkSize, chunkSize, arena))); + } + } + state.SetBytesProcessed(bytes * static_cast(state.iterations())); +} + +BENCHMARK(bench_encrypt)->Ranges({ { 1 << 12, 1 << 20 }, { 1, 1 << 12 } }); +BENCHMARK(bench_decrypt)->Ranges({ { 1 << 12, 1 << 20 }, { 1, 1 << 12 } }); diff --git a/flowbench/CMakeLists.txt b/flowbench/CMakeLists.txt index d1f10037ae..8caad0ce02 100644 --- a/flowbench/CMakeLists.txt +++ b/flowbench/CMakeLists.txt @@ -11,6 +11,12 @@ set(FLOWBENCH_SRCS GlobalData.h GlobalData.cpp) +if(WITH_TLS AND NOT WIN32) + set(FLOWBENCH_SRCS + ${FLOWBENCH_SRCS} + BenchEncrypt.cpp) +endif() + project (flowbench) # include the configurations from benchmark.cmake configure_file(benchmark.cmake googlebenchmark-download/CMakeLists.txt) From bbf82304627778f55390456d3cb7659c7239e071 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 29 Jan 2021 23:57:24 -0800 Subject: [PATCH 006/426] Use random eviction policy for cached AsyncFileEncrypted blocks --- fdbrpc/AsyncFileEncrypted.actor.cpp | 16 +++++------- fdbrpc/AsyncFileEncrypted.h | 40 ++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 2fadd545af..856c1849a0 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -55,17 +55,13 @@ public: state int bytesRead = 0; for (block = firstBlock; block <= lastBlock; ++block) { state StringRef plaintext; - auto it = self->readBuffers.find(block); - if (it != self->readBuffers.end()) { - plaintext = it->second; + + auto cachedBlock = self->readBuffers.get(block); + if (cachedBlock.present()) { + plaintext = cachedBlock.get(); } else { Standalone _plaintext = wait(readBlock(self, block)); - ASSERT(_plaintext.size() == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); - if (self->readBuffers.size() == FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { - // TODO: Improve eviction policy - self->readBuffers.erase(self->readBuffers.begin()); - } - self->readBuffers[block] = _plaintext; + self->readBuffers.insert(block, _plaintext); plaintext = _plaintext; } ASSERT(plaintext.size() == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); @@ -140,7 +136,7 @@ public: }; AsyncFileEncrypted::AsyncFileEncrypted(Reference file, bool canWrite) - : file(file), canWrite(canWrite), currentBlock(0) { + : file(file), canWrite(canWrite), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename()); if (canWrite) { encryptor = std::make_unique(AsyncFileEncrypted::getKey(), getIV(currentBlock)); diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index d5807d0ccf..d63bd408ab 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -42,8 +42,46 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted key; static StreamCipher::Key getKey(); + template + class RandomCache { + size_t maxSize; + std::vector vec; + std::unordered_map hashMap; + + size_t evict() { + ASSERT(vec.size() == maxSize); + auto index = deterministicRandom()->randomInt(0, maxSize); + hashMap.erase(vec[index]); + return index; + } + + public: + RandomCache(size_t maxSize) : maxSize(maxSize) { vec.reserve(maxSize); } + + void insert(const K& key, const V& value) { + auto [it, found] = hashMap.insert({ key, value }); + if (found) { + return; + } else if (vec.size() < maxSize) { + vec.push_back(key); + } else { + auto index = evict(); + vec[index] = key; + } + } + + Optional get(const K& key) const { + auto it = hashMap.find(key); + if (it == hashMap.end()) { + return {}; + } else { + return it->second; + } + } + }; + // Reading: - std::map> readBuffers; + RandomCache> readBuffers; // Writing (append only): std::unique_ptr encryptor; From 4563eeba5b14295192fa8b604392da91cc83a04c Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 30 Jan 2021 00:12:05 -0800 Subject: [PATCH 007/426] Moved AsyncFileEncrypted::RandomCache implementation to cpp file --- fdbrpc/AsyncFileEncrypted.actor.cpp | 32 +++++++++++++++++++++ fdbrpc/AsyncFileEncrypted.h | 44 ++++++----------------------- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 856c1849a0..029b9ae1ae 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -219,6 +219,38 @@ Future AsyncFileEncrypted::initializeKey(const Reference& keyF return AsyncFileEncryptedImpl::initializeKey(keyFile, offset); } +size_t AsyncFileEncrypted::RandomCache::evict() { + ASSERT(vec.size() == maxSize); + auto index = deterministicRandom()->randomInt(0, maxSize); + hashMap.erase(vec[index]); + return index; +} + +AsyncFileEncrypted::RandomCache::RandomCache(size_t maxSize) : maxSize(maxSize) { + vec.reserve(maxSize); +} + +void AsyncFileEncrypted::RandomCache::insert(uint16_t block, const Standalone& value) { + auto [_, found] = hashMap.insert({ block, value }); + if (found) { + return; + } else if (vec.size() < maxSize) { + vec.push_back(block); + } else { + auto index = evict(); + vec[index] = block; + } +} + +Optional> AsyncFileEncrypted::RandomCache::get(uint16_t block) const { + auto it = hashMap.find(block); + if (it == hashMap.end()) { + return {}; + } else { + return it->second; + } +} + TEST_CASE("fdbrpc/AsyncFileEncrypted") { state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000); state std::vector writeBuffer(bytes, 0); diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index d63bd408ab..45427b3900 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -42,46 +42,18 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted key; static StreamCipher::Key getKey(); - template + // Reading: class RandomCache { size_t maxSize; - std::vector vec; - std::unordered_map hashMap; - - size_t evict() { - ASSERT(vec.size() == maxSize); - auto index = deterministicRandom()->randomInt(0, maxSize); - hashMap.erase(vec[index]); - return index; - } + std::vector vec; + std::unordered_map> hashMap; + size_t evict(); public: - RandomCache(size_t maxSize) : maxSize(maxSize) { vec.reserve(maxSize); } - - void insert(const K& key, const V& value) { - auto [it, found] = hashMap.insert({ key, value }); - if (found) { - return; - } else if (vec.size() < maxSize) { - vec.push_back(key); - } else { - auto index = evict(); - vec[index] = key; - } - } - - Optional get(const K& key) const { - auto it = hashMap.find(key); - if (it == hashMap.end()) { - return {}; - } else { - return it->second; - } - } - }; - - // Reading: - RandomCache> readBuffers; + RandomCache(size_t maxSize); + void insert(uint16_t block, const Standalone& value); + Optional> get(uint16_t block) const; + } readBuffers; // Writing (append only): std::unique_ptr encryptor; From 5be4df6f84ec2890deb4c91e5e2843b6d57c6366 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 3 Feb 2021 17:39:59 -0800 Subject: [PATCH 008/426] Move global key into StreamCipher --- fdbrpc/AsyncFileEncrypted.actor.cpp | 33 +++++------------------------ fdbrpc/AsyncFileEncrypted.h | 3 --- flow/StreamCipher.cpp | 16 ++++++++++++++ flow/StreamCipher.h | 4 ++++ 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 029b9ae1ae..37747aa6f9 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -42,7 +42,7 @@ public: state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE]; int bytes = wait( self->file->read(encrypted, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * block)); - DecryptionStreamCipher decryptor(AsyncFileEncrypted::getKey(), self->getIV(block)); + DecryptionStreamCipher decryptor(StreamCipher::getKey(), self->getIV(block)); auto decrypted = decryptor.decrypt(encrypted, bytes, arena); return Standalone(decrypted, arena); } @@ -98,7 +98,8 @@ public: self->offsetInBlock = 0; ASSERT(self->currentBlock < std::numeric_limits::max()); ++self->currentBlock; - self->encryptor = std::make_unique(AsyncFileEncrypted::getKey(), self->getIV(self->currentBlock)); + self->encryptor = + std::make_unique(StreamCipher::getKey(), self->getIV(self->currentBlock)); } } return Void(); @@ -111,20 +112,6 @@ public: return Void(); } - ACTOR static Future initializeKey(Reference keyFile, int64_t offset) { - ASSERT(!AsyncFileEncrypted::key.present()); - AsyncFileEncrypted::key = StreamCipher::Key{}; - state int keySize = AsyncFileEncrypted::key.get().size(); - if (g_network->isSimulated()) { - generateRandomData(AsyncFileEncrypted::key.get().data(), keySize); - return Void(); - } else { - int bytesRead = wait(keyFile->read(AsyncFileEncrypted::key.get().data(), keySize, offset)); - ASSERT(bytesRead == keySize); - return Void(); - } - } - ACTOR static Future zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) { // TODO: Could optimize this Arena arena; @@ -139,7 +126,7 @@ AsyncFileEncrypted::AsyncFileEncrypted(Reference file, bool canWrite : file(file), canWrite(canWrite), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename()); if (canWrite) { - encryptor = std::make_unique(AsyncFileEncrypted::getKey(), getIV(currentBlock)); + encryptor = std::make_unique(StreamCipher::getKey(), getIV(currentBlock)); writeBuffer = std::vector(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0); } } @@ -209,16 +196,6 @@ Future AsyncFileEncrypted::writeLastBlockToFile() { return file->write(&writeBuffer[0], offsetInBlock, currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); } -Optional AsyncFileEncrypted::key; - -StreamCipher::Key AsyncFileEncrypted::getKey() { - return key.get(); -} - -Future AsyncFileEncrypted::initializeKey(const Reference& keyFile, int64_t offset) { - return AsyncFileEncryptedImpl::initializeKey(keyFile, offset); -} - size_t AsyncFileEncrypted::RandomCache::evict() { ASSERT(vec.size() == maxSize); auto index = deterministicRandom()->randomInt(0, maxSize); @@ -257,7 +234,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") { generateRandomData(&writeBuffer.front(), bytes); state std::vector readBuffer(bytes, 0); ASSERT(g_network->isSimulated()); - wait(AsyncFileEncrypted::initializeKey(Reference{})); + StreamCipher::initializeRandomKey(); int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO; diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index 45427b3900..dc7f15f299 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -39,8 +39,6 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted writeLastBlockToFile(); friend class AsyncFileEncryptedImpl; - static Optional key; - static StreamCipher::Key getKey(); // Reading: class RandomCache { @@ -76,7 +74,6 @@ public: Future readZeroCopy(void** data, int* length, int64_t offset) override; void releaseZeroCopy(void* data, int length, int64_t offset) override; int64_t debugFD() const override; - static Future initializeKey(const Reference& keyFile, int64_t offset = 0); }; #endif diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index ef2c8cf0d6..a45ea6a5f7 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -21,6 +21,8 @@ #include "flow/StreamCipher.h" #include "flow/UnitTest.h" +#include + EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) : ctx(EVP_CIPHER_CTX_new()) { EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); @@ -74,6 +76,20 @@ StringRef DecryptionStreamCipher::finish(Arena& arena) { return StringRef(plaintext, finalBlockBytes); } +static std::unordered_set ciphers; +static Optional globalKey; + +void StreamCipher::initializeRandomKey() { + ASSERT(g_network->isSimulated()); + ASSERT(!globalKey.present()); + globalKey = StreamCipher::Key{}; + generateRandomData(globalKey.get().data(), globalKey.get().size()); +} + +StreamCipher::Key StreamCipher::getKey() { + return globalKey.get(); +} + void forceLinkStreamCipherTests() {} TEST_CASE("flow/StreamCipher") { diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index bc734854e9..5443c48756 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -33,6 +33,10 @@ namespace StreamCipher { using Key = std::array; using IV = std::array; +void initializeRandomKey(); +Key getKey(); +void registerCipherForCleanup(EVP_CIPHER_CTX*) noexcept; +void deregisterCipherForCleanup(EVP_CIPHER_CTX*) noexcept; }; // namespace StreamCipher class EncryptionStreamCipher final : NonCopyable, public ReferenceCounted { From 7c0e331e07522853033f2f599ab53f44111efda8 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 3 Feb 2021 18:13:39 -0800 Subject: [PATCH 009/426] Disable copying StreamCipher::Key --- fdbrpc/AsyncFileEncrypted.actor.cpp | 10 +++++----- flow/StreamCipher.cpp | 19 ++++++++++--------- flow/StreamCipher.h | 13 ++++++++++--- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 37747aa6f9..7b13f3e804 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -42,7 +42,7 @@ public: state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE]; int bytes = wait( self->file->read(encrypted, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * block)); - DecryptionStreamCipher decryptor(StreamCipher::getKey(), self->getIV(block)); + DecryptionStreamCipher decryptor(StreamCipher::Key::getKey(), self->getIV(block)); auto decrypted = decryptor.decrypt(encrypted, bytes, arena); return Standalone(decrypted, arena); } @@ -98,8 +98,8 @@ public: self->offsetInBlock = 0; ASSERT(self->currentBlock < std::numeric_limits::max()); ++self->currentBlock; - self->encryptor = - std::make_unique(StreamCipher::getKey(), self->getIV(self->currentBlock)); + self->encryptor = std::make_unique(StreamCipher::Key::getKey(), + self->getIV(self->currentBlock)); } } return Void(); @@ -126,7 +126,7 @@ AsyncFileEncrypted::AsyncFileEncrypted(Reference file, bool canWrite : file(file), canWrite(canWrite), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename()); if (canWrite) { - encryptor = std::make_unique(StreamCipher::getKey(), getIV(currentBlock)); + encryptor = std::make_unique(StreamCipher::Key::getKey(), getIV(currentBlock)); writeBuffer = std::vector(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0); } } @@ -234,7 +234,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") { generateRandomData(&writeBuffer.front(), bytes); state std::vector readBuffer(bytes, 0); ASSERT(g_network->isSimulated()); - StreamCipher::initializeRandomKey(); + StreamCipher::Key::initializeRandomKey(); int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO; diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index a45ea6a5f7..559fc304e4 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -76,25 +76,26 @@ StringRef DecryptionStreamCipher::finish(Arena& arena) { return StringRef(plaintext, finalBlockBytes); } +std::unique_ptr StreamCipher::Key::globalKey; static std::unordered_set ciphers; -static Optional globalKey; -void StreamCipher::initializeRandomKey() { +void StreamCipher::Key::initializeRandomKey() { ASSERT(g_network->isSimulated()); - ASSERT(!globalKey.present()); - globalKey = StreamCipher::Key{}; - generateRandomData(globalKey.get().data(), globalKey.get().size()); + if (globalKey) return; + globalKey = std::make_unique(); + generateRandomData(globalKey.get()->arr.data(), globalKey.get()->arr.size()); } -StreamCipher::Key StreamCipher::getKey() { - return globalKey.get(); +const StreamCipher::Key& StreamCipher::Key::getKey() { + ASSERT(globalKey); + return *globalKey; } void forceLinkStreamCipherTests() {} TEST_CASE("flow/StreamCipher") { - StreamCipher::Key key; - generateRandomData(key.data(), key.size()); + StreamCipher::Key::initializeRandomKey(); + const auto& key = StreamCipher::Key::getKey(); StreamCipher::IV iv; generateRandomData(iv.data(), iv.size()); diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index 5443c48756..f70bea3678 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -31,10 +31,17 @@ #include namespace StreamCipher { -using Key = std::array; +class Key : NonCopyable { + std::array arr; + static std::unique_ptr globalKey; + +public: + Key() = default; // TODO: Make private + const unsigned char* data() const { return arr.data(); } + static void initializeRandomKey(); + static const Key& getKey(); +}; using IV = std::array; -void initializeRandomKey(); -Key getKey(); void registerCipherForCleanup(EVP_CIPHER_CTX*) noexcept; void deregisterCipherForCleanup(EVP_CIPHER_CTX*) noexcept; }; // namespace StreamCipher From b5ed7dcdf8a528fc95e28e9bdbcaf814445141bd Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 3 Feb 2021 18:21:04 -0800 Subject: [PATCH 010/426] Make StreamCipher::Key constructor effectively private --- flow/StreamCipher.cpp | 2 +- flow/StreamCipher.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index 559fc304e4..2bd0d964ac 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -82,7 +82,7 @@ static std::unordered_set ciphers; void StreamCipher::Key::initializeRandomKey() { ASSERT(g_network->isSimulated()); if (globalKey) return; - globalKey = std::make_unique(); + globalKey = std::make_unique(ConstructorTag{}); generateRandomData(globalKey.get()->arr.data(), globalKey.get()->arr.size()); } diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index f70bea3678..2de904cde2 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -34,9 +34,10 @@ namespace StreamCipher { class Key : NonCopyable { std::array arr; static std::unique_ptr globalKey; + struct ConstructorTag {}; public: - Key() = default; // TODO: Make private + Key(ConstructorTag) {} const unsigned char* data() const { return arr.data(); } static void initializeRandomKey(); static const Key& getKey(); From 8470a326a211994746c890444e6d01b345f800a6 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 3 Feb 2021 18:49:51 -0800 Subject: [PATCH 011/426] Clean up StreamCipher::Key::globalKey in crashHandler --- flow/Platform.actor.cpp | 3 +++ flow/StreamCipher.cpp | 8 ++++++++ flow/StreamCipher.h | 5 +++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 7c05bdf7ee..40f5ebd3c2 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -29,6 +29,7 @@ #include "flow/Platform.actor.h" #include "flow/Arena.h" +#include "flow/StreamCipher.h" #include "flow/Trace.h" #include "flow/Error.h" @@ -3237,6 +3238,8 @@ void crashHandler(int sig) { bool error = (sig != SIGUSR2); + StreamCipher::Key::cleanup(); + fflush(stdout); TraceEvent(error ? SevError : SevInfo, error ? "Crash" : "ProcessTerminated") .detail("Signal", sig) diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index 2bd0d964ac..6be6ef5b4c 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -91,6 +91,14 @@ const StreamCipher::Key& StreamCipher::Key::getKey() { return *globalKey; } +StreamCipher::Key::~Key() { + memset(arr.data(), 0, arr.size()); +} + +void StreamCipher::Key::cleanup() { + globalKey.reset(); +} + void forceLinkStreamCipherTests() {} TEST_CASE("flow/StreamCipher") { diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index 2de904cde2..627626913b 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -35,12 +35,13 @@ class Key : NonCopyable { std::array arr; static std::unique_ptr globalKey; struct ConstructorTag {}; - public: Key(ConstructorTag) {} - const unsigned char* data() const { return arr.data(); } + ~Key(); + unsigned char const* data() const { return arr.data(); } static void initializeRandomKey(); static const Key& getKey(); + static void cleanup(); }; using IV = std::array; void registerCipherForCleanup(EVP_CIPHER_CTX*) noexcept; From b601a73a25173e661d09850bb45b7bb2c91cc891 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Wed, 3 Feb 2021 21:30:20 -0800 Subject: [PATCH 012/426] Clean up all cipher contexts in crashHandler --- flow/Platform.actor.cpp | 2 +- flow/StreamCipher.cpp | 121 +++++++++++++++++++++------------------- flow/StreamCipher.h | 46 ++++++++------- 3 files changed, 92 insertions(+), 77 deletions(-) diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 40f5ebd3c2..f258ed9789 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -3238,7 +3238,7 @@ void crashHandler(int sig) { bool error = (sig != SIGUSR2); - StreamCipher::Key::cleanup(); + StreamCipher::cleanup(); fflush(stdout); TraceEvent(error ? SevError : SevInfo, error ? "Crash" : "ProcessTerminated") diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index 6be6ef5b4c..6afdbf2309 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -21,63 +21,28 @@ #include "flow/StreamCipher.h" #include "flow/UnitTest.h" -#include - -EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) - : ctx(EVP_CIPHER_CTX_new()) { - EVP_EncryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); - EVP_EncryptInit_ex(ctx, nullptr, nullptr, key.data(), iv.data()); -} - -EncryptionStreamCipher::~EncryptionStreamCipher() { - EVP_CIPHER_CTX_free(ctx); -} - -StringRef EncryptionStreamCipher::encrypt(unsigned char const* plaintext, int len, Arena& arena) { - auto ciphertext = new (arena) unsigned char[len + AES_BLOCK_SIZE]; - int bytes{ 0 }; - EVP_EncryptUpdate(ctx, ciphertext, &bytes, plaintext, len); - return StringRef(ciphertext, bytes); -} - -StringRef EncryptionStreamCipher::finish(Arena& arena) { - auto ciphertext = new (arena) unsigned char[AES_BLOCK_SIZE]; - int bytes{ 0 }; - EVP_EncryptFinal_ex(ctx, ciphertext, &bytes); - return StringRef(ciphertext, bytes); -} - -DecryptionStreamCipher::DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) - : ctx(EVP_CIPHER_CTX_new()) { - - EVP_DecryptInit_ex(ctx, EVP_aes_128_gcm(), nullptr, nullptr, nullptr); - EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); - EVP_DecryptInit_ex(ctx, nullptr, nullptr, key.data(), iv.data()); -} - -DecryptionStreamCipher::~DecryptionStreamCipher() { - EVP_CIPHER_CTX_free(ctx); -} - -StringRef DecryptionStreamCipher::decrypt(unsigned char const* ciphertext, int len, Arena& arena) { - auto plaintext = new (arena) unsigned char[len]; - int bytesDecrypted{ 0 }; - EVP_DecryptUpdate(ctx, plaintext, &bytesDecrypted, ciphertext, len); - int finalBlockBytes{ 0 }; - EVP_DecryptFinal_ex(ctx, plaintext + bytesDecrypted, &finalBlockBytes); - return StringRef(plaintext, bytesDecrypted + finalBlockBytes); -} - -StringRef DecryptionStreamCipher::finish(Arena& arena) { - auto plaintext = new (arena) unsigned char[AES_BLOCK_SIZE]; - int finalBlockBytes{ 0 }; - EVP_DecryptFinal_ex(ctx, plaintext, &finalBlockBytes); - return StringRef(plaintext, finalBlockBytes); -} - +std::unordered_set StreamCipher::ctxs; std::unique_ptr StreamCipher::Key::globalKey; -static std::unordered_set ciphers; + +StreamCipher::StreamCipher() : ctx(EVP_CIPHER_CTX_new()) { + ctxs.insert(ctx); +} + +StreamCipher::~StreamCipher() { + EVP_CIPHER_CTX_free(ctx); + ctxs.erase(ctx); +} + +EVP_CIPHER_CTX* StreamCipher::getCtx() { + return ctx; +} + +void StreamCipher::cleanup() noexcept { + Key::cleanup(); + for (auto ctx : ctxs) { + EVP_CIPHER_CTX_free(ctx); + } +} void StreamCipher::Key::initializeRandomKey() { ASSERT(g_network->isSimulated()); @@ -95,10 +60,52 @@ StreamCipher::Key::~Key() { memset(arr.data(), 0, arr.size()); } -void StreamCipher::Key::cleanup() { +void StreamCipher::Key::cleanup() noexcept { globalKey.reset(); } +EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) { + EVP_EncryptInit_ex(cipher.getCtx(), EVP_aes_128_gcm(), nullptr, nullptr, nullptr); + EVP_CIPHER_CTX_ctrl(cipher.getCtx(), EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); + EVP_EncryptInit_ex(cipher.getCtx(), nullptr, nullptr, key.data(), iv.data()); +} + +StringRef EncryptionStreamCipher::encrypt(unsigned char const* plaintext, int len, Arena& arena) { + auto ciphertext = new (arena) unsigned char[len + AES_BLOCK_SIZE]; + int bytes{ 0 }; + EVP_EncryptUpdate(cipher.getCtx(), ciphertext, &bytes, plaintext, len); + return StringRef(ciphertext, bytes); +} + +StringRef EncryptionStreamCipher::finish(Arena& arena) { + auto ciphertext = new (arena) unsigned char[AES_BLOCK_SIZE]; + int bytes{ 0 }; + EVP_EncryptFinal_ex(cipher.getCtx(), ciphertext, &bytes); + return StringRef(ciphertext, bytes); +} + +DecryptionStreamCipher::DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) { + EVP_DecryptInit_ex(cipher.getCtx(), EVP_aes_128_gcm(), nullptr, nullptr, nullptr); + EVP_CIPHER_CTX_ctrl(cipher.getCtx(), EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); + EVP_DecryptInit_ex(cipher.getCtx(), nullptr, nullptr, key.data(), iv.data()); +} + +StringRef DecryptionStreamCipher::decrypt(unsigned char const* ciphertext, int len, Arena& arena) { + auto plaintext = new (arena) unsigned char[len]; + int bytesDecrypted{ 0 }; + EVP_DecryptUpdate(cipher.getCtx(), plaintext, &bytesDecrypted, ciphertext, len); + int finalBlockBytes{ 0 }; + EVP_DecryptFinal_ex(cipher.getCtx(), plaintext + bytesDecrypted, &finalBlockBytes); + return StringRef(plaintext, bytesDecrypted + finalBlockBytes); +} + +StringRef DecryptionStreamCipher::finish(Arena& arena) { + auto plaintext = new (arena) unsigned char[AES_BLOCK_SIZE]; + int finalBlockBytes{ 0 }; + EVP_DecryptFinal_ex(cipher.getCtx(), plaintext, &finalBlockBytes); + return StringRef(plaintext, finalBlockBytes); +} + void forceLinkStreamCipherTests() {} TEST_CASE("flow/StreamCipher") { diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index 627626913b..8e47f5f17c 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -28,40 +28,48 @@ #include #include #include +#include #include -namespace StreamCipher { -class Key : NonCopyable { - std::array arr; - static std::unique_ptr globalKey; - struct ConstructorTag {}; +class StreamCipher final : NonCopyable { + static std::unordered_set ctxs; + EVP_CIPHER_CTX* ctx; + public: - Key(ConstructorTag) {} - ~Key(); - unsigned char const* data() const { return arr.data(); } - static void initializeRandomKey(); - static const Key& getKey(); - static void cleanup(); + StreamCipher(); + ~StreamCipher(); + EVP_CIPHER_CTX* getCtx(); + class Key : NonCopyable { + std::array arr; + static std::unique_ptr globalKey; + struct ConstructorTag {}; + + public: + Key(ConstructorTag) {} + ~Key(); + unsigned char const* data() const { return arr.data(); } + static void initializeRandomKey(); + static const Key& getKey(); + static void cleanup() noexcept; + }; + static void cleanup() noexcept; + using IV = std::array; }; -using IV = std::array; -void registerCipherForCleanup(EVP_CIPHER_CTX*) noexcept; -void deregisterCipherForCleanup(EVP_CIPHER_CTX*) noexcept; -}; // namespace StreamCipher class EncryptionStreamCipher final : NonCopyable, public ReferenceCounted { - EVP_CIPHER_CTX* ctx; + StreamCipher cipher; + public: EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv); - ~EncryptionStreamCipher(); StringRef encrypt(unsigned char const* plaintext, int len, Arena&); StringRef finish(Arena&); }; class DecryptionStreamCipher final : NonCopyable, public ReferenceCounted { - EVP_CIPHER_CTX* ctx; + StreamCipher cipher; + public: DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv); - ~DecryptionStreamCipher(); StringRef decrypt(unsigned char const* ciphertext, int len, Arena&); StringRef finish(Arena&); }; From 0516b3823d1d19b3ee993bdd2520dfd6ca8bcd34 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 5 Feb 2021 00:08:08 -0800 Subject: [PATCH 013/426] Fix flowbench build --- flowbench/BenchEncrypt.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/flowbench/BenchEncrypt.cpp b/flowbench/BenchEncrypt.cpp index 625ad800f6..91cf5fd978 100644 --- a/flowbench/BenchEncrypt.cpp +++ b/flowbench/BenchEncrypt.cpp @@ -23,12 +23,6 @@ #include "flow/StreamCipher.h" #include "flowbench/GlobalData.h" -static StreamCipher::Key getRandomKey() { - StreamCipher::Key key; - generateRandomData(key.data(), key.size()); - return key; -} - static StreamCipher::IV getRandomIV() { StreamCipher::IV iv; generateRandomData(iv.data(), iv.size()); @@ -47,7 +41,8 @@ static void bench_encrypt(benchmark::State& state) { auto bytes = state.range(0); auto chunks = state.range(1); auto chunkSize = bytes / chunks; - auto key = getRandomKey(); + StreamCipher::Key::initializeRandomKey(); + const auto& key = StreamCipher::Key::getKey(); auto iv = getRandomIV(); auto data = getKey(bytes); while (state.KeepRunning()) { @@ -62,7 +57,8 @@ static void bench_decrypt(benchmark::State& state) { auto bytes = state.range(0); auto chunks = state.range(1); auto chunkSize = bytes / chunks; - auto key = getRandomKey(); + StreamCipher::Key::initializeRandomKey(); + const auto& key = StreamCipher::Key::getKey(); auto iv = getRandomIV(); auto data = getKey(bytes); auto encrypted = encrypt(key, iv, data.begin(), data.size()); From dea735a3222d5004b1e3bacf22bde09ae4a5d7b7 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 5 Feb 2021 00:08:29 -0800 Subject: [PATCH 014/426] Fix Windows build --- flow/Platform.actor.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index f258ed9789..588ed32f6f 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -29,7 +29,9 @@ #include "flow/Platform.actor.h" #include "flow/Arena.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) #include "flow/StreamCipher.h" +#endif #include "flow/Trace.h" #include "flow/Error.h" @@ -3238,7 +3240,9 @@ void crashHandler(int sig) { bool error = (sig != SIGUSR2); +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) StreamCipher::cleanup(); +#endif fflush(stdout); TraceEvent(error ? SevError : SevInfo, error ? "Crash" : "ProcessTerminated") From d4191899d997f80608d86edadf3bfe322edd1751 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 28 Mar 2021 22:14:37 -0700 Subject: [PATCH 015/426] Add comments for AsyncFileEncrypted changes --- fdbrpc/AsyncFileEncrypted.actor.cpp | 6 ++++++ flow/StreamCipher.cpp | 2 ++ flow/StreamCipher.h | 2 ++ 3 files changed, 10 insertions(+) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 7b13f3e804..948630a700 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -26,6 +26,8 @@ class AsyncFileEncryptedImpl { public: + // Determine the initialization for the first block of a file based on a hash of + // the filename. static auto getFirstBlockIV(const std::string& filename) { StreamCipher::IV iv; auto hash = XXH3_128bits(filename.c_str(), filename.size()); @@ -37,6 +39,7 @@ public: return iv; } + // Read a single block of size ENCRYPTION_BLOCK_SIZE bytes, and decrypt. ACTOR static Future> readBlock(AsyncFileEncrypted* self, uint16_t block) { state Arena arena; state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE]; @@ -228,6 +231,9 @@ Optional> AsyncFileEncrypted::RandomCache::get(uint16_t bl } } +// This test writes random data into an encrypted file in random increments, +// then reads this data back from the file in random increments, then confirms that +// the bytes read match the bytes written. TEST_CASE("fdbrpc/AsyncFileEncrypted") { state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000); state std::vector writeBuffer(bytes, 0); diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index 6afdbf2309..add9c8988a 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -108,6 +108,8 @@ StringRef DecryptionStreamCipher::finish(Arena& arena) { void forceLinkStreamCipherTests() {} +// Tests both encryption and decryption of random data +// using the StreamCipher class TEST_CASE("flow/StreamCipher") { StreamCipher::Key::initializeRandomKey(); const auto& key = StreamCipher::Key::getKey(); diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index 8e47f5f17c..0618b7585f 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -31,6 +31,8 @@ #include #include +// Wrapper class for openssl implementation of AES-128-GCM +// encryption/decryption class StreamCipher final : NonCopyable { static std::unordered_set ctxs; EVP_CIPHER_CTX* ctx; From d56906cd540c48e8ebc0097ca7fc7aba92f2a28e Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 3 May 2021 15:26:27 -0700 Subject: [PATCH 016/426] Addressed review comments --- fdbrpc/AsyncFileEncrypted.actor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 948630a700..4fda5a77e4 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -85,6 +85,7 @@ public: ACTOR static Future write(AsyncFileEncrypted* self, void const* data, int length, int64_t offset) { ASSERT(self->canWrite); + // All writes must append to the end of the file: ASSERT(offset == self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock); state unsigned char const* input = reinterpret_cast(data); while (length > 0) { @@ -176,12 +177,12 @@ std::string AsyncFileEncrypted::getFilename() const { } Future AsyncFileEncrypted::readZeroCopy(void** data, int* length, int64_t offset) { - ASSERT(false); // Not implemented + throw io_error(); return Void(); } void AsyncFileEncrypted::releaseZeroCopy(void* data, int length, int64_t offset) { - ASSERT(false); // Not implemented + throw io_error(); } int64_t AsyncFileEncrypted::debugFD() const { From 56dadaa428634c5d9a8b8caec320ff882988f1d3 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Fri, 11 Jun 2021 16:20:38 -0500 Subject: [PATCH 017/426] TSS Mismatch Changes --- fdbclient/DatabaseContext.h | 10 +- fdbclient/NativeAPI.actor.cpp | 30 ++- fdbclient/StorageServerInterface.cpp | 382 ++++++++++++++------------- fdbclient/SystemData.cpp | 2 + fdbclient/SystemData.h | 4 + fdbrpc/LoadBalance.actor.h | 139 +++++++--- fdbrpc/TSSComparison.h | 33 ++- flow/Knobs.cpp | 2 + flow/Knobs.h | 2 + flow/Trace.h | 8 +- 10 files changed, 380 insertions(+), 232 deletions(-) diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 52c1945f8e..569b311c64 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -275,7 +275,7 @@ public: Future monitorProxiesInfoChange; Future monitorTssInfoChange; Future tssMismatchHandler; - PromiseStream tssMismatchStream; + PromiseStream>> tssMismatchStream; Reference commitProxies; Reference grvProxies; bool proxyProvisional; // Provisional commit proxy and grv proxy are used at the same time. @@ -428,12 +428,12 @@ public: static const std::vector debugTransactionTagChoices; std::unordered_map> watchMap; - // Adds or updates the specified (SS, TSS) pair in the TSS mapping (if not already present). - // Requests to the storage server will be duplicated to the TSS. + // Adds or updates the specified (SS, TSS) pair in the TSS mapping (if not already present). + // Requests to the storage server will be duplicated to the TSS. void addTssMapping(StorageServerInterface const& ssi, StorageServerInterface const& tssi); - // Removes the storage server and its TSS pair from the TSS mapping (if present). - // Requests to the storage server will no longer be duplicated to its pair TSS. + // Removes the storage server and its TSS pair from the TSS mapping (if present). + // Requests to the storage server will no longer be duplicated to its pair TSS. void removeTssMapping(StorageServerInterface const& ssi); }; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 3f6ed7530b..b5e3181a67 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -383,10 +383,11 @@ ACTOR Future databaseLogger(DatabaseContext* cx) { cx->bytesPerCommit.clear(); for (const auto& it : cx->tssMetrics) { - // TODO could skip this tss if request counter is zero? would potentially complicate elapsed calculation - // though + // TODO could skip this whole thing if tss if request counter is zero? + // That would potentially complicate elapsed calculation though if (it.second->mismatches.getIntervalDelta()) { - cx->tssMismatchStream.send(it.first); + cx->tssMismatchStream.send( + std::pair>(it.first, it.second->detailedMismatches)); } // do error histograms as separate event @@ -825,13 +826,15 @@ ACTOR Future monitorCacheList(DatabaseContext* self) { ACTOR static Future handleTssMismatches(DatabaseContext* cx) { state Reference tr; state KeyBackedMap tssMapDB = KeyBackedMap(tssMappingKeys.begin); + state KeyBackedMap tssMismatchDB = KeyBackedMap(tssMismatchKeys.begin); loop { - state UID tssID = waitNext(cx->tssMismatchStream.getFuture()); + // + state std::pair> data = waitNext(cx->tssMismatchStream.getFuture()); // find ss pair id so we can remove it from the mapping state UID tssPairID; bool found = false; for (const auto& it : cx->tssMapping) { - if (it.second.id() == tssID) { + if (it.second.id() == data.first) { tssPairID = it.first; found = true; break; @@ -840,7 +843,7 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { if (found) { state bool quarantine = CLIENT_KNOBS->QUARANTINE_TSS_ON_MISMATCH; TraceEvent(SevWarnAlways, quarantine ? "TSS_QuarantineMismatch" : "TSS_KillMismatch") - .detail("TSSID", tssID.toString()); + .detail("TSSID", data.first.toString()); TEST(quarantine); // Quarantining TSS because it got mismatch TEST(!quarantine); // Killing TSS because it got mismatch @@ -850,14 +853,21 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { try { tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (quarantine) { - tr->set(tssQuarantineKeyFor(tssID), LiteralStringRef("")); + tr->set(tssQuarantineKeyFor(data.first), LiteralStringRef("")); } else { - tr->clear(serverTagKeyFor(tssID)); + tr->clear(serverTagKeyFor(data.first)); } tssMapDB.erase(tr, tssPairID); + for (const DetailedTSSMismatch& d : data.second) { + // -> mismatch data + tssMismatchDB.set( + tr, + Tuple().append(data.first.toString()).append(d.timestamp).append(d.mismatchId.toString()), + d.traceString); + } + wait(tr->commit()); break; @@ -867,7 +877,7 @@ ACTOR static Future handleTssMismatches(DatabaseContext* cx) { tries++; if (tries > 10) { // Give up, it'll get another mismatch or a human will investigate eventually - TraceEvent("TSS_MismatchGaveUp").detail("TSSID", tssID.toString()); + TraceEvent("TSS_MismatchGaveUp").detail("TSSID", data.first.toString()); break; } } diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp index 79f2e2bc4b..404322e7aa 100644 --- a/fdbclient/StorageServerInterface.cpp +++ b/fdbclient/StorageServerInterface.cpp @@ -30,32 +30,31 @@ std::string traceChecksumValue(ValueRef s) { return s.size() > 12 ? format("(%d)%08x", s.size(), crc32c_append(0, s.begin(), s.size())) : s.toString(); } +// point reads template <> -bool TSS_doCompare(const GetValueRequest& req, - const GetValueReply& src, - const GetValueReply& tss, - Severity traceSeverity, - UID tssId) { - if (src.value.present() != tss.value.present() || (src.value.present() && src.value.get() != tss.value.get())) { - TraceEvent(traceSeverity, "TSSMismatchGetValue") - .suppressFor(1.0) - .detail("TSSID", tssId) - .detail("Key", req.key.printable()) - .detail("Version", req.version) - .detail("SSReply", src.value.present() ? traceChecksumValue(src.value.get()) : "missing") - .detail("TSSReply", tss.value.present() ? traceChecksumValue(tss.value.get()) : "missing"); - - return false; - } - return true; +bool TSS_doCompare(const GetValueReply& src, const GetValueReply& tss) { + return src.value.present() == tss.value.present() && (!src.value.present() || src.value.get() == tss.value.get()); } template <> -bool TSS_doCompare(const GetKeyRequest& req, - const GetKeyReply& src, - const GetKeyReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const GetValueRequest& req) { + return "TSSMismatchGetValue"; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const GetValueRequest& req, + const GetValueReply& src, + const GetValueReply& tss) { + event.detail("Key", req.key.printable()) + .detail("Version", req.version) + .detail("SSReply", src.value.present() ? traceChecksumValue(src.value.get()) : "missing") + .detail("TSSReply", tss.value.present() ? traceChecksumValue(tss.value.get()) : "missing"); +} + +// key selector reads +template <> +bool TSS_doCompare(const GetKeyReply& src, const GetKeyReply& tss) { // This process is a bit complicated. Since the tss and ss can return different results if neighboring shards to // req.sel.key are currently being moved, We validate that the results are the same IF the returned key selectors // are final. Otherwise, we only mark the request as a mismatch if the difference between the two returned key @@ -92,107 +91,170 @@ bool TSS_doCompare(const GetKeyRequest& req, bool tssOffsetLarger = (src.sel.offset == tss.sel.offset) ? tss.sel.orEqual : src.sel.offset < tss.sel.offset; matches = tssKeyLarger != tssOffsetLarger; } - if (!matches) { - TraceEvent(traceSeverity, "TSSMismatchGetKey") - .suppressFor(1.0) - .detail("TSSID", tssId) - .detail("KeySelector", - format("%s%s:%d", req.sel.orEqual ? "=" : "", req.sel.getKey().printable().c_str(), req.sel.offset)) - .detail("Version", req.version) - .detail("SSReply", - format("%s%s:%d", src.sel.orEqual ? "=" : "", src.sel.getKey().printable().c_str(), src.sel.offset)) - .detail( - "TSSReply", - format("%s%s:%d", tss.sel.orEqual ? "=" : "", tss.sel.getKey().printable().c_str(), tss.sel.offset)); - } return matches; } template <> -bool TSS_doCompare(const GetKeyValuesRequest& req, - const GetKeyValuesReply& src, - const GetKeyValuesReply& tss, - Severity traceSeverity, - UID tssId) { - if (src.more != tss.more || src.data != tss.data) { +const char* TSS_mismatchTraceName(const GetKeyRequest& req) { + return "TSSMismatchGetKey"; +} - std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : ""); - for (auto& it : src.data) { - ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); - } +template <> +void TSS_traceMismatch(TraceEvent& event, const GetKeyRequest& req, const GetKeyReply& src, const GetKeyReply& tss) { + event + .detail("KeySelector", + format("%s%s:%d", req.sel.orEqual ? "=" : "", req.sel.getKey().printable().c_str(), req.sel.offset)) + .detail("Version", req.version) + .detail("SSReply", + format("%s%s:%d", src.sel.orEqual ? "=" : "", src.sel.getKey().printable().c_str(), src.sel.offset)) + .detail("TSSReply", + format("%s%s:%d", tss.sel.orEqual ? "=" : "", tss.sel.getKey().printable().c_str(), tss.sel.offset)); +} - std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : ""); - for (auto& it : tss.data) { - tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); - } +// range reads +template <> +bool TSS_doCompare(const GetKeyValuesReply& src, const GetKeyValuesReply& tss) { + return src.more == tss.more && src.data == tss.data; +} - TraceEvent(traceSeverity, "TSSMismatchGetKeyValues") - .suppressFor(1.0) - .detail("TSSID", tssId) - .detail( - "Begin", - format( - "%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset)) - .detail("End", - format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset)) - .detail("Version", req.version) - .detail("Limit", req.limit) - .detail("LimitBytes", req.limitBytes) - .detail("SSReply", ssResultsString) - .detail("TSSReply", tssResultsString); +template <> +const char* TSS_mismatchTraceName(const GetKeyValuesRequest& req) { + return "TSSMismatchGetKeyValues"; +} - return false; +template <> +void TSS_traceMismatch(TraceEvent& event, + const GetKeyValuesRequest& req, + const GetKeyValuesReply& src, + const GetKeyValuesReply& tss) { + std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : ""); + for (auto& it : src.data) { + ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); } + + std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : ""); + for (auto& it : tss.data) { + tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); + } + event + .detail( + "Begin", + format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset)) + .detail("End", + format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset)) + .detail("Version", req.version) + .detail("Limit", req.limit) + .detail("LimitBytes", req.limitBytes) + .detail("SSReply", ssResultsString) + .detail("TSSReply", tssResultsString); +} + +template <> +bool TSS_doCompare(const WatchValueReply& src, const WatchValueReply& tss) { + // We duplicate watches just for load, no need to validate replies. return true; } template <> -bool TSS_doCompare(const WatchValueRequest& req, - const WatchValueReply& src, - const WatchValueReply& tss, - Severity traceSeverity, - UID tssId) { - // We duplicate watches just for load, no need to validte replies. - return true; +const char* TSS_mismatchTraceName(const WatchValueRequest& req) { + ASSERT(false); + return ""; } -// no-op template specializations for metrics replies template <> -bool TSS_doCompare(const WaitMetricsRequest& req, - const StorageMetrics& src, - const StorageMetrics& tss, - Severity traceSeverity, - UID tssId) { +void TSS_traceMismatch(TraceEvent& event, + const WatchValueRequest& req, + const WatchValueReply& src, + const WatchValueReply& tss) { + ASSERT(false); +} + +// template specializations for metrics replies that should never be called because these requests aren't duplicated + +// storage metrics +template <> +bool TSS_doCompare(const StorageMetrics& src, const StorageMetrics& tss) { + ASSERT(false); return true; } template <> -bool TSS_doCompare(const SplitMetricsRequest& req, - const SplitMetricsReply& src, - const SplitMetricsReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const WaitMetricsRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const WaitMetricsRequest& req, + const StorageMetrics& src, + const StorageMetrics& tss) { + ASSERT(false); +} + +// split metrics +template <> +bool TSS_doCompare(const SplitMetricsReply& src, const SplitMetricsReply& tss) { + ASSERT(false); return true; } template <> -bool TSS_doCompare(const ReadHotSubRangeRequest& req, - const ReadHotSubRangeReply& src, - const ReadHotSubRangeReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const SplitMetricsRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const SplitMetricsRequest& req, + const SplitMetricsReply& src, + const SplitMetricsReply& tss) { + ASSERT(false); +} + +// read hot sub range +template <> +bool TSS_doCompare(const ReadHotSubRangeReply& src, const ReadHotSubRangeReply& tss) { + ASSERT(false); return true; } template <> -bool TSS_doCompare(const SplitRangeRequest& req, - const SplitRangeReply& src, - const SplitRangeReply& tss, - Severity traceSeverity, - UID tssId) { +const char* TSS_mismatchTraceName(const ReadHotSubRangeRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const ReadHotSubRangeRequest& req, + const ReadHotSubRangeReply& src, + const ReadHotSubRangeReply& tss) { + ASSERT(false); +} + +// split range +template <> +bool TSS_doCompare(const SplitRangeReply& src, const SplitRangeReply& tss) { + ASSERT(false); return true; } +template <> +const char* TSS_mismatchTraceName(const SplitRangeRequest& req) { + ASSERT(false); + return ""; +} + +template <> +void TSS_traceMismatch(TraceEvent& event, + const SplitRangeRequest& req, + const SplitRangeReply& src, + const SplitRangeReply& tss) { + ASSERT(false); +} + // only record metrics for data reads template <> @@ -240,32 +302,20 @@ TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { std::string s_d = "d"; std::string s_e = "e"; - // test getValue - GetValueRequest gvReq; - gvReq.key = StringRef(s_a); - gvReq.version = 5; - UID tssId; GetValueReply gvReplyMissing; GetValueReply gvReplyA(Optional(StringRef(s_a)), false); GetValueReply gvReplyB(Optional(StringRef(s_b)), false); - ASSERT(TSS_doCompare(gvReq, gvReplyMissing, gvReplyMissing, SevInfo, tssId)); - ASSERT(TSS_doCompare(gvReq, gvReplyA, gvReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gvReq, gvReplyB, gvReplyB, SevInfo, tssId)); + ASSERT(TSS_doCompare(gvReplyMissing, gvReplyMissing)); + ASSERT(TSS_doCompare(gvReplyA, gvReplyA)); + ASSERT(TSS_doCompare(gvReplyB, gvReplyB)); - ASSERT(!TSS_doCompare(gvReq, gvReplyMissing, gvReplyA, SevInfo, tssId)); - ASSERT(!TSS_doCompare(gvReq, gvReplyA, gvReplyB, SevInfo, tssId)); + ASSERT(!TSS_doCompare(gvReplyMissing, gvReplyA)); + ASSERT(!TSS_doCompare(gvReplyA, gvReplyB)); // test GetKeyValues - Arena a; // for all of the refs. ASAN complains if this isn't done. Could also make them all standalone i guess - GetKeyValuesRequest gkvReq; - gkvReq.begin = firstGreaterOrEqual(StringRef(a, s_a)); - gkvReq.end = firstGreaterOrEqual(StringRef(a, s_b)); - gkvReq.version = 5; - gkvReq.limit = 100; - gkvReq.limitBytes = 1000; - + Arena a; GetKeyValuesReply gkvReplyEmpty; GetKeyValuesReply gkvReplyOne; KeyValueRef v; @@ -276,16 +326,11 @@ TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { gkvReplyOneMore.data.push_back_deep(gkvReplyOneMore.arena, v); gkvReplyOneMore.more = true; - ASSERT(TSS_doCompare(gkvReq, gkvReplyEmpty, gkvReplyEmpty, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkvReq, gkvReplyOne, gkvReplyOne, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkvReq, gkvReplyOneMore, gkvReplyOneMore, SevInfo, tssId)); - ASSERT(!TSS_doCompare(gkvReq, gkvReplyEmpty, gkvReplyOne, SevInfo, tssId)); - ASSERT(!TSS_doCompare(gkvReq, gkvReplyOne, gkvReplyOneMore, SevInfo, tssId)); - - // test GetKey - GetKeyRequest gkReq; - gkReq.sel = KeySelectorRef(StringRef(a, s_a), false, 1); - gkReq.version = 5; + ASSERT(TSS_doCompare(gkvReplyEmpty, gkvReplyEmpty)); + ASSERT(TSS_doCompare(gkvReplyOne, gkvReplyOne)); + ASSERT(TSS_doCompare(gkvReplyOneMore, gkvReplyOneMore)); + ASSERT(!TSS_doCompare(gkvReplyEmpty, gkvReplyOne)); + ASSERT(!TSS_doCompare(gkvReplyOne, gkvReplyOneMore)); GetKeyReply gkReplyA(KeySelectorRef(StringRef(a, s_a), false, 20), false); GetKeyReply gkReplyB(KeySelectorRef(StringRef(a, s_b), false, 10), false); @@ -294,85 +339,58 @@ TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { GetKeyReply gkReplyE(KeySelectorRef(StringRef(a, s_e), false, -20), false); // identical cases - ASSERT(TSS_doCompare(gkReq, gkReplyA, gkReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyB, gkReplyB, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyD, gkReplyD, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyE, gkReplyE, SevInfo, tssId)); + ASSERT(TSS_doCompare(gkReplyA, gkReplyA)); + ASSERT(TSS_doCompare(gkReplyB, gkReplyB)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyD, gkReplyD)); + ASSERT(TSS_doCompare(gkReplyE, gkReplyE)); // relative offset cases - ASSERT(TSS_doCompare(gkReq, gkReplyA, gkReplyB, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyB, gkReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyA, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyA, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyB, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyB, SevInfo, tssId)); + ASSERT(TSS_doCompare(gkReplyA, gkReplyB)); + ASSERT(TSS_doCompare(gkReplyB, gkReplyA)); + ASSERT(TSS_doCompare(gkReplyA, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyA)); + ASSERT(TSS_doCompare(gkReplyB, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyB)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyD, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyD, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyC, gkReplyE, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyE, gkReplyC, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyD, gkReplyE, SevInfo, tssId)); - ASSERT(TSS_doCompare(gkReq, gkReplyE, gkReplyD, SevInfo, tssId)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyD)); + ASSERT(TSS_doCompare(gkReplyD, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyC, gkReplyE)); + ASSERT(TSS_doCompare(gkReplyE, gkReplyC)); + ASSERT(TSS_doCompare(gkReplyD, gkReplyE)); + ASSERT(TSS_doCompare(gkReplyE, gkReplyD)); // test same offset/orEqual wrong key - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false), - SevInfo, - tssId)); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false))); // this could be from different shard boundaries, so don't say it's a mismatch - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 10), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 10), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false))); // test offsets and key difference don't match - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false), - SevInfo, - tssId)); - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, -10), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false), - SevInfo, - tssId)); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 10), false))); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, -10), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false))); // test key is next over in one shard, one found it and other didn't // positive // one that didn't find is +1 - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false), - SevInfo, - tssId)); - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 1), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false))); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 1), false))); // negative will have zero offset but not equal set - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false), - SevInfo, - tssId)); - ASSERT(!TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), false, 0), false))); + ASSERT(!TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_b), true, 0), false))); // test shard boundary key returned by incomplete query is the same as the key found by the other (only possible in // positive direction) - ASSERT(TSS_doCompare(gkReq, - GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), - GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false), - SevInfo, - tssId)); + ASSERT(TSS_doCompare(GetKeyReply(KeySelectorRef(StringRef(a, s_a), true, 0), false), + GetKeyReply(KeySelectorRef(StringRef(a, s_a), false, 1), false))); // explictly test checksum function std::string s12 = "ABCDEFGHIJKL"; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 2d827c199c..14bea63d42 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -364,6 +364,8 @@ UID decodeTssQuarantineKey(KeyRef const& key) { return serverID; } +const KeyRangeRef tssMismatchKeys(LiteralStringRef("\xff/tssMismatch/"), LiteralStringRef("\xff/tssMismatch0")); + const KeyRangeRef serverTagKeys(LiteralStringRef("\xff/serverTag/"), LiteralStringRef("\xff/serverTag0")); const KeyRef serverTagPrefix = serverTagKeys.begin; diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 8a5342da83..6225277e3f 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -124,6 +124,10 @@ extern const KeyRangeRef tssQuarantineKeys; const Key tssQuarantineKeyFor(UID serverID); UID decodeTssQuarantineKey(KeyRef const&); +// \xff/tssMismatch/[[Tuple]] := [[TraceEventString]] +// For recording tss mismatch details in the system keyspace +extern const KeyRangeRef tssMismatchKeys; + // "\xff/serverTag/[[serverID]]" = "[[Tag]]" // Provides the Tag for the given serverID. Used to access a // storage server's corresponding TLog in order to apply mutations. diff --git a/fdbrpc/LoadBalance.actor.h b/fdbrpc/LoadBalance.actor.h index 4bbaa03005..a1a70c1d19 100644 --- a/fdbrpc/LoadBalance.actor.h +++ b/fdbrpc/LoadBalance.actor.h @@ -77,16 +77,22 @@ struct LoadBalancedReply { Optional getLoadBalancedReply(const LoadBalancedReply* reply); Optional getLoadBalancedReply(const void*); -ACTOR template +ACTOR template Future tssComparison(Req req, Future> fSource, Future> fTss, - TSSEndpointData tssData) { + TSSEndpointData tssData, + uint64_t srcEndpointId, + Reference> ssTeam, + RequestStream Interface::*channel) { state double startTime = now(); state Future>> fTssWithTimeout = timeout(fTss, FLOW_KNOBS->LOAD_BALANCE_TSS_TIMEOUT); state int finished = 0; state double srcEndTime; state double tssEndTime; + // we want to record ss/tss errors to metrics + state int srcErrorCode = error_code_success; + state int tssErrorCode = error_code_success; loop { choose { @@ -108,11 +114,6 @@ Future tssComparison(Req req, } } } - - // we want to record ss/tss errors to metrics - int srcErrorCode = error_code_success; - int tssErrorCode = error_code_success; - ++tssData.metrics->requests; if (src.isError()) { @@ -137,15 +138,82 @@ Future tssComparison(Req req, // apples tssData.metrics->recordLatency(req, srcEndTime - startTime, tssEndTime - startTime); - // expect mismatches in drop mutations mode. - Severity traceSeverity = - (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) - ? SevWarnAlways - : SevError; - - if (!TSS_doCompare(req, src.get(), tss.get().get(), traceSeverity, tssData.tssId)) { + if (!TSS_doCompare(src.get(), tss.get().get())) { TEST(true); // TSS Mismatch - ++tssData.metrics->mismatches; + state TraceEvent mismatchEvent( + (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) + ? SevWarnAlways + : SevError, + TSS_mismatchTraceName(req)); + mismatchEvent.detail("TSSID", tssData.tssId); + + if (FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS && ssTeam->size() > 1) { + TEST(true); // checking TSS mismatch against rest of storage team + + // if there is more than 1 SS in the team, attempt to verify that the other SS servers have the same + // data + state std::vector>> restOfTeamFutures; + restOfTeamFutures.reserve(ssTeam->size() - 1); + for (int i = 0; i < ssTeam->size(); i++) { + RequestStream const* si = &ssTeam->get(i, channel); + if (si->getEndpoint().token.first() != + srcEndpointId) { // don't re-request to SS we already have a response from + resetReply(req); + restOfTeamFutures.push_back(si->tryGetReply(req)); + } + } + + wait(waitForAllReady(restOfTeamFutures)); + + int numError = 0; + int numMatchSS = 0; + int numMatchTSS = 0; + int numMatchNeither = 0; + for (Future> f : restOfTeamFutures) { + if (!f.canGet() || f.get().isError()) { + numError++; + } else { + Optional fLB = getLoadBalancedReply(&f.get().get()); + if (fLB.present() && fLB.get().error.present()) { + numError++; + } else if (TSS_doCompare(src.get(), f.get().get())) { + numMatchSS++; + } else if (TSS_doCompare(tss.get().get(), f.get().get())) { + numMatchTSS++; + } else { + numMatchNeither++; + } + } + } + mismatchEvent.detail("TeamCheckErrors", numError) + .detail("TeamCheckMatchSS", numMatchSS) + .detail("TeamCheckMatchTSS", numMatchTSS) + .detail("TeamCheckMatchNeither", numMatchNeither); + } + if (tssData.metrics->shouldRecordDetailedMismatch()) { + TSS_traceMismatch(mismatchEvent, req, src.get(), tss.get().get()); + + TEST(FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Full TSS Mismatch + TEST(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Partial TSS Mismatch and storing + // the rest in FDB + + if (!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL) { + mismatchEvent.disable(); + UID mismatchUID = deterministicRandom()->randomUniqueID(); + tssData.metrics->recordDetailedMismatchData(mismatchUID, mismatchEvent.getFields().toString()); + + // record a summarized trace event instead + TraceEvent summaryEvent((g_network->isSimulated() && + g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) + ? SevWarnAlways + : SevError, + TSS_mismatchTraceName(req)); + summaryEvent.detail("TSSID", tssData.tssId).detail("MismatchId", mismatchUID); + } + } else { + // don't record trace event + mismatchEvent.disable(); + } } } else if (tssLB.present() && tssLB.get().error.present()) { tssErrorCode = tssLB.get().error.get().code(); @@ -169,7 +237,7 @@ Future tssComparison(Req req, } // Stores state for a request made by the load balancer -template +template struct RequestData : NonCopyable { typedef ErrorOr Reply; @@ -187,7 +255,9 @@ struct RequestData : NonCopyable { static void maybeDuplicateTSSRequest(RequestStream const* stream, Request& request, QueueModel* model, - Future ssResponse) { + Future ssResponse, + Reference> alternatives, + RequestStream Interface::*channel) { if (model) { // Send parallel request to TSS pair, if it exists Optional tssData = model->getTssData(stream->getEndpoint().token.first()); @@ -198,34 +268,43 @@ struct RequestData : NonCopyable { // FIXME: optimize to avoid creating new netNotifiedQueue for each message RequestStream tssRequestStream(tssData.get().endpoint); Future> fTssResult = tssRequestStream.tryGetReply(request); - model->addActor.send(tssComparison(request, ssResponse, fTssResult, tssData.get())); + model->addActor.send(tssComparison(request, + ssResponse, + fTssResult, + tssData.get(), + stream->getEndpoint().token.first(), + alternatives, + channel)); } } } // Initializes the request state and starts it, possibly after a backoff delay - void startRequest(double backoff, - bool triedAllOptions, - RequestStream const* stream, - Request& request, - QueueModel* model) { + void startRequest( + double backoff, + bool triedAllOptions, + RequestStream const* stream, + Request& request, + QueueModel* model, + Reference> alternatives, // alternatives and channel passed through for TSS check + RequestStream Interface::*channel) { modelHolder = Reference(); requestStarted = false; if (backoff > 0) { response = mapAsync(Void)>, Reply>( - delay(backoff), [this, stream, &request, model](Void _) { + delay(backoff), [this, stream, &request, model, alternatives, channel](Void _) { requestStarted = true; modelHolder = Reference(new ModelHolder(model, stream->getEndpoint().token.first())); Future resp = stream->tryGetReply(request); - maybeDuplicateTSSRequest(stream, request, model, resp); + maybeDuplicateTSSRequest(stream, request, model, resp, alternatives, channel); return resp; }); } else { requestStarted = true; modelHolder = Reference(new ModelHolder(model, stream->getEndpoint().token.first())); response = stream->tryGetReply(request); - maybeDuplicateTSSRequest(stream, request, model, response); + maybeDuplicateTSSRequest(stream, request, model, response, alternatives, channel); } requestProcessed = false; @@ -363,8 +442,8 @@ Future loadBalance( bool atMostOnce = false, // if true, throws request_maybe_delivered() instead of retrying automatically QueueModel* model = nullptr) { - state RequestData firstRequestData; - state RequestData secondRequestData; + state RequestData firstRequestData; + state RequestData secondRequestData; state Optional firstRequestEndpoint; state Future secondDelay = Never(); @@ -577,7 +656,7 @@ Future loadBalance( firstRequestEndpoint = Optional(); } else if (firstRequestData.isValid()) { // Issue a second request, the first one is taking a long time. - secondRequestData.startRequest(backoff, triedAllOptions, stream, request, model); + secondRequestData.startRequest(backoff, triedAllOptions, stream, request, model, alternatives, channel); state bool firstFinished = false; loop choose { @@ -606,7 +685,7 @@ Future loadBalance( } } else { // Issue a request, if it takes too long to get a reply, go around the loop - firstRequestData.startRequest(backoff, triedAllOptions, stream, request, model); + firstRequestData.startRequest(backoff, triedAllOptions, stream, request, model, alternatives, channel); firstRequestEndpoint = stream->getEndpoint().token.first(); loop { diff --git a/fdbrpc/TSSComparison.h b/fdbrpc/TSSComparison.h index 335e8ae68e..650355e696 100644 --- a/fdbrpc/TSSComparison.h +++ b/fdbrpc/TSSComparison.h @@ -29,6 +29,15 @@ #include "fdbrpc/Stats.h" // refcounted + noncopyable because both DatabaseContext and individual endpoints share ownership +struct DetailedTSSMismatch { + UID mismatchId; + double timestamp; + std::string traceString; + + DetailedTSSMismatch(UID mismatchId, double timestamp, std::string traceString) + : mismatchId(mismatchId), timestamp(timestamp), traceString(traceString) {} +}; + struct TSSMetrics : ReferenceCounted, NonCopyable { CounterCollection cc; Counter requests; @@ -49,6 +58,8 @@ struct TSSMetrics : ReferenceCounted, NonCopyable { std::unordered_map ssErrorsByCode; std::unordered_map tssErrorsByCode; + std::vector detailedMismatches; + void ssError(int code) { ++ssErrors; ssErrorsByCode[code]++; @@ -62,6 +73,16 @@ struct TSSMetrics : ReferenceCounted, NonCopyable { template void recordLatency(const Req& req, double ssLatency, double tssLatency); + // only record a small number of the detailed mismatches per client per metrics window + bool shouldRecordDetailedMismatch() { + ++mismatches; + return (mismatches.getIntervalDelta() < 5); + } + + void recordDetailedMismatchData(UID mismatchUID, std::string traceString) { + detailedMismatches.push_back(DetailedTSSMismatch(mismatchUID, now(), traceString)); + } + void clear() { SSgetValueLatency.clear(); SSgetKeyLatency.clear(); @@ -73,6 +94,8 @@ struct TSSMetrics : ReferenceCounted, NonCopyable { tssErrorsByCode.clear(); ssErrorsByCode.clear(); + + detailedMismatches.clear(); } TSSMetrics() @@ -81,9 +104,13 @@ struct TSSMetrics : ReferenceCounted, NonCopyable { SSgetKeyValuesLatency(1000), TSSgetValueLatency(1000), TSSgetKeyLatency(1000), TSSgetKeyValuesLatency(1000) {} }; -// part of the contract of this function is that if there is a mismatch, the implementation needs to record a trace -// event with the specified severity and tssId in the event. +template +bool TSS_doCompare(const Rep& src, const Rep& tss); + +template +const char* TSS_mismatchTraceName(const Req& req); + template -bool TSS_doCompare(const Req& req, const Rep& src, const Rep& tss, Severity traceSeverity, UID tssId); +void TSS_traceMismatch(TraceEvent& event, const Req& req, const Rep& src, const Rep& tss); #endif diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 1d287feed3..2f06498084 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -235,6 +235,8 @@ void FlowKnobs::initialize(bool randomize, bool isSimulated) { init( BASIC_LOAD_BALANCE_BUCKETS, 40 ); //proxies bin recent GRV requests into 40 time bins init( BASIC_LOAD_BALANCE_COMPUTE_PRECISION, 10000 ); //determines how much of the LB usage is holding the CPU usage of the proxy init( LOAD_BALANCE_TSS_TIMEOUT, 5.0 ); + init( LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS, true ); if( randomize && BUGGIFY ) LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS = false; // Whether the client should validate the SS teams all agree on TSS mismatch + init( LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL, false ); if( randomize && BUGGIFY ) LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL = true; // If true, saves the full details of the mismatch in a trace event. If false, saves them in the DB and the trace event references the DB row. // Health Monitor init( FAILURE_DETECTION_DELAY, 4.0 ); if( randomize && BUGGIFY ) FAILURE_DETECTION_DELAY = 1.0; diff --git a/flow/Knobs.h b/flow/Knobs.h index 9b700613f3..bf465683d0 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -251,6 +251,8 @@ public: double BASIC_LOAD_BALANCE_MIN_REQUESTS; double BASIC_LOAD_BALANCE_MIN_CPU; double LOAD_BALANCE_TSS_TIMEOUT; + bool LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS; + bool LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL; // Health Monitor int FAILURE_DETECTION_DELAY; diff --git a/flow/Trace.h b/flow/Trace.h index 4c2eadb215..9a79383781 100644 --- a/flow/Trace.h +++ b/flow/Trace.h @@ -463,12 +463,14 @@ public: bool isEnabled() const { return enabled; } - TraceEvent &setErrorKind(ErrorKind errorKind); + TraceEvent& setErrorKind(ErrorKind errorKind); explicit operator bool() const { return enabled; } void log(); + void disable() { enabled = false; } // Disables the trace event so it doesn't get + ~TraceEvent(); // Actually logs the event // Return the number of invocations of TraceEvent() at the specified logging level. @@ -476,6 +478,8 @@ public: std::unique_ptr tmpEventMetric; // This just just a place to store fields + const TraceEventFields& getFields() const { return fields; } + private: bool initialized; bool enabled; @@ -491,7 +495,7 @@ private: int maxFieldLength; int maxEventLength; int timeIndex; - int errorKindIndex { -1 }; + int errorKindIndex{ -1 }; void setSizeLimits(); From a424abe5c2e46dfad4b4fc89f3461c200cd3fb5c Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 21 Jun 2021 13:38:12 -0700 Subject: [PATCH 018/426] Remove some boost dependencies --- bindings/flow/Tuple.cpp | 5 ++--- flow/Error.h | 3 +-- flow/ObjectSerializerTraits.h | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/bindings/flow/Tuple.cpp b/bindings/flow/Tuple.cpp index bef7775cca..337792e508 100644 --- a/bindings/flow/Tuple.cpp +++ b/bindings/flow/Tuple.cpp @@ -19,12 +19,11 @@ */ #include "Tuple.h" -#include namespace FDB { // The floating point operations depend on this using the IEEE 754 standard. -BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); -BOOST_STATIC_ASSERT(std::numeric_limits::is_iec559); +static_assert(std::numeric_limits::is_iec559); +static_assert(std::numeric_limits::is_iec559); const size_t Uuid::SIZE = 16; diff --git a/flow/Error.h b/flow/Error.h index d3612f72dd..56450ac467 100644 --- a/flow/Error.h +++ b/flow/Error.h @@ -24,7 +24,6 @@ #include #include -#include #include #include #include "flow/Platform.h" @@ -221,7 +220,7 @@ void assert_num_impl(char const* a_nm, EXTERNC void breakpoint_me(); #ifdef FDB_CLEAN_BUILD -#define NOT_IN_CLEAN BOOST_STATIC_ASSERT_MSG(0, "This code can not be enabled in a clean build."); +#define NOT_IN_CLEAN static_assert(false, "This code can not be enabled in a clean build."); #else #define NOT_IN_CLEAN #endif diff --git a/flow/ObjectSerializerTraits.h b/flow/ObjectSerializerTraits.h index d0419a1e9d..b191eba232 100644 --- a/flow/ObjectSerializerTraits.h +++ b/flow/ObjectSerializerTraits.h @@ -27,7 +27,6 @@ #include #include #include -#include template struct is_fb_function_t : std::false_type {}; From af0f286e17bacffb1b0302dbabfb79cbac063238 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Tue, 22 Jun 2021 21:44:59 -0700 Subject: [PATCH 019/426] Fix a few instances of UB Member access to address which does not point to correct type --- fdbclient/ISingleThreadTransaction.cpp | 7 +++---- fdbclient/PaxosConfigTransaction.actor.cpp | 2 ++ fdbclient/PaxosConfigTransaction.h | 3 +++ fdbclient/SimpleConfigTransaction.actor.cpp | 2 ++ fdbclient/SimpleConfigTransaction.h | 3 +++ flow/Histogram.cpp | 2 +- 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fdbclient/ISingleThreadTransaction.cpp b/fdbclient/ISingleThreadTransaction.cpp index 0b9a8f5abb..4a0b686540 100644 --- a/fdbclient/ISingleThreadTransaction.cpp +++ b/fdbclient/ISingleThreadTransaction.cpp @@ -26,15 +26,14 @@ ISingleThreadTransaction* ISingleThreadTransaction::allocateOnForeignThread(Type type) { if (type == Type::RYW) { - auto tr = - (ReadYourWritesTransaction*)(ReadYourWritesTransaction::operator new(sizeof(ReadYourWritesTransaction))); + auto tr = new ReadYourWritesTransaction; tr->preinitializeOnForeignThread(); return tr; } else if (type == Type::SIMPLE_CONFIG) { - auto tr = (SimpleConfigTransaction*)(SimpleConfigTransaction::operator new(sizeof(SimpleConfigTransaction))); + auto tr = new SimpleConfigTransaction; return tr; } else if (type == Type::PAXOS_CONFIG) { - auto tr = (PaxosConfigTransaction*)(PaxosConfigTransaction::operator new(sizeof(PaxosConfigTransaction))); + auto tr = new PaxosConfigTransaction; return tr; } ASSERT(false); diff --git a/fdbclient/PaxosConfigTransaction.actor.cpp b/fdbclient/PaxosConfigTransaction.actor.cpp index 8b7ef9f06d..2843cc049c 100644 --- a/fdbclient/PaxosConfigTransaction.actor.cpp +++ b/fdbclient/PaxosConfigTransaction.actor.cpp @@ -128,3 +128,5 @@ PaxosConfigTransaction::PaxosConfigTransaction(Database const& cx) { } PaxosConfigTransaction::~PaxosConfigTransaction() = default; + +PaxosConfigTransaction::PaxosConfigTransaction() = default; diff --git a/fdbclient/PaxosConfigTransaction.h b/fdbclient/PaxosConfigTransaction.h index 884afdb2d1..e8bab06930 100644 --- a/fdbclient/PaxosConfigTransaction.h +++ b/fdbclient/PaxosConfigTransaction.h @@ -32,6 +32,9 @@ class PaxosConfigTransaction final : public IConfigTransaction, public FastAlloc PaxosConfigTransactionImpl const& impl() const { return *_impl; } PaxosConfigTransactionImpl& impl() { return *_impl; } + PaxosConfigTransaction(); + friend class ISingleThreadTransaction; + public: PaxosConfigTransaction(Database const&); ~PaxosConfigTransaction(); diff --git a/fdbclient/SimpleConfigTransaction.actor.cpp b/fdbclient/SimpleConfigTransaction.actor.cpp index 8b03fb0d91..94629a9215 100644 --- a/fdbclient/SimpleConfigTransaction.actor.cpp +++ b/fdbclient/SimpleConfigTransaction.actor.cpp @@ -297,3 +297,5 @@ SimpleConfigTransaction::SimpleConfigTransaction(ConfigTransactionInterface cons : _impl(std::make_unique(cti)) {} SimpleConfigTransaction::~SimpleConfigTransaction() = default; + +SimpleConfigTransaction::SimpleConfigTransaction() = default; diff --git a/fdbclient/SimpleConfigTransaction.h b/fdbclient/SimpleConfigTransaction.h index dd779922bd..ea3ddf25b6 100644 --- a/fdbclient/SimpleConfigTransaction.h +++ b/fdbclient/SimpleConfigTransaction.h @@ -40,6 +40,9 @@ class SimpleConfigTransaction final : public IConfigTransaction, public FastAllo SimpleConfigTransactionImpl const& impl() const { return *_impl; } SimpleConfigTransactionImpl& impl() { return *_impl; } + SimpleConfigTransaction(); + friend class ISingleThreadTransaction; + public: SimpleConfigTransaction(ConfigTransactionInterface const&); SimpleConfigTransaction(Database const&); diff --git a/flow/Histogram.cpp b/flow/Histogram.cpp index 595dcc8fef..7814b89d60 100644 --- a/flow/Histogram.cpp +++ b/flow/Histogram.cpp @@ -45,7 +45,7 @@ static HistogramRegistry* globalHistograms = nullptr; #pragma region HistogramRegistry HistogramRegistry& GetHistogramRegistry() { - ISimulator::ProcessInfo* info = g_simulator.getCurrentProcess(); + ISimulator::ProcessInfo* info = g_network && g_network->isSimulated() ? g_simulator.getCurrentProcess() : nullptr; if (info) { // in simulator; scope histograms to simulated process From 0e9eabdb189f520441cb0a0161df9ca3881f89b1 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 23 Jun 2021 09:24:52 -0700 Subject: [PATCH 020/426] Cherry-pick UBSAN fix from https://github.com/Tencent/rapidjson/commit/16872af88915176f49e389defb167f899e2c230a --- fdbclient/rapidjson/internal/stack.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdbclient/rapidjson/internal/stack.h b/fdbclient/rapidjson/internal/stack.h index 7ab15d42a0..fa43aa0171 100644 --- a/fdbclient/rapidjson/internal/stack.h +++ b/fdbclient/rapidjson/internal/stack.h @@ -17,6 +17,7 @@ #include "../allocators.h" #include "swap.h" +#include #if defined(__clang__) RAPIDJSON_DIAG_PUSH @@ -106,7 +107,7 @@ public: template RAPIDJSON_FORCEINLINE void Reserve(size_t count = 1) { // Expand the stack if needed - if (RAPIDJSON_UNLIKELY(stackTop_ + sizeof(T) * count > stackEnd_)) + if (RAPIDJSON_UNLIKELY(static_cast(sizeof(T) * count) > (stackEnd_ - stackTop_))) Expand(count); } @@ -118,7 +119,7 @@ public: template RAPIDJSON_FORCEINLINE T* PushUnsafe(size_t count = 1) { - RAPIDJSON_ASSERT(stackTop_ + sizeof(T) * count <= stackEnd_); + RAPIDJSON_ASSERT(static_cast(sizeof(T) * count) <= (stackEnd_ - stackTop_)); T* ret = reinterpret_cast(stackTop_); stackTop_ += sizeof(T) * count; return ret; From c9b06839fc6c76a4ab1e0d3295d26179957c5817 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 23 Jun 2021 09:25:51 -0700 Subject: [PATCH 021/426] Halt on error in ubsan for ctest tests --- cmake/AddFdbTest.cmake | 2 ++ tests/CMakeLists.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index 7e0502c52e..3041f14c10 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -136,6 +136,7 @@ function(add_fdb_test) ${VALGRIND_OPTION} ${ADD_FDB_TEST_TEST_FILES} WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + set_tests_properties("${test_name}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) get_filename_component(test_dir_full ${first_file} DIRECTORY) if(NOT ${test_dir_full} STREQUAL "") get_filename_component(test_dir ${test_dir_full} NAME) @@ -402,6 +403,7 @@ function(add_fdbclient_test) -- ${T_COMMAND}) set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) + set_tests_properties("${T_NAME}" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) endfunction() function(add_java_test) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 55d64485b0..62735a4b39 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -272,10 +272,12 @@ if(WITH_PYTHON) NAME multiversion_client/unit_tests COMMAND $ -r unittests -f /fdbclient/multiversionclient/ ) + set_tests_properties("multiversion_client/unit_tests" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) add_test( NAME threadsafe_threadfuture_to_future/unit_tests COMMAND $ -r unittests -f /flow/safeThreadFutureToFuture/ ) + set_tests_properties("threadsafe_threadfuture_to_future/unit_tests" PROPERTIES ENVIRONMENT UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1) endif() verify_testing() From 2c19940a42f8f0adf03a22e370c18e0f88706f69 Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 23 Jun 2021 10:36:01 -0700 Subject: [PATCH 022/426] Make hidden options exist Otherwise we can't set EXTERNAL_CLIENT without invoking UB --- fdbclient/vexillographer/c.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbclient/vexillographer/c.cs b/fdbclient/vexillographer/c.cs index 2ea6675dff..dab01ff5cf 100644 --- a/fdbclient/vexillographer/c.cs +++ b/fdbclient/vexillographer/c.cs @@ -52,7 +52,7 @@ namespace vexillographer { string parameterComment = ""; if (o.scope.ToString().EndsWith("Option")) - parameterComment = String.Format("{0}/* {1} */\n", indent, "Parameter: " + o.getParameterComment()); + parameterComment = String.Format("{0}/* {1} {2}*/\n", indent, "Parameter: " + o.getParameterComment(), o.hidden ? "This is a hidden parameter and should not be used directly by applications." : ""); return String.Format("{0}/* {2} */\n{5}{0}{1}{3}={4}", indent, prefix, o.comment, o.name.ToUpper(), o.code, parameterComment); } @@ -64,7 +64,7 @@ namespace vexillographer options = new Option[] { new Option{ scope = scope, comment = "This option is only a placeholder for C compatibility and should not be used", code = -1, name = "DUMMY_DO_NOT_USE", paramDesc = null } }; - outFile.WriteLine(string.Join(",\n\n", options.Where(f => !f.hidden).Select(f => getCLine(f, " ", prefix)).ToArray())); + outFile.WriteLine(string.Join(",\n\n", options.Select(f => getCLine(f, " ", prefix)).ToArray())); outFile.WriteLine("}} FDB{0};", scope.ToString()); outFile.WriteLine(); } From 7f35a663831e307b8ebd0ed93f97bc096af1e13e Mon Sep 17 00:00:00 2001 From: Andrew Noyes Date: Wed, 23 Jun 2021 10:38:04 -0700 Subject: [PATCH 023/426] Fix the types of dl functions in the multi-version client This fixes undefined behavior of invoking a function pointer with the wrong type. --- fdbclient/MultiVersionTransaction.actor.cpp | 27 ++++++++++++--------- fdbclient/MultiVersionTransaction.h | 23 +++++++++--------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 1c5124c12a..1023169029 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -113,7 +113,7 @@ ThreadFuture DLTransaction::getRange(const KeySelectorRef& begin, end.offset, limits.rows, limits.bytes, - FDBStreamingModes::EXACT, + FDB_STREAMING_MODE_EXACT, 0, snapshot, reverse); @@ -207,12 +207,12 @@ ThreadFuture>> DLTransaction::getRangeSplitPoints(c void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ)); + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ)); } void DLTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) { api->transactionAtomicOp( - tr, key.begin(), key.size(), value.begin(), value.size(), (FDBMutationTypes::Option)operationType); + tr, key.begin(), key.size(), value.begin(), value.size(), static_cast(operationType)); } void DLTransaction::set(const KeyRef& key, const ValueRef& value) { @@ -239,7 +239,7 @@ ThreadFuture DLTransaction::watch(const KeyRef& key) { void DLTransaction::addWriteConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::WRITE)); + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE)); } ThreadFuture DLTransaction::commit() { @@ -269,8 +269,10 @@ ThreadFuture DLTransaction::getApproximateSize() { } void DLTransaction::setOption(FDBTransactionOptions::Option option, Optional value) { - throwIfError(api->transactionSetOption( - tr, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); + throwIfError(api->transactionSetOption(tr, + static_cast(option), + value.present() ? value.get().begin() : nullptr, + value.present() ? value.get().size() : 0)); } ThreadFuture DLTransaction::onError(Error const& e) { @@ -309,8 +311,10 @@ Reference DLDatabase::createTransaction() { } void DLDatabase::setOption(FDBDatabaseOptions::Option option, Optional value) { - throwIfError(api->databaseSetOption( - db, option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); + throwIfError(api->databaseSetOption(db, + static_cast(option), + value.present() ? value.get().begin() : nullptr, + value.present() ? value.get().size() : 0)); } ThreadFuture DLDatabase::rebootWorker(const StringRef& address, bool check, int duration) { @@ -504,7 +508,7 @@ void DLApi::selectApiVersion(int apiVersion) { init(); throwIfError(api->selectApiVersion(apiVersion, headerVersion)); - throwIfError(api->setNetworkOption(FDBNetworkOptions::EXTERNAL_CLIENT, nullptr, 0)); + throwIfError(api->setNetworkOption(FDB_NET_OPTION_EXTERNAL_CLIENT, nullptr, 0)); } const char* DLApi::getClientVersion() { @@ -516,8 +520,9 @@ const char* DLApi::getClientVersion() { } void DLApi::setNetworkOption(FDBNetworkOptions::Option option, Optional value) { - throwIfError(api->setNetworkOption( - option, value.present() ? value.get().begin() : nullptr, value.present() ? value.get().size() : 0)); + throwIfError(api->setNetworkOption(static_cast(option), + value.present() ? value.get().begin() : nullptr, + value.present() ? value.get().size() : 0)); } void DLApi::setupNetwork() { diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index a98e16b440..65892a8dbf 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -22,6 +22,7 @@ #define FDBCLIENT_MULTIVERSIONTRANSACTION_H #pragma once +#include "bindings/c/foundationdb/fdb_c_options.g.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/IClientApi.h" @@ -31,10 +32,10 @@ // FdbCApi is used as a wrapper around the FoundationDB C API that gets loaded from an external client library. // All of the required functions loaded from that external library are stored in function pointers in this struct. struct FdbCApi : public ThreadSafeReferenceCounted { - typedef struct future FDBFuture; - typedef struct cluster FDBCluster; - typedef struct database FDBDatabase; - typedef struct transaction FDBTransaction; + typedef struct FDB_future FDBFuture; + typedef struct FDB_cluster FDBCluster; + typedef struct FDB_database FDBDatabase; + typedef struct FDB_transaction FDBTransaction; #pragma pack(push, 4) typedef struct key { @@ -57,16 +58,16 @@ struct FdbCApi : public ThreadSafeReferenceCounted { // Network fdb_error_t (*selectApiVersion)(int runtimeVersion, int headerVersion); const char* (*getClientVersion)(); - fdb_error_t (*setNetworkOption)(FDBNetworkOptions::Option option, uint8_t const* value, int valueLength); + fdb_error_t (*setNetworkOption)(FDBNetworkOption option, uint8_t const* value, int valueLength); fdb_error_t (*setupNetwork)(); fdb_error_t (*runNetwork)(); fdb_error_t (*stopNetwork)(); - fdb_error_t* (*createDatabase)(const char* clusterFilePath, FDBDatabase** db); + fdb_error_t (*createDatabase)(const char* clusterFilePath, FDBDatabase** db); // Database fdb_error_t (*databaseCreateTransaction)(FDBDatabase* database, FDBTransaction** tr); fdb_error_t (*databaseSetOption)(FDBDatabase* database, - FDBDatabaseOptions::Option option, + FDBDatabaseOption option, uint8_t const* value, int valueLength); void (*databaseDestroy)(FDBDatabase* database); @@ -86,7 +87,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { // Transaction fdb_error_t (*transactionSetOption)(FDBTransaction* tr, - FDBTransactionOptions::Option option, + FDBTransactionOption option, uint8_t const* value, int valueLength); void (*transactionDestroy)(FDBTransaction* tr); @@ -113,7 +114,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int endOffset, int limit, int targetBytes, - FDBStreamingModes::Option mode, + FDBStreamingMode mode, int iteration, fdb_bool_t snapshot, fdb_bool_t reverse); @@ -135,7 +136,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int keyNameLength, uint8_t const* param, int paramLength, - FDBMutationTypes::Option operationType); + FDBMutationType operationType); FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name, @@ -163,7 +164,7 @@ struct FdbCApi : public ThreadSafeReferenceCounted { int beginKeyNameLength, uint8_t const* endKeyName, int endKeyNameLength, - FDBConflictRangeTypes::Option); + FDBConflictRangeType); // Future fdb_error_t (*futureGetDatabase)(FDBFuture* f, FDBDatabase** outDb); From 5f9d48b2ff0b10cf393c6b49836c5dbebdaa7e91 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Wed, 23 Jun 2021 13:46:17 -0700 Subject: [PATCH 024/426] initial draft on RedWoodMetrics changes --- fdbserver/IPager.h | 9 +- fdbserver/VersionedBTree.actor.cpp | 275 ++++++++++++++++++++++------- flow/Histogram.cpp | 1 + flow/Histogram.h | 6 +- 4 files changed, 224 insertions(+), 67 deletions(-) diff --git a/fdbserver/IPager.h b/fdbserver/IPager.h index aea7bbf9d2..e72e730e71 100644 --- a/fdbserver/IPager.h +++ b/fdbserver/IPager.h @@ -40,6 +40,9 @@ typedef uint32_t PhysicalPageID; typedef uint32_t QueueID; #define invalidQueueID std::numeric_limits::max() +// Reasons for page levle events. +enum class pagerEventReasons{ pointRead, rangeRead, rangePrefetch, commit, lazyClear, metaData}; + // Represents a block of memory in a 4096-byte aligned location held by an Arena. class ArenaPage : public ReferenceCounted, public FastAllocated { public: @@ -128,7 +131,7 @@ public: class IPagerSnapshot { public: - virtual Future> getPhysicalPage(LogicalPageID pageID, bool cacheable, bool nohit) = 0; + virtual Future> getPhysicalPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable, bool nohit) = 0; virtual bool tryEvictPage(LogicalPageID id) = 0; virtual Version getVersion() const = 0; @@ -188,8 +191,8 @@ public: // Cacheable indicates that the page should be added to the page cache (if applicable?) as a result of this read. // NoHit indicates that the read should not be considered a cache hit, such as when preloading pages that are // considered likely to be needed soon. - virtual Future> readPage(LogicalPageID pageID, bool cacheable = true, bool noHit = false) = 0; - virtual Future> readExtent(LogicalPageID pageID) = 0; + virtual Future> readPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable = true, bool noHit = false) = 0; + virtual Future> readExtent(pagerEventReasons r, LogicalPageID pageID) = 0; virtual void releaseExtentReadLock() = 0; // Temporary methods for testing diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 31f74084a4..c8bea9e43e 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -19,6 +19,9 @@ */ #include "flow/flow.h" +#include "flow/Histogram.h" +#include +#include "fdbrpc/ContinuousSample.h" #include "fdbserver/IPager.h" #include "fdbclient/Tuple.h" #include "flow/serialize.h" @@ -466,13 +469,13 @@ public: nextPageID = id; debug_printf( "FIFOQueue::Cursor(%s) loadPage start id=%s\n", toString().c_str(), ::toString(nextPageID).c_str()); - nextPageReader = waitOrError(queue->pager->readPage(nextPageID, true), queue->pagerError); + nextPageReader = waitOrError(queue->pager->readPage(pagerEventReasons::rangePrefetch, nextPageID, true), queue->pagerError); } Future loadExtent() { ASSERT(mode == POP | mode == READONLY); debug_printf("FIFOQueue::Cursor(%s) loadExtent\n", toString().c_str()); - return map(queue->pager->readExtent(pageID), [=](Reference p) { + return map(queue->pager->readExtent(pagerEventReasons::metaData, pageID), [=](Reference p) { page = p; debug_printf("FIFOQueue::Cursor(%s) loadExtent done. Page: %p\n", toString().c_str(), page->begin()); return Void(); @@ -1291,12 +1294,35 @@ struct RedwoodMetrics { void clear() { memset(this, 0, sizeof(RedwoodMetrics)); + int levelCounter = 0; for (auto& level : levels) { - level = {}; + level = { + .buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), + .modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), + .buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), + .modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), + .buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), + .modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes) + }; + ++levelCounter; } + + const events eventsVector[] = {events::pagerCacheLookup, events::pagerCacheHit, events::pagerCacheMiss, events::pagerWrite}; + const pagerEventReasons reasonsVector[] = {pagerEventReasons::pointRead, pagerEventReasons::rangeRead, pagerEventReasons::rangePrefetch, pagerEventReasons::commit, pagerEventReasons::lazyClear, pagerEventReasons::metaData}; + for(events e : eventsVector){ + for(pagerEventReasons r: reasonsVector){ + eventsReasons[getIndex(e)][getIndex(r)] = 0; + } + } + kvSizeWritten = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("Written"), Histogram::Unit::bytes); + kvSizeReadByGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByGet "), Histogram::Unit::bytes); + kvSizeReadByRangeGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByRangeGet"), Histogram::Unit::bytes); startTime = g_network ? now() : 0; } + // Page levle events + enum class events{ pagerCacheLookup, pagerCacheHit, pagerCacheMiss, pagerWrite, Update, Build}; + struct Level { unsigned int pageRead; unsigned int pageReadExt; @@ -1317,6 +1343,13 @@ struct RedwoodMetrics { double modifyStoredPct; double modifyFillPct; unsigned int modifyItemCount; + + Reference buildFillPctSketch; + Reference modifyFillPctSketch; + Reference buildStoredPctSketch; + Reference modifyStoredPctSketch; + Reference buildItemCountSketch; + Reference modifyItemCountSketch; }; Level levels[btreeLevels]; @@ -1343,6 +1376,28 @@ struct RedwoodMetrics { unsigned int btreeLeafPreload; unsigned int btreeLeafPreloadExt; + unsigned int eventsReasons[4][6]; + + Reference kvSizeWritten; + Reference kvSizeReadByGet; + Reference kvSizeReadByRangeGet; + + std::string getName(events e){ + std::map names = {{events::pagerCacheLookup, "pagerCacheLookup"}, {events::pagerCacheHit, "pagerCacheHit"}, {events::pagerCacheMiss, "pagerCacheMiss"}, {events::pagerWrite, "pagerWrite"}}; + return names[e]; + } + int getIndex(events e){ + std::map indices = {{events::pagerCacheLookup, 0}, {events::pagerCacheHit, 1}, {events::pagerCacheMiss, 2}, {events::pagerWrite, 3},{events::Update, 0},{events::Build, 1}}; + return indices[e]; + } + std::string getName(pagerEventReasons r){ + std::map names = {{pagerEventReasons::pointRead, "pointRead"}, {pagerEventReasons::rangeRead, "rangeRead"}, {pagerEventReasons::rangePrefetch, "rangePrefetch"}, {pagerEventReasons::commit, "commit"}, {pagerEventReasons::lazyClear, "lazyClear"}, {pagerEventReasons::metaData, "metaData"}}; + return names[r]; + } + int getIndex(pagerEventReasons r){ + std::map indices = {{pagerEventReasons::pointRead, 0}, {pagerEventReasons::rangeRead, 1}, {pagerEventReasons::rangePrefetch, 2}, {pagerEventReasons::commit, 3}, {pagerEventReasons::lazyClear, 4}, {pagerEventReasons::metaData, 5}}; + return indices[r]; + } // Return number of pages read or written, from cache or disk unsigned int pageOps() const { // All page reads are either a cache hit, probe hit, or a disk read @@ -1408,6 +1463,24 @@ struct RedwoodMetrics { } } } + + const events eventsVector[] = {events::pagerCacheLookup, events::pagerCacheHit, events::pagerCacheMiss, events::pagerWrite}; + const vector reasonsVector = {pagerEventReasons::pointRead, pagerEventReasons::rangeRead, pagerEventReasons::rangePrefetch, pagerEventReasons::commit, pagerEventReasons::lazyClear, pagerEventReasons::metaData}; + for(events e : eventsVector){ + std::cout<<"\nevents: "+getName(e)+" {"; + for(auto r = reasonsVector.begin() ; r != reasonsVector.end(); ++r){ + std::string temp = ""+getName(*r)+": "+std::to_string(eventsReasons[getIndex(e)][getIndex(*r)]); + temp += (std::next(r) != reasonsVector.end() ? ", " : "}"); + std::cout<buckets[i]<<"; "; + std::cout<<"kvSizeReadByRangeGet: "<buckets[i]<<"; "; + std::cout<<"kvSizeWritten: "<buckets[i]<<";\n"; + } for (int i = 0; i < btreeLevels; ++i) { auto& level = levels[i]; @@ -1527,14 +1600,16 @@ public: // Get the object for i if it exists, else return nullptr. // If the object exists, its eviction order will NOT change as this is not a cache hit. - ObjectType* getIfExists(const IndexType& index) { + ObjectType* getIfExists(pagerEventReasons r, const IndexType& index) { auto i = cache.find(index); if (i != cache.end()) { ++i->second.hits; ++g_redwoodMetrics.pagerProbeHit; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; return &i->second.item; } ++g_redwoodMetrics.pagerProbeMiss; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; return nullptr; } @@ -1568,7 +1643,7 @@ public: // After a get(), the object for i is the last in evictionOrder. // If noHit is set, do not consider this access to be cache hit if the object is present // If noMiss is set, do not consider this access to be a cache miss if the object is not present - ObjectType& get(const IndexType& index, bool noHit = false, bool noMiss = false) { + ObjectType& get(pagerEventReasons r, const IndexType& index, bool noHit = false, bool noMiss = false) { Entry& entry = cache[index]; // If entry is linked into evictionOrder then move it to the back of the order @@ -1576,6 +1651,8 @@ public: if (!noHit) { ++entry.hits; ++g_redwoodMetrics.pagerCacheHit; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheHit)][g_redwoodMetrics.getIndex(r)] += 1; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; // Move the entry to the back of the eviction order evictionOrder.erase(evictionOrder.iterator_to(entry)); @@ -1585,6 +1662,8 @@ public: // Otherwise it was a cache miss if (!noMiss) { ++g_redwoodMetrics.pagerCacheMiss; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheMiss)][g_redwoodMetrics.getIndex(r)] += 1; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; } // Finish initializing entry entry.index = index; @@ -1921,7 +2000,7 @@ public: if (extents[i].queueID == remapQueueID) { LogicalPageID extID = extents[i].extentID; debug_printf("DWALPager Extents: ID: %s ", toString(extID).c_str()); - self->readExtent(extID); + self->readExtent(pagerEventReasons::metaData, extID); } } } @@ -1962,7 +2041,7 @@ public: // If this fails, the backup header is still in tact for the next recovery attempt. if (recoveredHeader) { // Write the header to page 0 - wait(self->writeHeaderPage(0, self->headerPage)); + wait(self->writeHeaderPage(pagerEventReasons::metaData, 0, self->headerPage)); // Wait for all outstanding writes to complete wait(self->operations.signalAndCollapse()); @@ -2185,7 +2264,7 @@ public: Future newExtentPageID(QueueID queueID) override { return newExtentPageID_impl(this, queueID); } - Future writePhysicalPage(PhysicalPageID pageID, Reference page, bool header = false) { + Future writePhysicalPage(pagerEventReasons r, PhysicalPageID pageID, Reference page, bool header = false) { debug_printf("DWALPager(%s) op=%s %s ptr=%p\n", filename.c_str(), (header ? "writePhysicalHeader" : "writePhysical"), @@ -2193,6 +2272,8 @@ public: page->begin()); ++g_redwoodMetrics.pagerDiskWrite; + g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerWrite)][g_redwoodMetrics.getIndex(r)] += 1; + VALGRIND_MAKE_MEM_DEFINED(page->begin(), page->size()); page->updateChecksum(pageID); debug_printf("DWALPager(%s) writePhysicalPage %s CalculatedChecksum=%d ChecksumInPage=%d\n", @@ -2221,14 +2302,15 @@ public: return f; } - Future writeHeaderPage(PhysicalPageID pageID, Reference page) { - return writePhysicalPage(pageID, page, true); + Future writeHeaderPage(pagerEventReasons r, PhysicalPageID pageID, Reference page) { + return writePhysicalPage(r, pageID, page, true); } void updatePage(LogicalPageID pageID, Reference data) override { // Get the cache entry for this page, without counting it as a cache hit as we're replacing its contents now // or as a cache miss because there is no benefit to the page already being in cache - PageCacheEntry& cacheEntry = pageCache.get(pageID, true, true); + // this metaData reason will not be accounted since its not a cache hit or cache miss + PageCacheEntry& cacheEntry = pageCache.get(pagerEventReasons::metaData, pageID, true, true); debug_printf("DWALPager(%s) op=write %s cached=%d reading=%d writing=%d\n", filename.c_str(), toString(pageID).c_str(), @@ -2244,11 +2326,11 @@ public: // future reads of the version are not allowed) and the write of the next newest version over top // of the original page begins. if (!cacheEntry.initialized()) { - cacheEntry.writeFuture = writePhysicalPage(pageID, data); + cacheEntry.writeFuture = writePhysicalPage(pagerEventReasons::metaData, pageID, data); } else if (cacheEntry.reading()) { // Wait for the read to finish, then start the write. cacheEntry.writeFuture = map(success(cacheEntry.readFuture), [=](Void) { - writePhysicalPage(pageID, data); + writePhysicalPage(pagerEventReasons::metaData, pageID, data); return Void(); }); } @@ -2256,11 +2338,11 @@ public: // writes happen in the correct order else if (cacheEntry.writing()) { cacheEntry.writeFuture = map(cacheEntry.writeFuture, [=](Void) { - writePhysicalPage(pageID, data); + writePhysicalPage(pagerEventReasons::metaData, pageID, data); return Void(); }); } else { - cacheEntry.writeFuture = writePhysicalPage(pageID, data); + cacheEntry.writeFuture = writePhysicalPage(pagerEventReasons::metaData, pageID, data); } // Always update the page contents immediately regardless of what happened above. @@ -2448,12 +2530,12 @@ public: // Reads the most recent version of pageID, either previously committed or written using updatePage() // in the current commit - Future> readPage(LogicalPageID pageID, bool cacheable, bool noHit = false) override { + Future> readPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable, bool noHit = false) override { // Use cached page if present, without triggering a cache hit. // Otherwise, read the page and return it but don't add it to the cache if (!cacheable) { debug_printf("DWALPager(%s) op=readUncached %s\n", filename.c_str(), toString(pageID).c_str()); - PageCacheEntry* pCacheEntry = pageCache.getIfExists(pageID); + PageCacheEntry* pCacheEntry = pageCache.getIfExists(r, pageID); if (pCacheEntry != nullptr) { debug_printf("DWALPager(%s) op=readUncachedHit %s\n", filename.c_str(), toString(pageID).c_str()); @@ -2464,7 +2546,7 @@ public: return forwardError(readPhysicalPage(this, (PhysicalPageID)pageID), errorPromise); } - PageCacheEntry& cacheEntry = pageCache.get(pageID, noHit); + PageCacheEntry& cacheEntry = pageCache.get(r, pageID, noHit); debug_printf("DWALPager(%s) op=read %s cached=%d reading=%d writing=%d noHit=%d\n", filename.c_str(), toString(pageID).c_str(), @@ -2511,9 +2593,9 @@ public: return (PhysicalPageID)pageID; } - Future> readPageAtVersion(LogicalPageID logicalID, Version v, bool cacheable, bool noHit) { + Future> readPageAtVersion(pagerEventReasons r, LogicalPageID logicalID, Version v, bool cacheable, bool noHit) { PhysicalPageID physicalID = getPhysicalPageID(logicalID, v); - return readPage(physicalID, cacheable, noHit); + return readPage(r, physicalID, cacheable, noHit); } void releaseExtentReadLock() override { concurrentExtentReads->release(); } @@ -2592,9 +2674,9 @@ public: return extent; } - Future> readExtent(LogicalPageID pageID) override { + Future> readExtent(pagerEventReasons r, LogicalPageID pageID) override { debug_printf("DWALPager(%s) op=readExtent %s\n", filename.c_str(), toString(pageID).c_str()); - PageCacheEntry* pCacheEntry = extentCache.getIfExists(pageID); + PageCacheEntry* pCacheEntry = extentCache.getIfExists(r, pageID); if (pCacheEntry != nullptr) { debug_printf("DWALPager(%s) Cache Entry exists for %s\n", filename.c_str(), toString(pageID).c_str()); return pCacheEntry->readFuture; @@ -2621,7 +2703,7 @@ public: else if (tailExt) readSize = (tailPageID - pageID + 1) * physicalPageSize; - PageCacheEntry& cacheEntry = extentCache.get(pageID); + PageCacheEntry& cacheEntry = extentCache.get(r, pageID); if (!cacheEntry.initialized()) { cacheEntry.writeFuture = Void(); cacheEntry.readFuture = @@ -2741,7 +2823,7 @@ public: debug_printf("DWALPager(%s) remapCleanup copy %s\n", self->filename.c_str(), p.toString().c_str()); // Read the data from the page that the original was mapped to - Reference data = wait(self->readPage(p.newPageID, false, true)); + Reference data = wait(self->readPage(pagerEventReasons::metaData, p.newPageID, false, true)); // Write the data to the original page so it can be read using its original pageID self->updatePage(p.originalPageID, data); @@ -2889,7 +2971,7 @@ public: debug_printf("DWALPager(%s) commit begin\n", self->filename.c_str()); // Write old committed header to Page 1 - self->writeHeaderPage(1, self->lastCommittedHeaderPage); + self->writeHeaderPage(pagerEventReasons::commit, 1, self->lastCommittedHeaderPage); // Trigger the remap eraser to stop and then wait for it. self->remapCleanupStop = true; @@ -2921,7 +3003,7 @@ public: } // Update header on disk and sync again. - wait(self->writeHeaderPage(0, self->headerPage)); + wait(self->writeHeaderPage(pagerEventReasons::commit, 0, self->headerPage)); if (g_network->getCurrentTask() > TaskPriority::DiskWrite) { wait(delay(0, TaskPriority::DiskWrite)); } @@ -3214,11 +3296,11 @@ public: : pager(pager), metaKey(meta), version(version), expired(expiredFuture) {} ~DWALPagerSnapshot() override {} - Future> getPhysicalPage(LogicalPageID pageID, bool cacheable, bool noHit) override { + Future> getPhysicalPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable, bool noHit) override { if (expired.isError()) { throw expired.getError(); } - return map(pager->readPageAtVersion(pageID, version, cacheable, noHit), + return map(pager->readPageAtVersion(r, pageID, version, cacheable, noHit), [=](Reference p) { return Reference(std::move(p)); }); } @@ -4155,7 +4237,7 @@ public: break; } // Start reading the page, without caching - entries.push_back(std::make_pair(q.get(), self->readPage(snapshot, q.get().pageID, true, false))); + entries.push_back(std::make_pair(q.get(), self->readPage(pagerEventReasons::lazyClear, snapshot, q.get().pageID, true, false))); --toPop; } @@ -4856,6 +4938,8 @@ private: btPage->height = height; btPage->kvBytes = p.kvBytes; + g_redwoodMetrics.kvSizeWritten->sample(p.kvBytes); + debug_printf("Building tree for %s\nlower: %s\nupper: %s\n", p.toString().c_str(), pageLowerBound.toString(false).c_str(), @@ -4883,6 +4967,10 @@ private: metrics.buildStoredPct += p.kvFraction(); metrics.buildItemCount += p.count; + metrics.buildFillPctSketch->sample(p.usedFraction()); + metrics.buildStoredPctSketch->sample(p.kvFraction()); + metrics.buildItemCountSketch->sample(p.count); + // Create chunked pages // TODO: Avoid copying page bytes, but this is not trivial due to how pager checksums are currently handled. if (p.blockCount != 1) { @@ -4986,7 +5074,8 @@ private: return pager->tryEvictPage(id.front()); } - ACTOR static Future> readPage(Reference snapshot, + ACTOR static Future> readPage(pagerEventReasons r, + Reference snapshot, BTreePageIDRef id, bool forLazyClear = false, bool cacheable = true) { @@ -4999,13 +5088,13 @@ private: state Reference page; if (id.size() == 1) { - Reference p = wait(snapshot->getPhysicalPage(id.front(), cacheable, false)); + Reference p = wait(snapshot->getPhysicalPage(r, id.front(), cacheable, false)); page = std::move(p); } else { ASSERT(!id.empty()); std::vector>> reads; for (auto& pageID : id) { - reads.push_back(snapshot->getPhysicalPage(pageID, cacheable, false)); + reads.push_back(snapshot->getPhysicalPage(r, pageID, cacheable, false)); } std::vector> pages = wait(getAll(reads)); // TODO: Cache reconstituted super pages somehow, perhaps with help from the Pager. @@ -5056,7 +5145,7 @@ private: g_redwoodMetrics.btreeLeafPreloadExt += (id.size() - 1); for (auto pageID : id) { - snapshot->getPhysicalPage(pageID, true, true); + snapshot->getPhysicalPage(pagerEventReasons::rangePrefetch, pageID, true, true); } } @@ -5203,6 +5292,12 @@ private: metrics.modifyStoredPct += (double)btPage->kvBytes / capacity; metrics.modifyItemCount += btPage->tree()->numItems; + metrics.modifyFillPctSketch->sample((double)btPage->size() / capacity); + metrics.modifyStoredPctSketch->sample((double)btPage->kvBytes / capacity); + metrics.modifyItemCountSketch->sample(btPage->tree()->numItems); + + g_redwoodMetrics.kvSizeWritten->sample(btPage->kvBytes); + // The boundaries can't have changed, but the child page link may have. if (maybeNewID != decodeLowerBound.getChildPage()) { // Add page's decode lower bound to newLinks set without its child page, intially @@ -5458,7 +5553,7 @@ private: debug_printf("%s -------------------------------------\n", context.c_str()); } - state Reference page = wait(readPage(snapshot, rootID, false, false)); + state Reference page = wait(readPage(pagerEventReasons::commit, snapshot, rootID, false, false)); state Version writeVersion = self->getLastCommittedVersion() + 1; // If the page exists in the cache, it must be copied before modification. @@ -6298,9 +6393,9 @@ public: PathEntry& back() { return path.back(); } void popPath() { path.pop_back(); } - Future pushPage(const BTreePage::BinaryTree::Cursor& link) { + Future pushPage(pagerEventReasons r, const BTreePage::BinaryTree::Cursor& link) { debug_printf("pushPage(link=%s)\n", link.get().toString(false).c_str()); - return map(readPage(pager, link.get().getChildPage()), [=](Reference p) { + return map(readPage(r, pager, link.get().getChildPage()), [=](Reference p) { #if REDWOOD_DEBUG path.push_back({ p, getCursor(p, link), link.get().getChildPage() }); #else @@ -6310,9 +6405,9 @@ public: }); } - Future pushPage(BTreePageIDRef id) { + Future pushPage(pagerEventReasons r, BTreePageIDRef id) { debug_printf("pushPage(root=%s)\n", ::toString(id).c_str()); - return map(readPage(pager, id), [=](Reference p) { + return map(readPage(r, pager, id), [=](Reference p) { #if REDWOOD_DEBUG path.push_back({ p, getCursor(p, dbBegin, dbEnd), id }); #else @@ -6329,7 +6424,7 @@ public: path.clear(); path.reserve(6); valid = false; - return pushPage(root); + return pushPage(pagerEventReasons::commit, root); } // Seeks cursor to query if it exists, the record before or after it, or an undefined and invalid @@ -6341,7 +6436,7 @@ public: // If there is a record in the tree > query then moveNext() will move to it. // If non-zero is returned then the cursor is valid and the return value is logically equivalent // to query.compare(cursor.get()) - ACTOR Future seek_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + ACTOR Future seek_impl(pagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { state RedwoodRecordRef internalPageQuery = query.withMaxPageID(); self->path.resize(1); debug_printf( @@ -6369,7 +6464,7 @@ public: query.toString().c_str(), prefetchBytes, self->toString().c_str()); - Future f = self->pushPage(entry.cursor); + Future f = self->pushPage(r, entry.cursor); // Prefetch siblings, at least prefetchBytes, at level 2 but without jumping to another level 2 // sibling @@ -6400,32 +6495,32 @@ public: } } - Future seek(RedwoodRecordRef query, int prefetchBytes) { return seek_impl(this, query, prefetchBytes); } + Future seek(pagerEventReasons r, RedwoodRecordRef query, int prefetchBytes) { return seek_impl(r, this, query, prefetchBytes); } - ACTOR Future seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + ACTOR Future seekGTE_impl(pagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { debug_printf("seekGTE(%s, %d) start\n", query.toString().c_str(), prefetchBytes); - int cmp = wait(self->seek(query, prefetchBytes)); + int cmp = wait(self->seek(r, query, prefetchBytes)); if (cmp > 0 || (cmp == 0 && !self->isValid())) { wait(self->moveNext()); } return Void(); } - Future seekGTE(RedwoodRecordRef query, int prefetchBytes) { - return seekGTE_impl(this, query, prefetchBytes); + Future seekGTE(pagerEventReasons r, RedwoodRecordRef query, int prefetchBytes) { + return seekGTE_impl(r, this, query, prefetchBytes); } - ACTOR Future seekLT_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + ACTOR Future seekLT_impl(pagerEventReasons r, BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { debug_printf("seekLT(%s, %d) start\n", query.toString().c_str(), prefetchBytes); - int cmp = wait(self->seek(query, prefetchBytes)); + int cmp = wait(self->seek(r, query, prefetchBytes)); if (cmp <= 0) { wait(self->movePrev()); } return Void(); } - Future seekLT(RedwoodRecordRef query, int prefetchBytes) { - return seekLT_impl(this, query, -prefetchBytes); + Future seekLT(pagerEventReasons r, RedwoodRecordRef query, int prefetchBytes) { + return seekLT_impl(r, this, query, -prefetchBytes); } ACTOR Future move_impl(BTreeCursor* self, bool forward) { @@ -6476,7 +6571,7 @@ public: ASSERT(entry.cursor.get().value.present()); } - wait(self->pushPage(entry.cursor)); + wait(self->pushPage(pagerEventReasons::metaData, entry.cursor)); auto& newEntry = self->path.back(); ASSERT(forward ? newEntry.cursor.moveFirst() : newEntry.cursor.moveLast()); } @@ -6623,7 +6718,7 @@ public: state int prefetchBytes = 0; if (rowLimit > 0) { - wait(cur.seekGTE(keys.begin, prefetchBytes)); + wait(cur.seekGTE(pagerEventReasons::rangeRead, keys.begin, prefetchBytes)); while (cur.isValid()) { // Read page contents without using waits BTreePage::BinaryTree::Cursor leafCursor = cur.back().cursor; @@ -6665,7 +6760,7 @@ public: wait(cur.moveNext()); } } else { - wait(cur.seekLT(keys.end, prefetchBytes)); + wait(cur.seekLT(pagerEventReasons::rangeRead, keys.end, prefetchBytes)); while (cur.isValid()) { // Read page contents without using waits BTreePage::BinaryTree::Cursor leafCursor = cur.back().cursor; @@ -6713,6 +6808,7 @@ public: ASSERT(result.size() > 0); result.readThrough = result[result.size() - 1].key; } + g_redwoodMetrics.kvSizeReadByRangeGet->sample(accumulatedBytes); return result; } @@ -6726,12 +6822,14 @@ public: state FlowLock::Releaser releaser(self->m_concurrentReads); ++g_redwoodMetrics.opGet; - wait(cur.seekGTE(key, 0)); + wait(cur.seekGTE(pagerEventReasons::pointRead, key, 0)); if (cur.isValid() && cur.get().key == key) { // Return a Value whose arena depends on the source page arena Value v; v.arena().dependsOn(cur.back().page->getArena()); v.contents() = cur.get().value.get(); + //std::cout<<"kvBytes for readValue impl: "<sample(cur.get().kvBytes()); return v; } @@ -6842,12 +6940,12 @@ ACTOR Future verifyRangeBTreeCursor(VersionedBTree* btree, start.printable().c_str(), end.printable().c_str(), randomKey.toString().c_str()); - wait(success(cur.seek(randomKey, 0))); + wait(success(cur.seek(pagerEventReasons::rangeRead, randomKey, 0))); } debug_printf( "VerifyRange(@%" PRId64 ", %s, %s): Actual seek\n", v, start.printable().c_str(), end.printable().c_str()); - wait(cur.seekGTE(start, 0)); + wait(cur.seekGTE(pagerEventReasons::rangeRead, start, 0)); state Standalone> results; @@ -6947,7 +7045,7 @@ ACTOR Future verifyRangeBTreeCursor(VersionedBTree* btree, } // Now read the range from the tree in reverse order and compare to the saved results - wait(cur.seekLT(end, 0)); + wait(cur.seekLT(pagerEventReasons::rangeRead, end, 0)); state std::reverse_iterator r = results.rbegin(); @@ -7024,7 +7122,7 @@ ACTOR Future seekAllBTreeCursor(VersionedBTree* btree, state Optional val = i->second; debug_printf("Verifying @%" PRId64 " '%s'\n", ver, key.c_str()); state Arena arena; - wait(cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key)), 0)); + wait(cur.seekGTE(pagerEventReasons::metaData, RedwoodRecordRef(KeyRef(arena, key)), 0)); bool foundKey = cur.isValid() && cur.get().key == key; bool hasValue = foundKey && cur.get().value.present(); @@ -7149,7 +7247,7 @@ ACTOR Future randomReader(VersionedBTree* btree) { } state KeyValue kv = randomKV(10, 0); - wait(cur.seekGTE(kv.key, 0)); + wait(cur.seekGTE(pagerEventReasons::pointRead, kv.key, 0)); state int c = deterministicRandom()->randomInt(0, 100); state bool direction = deterministicRandom()->coinflip(); while (cur.isValid() && c-- > 0) { @@ -8729,7 +8827,7 @@ ACTOR Future randomSeeks(VersionedBTree* btree, int count, char firstChar, wait(btree->initBTreeCursor(&cur, readVer)); while (c < count) { state Key k = randomString(20, firstChar, lastChar); - wait(cur.seekGTE(k, 0)); + wait(cur.seekGTE(pagerEventReasons::pointRead, k, 0)); ++c; } double elapsed = timer() - readStart; @@ -8753,7 +8851,7 @@ ACTOR Future randomScans(VersionedBTree* btree, state int totalScanBytes = 0; while (c++ < count) { state Key k = randomString(20, firstChar, lastChar); - wait(cur.seekGTE(k, readAhead)); + wait(cur.seekGTE(pagerEventReasons::pointRead, k, readAhead)); if (adaptive) { readAhead = totalScanBytes / c; } @@ -8792,7 +8890,7 @@ TEST_CASE(":/redwood/correctness/pager/cow") { pager->updatePage(id, p); pager->setMetaKey(LiteralStringRef("asdfasdf")); wait(pager->commit()); - Reference p2 = wait(pager->readPage(id, true)); + Reference p2 = wait(pager->readPage(pagerEventReasons::pointRead, id, true)); printf("%s\n", StringRef(p2->begin(), p2->size()).toHexString().c_str()); // TODO: Verify reads, do more writes and reads to make this a real pager validator @@ -8922,7 +9020,7 @@ TEST_CASE(":/redwood/performance/extentQueue") { state int i; for (i = 1; i < extentIDs.size() - 1; i++) { LogicalPageID extID = extentIDs[i]; - pager->readExtent(extID); + pager->readExtent(pagerEventReasons::rangeRead, extID); } state PromiseStream>>> resultStream; @@ -9632,3 +9730,54 @@ TEST_CASE("!/redwood/performance/randomRangeScans") { return Void(); } + + +TEST_CASE(":/redwood/performance/histogramThroughput") { + std::default_random_engine generator; + std::uniform_int_distribution distribution(0,pow(2,32)); + state size_t inputSize = pow(10, 8); + state vector uniform; + for(int i=0; i h = + Histogram::getHistogram(LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::bytes); + std::cout<<"histogramTest is okay"<sample(uniform[i]); + } + GetHistogramRegistry().logReport(); + auto t_end = std::chrono::high_resolution_clock::now(); + double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); + std::cout<<"size of input: "< distribution(0,pow(2,32)); + state size_t inputSize = pow(10, 8); + state vector uniform; + for(int i=0; i s = ContinuousSample(pow(10,3)); + auto t_start = std::chrono::high_resolution_clock::now(); + ASSERT(uniform.size() == inputSize); + for(size_t i=0; i(t_end-t_start).count(); + std::cout<<"size of input: "< #include #include +#include #ifdef _WIN32 #include @@ -66,7 +67,10 @@ private: Histogram(std::string const& group, std::string const& op, Unit unit, HistogramRegistry& registry) : group(group), op(op), unit(unit), registry(registry), ReferenceCounted() { - ASSERT(UnitToStringMapper.find(unit) != UnitToStringMapper.end()); + for(const auto & [ key, value ] : UnitToStringMapper){ + std::cout< Date: Wed, 23 Jun 2021 14:33:41 -0700 Subject: [PATCH 025/426] Remove preinitializeOnForeignThread --- fdbclient/ISingleThreadTransaction.cpp | 4 +++- fdbclient/NativeAPI.actor.h | 4 +--- fdbclient/ReadYourWrites.actor.cpp | 4 ---- fdbclient/ReadYourWrites.h | 2 -- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/fdbclient/ISingleThreadTransaction.cpp b/fdbclient/ISingleThreadTransaction.cpp index 4a0b686540..a36f92fe9a 100644 --- a/fdbclient/ISingleThreadTransaction.cpp +++ b/fdbclient/ISingleThreadTransaction.cpp @@ -27,7 +27,6 @@ ISingleThreadTransaction* ISingleThreadTransaction::allocateOnForeignThread(Type type) { if (type == Type::RYW) { auto tr = new ReadYourWritesTransaction; - tr->preinitializeOnForeignThread(); return tr; } else if (type == Type::SIMPLE_CONFIG) { auto tr = new SimpleConfigTransaction; @@ -43,12 +42,15 @@ ISingleThreadTransaction* ISingleThreadTransaction::allocateOnForeignThread(Type void ISingleThreadTransaction::create(ISingleThreadTransaction* tr, Type type, Database db) { switch (type) { case Type::RYW: + dynamic_cast(tr)->~ReadYourWritesTransaction(); new (tr) ReadYourWritesTransaction(db); break; case Type::SIMPLE_CONFIG: + dynamic_cast(tr)->~SimpleConfigTransaction(); new (tr) SimpleConfigTransaction(db); break; case Type::PAXOS_CONFIG: + dynamic_cast(tr)->~PaxosConfigTransaction(); new (tr) PaxosConfigTransaction(db); break; default: diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 043bcaf4f2..b0f589e97a 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -241,8 +241,6 @@ public: explicit Transaction(Database const& cx); ~Transaction(); - void preinitializeOnForeignThread() { committedVersion = invalidVersion; } - void setVersion(Version v); Future getReadVersion() { return getReadVersion(0); } Future getRawReadVersion(); @@ -418,7 +416,7 @@ private: Database cx; double backoff; - Version committedVersion; + Version committedVersion{ invalidVersion }; CommitTransactionRequest tr; Future readVersion; Promise> metadataVersion; diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 4db07f527b..8b0ef80754 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -1729,10 +1729,6 @@ void ReadYourWritesTransaction::getWriteConflicts(KeyRangeMap* result) { } } -void ReadYourWritesTransaction::preinitializeOnForeignThread() { - tr.preinitializeOnForeignThread(); -} - void ReadYourWritesTransaction::setTransactionID(uint64_t id) { tr.setTransactionID(id); } diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index 65bb972da9..4f4827792d 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -153,8 +153,6 @@ public: void getWriteConflicts(KeyRangeMap* result) override; - void preinitializeOnForeignThread(); - Database getDatabase() const { return tr.getDatabase(); } const TransactionInfo& getTransactionInfo() const { return tr.info; } From ed2f8c928224c7940bbeb2c2e6407b3214f23882 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Wed, 23 Jun 2021 14:54:41 -0700 Subject: [PATCH 026/426] add percentage unit to histogram class --- .gitignore | 1 + fdbserver/VersionedBTree.actor.cpp | 16 ++++++++-------- flow/Histogram.cpp | 6 +++++- flow/Histogram.h | 13 ++++++++++++- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index bb16f145de..01bf5d30ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Build artifacts +/my_build/ /bin/ /lib/ /packages/ diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index c8bea9e43e..7232ee3a43 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1297,10 +1297,10 @@ struct RedwoodMetrics { int levelCounter = 0; for (auto& level : levels) { level = { - .buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), - .modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), - .buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), - .modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), + .buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + .modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + .buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + .modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), .buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), .modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes) }; @@ -4967,8 +4967,8 @@ private: metrics.buildStoredPct += p.kvFraction(); metrics.buildItemCount += p.count; - metrics.buildFillPctSketch->sample(p.usedFraction()); - metrics.buildStoredPctSketch->sample(p.kvFraction()); + metrics.buildFillPctSketch->samplePercentage(p.usedFraction()); + metrics.buildStoredPctSketch->samplePercentage(p.kvFraction()); metrics.buildItemCountSketch->sample(p.count); // Create chunked pages @@ -5292,8 +5292,8 @@ private: metrics.modifyStoredPct += (double)btPage->kvBytes / capacity; metrics.modifyItemCount += btPage->tree()->numItems; - metrics.modifyFillPctSketch->sample((double)btPage->size() / capacity); - metrics.modifyStoredPctSketch->sample((double)btPage->kvBytes / capacity); + metrics.modifyFillPctSketch->samplePercentage((double)btPage->size() / capacity); + metrics.modifyStoredPctSketch->samplePercentage((double)btPage->kvBytes / capacity); metrics.modifyItemCountSketch->sample(btPage->tree()->numItems); g_redwoodMetrics.kvSizeWritten->sample(btPage->kvBytes); diff --git a/flow/Histogram.cpp b/flow/Histogram.cpp index 8ac2437503..c21f4d228e 100644 --- a/flow/Histogram.cpp +++ b/flow/Histogram.cpp @@ -99,7 +99,8 @@ void HistogramRegistry::logReport() { const std::unordered_map Histogram::UnitToStringMapper = { { Histogram::Unit::microseconds, "microseconds" }, { Histogram::Unit::bytes, "bytes" }, - { Histogram::Unit::bytes_per_second, "bytes_per_second" } + { Histogram::Unit::bytes_per_second, "bytes_per_second" }, + { Histogram::Unit::percentage, "percentage" }, }; void Histogram::writeToLog() { @@ -130,6 +131,9 @@ void Histogram::writeToLog() { case Unit::bytes_per_second: e.detail(format("LessThan%u", value), buckets[i]); break; + case Unit::percentage: + e.detail(format("LessThan%f", (i+1)*0.04), buckets[i]); + break; default: ASSERT(false); } diff --git a/flow/Histogram.h b/flow/Histogram.h index 4573563f17..22bf8a02a4 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -59,7 +59,7 @@ HistogramRegistry& GetHistogramRegistry(); */ class Histogram final : public ReferenceCounted { public: - enum class Unit { microseconds, bytes, bytes_per_second }; + enum class Unit { microseconds, bytes, bytes_per_second, percentage }; private: static const std::unordered_map UnitToStringMapper; @@ -121,6 +121,17 @@ public: sample((uint32_t)(delta * 1000000)); // convert to microseconds and truncate to integer } } + // This histogram buckets samples into linear interval of size 4 percent. + inline void samplePercentage(double pct) { + ASSERT(unit==Histogram::Unit::percentage); + ASSERT(pct>=0.0); + if (pct >= 1.28){ + pct = 1.24; + } + size_t idx = (pct*100) / 4; + ASSERT(idx < 32); + buckets[idx]++; + } void clear() { for (uint32_t& i : buckets) { From 109cb652ab1c6aefd9197d9469b693c863e3507b Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Wed, 23 Jun 2021 16:02:55 -0700 Subject: [PATCH 027/426] add recordCounter in histogram to track data from small ranges --- fdbserver/VersionedBTree.actor.cpp | 15 +++++++++------ flow/Histogram.cpp | 1 + flow/Histogram.h | 26 ++++++++++++++++++++------ 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 7232ee3a43..847f84855d 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1289,6 +1289,7 @@ int nextPowerOf2(uint32_t x) { struct RedwoodMetrics { static constexpr int btreeLevels = 5; + static constexpr int maxRecordCount = 315; RedwoodMetrics() { clear(); } @@ -1301,8 +1302,8 @@ struct RedwoodMetrics { .modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), .buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), .modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - .buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes), - .modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::bytes) + .buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount), + .modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount) }; ++levelCounter; } @@ -4969,7 +4970,8 @@ private: metrics.buildFillPctSketch->samplePercentage(p.usedFraction()); metrics.buildStoredPctSketch->samplePercentage(p.kvFraction()); - metrics.buildItemCountSketch->sample(p.count); + //std::cout<<"build item count: "<sampleRecordCounter(p.count); // Create chunked pages // TODO: Avoid copying page bytes, but this is not trivial due to how pager checksums are currently handled. @@ -5294,7 +5296,8 @@ private: metrics.modifyFillPctSketch->samplePercentage((double)btPage->size() / capacity); metrics.modifyStoredPctSketch->samplePercentage((double)btPage->kvBytes / capacity); - metrics.modifyItemCountSketch->sample(btPage->tree()->numItems); + //std::cout<<"modify item count: "<tree()->numItems<sampleRecordCounter(btPage->tree()->numItems); g_redwoodMetrics.kvSizeWritten->sample(btPage->kvBytes); @@ -9734,7 +9737,7 @@ TEST_CASE("!/redwood/performance/randomRangeScans") { TEST_CASE(":/redwood/performance/histogramThroughput") { std::default_random_engine generator; - std::uniform_int_distribution distribution(0,pow(2,32)); + std::uniform_int_distribution distribution(0,UINT32_MAX); state size_t inputSize = pow(10, 8); state vector uniform; for(int i=0; i distribution(0,pow(2,32)); + std::uniform_int_distribution distribution(0,UINT32_MAX); state size_t inputSize = pow(10, 8); state vector uniform; for(int i=0; i Histogram::UnitToStringMa { Histogram::Unit::bytes, "bytes" }, { Histogram::Unit::bytes_per_second, "bytes_per_second" }, { Histogram::Unit::percentage, "percentage" }, + { Histogram::Unit::record_counter, "record_counter" }, }; void Histogram::writeToLog() { diff --git a/flow/Histogram.h b/flow/Histogram.h index 22bf8a02a4..a7ef9d9aa6 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -59,13 +59,13 @@ HistogramRegistry& GetHistogramRegistry(); */ class Histogram final : public ReferenceCounted { public: - enum class Unit { microseconds, bytes, bytes_per_second, percentage }; + enum class Unit { microseconds, bytes, bytes_per_second, percentage, record_counter }; private: static const std::unordered_map UnitToStringMapper; - Histogram(std::string const& group, std::string const& op, Unit unit, HistogramRegistry& registry) - : group(group), op(op), unit(unit), registry(registry), ReferenceCounted() { + Histogram(std::string const& group, std::string const& op, Unit unit, HistogramRegistry& registry, uint32_t lower, uint32_t upper) + : group(group), op(op), unit(unit), registry(registry), lowerBound(lower), upperBound(upper), ReferenceCounted() { for(const auto & [ key, value ] : UnitToStringMapper){ std::cout< getHistogram(StringRef group, StringRef op, Unit unit) { + static Reference getHistogram(StringRef group, StringRef op, Unit unit, uint32_t lower = 0, uint32_t upper = UINT32_MAX) { std::string group_str = group.toString(); std::string op_str = op.toString(); std::string name = generateName(group_str, op_str); HistogramRegistry& registry = GetHistogramRegistry(); Histogram* h = registry.lookupHistogram(name); if (!h) { - h = new Histogram(group_str, op_str, unit, registry); + h = new Histogram(group_str, op_str, unit, registry, lower, upper); registry.registerHistogram(h); return Reference(h); } else { @@ -121,7 +121,7 @@ public: sample((uint32_t)(delta * 1000000)); // convert to microseconds and truncate to integer } } - // This histogram buckets samples into linear interval of size 4 percent. + // Histogram buckets samples into linear interval of size 4 percent. inline void samplePercentage(double pct) { ASSERT(unit==Histogram::Unit::percentage); ASSERT(pct>=0.0); @@ -133,6 +133,18 @@ public: buckets[idx]++; } + // Histogram buckets samples into one of the same sized buckets + // This is used when the distance b/t upperBound and lowerBound are relativly small + inline void sampleRecordCounter(uint32_t sample) { + ASSERT(unit==Histogram::Unit::record_counter); + if(sample == upperBound){ + sample = upperBound - 1; + } + size_t idx = sample * 32 / (upperBound - lowerBound); + ASSERT(idx < 32); + buckets[idx]++; + } + void clear() { for (uint32_t& i : buckets) { i = 0; @@ -147,6 +159,8 @@ public: Unit const unit; HistogramRegistry& registry; uint32_t buckets[32]; + uint32_t lowerBound; + uint32_t upperBound; }; #endif // FLOW_HISTOGRAM_H From acde8134f31caf74809163fa82c40d9171d0a1c8 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Thu, 24 Jun 2021 09:48:26 -0700 Subject: [PATCH 028/426] change histogram class static member variable to non-static --- fdbserver/VersionedBTree.actor.cpp | 23 ++++++++++++++++++----- flow/Histogram.cpp | 9 --------- flow/Histogram.h | 15 +++++++++------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 847f84855d..01b0fd6cce 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1294,7 +1294,9 @@ struct RedwoodMetrics { RedwoodMetrics() { clear(); } void clear() { + std::cout<<"in clear"<samplePercentage(p.usedFraction()); metrics.buildStoredPctSketch->samplePercentage(p.kvFraction()); - //std::cout<<"build item count: "<sampleRecordCounter(p.count); // Create chunked pages @@ -5296,7 +5297,6 @@ private: metrics.modifyFillPctSketch->samplePercentage((double)btPage->size() / capacity); metrics.modifyStoredPctSketch->samplePercentage((double)btPage->kvBytes / capacity); - //std::cout<<"modify item count: "<tree()->numItems<sampleRecordCounter(btPage->tree()->numItems); g_redwoodMetrics.kvSizeWritten->sample(btPage->kvBytes); @@ -6831,7 +6831,6 @@ public: Value v; v.arena().dependsOn(cur.back().page->getArena()); v.contents() = cur.get().value.get(); - //std::cout<<"kvBytes for readValue impl: "<sample(cur.get().kvBytes()); return v; } @@ -9743,11 +9742,12 @@ TEST_CASE(":/redwood/performance/histogramThroughput") { for(int i=0; i h = Histogram::getHistogram(LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::bytes); - std::cout<<"histogramTest is okay"<sample(uniform[i]); @@ -9755,7 +9755,20 @@ TEST_CASE(":/redwood/performance/histogramThroughput") { GetHistogramRegistry().logReport(); auto t_end = std::chrono::high_resolution_clock::now(); double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); - std::cout<<"size of input: "< h = + Histogram::getHistogram(LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::percentage); + ASSERT(uniform.size() == inputSize); + for(size_t i=0; isample((double)uniform[i]/UINT32_MAX); + } + GetHistogramRegistry().logReport(); + auto t_end = std::chrono::high_resolution_clock::now(); + double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); std::cout<<"Time in millisecond: "< Histogram::UnitToStringMapper = { - { Histogram::Unit::microseconds, "microseconds" }, - { Histogram::Unit::bytes, "bytes" }, - { Histogram::Unit::bytes_per_second, "bytes_per_second" }, - { Histogram::Unit::percentage, "percentage" }, - { Histogram::Unit::record_counter, "record_counter" }, -}; - void Histogram::writeToLog() { bool active = false; for (uint32_t i = 0; i < 32; i++) { @@ -117,7 +109,6 @@ void Histogram::writeToLog() { } TraceEvent e(SevInfo, "Histogram"); - ASSERT(UnitToStringMapper.find(unit) != UnitToStringMapper.end()); e.detail("Group", group).detail("Op", op).detail("Unit", UnitToStringMapper.at(unit)); for (uint32_t i = 0; i < 32; i++) { diff --git a/flow/Histogram.h b/flow/Histogram.h index a7ef9d9aa6..03ae928013 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -62,15 +62,18 @@ public: enum class Unit { microseconds, bytes, bytes_per_second, percentage, record_counter }; private: - static const std::unordered_map UnitToStringMapper; + //static const std::unordered_map UnitToStringMapper; + const std::unordered_map UnitToStringMapper; Histogram(std::string const& group, std::string const& op, Unit unit, HistogramRegistry& registry, uint32_t lower, uint32_t upper) - : group(group), op(op), unit(unit), registry(registry), lowerBound(lower), upperBound(upper), ReferenceCounted() { + : UnitToStringMapper ( + { { Histogram::Unit::microseconds, "microseconds" },{ Histogram::Unit::bytes, "bytes" }, + { Histogram::Unit::bytes_per_second, "bytes_per_second" }, + { Histogram::Unit::percentage, "percentage" }, + { Histogram::Unit::record_counter, "record_counter" }, + }), group(group), op(op), unit(unit), registry(registry), lowerBound(lower), upperBound(upper), ReferenceCounted() { - for(const auto & [ key, value ] : UnitToStringMapper){ - std::cout< Date: Thu, 24 Jun 2021 13:08:40 -0700 Subject: [PATCH 029/426] add drawHistogram method in histogram class --- fdbserver/VersionedBTree.actor.cpp | 8 +++-- flow/Histogram.cpp | 57 ++++++++++++++++++++++++++++++ flow/Histogram.h | 3 ++ 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 01b0fd6cce..1897f3488d 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -9752,8 +9752,9 @@ TEST_CASE(":/redwood/performance/histogramThroughput") { for(size_t i=0; isample(uniform[i]); } - GetHistogramRegistry().logReport(); auto t_end = std::chrono::high_resolution_clock::now(); + h->drawHistogram(); + GetHistogramRegistry().logReport(); double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); std::cout<<"Time in millisecond: "<sample((double)uniform[i]/UINT32_MAX); + h->samplePercentage((double)uniform[i]/UINT32_MAX); } - GetHistogramRegistry().logReport(); auto t_end = std::chrono::high_resolution_clock::now(); + h->drawHistogram(); + GetHistogramRegistry().logReport(); double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); std::cout<<"Time in millisecond: "< currHeight) std::cout << fullCell; + else if (pct > halfFullHeight) std::cout << halfCell; + else std::cout << emptyCell; + } + std::cout << lineEnd << "\n"; + } + + std::cout<<" 0.00 "< intervalSize/4) std::cout << xFull; + else std::cout << xEmpty; + } + std::cout << lineEnd << "\n"; + + std::cout << std::string(9, ' '); + for (int i = 0; i<32; i++){ + std::cout< #include #include +#include #ifdef _WIN32 #include @@ -157,6 +158,8 @@ public: std::string name() const { return generateName(this->group, this->op); } + void drawHistogram(); + std::string const group; std::string const op; Unit const unit; From c0b539c49546567ae57fc4c3a22c9ae9ed73d6cd Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Thu, 24 Jun 2021 13:24:48 -0700 Subject: [PATCH 030/426] initialize fields in redWoodMetrics struct explicitly --- fdbserver/VersionedBTree.actor.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 1897f3488d..84565b3b78 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1294,9 +1294,7 @@ struct RedwoodMetrics { RedwoodMetrics() { clear(); } void clear() { - std::cout<<"in clear"< kvSizeWritten; @@ -1407,8 +1411,6 @@ struct RedwoodMetrics { return pagerDiskWrite + pagerDiskRead + pagerCacheHit + pagerProbeHit; } - double startTime; - Level& level(unsigned int level) { static Level outOfBound; if (level == 0 || level > btreeLevels) { From 7b84b8fae6983eecf5eb89c8b805e7315778a946 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Thu, 24 Jun 2021 13:53:40 -0700 Subject: [PATCH 031/426] add redWoodMetrics::Level struct constructor --- fdbserver/VersionedBTree.actor.cpp | 31 ++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 84565b3b78..b5239ead25 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1298,12 +1298,12 @@ struct RedwoodMetrics { int levelCounter = 0; for (auto& level : levels) { level = { - .buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - .modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - .buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - .modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - .buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount), - .modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount) + Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), + Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount), + Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount) }; ++levelCounter; } @@ -1355,6 +1355,25 @@ struct RedwoodMetrics { Reference modifyStoredPctSketch; Reference buildItemCountSketch; Reference modifyItemCountSketch; + + Level( Reference a, + Reference b, + Reference c, + Reference d, + Reference e, + Reference f + ) : + pageRead(0), pageReadExt(0), pageBuild(0), pageBuildExt(0), pageCommitStart(0), pageModify(0), pageModifyExt(0), + lazyClearRequeue(0), lazyClearRequeueExt(0), lazyClearFree(0), lazyClearFreeExt(0), forceUpdate(0), detachChild(0), + buildStoredPct(0), buildFillPct(0), buildItemCount(0), modifyStoredPct(0), modifyFillPct(0), modifyItemCount(0), + buildFillPctSketch(a), modifyFillPctSketch(b), buildStoredPctSketch(c), modifyStoredPctSketch(d), buildItemCountSketch(e), modifyItemCountSketch(f) + {} + Level() : + pageRead(0), pageReadExt(0), pageBuild(0), pageBuildExt(0), pageCommitStart(0), pageModify(0), pageModifyExt(0), + lazyClearRequeue(0), lazyClearRequeueExt(0), lazyClearFree(0), lazyClearFreeExt(0), forceUpdate(0), detachChild(0), + buildStoredPct(0), buildFillPct(0), buildItemCount(0), modifyStoredPct(0), modifyFillPct(0), modifyItemCount(0), + buildFillPctSketch(), modifyFillPctSketch(), buildStoredPctSketch(), modifyStoredPctSketch(), buildItemCountSketch(), modifyItemCountSketch() + {} }; Level levels[btreeLevels]; From 1e0c233d4003235e12070866fdafabb0492367f9 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Thu, 24 Jun 2021 17:44:54 -0400 Subject: [PATCH 032/426] Add some documentation for the onMainThread functions --- flow/ThreadHelper.actor.h | 69 +++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 24 deletions(-) diff --git a/flow/ThreadHelper.actor.h b/flow/ThreadHelper.actor.h index 7bf87a57d4..263850d41d 100644 --- a/flow/ThreadHelper.actor.h +++ b/flow/ThreadHelper.actor.h @@ -33,17 +33,38 @@ #include "flow/flow.h" #include "flow/actorcompiler.h" // This must be the last #include. -// template -// void onMainThreadVoid( F f ) { -// Promise signal; -// doOnMainThreadVoid( signal.getFuture(), f ); -// g_network->onMainThread( std::move(signal), TaskPriority::DefaultOnMainThread ); -// } +// Helper actor. Do not use directly! +namespace internal_thread_helper { +ACTOR template +void doOnMainThreadVoid(Future signal, F f, Error* err) { + wait(signal); + if (err && err->code() != invalid_error_code) + return; + try { + f(); + } catch (Error& e) { + if (err) + *err = e; + } +} + +} // namespace internal_thread_helper + +// onMainThreadVoid runs a functor on the FDB network thread. The value returned by the functor is ignored. +// There is no way wait for the functor run to finish. For cases where you need a result back or simply need +// to know when the functor has finished running, use `onMainThread`. +// +// WARNING: Successive invocations of `onMainThreadVoid` with different task priorities may run out of order. +// +// WARNING: The error returned in `err` can only be read on the FDB network thread because there is no way to +// order the write to `err` with actions on other threads. +// +// `onMainThreadVoid` is defined here because of the dependency in `ThreadSingleAssignmentVarBase`. template -void onMainThreadVoid(F f, Error* err, TaskPriority taskID = TaskPriority::DefaultOnMainThread) { +void onMainThreadVoid(F f, Error* err = nullptr, TaskPriority taskID = TaskPriority::DefaultOnMainThread) { Promise signal; - doOnMainThreadVoid(signal.getFuture(), f, err); + internal_thread_helper::doOnMainThreadVoid(signal.getFuture(), f, err); g_network->onMainThread(std::move(signal), taskID); } @@ -318,8 +339,7 @@ public: [this]() { this->cancelFuture.cancel(); this->delref(); - }, - nullptr); + }); } void releaseMemory() { @@ -582,6 +602,9 @@ Future safeThreadFutureToFuture(ThreadFuture threadFuture) { return threadFuture.get(); } +// Helper actor. Do not use directly! +namespace internal_thread_helper { + ACTOR template Future doOnMainThread(Future signal, F f, ThreadSingleAssignmentVar* result) { try { @@ -600,26 +623,24 @@ Future doOnMainThread(Future signal, F f, ThreadSingleAssignmentVar< return Void(); } -ACTOR template -void doOnMainThreadVoid(Future signal, F f, Error* err) { - wait(signal); - if (err && err->code() != invalid_error_code) - return; - try { - f(); - } catch (Error& e) { - if (err) - *err = e; - } -} +} // namespace internal_thread_helper +// `onMainThread` runs a functor returning a `Future` on the main thread, waits for the future, and sends either the +// value returned from the waited `Future` or an error through the `ThreadFuture` returned from the function call. +// +// A workaround for cases where your functor returns a non-`Future` value is to wrap the value in an immediately +// filled `Future`. In cases where the functor returns void, a workaround is to return a `Future(true)` that +// can be waited on. +// +// TODO: Add SFINAE overloads for functors returning void or a non-Future type. template ThreadFuture()().getValue())> onMainThread(F f) { Promise signal; auto returnValue = new ThreadSingleAssignmentVar()().getValue())>(); returnValue->addref(); // For the ThreadFuture we return - Future cancelFuture = - doOnMainThread()().getValue()), F>(signal.getFuture(), f, returnValue); + // TODO: Is this cancellation logic actually needed? + Future cancelFuture = internal_thread_helper::doOnMainThread()().getValue()), F>( + signal.getFuture(), f, returnValue); returnValue->setCancel(std::move(cancelFuture)); g_network->onMainThread(std::move(signal), TaskPriority::DefaultOnMainThread); return ThreadFuture()().getValue())>(returnValue); From 3d83f2fbc8f7348483d8b4a0181208581a26a041 Mon Sep 17 00:00:00 2001 From: Xiaoxi Wang Date: Thu, 24 Jun 2021 23:36:59 +0000 Subject: [PATCH 033/426] add setByteLimit in tutorial --- documentation/tutorial/tutorial.actor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp index 50d6fbb7ac..87fee7f2ce 100644 --- a/documentation/tutorial/tutorial.actor.cpp +++ b/documentation/tutorial/tutorial.actor.cpp @@ -183,6 +183,7 @@ ACTOR Future echoServer() { req.reply.send(std::string(req.message.rbegin(), req.message.rend())); } when(state StreamRequest req = waitNext(echoServer.stream.getFuture())) { + req.reply.setByteLimit(1024); state int i = 0; for (; i < 100; ++i) { wait(req.reply.onReady()); From 66a97b76f12d7425b88fabbea42a0c4b83b591b3 Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Thu, 24 Jun 2021 21:20:21 -0700 Subject: [PATCH 034/426] Add metric to track the ratio of empty messages to tlogs This metric is to understand the impact of batching on empty commit messages. If the ratio is high, it means proxy sends many empty commit messages, which can potentially be optimized for better performance. If the ratio is low, which means the spread factor is large, thus we need to optimize the proxy to reduce such a factor. --- fdbserver/CommitProxyServer.actor.cpp | 3 +++ fdbserver/LogSystem.h | 20 +++++++++++++++++++- fdbserver/ProxyCommitData.actor.h | 7 +++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index c88d4c054b..cd69a79f92 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -1186,6 +1186,9 @@ ACTOR Future postResolution(CommitBatchContext* self) { span.context, self->debugID); + float ratio = self->toCommit.getEmptyLocationRatio(); + pProxyCommitData->stats.commitBatchingEmptyMessageRatio.addMeasurement(ratio); + if (!self->forceRecovery) { ASSERT(pProxyCommitData->latestLocalCommitBatchLogging.get() == self->localBatchNumber - 1); pProxyCommitData->latestLocalCommitBatchLogging.set(self->localBatchNumber); diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index da2fbcf5f2..39d4a16871 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -970,6 +970,7 @@ struct LogPushData : NonCopyable { } } } + written = std::vector(messagesWriter.size(), false); } void addTxsTag() { @@ -1087,13 +1088,30 @@ struct LogPushData : NonCopyable { next_message_tags.clear(); } - Standalone getMessages(int loc) { return messagesWriter[loc].toValue(); } + Standalone getMessages(int loc) { + // Update written here because this is called less frequently. + Standalone value = messagesWriter[loc].toValue(); + if (!written[loc]) { + BinaryWriter w(AssumeVersion(g_network->protocolVersion())); + Standalone v = w.toValue(); + if (value.size() > v.size()) { + written[loc] = true; + } + } + return value; + } + + float getEmptyLocationRatio() const { + auto count = std::count(written.begin(), written.end(), false); + return 1.0 * count / written.size(); + } private: Reference logSystem; std::vector next_message_tags; std::vector prev_tags; std::vector messagesWriter; + std::vector written; // if messagesWriter has written anything std::vector msg_locations; // Stores message locations that have had span information written to them // for the current transaction. Adding transaction info will reset this diff --git a/fdbserver/ProxyCommitData.actor.h b/fdbserver/ProxyCommitData.actor.h index 9ef0f83778..99d210bc6e 100644 --- a/fdbserver/ProxyCommitData.actor.h +++ b/fdbserver/ProxyCommitData.actor.h @@ -64,6 +64,9 @@ struct ProxyStats { LatencySample commitLatencySample; LatencyBands commitLatencyBands; + // Ratio of tlogs receiving empty commit messages. + LatencySample commitBatchingEmptyMessageRatio; + LatencySample commitBatchingWindowSize; Future logger; @@ -102,6 +105,10 @@ struct ProxyStats { SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL, SERVER_KNOBS->LATENCY_SAMPLE_SIZE), commitLatencyBands("CommitLatencyMetrics", id, SERVER_KNOBS->STORAGE_LOGGING_DELAY), + commitBatchingEmptyMessageRatio("CommitBatchingEmptyMessageRatio", + id, + SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL, + SERVER_KNOBS->LATENCY_SAMPLE_SIZE), commitBatchingWindowSize("CommitBatchingWindowSize", id, SERVER_KNOBS->LATENCY_METRICS_LOGGING_INTERVAL, From 5529c96e51d0121efd8bae06dadcaaa7c7f04da5 Mon Sep 17 00:00:00 2001 From: Daniel Smith Date: Thu, 24 Jun 2021 16:26:16 -0400 Subject: [PATCH 035/426] Clean up a few memory leaks --- fdbmonitor/fdbmonitor.cpp | 44 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp index 2c97b7a1d8..3f413cd47d 100644 --- a/fdbmonitor/fdbmonitor.cpp +++ b/fdbmonitor/fdbmonitor.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include #include @@ -371,7 +372,7 @@ private: fdb_fd_set fds; public: - const char** argv; + char** argv; std::string section, ssection; uint32_t initial_restart_delay; uint32_t max_restart_delay; @@ -534,7 +535,7 @@ public: commands.push_back(std::string("--").append(i.pItem).append("=").append(opt)); } - argv = new const char*[commands.size() + 1]; + argv = new char*[commands.size() + 1]; int i = 0; for (auto itr : commands) { argv[i++] = strdup(itr.c_str()); @@ -542,6 +543,9 @@ public: argv[i] = nullptr; } ~Command() { + for (int i = 0; i < commands.size(); ++i) { + free(argv[i]); + } delete[] argv; for (auto p : pipes) { if (p[0] >= 0 && p[1] >= 0) { @@ -589,7 +593,7 @@ public: } }; -std::unordered_map id_command; +std::unordered_map> id_command; std::unordered_map pid_id; std::unordered_map id_pid; @@ -819,7 +823,6 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb } for (auto i : kill_ids) { kill_process(i); - delete id_command[i]; id_command.erase(i); } } @@ -840,28 +843,26 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb if (id_command[i.first]->kill_on_configuration_change) { kill_ids.push_back(i.first); - delete id_command[i.first]; id_command.erase(i.first); } } else { - Command* cmd = new Command(ini, id_command[i.first]->section, i.first, rfds, maxfd); + auto cmd = std::make_unique(ini, id_command[i.first]->section, i.first, rfds, maxfd); // If we just turned on 'kill_on_configuration_change', then kill the process to make sure we pick up any of // its pending config changes if (*(id_command[i.first]) != *cmd || (cmd->kill_on_configuration_change && !id_command[i.first]->kill_on_configuration_change)) { log_msg(SevInfo, "Found new configuration for %s\n", id_command[i.first]->ssection.c_str()); - delete id_command[i.first]; - id_command[i.first] = cmd; + auto* c = cmd.get(); + id_command[i.first] = std::move(cmd); - if (id_command[i.first]->kill_on_configuration_change) { + if (c->kill_on_configuration_change) { kill_ids.push_back(i.first); - start_ids.emplace_back(i.first, cmd); + start_ids.emplace_back(i.first, c); } } else { log_msg(SevInfo, "Updated configuration for %s\n", id_command[i.first]->ssection.c_str()); id_command[i.first]->update(*cmd); - delete cmd; } } } @@ -893,11 +894,12 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb auto itr = id_command.find(id); if (itr != id_command.end()) { - cmd = itr->second; + cmd = itr->second.get(); } else { std::string section(i.pItem, dot - i.pItem); - cmd = new Command(ini, section, id, rfds, maxfd); - id_command[id] = cmd; + auto p = std::make_unique(ini, section, id, rfds, maxfd); + cmd = p.get(); + id_command[id] = std::move(p); } if (cmd->fork_retry_time <= timer()) { @@ -1271,13 +1273,14 @@ int main(int argc, char** argv) { // Guaranteed (if non-nullptr) to be an absolute path with no // symbolic link, /./ or /../ components - const char* p = realpath(_confpath.c_str(), nullptr); + char* p = realpath(_confpath.c_str(), nullptr); if (!p) { log_msg(SevError, "No configuration file at %s\n", _confpath.c_str()); exit(1); } std::string confpath = p; + free(p); // Will always succeed given an absolute path std::string confdir = parentDirectory(confpath, false); @@ -1499,7 +1502,7 @@ int main(int argc, char** argv) { } double end_time = std::numeric_limits::max(); - for (auto i : id_command) { + for (auto& i : id_command) { if (i.second->fork_retry_time >= 0) { end_time = std::min(i.second->fork_retry_time, end_time); } @@ -1590,7 +1593,7 @@ int main(int argc, char** argv) { if (exit_signal > 0) { switch (exit_signal) { case SIGHUP: - for (auto i : id_command) { + for (auto& i : id_command) { i.second->current_restart_delay = i.second->initial_restart_delay; i.second->fork_retry_time = -1; } @@ -1643,10 +1646,10 @@ int main(int argc, char** argv) { char buf[4096]; - for (auto itr : id_command) { + for (auto& itr : id_command) { for (int i = 0; i < 2; i++) { if (FD_ISSET((itr.second)->pipes[i][0], &srfds)) { - read_child_output(itr.second, i, watched_fds); + read_child_output(itr.second.get(), i, watched_fds); } } } @@ -1722,13 +1725,12 @@ int main(int argc, char** argv) { } uint64_t id = pid_id[pid]; - Command* cmd = id_command[id]; + Command* cmd = id_command[id].get(); pid_id.erase(pid); id_pid.erase(id); if (cmd->deconfigured) { - delete cmd; id_command.erase(id); } else { int delay = cmd->get_and_update_current_restart_delay(); From 5858ca3c62b7d6fbc9df3892b77b068cb122a5b9 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 25 Jun 2021 12:23:08 -0700 Subject: [PATCH 036/426] Use unit test data directory for fdbrpc/AsyncFileEncrypted test --- fdbrpc/AsyncFileEncrypted.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 4fda5a77e4..0589f68a76 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -245,7 +245,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") { int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO; - state Reference file = wait(IAsyncFileSystem::filesystem()->open("/tmp/test", flags, 0600)); + state Reference file = wait(IAsyncFileSystem::filesystem()->open(params.getDataDir(), flags, 0600)); state int bytesWritten = 0; while (bytesWritten < bytes) { chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten); From 53f5cd24534eb397afcd4651b8673c318df48f61 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 25 Jun 2021 14:11:21 -0700 Subject: [PATCH 037/426] Support passing encryption file to BackupContainer::openContainer --- fdbclient/BackupAgent.actor.h | 13 +++++--- fdbclient/BackupContainer.actor.cpp | 5 +-- fdbclient/BackupContainer.h | 3 +- .../BackupContainerLocalDirectory.actor.cpp | 31 ++++++++++++++++++- fdbclient/BackupContainerLocalDirectory.h | 4 ++- fdbclient/FileBackupAgent.actor.cpp | 18 ++++++----- fdbrpc/AsyncFileEncrypted.actor.cpp | 5 +-- fdbrpc/AsyncFileEncrypted.h | 5 +-- flow/StreamCipher.cpp | 23 ++++++++++++-- flow/StreamCipher.h | 10 +++--- 10 files changed, 86 insertions(+), 31 deletions(-) diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index c8903b9fe4..a0222fbf65 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -362,20 +362,22 @@ public: Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, bool stopWhenDone = true, bool partitionedLog = false, - bool incrementalBackupOnly = false); + bool incrementalBackupOnly = false, + Optional const& encryptionKeyFileName = {}); Future submitBackup(Database cx, Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, bool stopWhenDone = true, bool partitionedLog = false, - bool incrementalBackupOnly = false) { + bool incrementalBackupOnly = false, + Optional const& encryptionKeyFileName = {}) { return runRYWTransactionFailIfLocked(cx, [=](Reference tr) { return submitBackup(tr, outContainer, @@ -385,7 +387,8 @@ public: backupRanges, stopWhenDone, partitionedLog, - incrementalBackupOnly); + incrementalBackupOnly, + encryptionKeyFileName); }); } diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a71ab0c6ff..56357a1467 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -253,7 +253,8 @@ std::vector IBackupContainer::getURLFormats() { } // Get an IBackupContainer based on a container URL string -Reference IBackupContainer::openContainer(const std::string& url) { +Reference IBackupContainer::openContainer(const std::string& url, + Optional const& encryptionKeyFileName) { static std::map> m_cache; Reference& r = m_cache[url]; @@ -263,7 +264,7 @@ Reference IBackupContainer::openContainer(const std::string& u try { StringRef u(url); if (u.startsWith(LiteralStringRef("file://"))) { - r = Reference(new BackupContainerLocalDirectory(url)); + r = makeReference(url, encryptionKeyFileName); } else if (u.startsWith(LiteralStringRef("blobstore://"))) { std::string resource; diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index 2da1e50985..5a9af3d1d9 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -293,7 +293,8 @@ public: Version beginVersion = -1) = 0; // Get an IBackupContainer based on a container spec string - static Reference openContainer(const std::string& url); + static Reference openContainer(const std::string& url, + const Optional& encryptionKeyFileName = {}); static std::vector getURLFormats(); static Future> listContainers(const std::string& baseURL); diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp index e0c78a31bf..d94c92441a 100644 --- a/fdbclient/BackupContainerLocalDirectory.actor.cpp +++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp @@ -23,6 +23,7 @@ #include "fdbrpc/IAsyncFile.h" #include "flow/Platform.actor.h" #include "flow/Platform.h" +#include "flow/StreamCipher.h" #include "fdbrpc/simulator.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -131,7 +132,29 @@ std::string BackupContainerLocalDirectory::getURLFormat() { return "file://"; } -BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url) { +ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { + state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + int64_t fileSize = wait(keyFile->size()); + // TODO: Use new error code and avoid hard-coding expected size + if (fileSize != 16) { + throw internal_error(); + } + state std::array key; + wait(success(keyFile->read(key.data(), key.size(), 0))); + StreamCipher::Key::initializeKey(std::move(key)); + return Void(); +} + +bool BackupContainerLocalDirectory::usesEncryption() const { + return encryptionSetupFuture.isValid(); +} + +BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url, + const Optional& encryptionKeyFileName) { + if (encryptionKeyFileName.present()) { + encryptionSetupFuture = readEncryptionKey(encryptionKeyFileName.get()); + } + std::string path; if (url.find("file://") != 0) { TraceEvent(SevWarn, "BackupContainerLocalDirectory") @@ -207,6 +230,9 @@ Future BackupContainerLocalDirectory::exists() { Future> BackupContainerLocalDirectory::readFile(const std::string& path) { int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED; + if (usesEncryption()) { + flags |= IAsyncFile::OPEN_ENCRYPTED; + } // Simulation does not properly handle opening the same file from multiple machines using a shared filesystem, // so create a symbolic link to make each file opening appear to be unique. This could also work in production // but only if the source directory is writeable which shouldn't be required for a restore. @@ -260,6 +286,9 @@ Future> BackupContainerLocalDirectory::readFile(const std: Future> BackupContainerLocalDirectory::writeFile(const std::string& path) { int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; + if (usesEncryption()) { + flags |= IAsyncFile::OPEN_ENCRYPTED; + } std::string fullPath = joinPath(m_path, path); platform::createDirectory(parentDirectory(fullPath)); std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp"; diff --git a/fdbclient/BackupContainerLocalDirectory.h b/fdbclient/BackupContainerLocalDirectory.h index 9db8e07aef..52cd810907 100644 --- a/fdbclient/BackupContainerLocalDirectory.h +++ b/fdbclient/BackupContainerLocalDirectory.h @@ -33,7 +33,7 @@ public: static std::string getURLFormat(); - BackupContainerLocalDirectory(const std::string& url); + BackupContainerLocalDirectory(const std::string& url, Optional const& encryptionKeyFileName); static Future> listURLs(const std::string& url); @@ -54,6 +54,8 @@ public: private: std::string m_path; + Future encryptionSetupFuture; + bool usesEncryption() const; }; #endif diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 35d6743821..a730822f96 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -4506,6 +4506,7 @@ public: } } + // TODO: Get rid of all of these confusing boolean flags ACTOR static Future submitBackup(FileBackupAgent* backupAgent, Reference tr, Key outContainer, @@ -4515,7 +4516,8 @@ public: Standalone> backupRanges, bool stopWhenDone, bool partitionedLog, - bool incrementalBackupOnly) { + bool incrementalBackupOnly, + Optional encryptionKeyFileName) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); @@ -4553,7 +4555,7 @@ public: backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString()); } - state Reference bc = IBackupContainer::openContainer(backupContainer); + state Reference bc = IBackupContainer::openContainer(backupContainer, encryptionKeyFileName); try { wait(timeoutError(bc->create(), 30)); } catch (Error& e) { @@ -5631,11 +5633,12 @@ Future FileBackupAgent::submitBackup(Reference Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, bool stopWhenDone, bool partitionedLog, - bool incrementalBackupOnly) { + bool incrementalBackupOnly, + Optional const& encryptionKeyFileName) { return FileBackupAgentImpl::submitBackup(this, tr, outContainer, @@ -5645,7 +5648,8 @@ Future FileBackupAgent::submitBackup(Reference backupRanges, stopWhenDone, partitionedLog, - incrementalBackupOnly); + incrementalBackupOnly, + encryptionKeyFileName); } Future FileBackupAgent::discontinueBackup(Reference tr, Key tagName) { @@ -5739,8 +5743,8 @@ ACTOR static Future writeKVs(Database cx, Standalone readBuffer(bytes, 0); ASSERT(g_network->isSimulated()); - StreamCipher::Key::initializeRandomKey(); + StreamCipher::Key::initializeRandomTestKey(); int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_NO_AIO; - state Reference file = wait(IAsyncFileSystem::filesystem()->open(params.getDataDir(), flags, 0600)); + state Reference file = + wait(IAsyncFileSystem::filesystem()->open(joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600)); state int bytesWritten = 0; while (bytesWritten < bytes) { chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten); diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index dc7f15f299..b474198e1c 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -18,8 +18,7 @@ * limitations under the License. */ -#ifndef __FDBRPC_ASYNC_FILE_ENCRYPTED_H__ -#define __FDBRPC_ASYNC_FILE_ENCRYPTED_H__ +#pragma once #include "fdbrpc/IAsyncFile.h" #include "flow/FastRef.h" @@ -75,5 +74,3 @@ public: void releaseZeroCopy(void* data, int length, int64_t offset) override; int64_t debugFD() const override; }; - -#endif diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index add9c8988a..e0066ae4a5 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -44,11 +44,18 @@ void StreamCipher::cleanup() noexcept { } } -void StreamCipher::Key::initializeRandomKey() { +void StreamCipher::Key::initializeKey(std::array&& arr) { + ASSERT(!globalKey); + globalKey = std::make_unique(ConstructorTag{}); + globalKey->arr = std::move(arr); + memset(arr.data(), 0, arr.size()); +} + +void StreamCipher::Key::initializeRandomTestKey() { ASSERT(g_network->isSimulated()); if (globalKey) return; globalKey = std::make_unique(ConstructorTag{}); - generateRandomData(globalKey.get()->arr.data(), globalKey.get()->arr.size()); + generateRandomData(globalKey->arr.data(), globalKey->arr.size()); } const StreamCipher::Key& StreamCipher::Key::getKey() { @@ -56,6 +63,16 @@ const StreamCipher::Key& StreamCipher::Key::getKey() { return *globalKey; } +StreamCipher::Key::Key(Key&& rhs) : arr(std::move(rhs.arr)) { + memset(arr.data(), 0, arr.size()); +} + +StreamCipher::Key& StreamCipher::Key::operator=(Key&& rhs) { + arr = std::move(rhs.arr); + memset(arr.data(), 0, arr.size()); + return *this; +} + StreamCipher::Key::~Key() { memset(arr.data(), 0, arr.size()); } @@ -111,7 +128,7 @@ void forceLinkStreamCipherTests() {} // Tests both encryption and decryption of random data // using the StreamCipher class TEST_CASE("flow/StreamCipher") { - StreamCipher::Key::initializeRandomKey(); + StreamCipher::Key::initializeRandomTestKey(); const auto& key = StreamCipher::Key::getKey(); StreamCipher::IV iv; diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index 0618b7585f..f91dc272ce 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -18,8 +18,7 @@ * limitations under the License. */ -#ifndef __FLOW_STREAM_CIPHER_H__ -#define __FLOW_STREAM_CIPHER_H__ +#pragma once #include "flow/Arena.h" #include "flow/FastRef.h" @@ -48,9 +47,12 @@ public: public: Key(ConstructorTag) {} + Key(Key&&); + Key& operator=(Key&&); ~Key(); unsigned char const* data() const { return arr.data(); } - static void initializeRandomKey(); + static void initializeKey(decltype(arr)&&); + static void initializeRandomTestKey(); static const Key& getKey(); static void cleanup() noexcept; }; @@ -75,5 +77,3 @@ public: StringRef decrypt(unsigned char const* ciphertext, int len, Arena&); StringRef finish(Arena&); }; - -#endif From f5aa3df917b90363fd1324b34bfd720ec0af085b Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 25 Jun 2021 15:11:03 -0700 Subject: [PATCH 038/426] Add --encryption_key_file command line argument to fdbbackup and fdbrestore --- fdbbackup/backup.actor.cpp | 66 +++++++++++++++++++---------- fdbclient/BackupAgent.actor.h | 13 +++--- fdbclient/FileBackupAgent.actor.cpp | 6 ++- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 04fc42cded..f69b731260 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -133,6 +133,7 @@ enum { OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER, OPT_INCREMENTALONLY, + OPT_ENCRYPTION_KEY_FILE, // Backup Modify OPT_MOD_ACTIVE_INTERVAL, @@ -259,6 +260,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = { { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -695,6 +697,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, { OPT_RESTORE_BEGIN_VERSION, "--begin_version", SO_REQ_SEP }, { OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent_snapshot_only", SO_NONE }, #ifndef TLS_DISABLED @@ -1089,6 +1092,8 @@ static void printBackupUsage(bool devhelp) { " Performs incremental backup without the base backup.\n" " This option indicates to the backup agent that it will only need to record the log files, " "and ignore the range files.\n"); + printf(" --encryption_key_file" + " The AES-128-GCM key in the provided file is used for encrypting backup files.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1162,6 +1167,8 @@ static void printRestoreUsage(bool devhelp) { " To be used in conjunction with incremental restore.\n" " Indicates to the backup agent to only begin replaying log files from a certain version, " "instead of the entire set.\n"); + printf(" --encryption_key_file" + " The AES-128-GCM key in the provided file is used for decrypting backup files.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -2220,7 +2227,9 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause return Void(); } -Reference openBackupContainer(const char* name, std::string destinationContainer) { +Reference openBackupContainer(const char* name, + std::string destinationContainer, + Optional const& encryptionKeyFile = {}) { // Error, if no dest container was specified if (destinationContainer.empty()) { fprintf(stderr, "ERROR: No backup destination was specified.\n"); @@ -2230,7 +2239,7 @@ Reference openBackupContainer(const char* name, std::string de Reference c; try { - c = IBackupContainer::openContainer(destinationContainer); + c = IBackupContainer::openContainer(destinationContainer, encryptionKeyFile); } catch (Error& e) { std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { @@ -2259,8 +2268,9 @@ ACTOR Future runRestore(Database db, bool waitForDone, std::string addPrefix, std::string removePrefix, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly) { + bool onlyApplyMutationLogs, + bool inconsistentSnapshotOnly, + Optional encryptionKeyFile) { if (ranges.empty()) { ranges.push_back_deep(ranges.arena(), normalKeys); } @@ -2296,7 +2306,8 @@ ACTOR Future runRestore(Database db, try { state FileBackupAgent backupAgent; - state Reference bc = openBackupContainer(exeRestore.toString().c_str(), container); + state Reference bc = + openBackupContainer(exeRestore.toString().c_str(), container, encryptionKeyFile); // If targetVersion is unset then use the maximum restorable version from the backup description if (targetVersion == invalidVersion) { @@ -2306,7 +2317,7 @@ ACTOR Future runRestore(Database db, BackupDescription desc = wait(bc->describeBackup()); - if (onlyAppyMutationLogs && desc.contiguousLogEnd.present()) { + if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { targetVersion = desc.contiguousLogEnd.get() - 1; } else if (desc.maxRestorableVersion.present()) { targetVersion = desc.maxRestorableVersion.get(); @@ -2331,9 +2342,10 @@ ACTOR Future runRestore(Database db, KeyRef(addPrefix), KeyRef(removePrefix), true, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, - beginVersion)); + beginVersion, + encryptionKeyFile)); if (waitForDone && verbose) { // If restore is now complete then report version restored @@ -2478,8 +2490,9 @@ ACTOR Future runFastRestoreTool(Database db, ACTOR Future dumpBackupData(const char* name, std::string destinationContainer, Version beginVersion, - Version endVersion) { - state Reference c = openBackupContainer(name, destinationContainer); + Version endVersion, + Optional encryptionKeyFile) { + state Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); if (beginVersion < 0 || endVersion < 0) { BackupDescription desc = wait(c->describeBackup()); @@ -2512,7 +2525,8 @@ ACTOR Future expireBackupData(const char* name, Database db, bool force, Version restorableAfterVersion, - std::string restorableAfterDatetime) { + std::string restorableAfterDatetime, + Optional encryptionKeyFile) { if (!endDatetime.empty()) { Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db)); endVersion = v; @@ -2531,7 +2545,7 @@ ACTOR Future expireBackupData(const char* name, } try { - Reference c = openBackupContainer(name, destinationContainer); + Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); state IBackupContainer::ExpireProgress progress; state std::string lastProgress; @@ -2613,9 +2627,10 @@ ACTOR Future describeBackup(const char* name, std::string destinationContainer, bool deep, Optional cx, - bool json) { + bool json, + Optional encryptionKeyFile) { try { - Reference c = openBackupContainer(name, destinationContainer); + Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); state BackupDescription desc = wait(c->describeBackup(deep)); if (cx.present()) wait(desc.resolveVersionTimes(cx.get())); @@ -3248,7 +3263,7 @@ int main(int argc, char* argv[]) { bool stopWhenDone = true; bool usePartitionedLog = false; // Set to true to use new backup system bool incrementalBackupOnly = false; - bool onlyAppyMutationLogs = false; + bool onlyApplyMutationLogs = false; bool inconsistentSnapshotOnly = false; bool forceAction = false; bool trace = false; @@ -3272,6 +3287,7 @@ int main(int argc, char* argv[]) { std::string restoreClusterFileOrig; bool jsonOutput = false; bool deleteData = false; + Optional encryptionKeyFile; BackupModifyOptions modifyOptions; @@ -3513,7 +3529,10 @@ int main(int argc, char* argv[]) { break; case OPT_INCREMENTALONLY: incrementalBackupOnly = true; - onlyAppyMutationLogs = true; + onlyApplyMutationLogs = true; + break; + case OPT_ENCRYPTION_KEY_FILE: + encryptionKeyFile = args->OptionArg(); break; case OPT_RESTORECONTAINER: restoreContainer = args->OptionArg(); @@ -3853,7 +3872,7 @@ int main(int argc, char* argv[]) { if (!initCluster()) return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. - openBackupContainer(argv[0], destinationContainer); + openBackupContainer(argv[0], destinationContainer, encryptionKeyFile); f = stopAfter(submitBackup(db, destinationContainer, initialSnapshotIntervalSeconds, @@ -3932,7 +3951,8 @@ int main(int argc, char* argv[]) { db, forceAction, expireRestorableAfterVersion, - expireRestorableAfterDatetime)); + expireRestorableAfterDatetime, + encryptionKeyFile)); break; case BackupType::DELETE_BACKUP: @@ -3952,7 +3972,8 @@ int main(int argc, char* argv[]) { destinationContainer, describeDeep, describeTimestamps ? Optional(db) : Optional(), - jsonOutput)); + jsonOutput, + encryptionKeyFile)); break; case BackupType::LIST: @@ -3973,7 +3994,7 @@ int main(int argc, char* argv[]) { case BackupType::DUMP: initTraceFile(); - f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd)); + f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd, encryptionKeyFile)); break; case BackupType::UNDEFINED: @@ -4033,8 +4054,9 @@ int main(int argc, char* argv[]) { waitForDone, addPrefix, removePrefix, - onlyAppyMutationLogs, - inconsistentSnapshotOnly)); + onlyApplyMutationLogs, + inconsistentSnapshotOnly, + encryptionKeyFile)); break; case RestoreType::WAIT: f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), true))); diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index a0222fbf65..b3cbff7bb3 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -289,20 +289,21 @@ public: Key url, Standalone> ranges, bool waitForComplete = true, - Version targetVersion = -1, + Version targetVersion = ::invalidVersion, bool verbose = true, Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true, bool onlyAppyMutationLogs = false, bool inconsistentSnapshotOnly = false, - Version beginVersion = -1); + Version beginVersion = ::invalidVersion, + Optional const& encryptionKeyFileName = {}); Future restore(Database cx, Optional cxOrig, Key tagName, Key url, bool waitForComplete = true, - Version targetVersion = -1, + Version targetVersion = ::invalidVersion, bool verbose = true, KeyRange range = normalKeys, Key addPrefix = Key(), @@ -310,7 +311,8 @@ public: bool lockDB = true, bool onlyAppyMutationLogs = false, bool inconsistentSnapshotOnly = false, - Version beginVersion = -1) { + Version beginVersion = ::invalidVersion, + Optional const& encryptionKeyFileName = {}) { Standalone> rangeRef; rangeRef.push_back_deep(rangeRef.arena(), range); return restore(cx, @@ -326,7 +328,8 @@ public: lockDB, onlyAppyMutationLogs, inconsistentSnapshotOnly, - beginVersion); + beginVersion, + encryptionKeyFileName); } Future atomicRestore(Database cx, Key tagName, diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index a730822f96..926bbc8886 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -5312,6 +5312,7 @@ public: bool onlyAppyMutationLogs, bool inconsistentSnapshotOnly, Version beginVersion, + Optional encryptionKeyFileName, UID randomUid) { // The restore command line tool won't allow ranges to be empty, but correctness workloads somehow might. if (ranges.empty()) { @@ -5525,6 +5526,7 @@ public: false, false, invalidVersion, + {}, randomUid)); return ver; } @@ -5586,7 +5588,8 @@ Future FileBackupAgent::restore(Database cx, bool lockDB, bool onlyAppyMutationLogs, bool inconsistentSnapshotOnly, - Version beginVersion) { + Version beginVersion, + Optional const& encryptionKeyFileName) { return FileBackupAgentImpl::restore(this, cx, cxOrig, @@ -5602,6 +5605,7 @@ Future FileBackupAgent::restore(Database cx, onlyAppyMutationLogs, inconsistentSnapshotOnly, beginVersion, + encryptionKeyFileName, deterministicRandom()->randomUniqueID()); } From 9c945253f65cb6896172c6448ad962f896489c52 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Fri, 25 Jun 2021 16:12:38 -0700 Subject: [PATCH 039/426] update redwoodmetrics structure; need to fix bug --- fdbserver/IPager.h | 14 +- fdbserver/VersionedBTree.actor.cpp | 588 +++++++++++++++-------------- flow/Histogram.cpp | 34 +- flow/Histogram.h | 4 +- 4 files changed, 339 insertions(+), 301 deletions(-) diff --git a/fdbserver/IPager.h b/fdbserver/IPager.h index e72e730e71..99e21db861 100644 --- a/fdbserver/IPager.h +++ b/fdbserver/IPager.h @@ -40,8 +40,10 @@ typedef uint32_t PhysicalPageID; typedef uint32_t QueueID; #define invalidQueueID std::numeric_limits::max() +// Pager Events +enum class events{ pagerCacheLookup = 0, pagerCacheHit, pagerCacheMiss, pagerWrite, MAXEVENTS}; // Reasons for page levle events. -enum class pagerEventReasons{ pointRead, rangeRead, rangePrefetch, commit, lazyClear, metaData}; +enum class pagerEventReasons{ pointRead, rangeRead, rangePrefetch, commit, lazyClear, metaData, MAXEVENTREASONS}; // Represents a block of memory in a 4096-byte aligned location held by an Arena. class ArenaPage : public ReferenceCounted, public FastAllocated { @@ -131,7 +133,7 @@ public: class IPagerSnapshot { public: - virtual Future> getPhysicalPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable, bool nohit) = 0; + virtual Future> getPhysicalPage(pagerEventReasons r, unsigned int l, LogicalPageID pageID, bool cacheable, bool nohit) = 0; virtual bool tryEvictPage(LogicalPageID id) = 0; virtual Version getVersion() const = 0; @@ -167,13 +169,13 @@ public: // Replace the contents of a page with new data across *all* versions. // Existing holders of a page reference for pageID, read from any version, // may see the effects of this write. - virtual void updatePage(LogicalPageID pageID, Reference data) = 0; + virtual void updatePage(pagerEventReasons r, unsigned int l, LogicalPageID pageID, Reference data) = 0; // Try to atomically update the contents of a page as of version v in the next commit. // If the pager is unable to do this at this time, it may choose to write the data to a new page ID // instead and return the new page ID to the caller. Otherwise the original pageID argument will be returned. // If a new page ID is returned, the old page ID will be freed as of version v - virtual Future atomicUpdatePage(LogicalPageID pageID, Reference data, Version v) = 0; + virtual Future atomicUpdatePage(unsigned int l, LogicalPageID pageID, Reference data, Version v) = 0; // Free pageID to be used again after the commit that moves oldestVersion past v virtual void freePage(LogicalPageID pageID, Version v) = 0; @@ -191,8 +193,8 @@ public: // Cacheable indicates that the page should be added to the page cache (if applicable?) as a result of this read. // NoHit indicates that the read should not be considered a cache hit, such as when preloading pages that are // considered likely to be needed soon. - virtual Future> readPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable = true, bool noHit = false) = 0; - virtual Future> readExtent(pagerEventReasons r, LogicalPageID pageID) = 0; + virtual Future> readPage(pagerEventReasons r, unsigned int l, LogicalPageID pageID, bool cacheable = true, bool noHit = false) = 0; + virtual Future> readExtent(pagerEventReasons r, unsigned int l, LogicalPageID pageID) = 0; virtual void releaseExtentReadLock() = 0; // Temporary methods for testing diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index b5239ead25..8a28b5d690 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -469,13 +469,13 @@ public: nextPageID = id; debug_printf( "FIFOQueue::Cursor(%s) loadPage start id=%s\n", toString().c_str(), ::toString(nextPageID).c_str()); - nextPageReader = waitOrError(queue->pager->readPage(pagerEventReasons::rangePrefetch, nextPageID, true), queue->pagerError); + nextPageReader = waitOrError(queue->pager->readPage(pagerEventReasons::metaData, 0, nextPageID, true), queue->pagerError); // check if 0 is correct } Future loadExtent() { ASSERT(mode == POP | mode == READONLY); debug_printf("FIFOQueue::Cursor(%s) loadExtent\n", toString().c_str()); - return map(queue->pager->readExtent(pagerEventReasons::metaData, pageID), [=](Reference p) { + return map(queue->pager->readExtent(pagerEventReasons::metaData, 0, pageID), [=](Reference p) { page = p; debug_printf("FIFOQueue::Cursor(%s) loadExtent done. Page: %p\n", toString().c_str(), page->begin()); return Void(); @@ -487,7 +487,7 @@ public: debug_printf("FIFOQueue::Cursor(%s) writePage\n", toString().c_str()); VALGRIND_MAKE_MEM_DEFINED(raw()->begin(), offset); VALGRIND_MAKE_MEM_DEFINED(raw()->begin() + offset, queue->dataBytesPerPage - raw()->endOffset); - queue->pager->updatePage(pageID, page); + queue->pager->updatePage(pagerEventReasons::commit, 0, pageID, page); // check if the level is 0 if (firstPageIDWritten == invalidLogicalPageID) { firstPageIDWritten = pageID; } @@ -1291,64 +1291,50 @@ struct RedwoodMetrics { static constexpr int btreeLevels = 5; static constexpr int maxRecordCount = 315; - RedwoodMetrics() { clear(); } + struct eventReasonsArray{ + unsigned int eventReasons[(size_t)events::MAXEVENTS][(size_t)pagerEventReasons::MAXEVENTREASONS]; - void clear() { - //memset(this, 0, sizeof(RedwoodMetrics)); - int levelCounter = 0; - for (auto& level : levels) { - level = { - Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage), - Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount), - Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount) - }; - ++levelCounter; - } - opSet = 0, opSetKeyBytes = 0, opSetValueBytes = 0, opClear = 0, opClearKey = 0, opCommit = 0, opGet = 0, opGetRange = 0; - pagerDiskWrite = 0, pagerDiskRead = 0, pagerRemapFree = 0, pagerRemapCopy = 0, pagerRemapSkip = 0; - pagerCacheHit = 0, pagerCacheMiss = 0, pagerProbeHit = 0, pagerProbeMiss = 0, pagerEvictUnhit = 0, pagerEvictFail = 0; - btreeLeafPreload = 0, btreeLeafPreloadExt = 0; - - const events eventsVector[] = {events::pagerCacheLookup, events::pagerCacheHit, events::pagerCacheMiss, events::pagerWrite}; - const pagerEventReasons reasonsVector[] = {pagerEventReasons::pointRead, pagerEventReasons::rangeRead, pagerEventReasons::rangePrefetch, pagerEventReasons::commit, pagerEventReasons::lazyClear, pagerEventReasons::metaData}; - for(events e : eventsVector){ - for(pagerEventReasons r: reasonsVector){ - eventsReasons[getIndex(e)][getIndex(r)] = 0; + eventReasonsArray(){clear();} + void clear(){ + for(size_t i = 0; i<(size_t)events::MAXEVENTS; i++){ + for(size_t j = 0; j<(size_t)pagerEventReasons::MAXEVENTREASONS; j++){ + eventReasons[i][j] = 0; + } } } - kvSizeWritten = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("Written"), Histogram::Unit::bytes); - kvSizeReadByGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByGet "), Histogram::Unit::bytes); - kvSizeReadByRangeGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByRangeGet"), Histogram::Unit::bytes); - startTime = g_network ? now() : 0; - } + void addEventReason(events e, pagerEventReasons r){ + eventReasons[(size_t)e][(size_t)r] += 1; + } + const unsigned int& getEventReason(events e, pagerEventReasons r){ + return eventReasons[(size_t)e][(size_t)r]; + } + }; // Page levle events - enum class events{ pagerCacheLookup, pagerCacheHit, pagerCacheMiss, pagerWrite, Update, Build}; - struct Level { - unsigned int pageRead; - unsigned int pageReadExt; - unsigned int pageBuild; - unsigned int pageBuildExt; - unsigned int pageCommitStart; - unsigned int pageModify; - unsigned int pageModifyExt; - unsigned int lazyClearRequeue; - unsigned int lazyClearRequeueExt; - unsigned int lazyClearFree; - unsigned int lazyClearFreeExt; - unsigned int forceUpdate; - unsigned int detachChild; - double buildStoredPct; - double buildFillPct; - unsigned int buildItemCount; - double modifyStoredPct; - double modifyFillPct; - unsigned int modifyItemCount; - + struct levelMetrics{ + unsigned int pageRead; + unsigned int pageReadExt; + unsigned int pageBuild; + unsigned int pageBuildExt; + unsigned int pageCommitStart; + unsigned int pageModify; + unsigned int pageModifyExt; + unsigned int lazyClearRequeue; + unsigned int lazyClearRequeueExt; + unsigned int lazyClearFree; + unsigned int lazyClearFreeExt; + unsigned int forceUpdate; + unsigned int detachChild; + double buildStoredPct; + double buildFillPct; + unsigned int buildItemCount; + double modifyStoredPct; + double modifyFillPct; + unsigned int modifyItemCount; + eventReasonsArray eventReasons; + }; + levelMetrics metric; Reference buildFillPctSketch; Reference modifyFillPctSketch; Reference buildStoredPctSketch; @@ -1356,82 +1342,102 @@ struct RedwoodMetrics { Reference buildItemCountSketch; Reference modifyItemCountSketch; - Level( Reference a, - Reference b, - Reference c, - Reference d, - Reference e, - Reference f - ) : - pageRead(0), pageReadExt(0), pageBuild(0), pageBuildExt(0), pageCommitStart(0), pageModify(0), pageModifyExt(0), - lazyClearRequeue(0), lazyClearRequeueExt(0), lazyClearFree(0), lazyClearFreeExt(0), forceUpdate(0), detachChild(0), - buildStoredPct(0), buildFillPct(0), buildItemCount(0), modifyStoredPct(0), modifyFillPct(0), modifyItemCount(0), - buildFillPctSketch(a), modifyFillPctSketch(b), buildStoredPctSketch(c), modifyStoredPctSketch(d), buildItemCountSketch(e), modifyItemCountSketch(f) - {} - Level() : - pageRead(0), pageReadExt(0), pageBuild(0), pageBuildExt(0), pageCommitStart(0), pageModify(0), pageModifyExt(0), - lazyClearRequeue(0), lazyClearRequeueExt(0), lazyClearFree(0), lazyClearFreeExt(0), forceUpdate(0), detachChild(0), - buildStoredPct(0), buildFillPct(0), buildItemCount(0), modifyStoredPct(0), modifyFillPct(0), modifyItemCount(0), - buildFillPctSketch(), modifyFillPctSketch(), buildStoredPctSketch(), modifyStoredPctSketch(), buildItemCountSketch(), modifyItemCountSketch() - {} + Level() { levelClear(); } + + void levelClear(unsigned int levelCounter = 0){ + metric = {}; + metric.eventReasons.clear(); + buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + buildFillPctSketch->clear(); + modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + modifyFillPctSketch->clear(); + buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + buildStoredPctSketch->clear(); + modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + modifyStoredPctSketch->clear(); + buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); + buildItemCountSketch->clear(); + modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); + modifyItemCountSketch->clear(); + } }; + struct metrics{ + unsigned int opSet; + unsigned int opSetKeyBytes; + unsigned int opSetValueBytes; + unsigned int opClear; + unsigned int opClearKey; + unsigned int opCommit; + unsigned int opGet; + unsigned int opGetRange; + unsigned int pagerDiskWrite; + unsigned int pagerDiskRead; + unsigned int pagerRemapFree; + unsigned int pagerRemapCopy; + unsigned int pagerRemapSkip; + unsigned int pagerCacheHit; + unsigned int pagerCacheMiss; + unsigned int pagerProbeHit; + unsigned int pagerProbeMiss; + unsigned int pagerEvictUnhit; + unsigned int pagerEvictFail; + unsigned int btreeLeafPreload; + unsigned int btreeLeafPreloadExt; + eventReasonsArray eventReasons; + }; + + RedwoodMetrics() { clear(); } + + void clear() { + unsigned int levelCounter = 1; + for (auto& level : levels) { + level.levelClear(levelCounter); + ++levelCounter; + } + metric = {}; + metric.eventReasons.clear(); + kvSizeWritten = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("Written"), Histogram::Unit::bytes); + kvSizeWritten->clear(); + kvSizeReadByGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByGet "), Histogram::Unit::bytes); + kvSizeReadByGet->clear(); + kvSizeReadByRangeGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByRangeGet"), Histogram::Unit::bytes); + kvSizeReadByRangeGet->clear(); + startTime = g_network ? now() : 0; + } + Level levels[btreeLevels]; - - unsigned int opSet; - unsigned int opSetKeyBytes; - unsigned int opSetValueBytes; - unsigned int opClear; - unsigned int opClearKey; - unsigned int opCommit; - unsigned int opGet; - unsigned int opGetRange; - unsigned int pagerDiskWrite; - unsigned int pagerDiskRead; - unsigned int pagerRemapFree; - unsigned int pagerRemapCopy; - unsigned int pagerRemapSkip; - unsigned int pagerCacheHit; - unsigned int pagerCacheMiss; - unsigned int pagerProbeHit; - unsigned int pagerProbeMiss; - unsigned int pagerEvictUnhit; - unsigned int pagerEvictFail; - unsigned int btreeLeafPreload; - unsigned int btreeLeafPreloadExt; - - double startTime; - - unsigned int eventsReasons[4][6]; - + metrics metric; Reference kvSizeWritten; Reference kvSizeReadByGet; Reference kvSizeReadByRangeGet; + double startTime; std::string getName(events e){ std::map names = {{events::pagerCacheLookup, "pagerCacheLookup"}, {events::pagerCacheHit, "pagerCacheHit"}, {events::pagerCacheMiss, "pagerCacheMiss"}, {events::pagerWrite, "pagerWrite"}}; + ASSERT(names.find(e) != names.end()); return names[e]; } - int getIndex(events e){ - std::map indices = {{events::pagerCacheLookup, 0}, {events::pagerCacheHit, 1}, {events::pagerCacheMiss, 2}, {events::pagerWrite, 3},{events::Update, 0},{events::Build, 1}}; - return indices[e]; - } + std::string getName(pagerEventReasons r){ std::map names = {{pagerEventReasons::pointRead, "pointRead"}, {pagerEventReasons::rangeRead, "rangeRead"}, {pagerEventReasons::rangePrefetch, "rangePrefetch"}, {pagerEventReasons::commit, "commit"}, {pagerEventReasons::lazyClear, "lazyClear"}, {pagerEventReasons::metaData, "metaData"}}; + ASSERT(names.find(r) != names.end()); return names[r]; } - int getIndex(pagerEventReasons r){ - std::map indices = {{pagerEventReasons::pointRead, 0}, {pagerEventReasons::rangeRead, 1}, {pagerEventReasons::rangePrefetch, 2}, {pagerEventReasons::commit, 3}, {pagerEventReasons::lazyClear, 4}, {pagerEventReasons::metaData, 5}}; - return indices[r]; - } + // Return number of pages read or written, from cache or disk unsigned int pageOps() const { // All page reads are either a cache hit, probe hit, or a disk read - return pagerDiskWrite + pagerDiskRead + pagerCacheHit + pagerProbeHit; + return metric.pagerDiskWrite + metric.pagerDiskRead + metric.pagerCacheHit + metric.pagerProbeHit; } Level& level(unsigned int level) { static Level outOfBound; + //ASSERT(level<=btreeLevels && level>0); + // modify this later! + if(level>btreeLevels || level<=0){ + level = 1; + } if (level == 0 || level > btreeLevels) { return outOfBound; } @@ -1441,32 +1447,32 @@ struct RedwoodMetrics { // This will populate a trace event and/or a string with Redwood metrics. // The string is a reasonably well formatted page of information void getFields(TraceEvent* e, std::string* s = nullptr, bool skipZeroes = false) { - std::pair metrics[] = { { "BTreePreload", btreeLeafPreload }, - { "BTreePreloadExt", btreeLeafPreloadExt }, + std::pair metrics[] = { { "BTreePreload", metric.btreeLeafPreload }, + { "BTreePreloadExt", metric.btreeLeafPreloadExt }, { "", 0 }, - { "OpSet", opSet }, - { "OpSetKeyBytes", opSetKeyBytes }, - { "OpSetValueBytes", opSetValueBytes }, - { "OpClear", opClear }, - { "OpClearKey", opClearKey }, + { "OpSet", metric.opSet }, + { "OpSetKeyBytes", metric.opSetKeyBytes }, + { "OpSetValueBytes", metric.opSetValueBytes }, + { "OpClear", metric.opClear }, + { "OpClearKey", metric.opClearKey }, { "", 0 }, - { "OpGet", opGet }, - { "OpGetRange", opGetRange }, - { "OpCommit", opCommit }, + { "OpGet", metric.opGet }, + { "OpGetRange", metric.opGetRange }, + { "OpCommit", metric.opCommit }, { "", 0 }, - { "PagerDiskWrite", pagerDiskWrite }, - { "PagerDiskRead", pagerDiskRead }, - { "PagerCacheHit", pagerCacheHit }, - { "PagerCacheMiss", pagerCacheMiss }, + { "PagerDiskWrite", metric.pagerDiskWrite }, + { "PagerDiskRead", metric.pagerDiskRead }, + { "PagerCacheHit", metric.pagerCacheHit }, + { "PagerCacheMiss", metric.pagerCacheMiss }, { "", 0 }, - { "PagerProbeHit", pagerProbeHit }, - { "PagerProbeMiss", pagerProbeMiss }, - { "PagerEvictUnhit", pagerEvictUnhit }, - { "PagerEvictFail", pagerEvictFail }, + { "PagerProbeHit", metric.pagerProbeHit }, + { "PagerProbeMiss", metric.pagerProbeMiss }, + { "PagerEvictUnhit", metric.pagerEvictUnhit }, + { "PagerEvictFail", metric.pagerEvictFail }, { "", 0 }, - { "PagerRemapFree", pagerRemapFree }, - { "PagerRemapCopy", pagerRemapCopy }, - { "PagerRemapSkip", pagerRemapSkip } }; + { "PagerRemapFree", metric.pagerRemapFree }, + { "PagerRemapCopy", metric.pagerRemapCopy }, + { "PagerRemapSkip", metric.pagerRemapSkip } }; double elapsed = now() - startTime; if (e != nullptr) { @@ -1486,53 +1492,46 @@ struct RedwoodMetrics { *s += format("%-15s %-8u %8" PRId64 "/s ", m.first, m.second, int64_t(m.second / elapsed)); } } - } - - const events eventsVector[] = {events::pagerCacheLookup, events::pagerCacheHit, events::pagerCacheMiss, events::pagerWrite}; - const vector reasonsVector = {pagerEventReasons::pointRead, pagerEventReasons::rangeRead, pagerEventReasons::rangePrefetch, pagerEventReasons::commit, pagerEventReasons::lazyClear, pagerEventReasons::metaData}; - for(events e : eventsVector){ - std::cout<<"\nevents: "+getName(e)+" {"; - for(auto r = reasonsVector.begin() ; r != reasonsVector.end(); ++r){ - std::string temp = ""+getName(*r)+": "+std::to_string(eventsReasons[getIndex(e)][getIndex(*r)]); - temp += (std::next(r) != reasonsVector.end() ? ", " : "}"); - std::cout<buckets[i]<<"; "; - std::cout<<"kvSizeReadByRangeGet: "<buckets[i]<<"; "; - std::cout<<"kvSizeWritten: "<buckets[i]<<";\n"; + const events eventsVector[] = {events::pagerCacheLookup, events::pagerCacheHit, events::pagerCacheMiss, events::pagerWrite}; + const vector reasonsVector = {pagerEventReasons::pointRead, pagerEventReasons::rangeRead, pagerEventReasons::rangePrefetch, pagerEventReasons::commit, pagerEventReasons::lazyClear, pagerEventReasons::metaData}; + for(events e : eventsVector){ + std::cout<<"\n"+getName(e)+": {"; + for(auto r = reasonsVector.begin() ; r != reasonsVector.end(); ++r){ + std::string temp = ""+getName(*r)+": "+std::to_string(metric.eventReasons.getEventReason(e,*r)); + temp += (std::next(r) != reasonsVector.end() ? ", " : "}"); + std::cout< metrics[] = { - { "PageBuild", level.pageBuild }, - { "PageBuildExt", level.pageBuildExt }, - { "PageModify", level.pageModify }, - { "PageModifyExt", level.pageModifyExt }, + { "PageBuild", level.metric.pageBuild }, + { "PageBuildExt", level.metric.pageBuildExt }, + { "PageModify", level.metric.pageModify }, + { "PageModifyExt", level.metric.pageModifyExt }, { "", 0 }, - { "PageRead", level.pageRead }, - { "PageReadExt", level.pageReadExt }, - { "PageCommitStart", level.pageCommitStart }, + { "PageRead", level.metric.pageRead }, + { "PageReadExt", level.metric.pageReadExt }, + { "PageCommitStart", level.metric.pageCommitStart }, { "", 0 }, - { "LazyClearInt", level.lazyClearRequeue }, - { "LazyClearIntExt", level.lazyClearRequeueExt }, - { "LazyClear", level.lazyClearFree }, - { "LazyClearExt", level.lazyClearFreeExt }, + { "LazyClearInt", level.metric.lazyClearRequeue }, + { "LazyClearIntExt", level.metric.lazyClearRequeueExt }, + { "LazyClear", level.metric.lazyClearFree }, + { "LazyClearExt", level.metric.lazyClearFreeExt }, { "", 0 }, - { "ForceUpdate", level.forceUpdate }, - { "DetachChild", level.detachChild }, + { "ForceUpdate", level.metric.forceUpdate }, + { "DetachChild", level.metric.detachChild }, { "", 0 }, - { "-BldAvgCount", level.pageBuild ? level.buildItemCount / level.pageBuild : 0 }, - { "-BldAvgFillPct", level.pageBuild ? level.buildFillPct / level.pageBuild * 100 : 0 }, - { "-BldAvgStoredPct", level.pageBuild ? level.buildStoredPct / level.pageBuild * 100 : 0 }, + { "-BldAvgCount", level.metric.pageBuild ? level.metric.buildItemCount / level.metric.pageBuild : 0 }, + { "-BldAvgFillPct", level.metric.pageBuild ? level.metric.buildFillPct / level.metric.pageBuild * 100 : 0 }, + { "-BldAvgStoredPct", level.metric.pageBuild ? level.metric.buildStoredPct / level.metric.pageBuild * 100 : 0 }, { "", 0 }, - { "-ModAvgCount", level.pageModify ? level.modifyItemCount / level.pageModify : 0 }, - { "-ModAvgFillPct", level.pageModify ? level.modifyFillPct / level.pageModify * 100 : 0 }, - { "-ModAvgStoredPct", level.pageModify ? level.modifyStoredPct / level.pageModify * 100 : 0 }, + { "-ModAvgCount", level.metric.pageModify ? level.metric.modifyItemCount / level.metric.pageModify : 0 }, + { "-ModAvgFillPct", level.metric.pageModify ? level.metric.modifyFillPct / level.metric.pageModify * 100 : 0 }, + { "-ModAvgStoredPct", level.metric.pageModify ? level.metric.modifyStoredPct / level.metric.pageModify * 100 : 0 }, { "", 0 }, }; @@ -1624,16 +1623,23 @@ public: // Get the object for i if it exists, else return nullptr. // If the object exists, its eviction order will NOT change as this is not a cache hit. - ObjectType* getIfExists(pagerEventReasons r, const IndexType& index) { + ObjectType* getIfExists(pagerEventReasons r, unsigned int l, const IndexType& index) { auto i = cache.find(index); if (i != cache.end()) { ++i->second.hits; - ++g_redwoodMetrics.pagerProbeHit; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; + ++g_redwoodMetrics.metric.pagerProbeHit; + if (l == 0){ + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } + else{ + auto& metrics = g_redwoodMetrics.level(l); + metrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } return &i->second.item; } - ++g_redwoodMetrics.pagerProbeMiss; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; + ++g_redwoodMetrics.metric.pagerProbeMiss; + (l == 0) ? g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r) + : g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheLookup, r); return nullptr; } @@ -1656,7 +1662,7 @@ public: } Entry& toEvict = i->second; if (toEvict.hits == 0) { - ++g_redwoodMetrics.pagerEvictUnhit; + ++g_redwoodMetrics.metric.pagerEvictUnhit; } evictionOrder.erase(evictionOrder.iterator_to(toEvict)); cache.erase(i); @@ -1667,16 +1673,22 @@ public: // After a get(), the object for i is the last in evictionOrder. // If noHit is set, do not consider this access to be cache hit if the object is present // If noMiss is set, do not consider this access to be a cache miss if the object is not present - ObjectType& get(pagerEventReasons r, const IndexType& index, bool noHit = false, bool noMiss = false) { + ObjectType& get(pagerEventReasons r, unsigned int l, const IndexType& index, bool noHit = false, bool noMiss = false) { Entry& entry = cache[index]; // If entry is linked into evictionOrder then move it to the back of the order if (entry.is_linked()) { if (!noHit) { ++entry.hits; - ++g_redwoodMetrics.pagerCacheHit; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheHit)][g_redwoodMetrics.getIndex(r)] += 1; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; + ++g_redwoodMetrics.metric.pagerCacheHit; + if(l==0){ + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheHit, r); + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } + else{ + g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheHit, r); + g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } // Move the entry to the back of the eviction order evictionOrder.erase(evictionOrder.iterator_to(entry)); @@ -1685,9 +1697,15 @@ public: } else { // Otherwise it was a cache miss if (!noMiss) { - ++g_redwoodMetrics.pagerCacheMiss; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheMiss)][g_redwoodMetrics.getIndex(r)] += 1; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerCacheLookup)][g_redwoodMetrics.getIndex(r)] += 1; + ++g_redwoodMetrics.metric.pagerCacheMiss; + if(l==0){ + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheMiss, r); + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } + else{ + g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheMiss, r); + g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } } // Finish initializing entry entry.index = index; @@ -1715,11 +1733,11 @@ public: if (!toEvict.item.evictable()) { // shift the front to the back evictionOrder.shift_forward(1); - ++g_redwoodMetrics.pagerEvictFail; + ++g_redwoodMetrics.metric.pagerEvictFail; break; } else { if (toEvict.hits == 0) { - ++g_redwoodMetrics.pagerEvictUnhit; + ++g_redwoodMetrics.metric.pagerEvictUnhit; } debug_printf( "Evicting %s to make room for %s\n", toString(toEvict.index).c_str(), toString(index).c_str()); @@ -2024,7 +2042,7 @@ public: if (extents[i].queueID == remapQueueID) { LogicalPageID extID = extents[i].extentID; debug_printf("DWALPager Extents: ID: %s ", toString(extID).c_str()); - self->readExtent(pagerEventReasons::metaData, extID); + self->readExtent(pagerEventReasons::metaData, 0, extID); // check if this should be 0? } } } @@ -2288,15 +2306,19 @@ public: Future newExtentPageID(QueueID queueID) override { return newExtentPageID_impl(this, queueID); } - Future writePhysicalPage(pagerEventReasons r, PhysicalPageID pageID, Reference page, bool header = false) { + Future writePhysicalPage(pagerEventReasons r, unsigned int l, PhysicalPageID pageID, Reference page, bool header = false) { debug_printf("DWALPager(%s) op=%s %s ptr=%p\n", filename.c_str(), (header ? "writePhysicalHeader" : "writePhysical"), toString(pageID).c_str(), page->begin()); - ++g_redwoodMetrics.pagerDiskWrite; - g_redwoodMetrics.eventsReasons[g_redwoodMetrics.getIndex(RedwoodMetrics::events::pagerWrite)][g_redwoodMetrics.getIndex(r)] += 1; + ++g_redwoodMetrics.metric.pagerDiskWrite; + if (l == 0){ g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerWrite, r);} + else{ + auto& metrics = g_redwoodMetrics.level(l); + metrics.metric.eventReasons.addEventReason(events::pagerWrite, r); + } VALGRIND_MAKE_MEM_DEFINED(page->begin(), page->size()); page->updateChecksum(pageID); @@ -2327,14 +2349,14 @@ public: } Future writeHeaderPage(pagerEventReasons r, PhysicalPageID pageID, Reference page) { - return writePhysicalPage(r, pageID, page, true); + return writePhysicalPage(r, 0, pageID, page, true); } - void updatePage(LogicalPageID pageID, Reference data) override { + void updatePage(pagerEventReasons r, unsigned int l, LogicalPageID pageID, Reference data) override { // Get the cache entry for this page, without counting it as a cache hit as we're replacing its contents now // or as a cache miss because there is no benefit to the page already being in cache // this metaData reason will not be accounted since its not a cache hit or cache miss - PageCacheEntry& cacheEntry = pageCache.get(pagerEventReasons::metaData, pageID, true, true); + PageCacheEntry& cacheEntry = pageCache.get(r, l, pageID, true, true); debug_printf("DWALPager(%s) op=write %s cached=%d reading=%d writing=%d\n", filename.c_str(), toString(pageID).c_str(), @@ -2350,11 +2372,11 @@ public: // future reads of the version are not allowed) and the write of the next newest version over top // of the original page begins. if (!cacheEntry.initialized()) { - cacheEntry.writeFuture = writePhysicalPage(pagerEventReasons::metaData, pageID, data); + cacheEntry.writeFuture = writePhysicalPage(r, l, pageID, data); } else if (cacheEntry.reading()) { // Wait for the read to finish, then start the write. cacheEntry.writeFuture = map(success(cacheEntry.readFuture), [=](Void) { - writePhysicalPage(pagerEventReasons::metaData, pageID, data); + writePhysicalPage(r, l, pageID, data); return Void(); }); } @@ -2362,21 +2384,21 @@ public: // writes happen in the correct order else if (cacheEntry.writing()) { cacheEntry.writeFuture = map(cacheEntry.writeFuture, [=](Void) { - writePhysicalPage(pagerEventReasons::metaData, pageID, data); + writePhysicalPage(r, l, pageID, data); return Void(); }); } else { - cacheEntry.writeFuture = writePhysicalPage(pagerEventReasons::metaData, pageID, data); + cacheEntry.writeFuture = writePhysicalPage(r, l, pageID, data); } // Always update the page contents immediately regardless of what happened above. cacheEntry.readFuture = data; } - Future atomicUpdatePage(LogicalPageID pageID, Reference data, Version v) override { + Future atomicUpdatePage(unsigned int l, LogicalPageID pageID, Reference data, Version v) override { debug_printf("DWALPager(%s) op=writeAtomic %s @%" PRId64 "\n", filename.c_str(), toString(pageID).c_str(), v); Future f = map(newPageID(), [=](LogicalPageID newPageID) { - updatePage(newPageID, data); + updatePage(pagerEventReasons::commit, l, newPageID, data); // TODO: Possibly limit size of remap queue since it must be recovered on cold start RemappedPage r{ v, pageID, newPageID }; remapQueue.pushBack(r); @@ -2499,7 +2521,7 @@ public: PhysicalPageID pageID, bool header = false) { ASSERT(!self->memoryOnly); - ++g_redwoodMetrics.pagerDiskRead; + ++g_redwoodMetrics.metric.pagerDiskRead; if (g_network->getCurrentTask() > TaskPriority::DiskRead) { wait(delay(0, TaskPriority::DiskRead)); @@ -2554,12 +2576,12 @@ public: // Reads the most recent version of pageID, either previously committed or written using updatePage() // in the current commit - Future> readPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable, bool noHit = false) override { + Future> readPage(pagerEventReasons r, unsigned int l, LogicalPageID pageID, bool cacheable, bool noHit = false) override { // Use cached page if present, without triggering a cache hit. // Otherwise, read the page and return it but don't add it to the cache if (!cacheable) { debug_printf("DWALPager(%s) op=readUncached %s\n", filename.c_str(), toString(pageID).c_str()); - PageCacheEntry* pCacheEntry = pageCache.getIfExists(r, pageID); + PageCacheEntry* pCacheEntry = pageCache.getIfExists(r, l, pageID); if (pCacheEntry != nullptr) { debug_printf("DWALPager(%s) op=readUncachedHit %s\n", filename.c_str(), toString(pageID).c_str()); @@ -2570,7 +2592,7 @@ public: return forwardError(readPhysicalPage(this, (PhysicalPageID)pageID), errorPromise); } - PageCacheEntry& cacheEntry = pageCache.get(r, pageID, noHit); + PageCacheEntry& cacheEntry = pageCache.get(r, l, pageID, noHit); debug_printf("DWALPager(%s) op=read %s cached=%d reading=%d writing=%d noHit=%d\n", filename.c_str(), toString(pageID).c_str(), @@ -2617,9 +2639,9 @@ public: return (PhysicalPageID)pageID; } - Future> readPageAtVersion(pagerEventReasons r, LogicalPageID logicalID, Version v, bool cacheable, bool noHit) { + Future> readPageAtVersion(pagerEventReasons r, unsigned int l, LogicalPageID logicalID, Version v, bool cacheable, bool noHit) { PhysicalPageID physicalID = getPhysicalPageID(logicalID, v); - return readPage(r, physicalID, cacheable, noHit); + return readPage(r, l, physicalID, cacheable, noHit); } void releaseExtentReadLock() override { concurrentExtentReads->release(); } @@ -2633,7 +2655,7 @@ public: wait(self->concurrentExtentReads->take()); ASSERT(!self->memoryOnly); - ++g_redwoodMetrics.pagerDiskRead; + ++g_redwoodMetrics.metric.pagerDiskRead; if (g_network->getCurrentTask() > TaskPriority::DiskRead) { wait(delay(0, TaskPriority::DiskRead)); @@ -2698,9 +2720,9 @@ public: return extent; } - Future> readExtent(pagerEventReasons r, LogicalPageID pageID) override { + Future> readExtent(pagerEventReasons r, unsigned int l, LogicalPageID pageID) override { debug_printf("DWALPager(%s) op=readExtent %s\n", filename.c_str(), toString(pageID).c_str()); - PageCacheEntry* pCacheEntry = extentCache.getIfExists(r, pageID); + PageCacheEntry* pCacheEntry = extentCache.getIfExists(r, l, pageID); if (pCacheEntry != nullptr) { debug_printf("DWALPager(%s) Cache Entry exists for %s\n", filename.c_str(), toString(pageID).c_str()); return pCacheEntry->readFuture; @@ -2727,7 +2749,7 @@ public: else if (tailExt) readSize = (tailPageID - pageID + 1) * physicalPageSize; - PageCacheEntry& cacheEntry = extentCache.get(r, pageID); + PageCacheEntry& cacheEntry = extentCache.get(r, l, pageID); if (!cacheEntry.initialized()) { cacheEntry.writeFuture = Void(); cacheEntry.readFuture = @@ -2847,13 +2869,13 @@ public: debug_printf("DWALPager(%s) remapCleanup copy %s\n", self->filename.c_str(), p.toString().c_str()); // Read the data from the page that the original was mapped to - Reference data = wait(self->readPage(pagerEventReasons::metaData, p.newPageID, false, true)); + Reference data = wait(self->readPage(pagerEventReasons::metaData, 0, p.newPageID, false, true)); // Write the data to the original page so it can be read using its original pageID - self->updatePage(p.originalPageID, data); - ++g_redwoodMetrics.pagerRemapCopy; + self->updatePage(pagerEventReasons::metaData, 0, p.originalPageID, data); // check if this should be 0 + ++g_redwoodMetrics.metric.pagerRemapCopy; } else if (firstType == RemappedPage::REMAP) { - ++g_redwoodMetrics.pagerRemapSkip; + ++g_redwoodMetrics.metric.pagerRemapSkip; } // Now that the page contents have been copied to the original page, if the corresponding map entry @@ -2883,13 +2905,13 @@ public: if (freeNewID) { debug_printf("DWALPager(%s) remapCleanup freeNew %s\n", self->filename.c_str(), p.toString().c_str()); self->freeUnmappedPage(p.newPageID, 0); - ++g_redwoodMetrics.pagerRemapFree; + ++g_redwoodMetrics.metric.pagerRemapFree; } if (freeOriginalID) { debug_printf("DWALPager(%s) remapCleanup freeOriginal %s\n", self->filename.c_str(), p.toString().c_str()); self->freeUnmappedPage(p.originalPageID, 0); - ++g_redwoodMetrics.pagerRemapFree; + ++g_redwoodMetrics.metric.pagerRemapFree; } return Void(); @@ -3320,11 +3342,11 @@ public: : pager(pager), metaKey(meta), version(version), expired(expiredFuture) {} ~DWALPagerSnapshot() override {} - Future> getPhysicalPage(pagerEventReasons r, LogicalPageID pageID, bool cacheable, bool noHit) override { + Future> getPhysicalPage(pagerEventReasons r, unsigned int l, LogicalPageID pageID, bool cacheable, bool noHit) override { if (expired.isError()) { throw expired.getError(); } - return map(pager->readPageAtVersion(r, pageID, version, cacheable, noHit), + return map(pager->readPageAtVersion(r, l, pageID, version, cacheable, noHit), [=](Reference p) { return Reference(std::move(p)); }); } @@ -4163,6 +4185,7 @@ public: ::toString(root.get()).c_str(), lazyDeleteQueue.toString().c_str()); } + }; #pragma pack(pop) @@ -4192,9 +4215,9 @@ public: // setWriteVersion() A write shall not become durable until the following call to commit() begins, and shall be // durable once the following call to commit() returns void set(KeyValueRef keyValue) { - ++g_redwoodMetrics.opSet; - g_redwoodMetrics.opSetKeyBytes += keyValue.key.size(); - g_redwoodMetrics.opSetValueBytes += keyValue.value.size(); + ++g_redwoodMetrics.metric.opSet; + g_redwoodMetrics.metric.opSetKeyBytes += keyValue.key.size(); + g_redwoodMetrics.metric.opSetValueBytes += keyValue.value.size(); m_pBuffer->insert(keyValue.key).mutation().setBoundaryValue(m_pBuffer->copyToArena(keyValue.value)); } @@ -4202,13 +4225,13 @@ public: // Optimization for single key clears to create just one mutation boundary instead of two if (clearedRange.begin.size() == clearedRange.end.size() - 1 && clearedRange.end[clearedRange.end.size() - 1] == 0 && clearedRange.end.startsWith(clearedRange.begin)) { - ++g_redwoodMetrics.opClear; - ++g_redwoodMetrics.opClearKey; + ++g_redwoodMetrics.metric.opClear; + ++g_redwoodMetrics.metric.opClearKey; m_pBuffer->insert(clearedRange.begin).mutation().clearBoundary(); return; } - ++g_redwoodMetrics.opClear; + ++g_redwoodMetrics.metric.opClear; MutationBuffer::iterator iBegin = m_pBuffer->insert(clearedRange.begin); MutationBuffer::iterator iEnd = m_pBuffer->insert(clearedRange.end); @@ -4261,7 +4284,8 @@ public: break; } // Start reading the page, without caching - entries.push_back(std::make_pair(q.get(), self->readPage(pagerEventReasons::lazyClear, snapshot, q.get().pageID, true, false))); + int currLevel = (self!=nullptr && self->m_pHeader!=nullptr) ? self->m_pHeader->height : 0; + entries.push_back(std::make_pair(q.get(), self->readPage(pagerEventReasons::lazyClear, currLevel, snapshot, q.get().pageID, true, false))); --toPop; } @@ -4292,14 +4316,14 @@ public: debug_printf("LazyClear: freeing child %s\n", toString(btChildPageID).c_str()); self->freeBTreePage(btChildPageID, v); freedPages += btChildPageID.size(); - metrics.lazyClearFree += 1; - metrics.lazyClearFreeExt += (btChildPageID.size() - 1); + metrics.metric.lazyClearFree += 1; + metrics.metric.lazyClearFreeExt += (btChildPageID.size() - 1); } else { // Otherwise, queue them for lazy delete. debug_printf("LazyClear: queuing child %s\n", toString(btChildPageID).c_str()); self->m_lazyClearQueue.pushFront(LazyClearQueueEntry{ v, btChildPageID }); - metrics.lazyClearRequeue += 1; - metrics.lazyClearRequeueExt += (btChildPageID.size() - 1); + metrics.metric.lazyClearRequeue += 1; + metrics.metric.lazyClearRequeueExt += (btChildPageID.size() - 1); } } if (!c.moveNext()) { @@ -4311,8 +4335,8 @@ public: debug_printf("LazyClear: freeing queue entry %s\n", toString(entry.pageID).c_str()); self->freeBTreePage(entry.pageID, v); freedPages += entry.pageID.size(); - metrics.lazyClearFree += 1; - metrics.lazyClearFreeExt += entry.pageID.size() - 1; + metrics.metric.lazyClearFree += 1; + metrics.metric.lazyClearFreeExt += entry.pageID.size() - 1; } // Stop if @@ -4357,7 +4381,7 @@ public: ++latest; Reference page = self->m_pager->newPageBuffer(); makeEmptyRoot(page); - self->m_pager->updatePage(id, page); + self->m_pager->updatePage(pagerEventReasons::metaData, 0, id, page); self->m_pager->setCommitVersion(latest); LogicalPageID newQueuePage = wait(self->m_pager->newPageID()); @@ -4935,7 +4959,6 @@ private: break; } } - // Use the next entry as the upper bound, or upperBound if there are no more entries beyond this page int endIndex = p.endIndex(); bool lastPage = endIndex == entries.size(); @@ -4985,11 +5008,12 @@ private: } auto& metrics = g_redwoodMetrics.level(btPage->height); - metrics.pageBuild += 1; - metrics.pageBuildExt += p.blockCount - 1; - metrics.buildFillPct += p.usedFraction(); - metrics.buildStoredPct += p.kvFraction(); - metrics.buildItemCount += p.count; + state unsigned int currLevel = btPage->height; + metrics.metric.pageBuild += 1; + metrics.metric.pageBuildExt += p.blockCount - 1; + metrics.metric.buildFillPct += p.usedFraction(); + metrics.metric.buildStoredPct += p.kvFraction(); + metrics.metric.buildItemCount += p.count; metrics.buildFillPctSketch->samplePercentage(p.usedFraction()); metrics.buildStoredPctSketch->samplePercentage(p.kvFraction()); @@ -5018,7 +5042,7 @@ private: // LogicalPageIDs in previousID and try to update them atomically. if (pagesToBuild.size() == 1 && previousID.size() == pages.size()) { for (k = 0; k < pages.size(); ++k) { - LogicalPageID id = wait(self->m_pager->atomicUpdatePage(previousID[k], pages[k], v)); + LogicalPageID id = wait(self->m_pager->atomicUpdatePage(previousID[k], currLevel , pages[k], v)); childPageID.push_back(records.arena(), id); } } else { @@ -5031,7 +5055,7 @@ private: } for (k = 0; k < pages.size(); ++k) { LogicalPageID id = wait(self->m_pager->newPageID()); - self->m_pager->updatePage(id, pages[k]); + self->m_pager->updatePage(pagerEventReasons::commit, height, id, pages[k]); childPageID.push_back(records.arena(), id); } } @@ -5099,6 +5123,7 @@ private: } ACTOR static Future> readPage(pagerEventReasons r, + unsigned int l, Reference snapshot, BTreePageIDRef id, bool forLazyClear = false, @@ -5112,13 +5137,13 @@ private: state Reference page; if (id.size() == 1) { - Reference p = wait(snapshot->getPhysicalPage(r, id.front(), cacheable, false)); + Reference p = wait(snapshot->getPhysicalPage(r, l, id.front(), cacheable, false)); page = std::move(p); } else { ASSERT(!id.empty()); std::vector>> reads; for (auto& pageID : id) { - reads.push_back(snapshot->getPhysicalPage(r, pageID, cacheable, false)); + reads.push_back(snapshot->getPhysicalPage(r, l, pageID, cacheable, false)); } std::vector> pages = wait(getAll(reads)); // TODO: Cache reconstituted super pages somehow, perhaps with help from the Pager. @@ -5128,8 +5153,8 @@ private: debug_printf("readPage() op=readComplete %s @%" PRId64 " \n", toString(id).c_str(), snapshot->getVersion()); const BTreePage* pTreePage = (const BTreePage*)page->begin(); auto& metrics = g_redwoodMetrics.level(pTreePage->height); - metrics.pageRead += 1; - metrics.pageReadExt += (id.size() - 1); + metrics.metric.pageRead += 1; + metrics.metric.pageReadExt += (id.size() - 1); return std::move(page); } @@ -5165,11 +5190,11 @@ private: } static void preLoadPage(IPagerSnapshot* snapshot, BTreePageIDRef id) { - g_redwoodMetrics.btreeLeafPreload += 1; - g_redwoodMetrics.btreeLeafPreloadExt += (id.size() - 1); + g_redwoodMetrics.metric.btreeLeafPreload += 1; + g_redwoodMetrics.metric.btreeLeafPreloadExt += (id.size() - 1); for (auto pageID : id) { - snapshot->getPhysicalPage(pagerEventReasons::rangePrefetch, pageID, true, true); + snapshot->getPhysicalPage(pagerEventReasons::rangePrefetch, 1, pageID, true, true); // prefetch btree leaf node } } @@ -5203,7 +5228,9 @@ private: } if (oldID.size() == 1) { - LogicalPageID id = wait(self->m_pager->atomicUpdatePage(oldID.front(), page, writeVersion)); + BTreePage* btPage = (BTreePage*)page->begin(); + int currLevel = btPage->height; + LogicalPageID id = wait(self->m_pager->atomicUpdatePage(oldID.front(), currLevel, page, writeVersion)); newID.front() = id; } else { state std::vector> pages; @@ -5222,7 +5249,9 @@ private: // Write pages, trying to reuse original page IDs state int i = 0; for (; i < pages.size(); ++i) { - LogicalPageID id = wait(self->m_pager->atomicUpdatePage(oldID[i], pages[i], writeVersion)); + BTreePage* btPage = (BTreePage*)page->begin(); + int currLevel = btPage->height; + LogicalPageID id = wait(self->m_pager->atomicUpdatePage(oldID[i], currLevel, pages[i], writeVersion)); newID[i] = id; } } @@ -5310,11 +5339,11 @@ private: void updatedInPlace(BTreePageIDRef maybeNewID, BTreePage* btPage, int capacity) { inPlaceUpdate = true; auto& metrics = g_redwoodMetrics.level(btPage->height); - metrics.pageModify += 1; - metrics.pageModifyExt += (maybeNewID.size() - 1); - metrics.modifyFillPct += (double)btPage->size() / capacity; - metrics.modifyStoredPct += (double)btPage->kvBytes / capacity; - metrics.modifyItemCount += btPage->tree()->numItems; + metrics.metric.pageModify += 1; + metrics.metric.pageModifyExt += (maybeNewID.size() - 1); + metrics.metric.modifyFillPct += (double)btPage->size() / capacity; + metrics.metric.modifyStoredPct += (double)btPage->kvBytes / capacity; + metrics.metric.modifyItemCount += btPage->tree()->numItems; metrics.modifyFillPctSketch->samplePercentage((double)btPage->size() / capacity); metrics.modifyStoredPctSketch->samplePercentage((double)btPage->kvBytes / capacity); @@ -5547,6 +5576,7 @@ private: }; ACTOR static Future commitSubtree( + unsigned int l, VersionedBTree* self, Reference snapshot, MutationBuffer* mutationBuffer, @@ -5577,7 +5607,7 @@ private: debug_printf("%s -------------------------------------\n", context.c_str()); } - state Reference page = wait(readPage(pagerEventReasons::commit, snapshot, rootID, false, false)); + state Reference page = wait(readPage(pagerEventReasons::commit, l, snapshot, rootID, false, false)); state Version writeVersion = self->getLastCommittedVersion() + 1; // If the page exists in the cache, it must be copied before modification. @@ -5588,7 +5618,8 @@ private: state BTreePage* btPage = (BTreePage*)page->begin(); ASSERT(isLeaf == btPage->isLeaf()); - g_redwoodMetrics.level(btPage->height).pageCommitStart += 1; + ASSERT(btPage != nullptr); + g_redwoodMetrics.level(btPage->height).metric.pageCommitStart += 1; // TODO: Decide if it is okay to update if the subtree boundaries are expanded. It can result in // records in a DeltaTree being outside its decode boundary range, which isn't actually invalid @@ -5625,7 +5656,6 @@ private: if (isLeaf) { bool updating = tryToUpdate; bool changesMade = false; - state Standalone> merged; auto switchToLinearMerge = [&]() { // Couldn't make changes in place, so now do a linear merge and build new pages. @@ -5895,7 +5925,6 @@ private: // Put new links into update and tell update that pages were rebuilt update->rebuilt(entries); - debug_printf("%s Merge complete, returning %s\n", context.c_str(), toString(*update).c_str()); return Void(); } else { @@ -6063,7 +6092,7 @@ private: // If this page has height of 2 then its children are leaf nodes recursions.push_back( - self->commitSubtree(self, snapshot, mutationBuffer, pageID, btPage->height == 2, mBegin, mEnd, &u)); + self->commitSubtree(btPage->height, self, snapshot, mutationBuffer, pageID, btPage->height == 2, mBegin, mEnd, &u)); } debug_printf( @@ -6123,8 +6152,8 @@ private: // Make sure the modifier cloned the page so we can update the child links in-place below. modifier.cloneForUpdate(); - - ++g_redwoodMetrics.level(btPage->height).forceUpdate; + ASSERT(btPage != nullptr); + ++g_redwoodMetrics.level(btPage->height).metric.forceUpdate; } // If the modifier cloned the page for updating, then update our local pageCopy, btPage, and cursor @@ -6165,7 +6194,7 @@ private: if (newID != invalidLogicalPageID) { debug_printf("%s Detach updated %u -> %u\n", context.c_str(), p, newID); p = newID; - ++stats.detachChild; + ++stats.metric.detachChild; ++detached; } } @@ -6223,7 +6252,7 @@ private: } debug_printf("%s Detach updated %u -> %u\n", context.c_str(), p, newID); newPages[i] = newID; - ++stats.detachChild; + ++stats.metric.detachChild; } } } @@ -6294,8 +6323,9 @@ private: MutationBuffer::const_iterator mBegin = mutations->upper_bound(all.subtreeLowerBound.key); --mBegin; MutationBuffer::const_iterator mEnd = mutations->lower_bound(all.subtreeUpperBound.key); - - wait(commitSubtree(self, + int currLevel = (self->m_pHeader != nullptr) ? self->m_pHeader->height : 0; + wait(commitSubtree(currLevel, + self, self->m_pager->getReadSnapshot(latestVersion), mutations, rootPageID, @@ -6312,7 +6342,7 @@ private: Reference page = self->m_pager->newPageBuffer(); makeEmptyRoot(page); self->m_pHeader->height = 1; - self->m_pager->updatePage(newRootID, page); + self->m_pager->updatePage(pagerEventReasons::commit, 1, newRootID, page); rootPageID = BTreePageIDRef((LogicalPageID*)&newRootID, 1); } else { Standalone> newRootLevel(all.newLinks, all.newLinks.arena()); @@ -6352,7 +6382,7 @@ private: self->m_mutationBuffers.erase(self->m_mutationBuffers.begin()); self->m_lastCommittedVersion = writeVersion; - ++g_redwoodMetrics.opCommit; + ++g_redwoodMetrics.metric.opCommit; self->m_lazyClearActor = incrementalLazyClear(self); committed.send(Void()); @@ -6387,6 +6417,10 @@ public: bool intialized() const { return pager.isValid(); } bool isValid() const { return valid; } + int getHeight(){ + if(!path.empty()) return path.back().btPage()->height; + return 0; + } std::string toString() const { std::string r = format("{ptr=%p %s ", this, ::toString(pager->getVersion()).c_str()); for (int i = 0; i < path.size(); ++i) { @@ -6417,9 +6451,9 @@ public: PathEntry& back() { return path.back(); } void popPath() { path.pop_back(); } - Future pushPage(pagerEventReasons r, const BTreePage::BinaryTree::Cursor& link) { + Future pushPage(pagerEventReasons r, unsigned int l, const BTreePage::BinaryTree::Cursor& link) { debug_printf("pushPage(link=%s)\n", link.get().toString(false).c_str()); - return map(readPage(r, pager, link.get().getChildPage()), [=](Reference p) { + return map(readPage(r, l, pager, link.get().getChildPage()), [=](Reference p) { #if REDWOOD_DEBUG path.push_back({ p, getCursor(p, link), link.get().getChildPage() }); #else @@ -6429,9 +6463,9 @@ public: }); } - Future pushPage(pagerEventReasons r, BTreePageIDRef id) { + Future pushPage(pagerEventReasons r, unsigned int l, BTreePageIDRef id) { debug_printf("pushPage(root=%s)\n", ::toString(id).c_str()); - return map(readPage(r, pager, id), [=](Reference p) { + return map(readPage(r, l, pager, id), [=](Reference p) { #if REDWOOD_DEBUG path.push_back({ p, getCursor(p, dbBegin, dbEnd), id }); #else @@ -6448,7 +6482,7 @@ public: path.clear(); path.reserve(6); valid = false; - return pushPage(pagerEventReasons::commit, root); + return pushPage(pagerEventReasons::commit, 1, root); } // Seeks cursor to query if it exists, the record before or after it, or an undefined and invalid @@ -6488,7 +6522,7 @@ public: query.toString().c_str(), prefetchBytes, self->toString().c_str()); - Future f = self->pushPage(r, entry.cursor); + Future f = self->pushPage(r, self->getHeight(), entry.cursor); // Prefetch siblings, at least prefetchBytes, at level 2 but without jumping to another level 2 // sibling @@ -6594,8 +6628,8 @@ public: ASSERT(entry.cursor.movePrev()); ASSERT(entry.cursor.get().value.present()); } - - wait(self->pushPage(pagerEventReasons::metaData, entry.cursor)); + int currLevel = (entry.btPage() != nullptr) ? entry.btPage()->height : 0; + wait(self->pushPage(pagerEventReasons::metaData, currLevel, entry.cursor)); auto& newEntry = self->path.back(); ASSERT(forward ? newEntry.cursor.moveFirst() : newEntry.cursor.moveLast()); } @@ -6728,7 +6762,7 @@ public: wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - ++g_redwoodMetrics.opGetRange; + ++g_redwoodMetrics.metric.opGetRange; state RangeResult result; state int accumulatedBytes = 0; @@ -6844,7 +6878,7 @@ public: wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - ++g_redwoodMetrics.opGet; + ++g_redwoodMetrics.metric.opGet; wait(cur.seekGTE(pagerEventReasons::pointRead, key, 0)); if (cur.isValid() && cur.get().key == key) { @@ -8910,10 +8944,10 @@ TEST_CASE(":/redwood/correctness/pager/cow") { state LogicalPageID id = wait(pager->newPageID()); Reference p = pager->newPageBuffer(); memset(p->mutate(), (char)id, p->size()); - pager->updatePage(id, p); + pager->updatePage(pagerEventReasons::commit, 0, id, p); pager->setMetaKey(LiteralStringRef("asdfasdf")); wait(pager->commit()); - Reference p2 = wait(pager->readPage(pagerEventReasons::pointRead, id, true)); + Reference p2 = wait(pager->readPage(pagerEventReasons::pointRead, 0, id, true)); // check if this level 0 is correct printf("%s\n", StringRef(p2->begin(), p2->size()).toHexString().c_str()); // TODO: Verify reads, do more writes and reads to make this a real pager validator @@ -9043,7 +9077,7 @@ TEST_CASE(":/redwood/performance/extentQueue") { state int i; for (i = 1; i < extentIDs.size() - 1; i++) { LogicalPageID extID = extentIDs[i]; - pager->readExtent(pagerEventReasons::rangeRead, extID); + pager->readExtent(pagerEventReasons::rangeRead, 1, extID); } state PromiseStream>>> resultStream; @@ -9774,7 +9808,7 @@ TEST_CASE(":/redwood/performance/histogramThroughput") { h->sample(uniform[i]); } auto t_end = std::chrono::high_resolution_clock::now(); - h->drawHistogram(); + std::cout<drawHistogram(); GetHistogramRegistry().logReport(); double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); std::cout<<"Time in millisecond: "<samplePercentage((double)uniform[i]/UINT32_MAX); } auto t_end = std::chrono::high_resolution_clock::now(); - h->drawHistogram(); + std::cout<drawHistogram(); GetHistogramRegistry().logReport(); double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); std::cout<<"Time in millisecond: "< currHeight) std::cout << fullCell; - else if (pct > halfFullHeight) std::cout << halfCell; - else std::cout << emptyCell; + if(pct > currHeight) result< halfFullHeight) result< intervalSize/4) std::cout << xFull; - else std::cout << xEmpty; + if (pct > intervalSize/4) result<group, this->op); } - void drawHistogram(); + std::string drawHistogram(); std::string const group; std::string const op; From 60f1a0a08fc2852d01789b863e80782d557a03ee Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Fri, 25 Jun 2021 16:37:03 -0700 Subject: [PATCH 040/426] fix minor error --- fdbserver/VersionedBTree.actor.cpp | 51 +++++++++++++++++++----------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 8a28b5d690..17c2eb9332 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1342,22 +1342,26 @@ struct RedwoodMetrics { Reference buildItemCountSketch; Reference modifyItemCountSketch; - Level() { levelClear(); } + Level() { + levelClear(); + } void levelClear(unsigned int levelCounter = 0){ metric = {}; - metric.eventReasons.clear(); + buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - buildFillPctSketch->clear(); modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - modifyFillPctSketch->clear(); buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - buildStoredPctSketch->clear(); modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - modifyStoredPctSketch->clear(); buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); - buildItemCountSketch->clear(); modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); + + metric.eventReasons.clear(); + buildFillPctSketch->clear(); + modifyFillPctSketch->clear(); + buildStoredPctSketch->clear(); + modifyStoredPctSketch->clear(); + buildItemCountSketch->clear(); modifyItemCountSketch->clear(); } }; @@ -1387,7 +1391,12 @@ struct RedwoodMetrics { eventReasonsArray eventReasons; }; - RedwoodMetrics() { clear(); } + RedwoodMetrics() { + kvSizeWritten = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("Written"), Histogram::Unit::bytes); + kvSizeReadByGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByGet "), Histogram::Unit::bytes); + kvSizeReadByRangeGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByRangeGet"), Histogram::Unit::bytes); + clear(); + } void clear() { unsigned int levelCounter = 1; @@ -1397,12 +1406,11 @@ struct RedwoodMetrics { } metric = {}; metric.eventReasons.clear(); - kvSizeWritten = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("Written"), Histogram::Unit::bytes); + kvSizeWritten->clear(); - kvSizeReadByGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByGet "), Histogram::Unit::bytes); kvSizeReadByGet->clear(); - kvSizeReadByRangeGet = Histogram::getHistogram(LiteralStringRef("kvSize"), LiteralStringRef("ReadByRangeGet"), Histogram::Unit::bytes); kvSizeReadByRangeGet->clear(); + startTime = g_network ? now() : 0; } @@ -1638,8 +1646,13 @@ public: return &i->second.item; } ++g_redwoodMetrics.metric.pagerProbeMiss; - (l == 0) ? g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r) - : g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + if (l == 0 ){ + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } + else{ + auto& metrics = g_redwoodMetrics.level(l); + metrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + } return nullptr; } @@ -1686,8 +1699,8 @@ public: g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); } else{ - g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheHit, r); - g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + g_redwoodMetrics.level(l).metric.eventReasons.addEventReason(events::pagerCacheHit, r); + g_redwoodMetrics.level(l).metric.eventReasons.addEventReason(events::pagerCacheLookup, r); } // Move the entry to the back of the eviction order @@ -1703,8 +1716,8 @@ public: g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerCacheLookup, r); } else{ - g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheMiss, r); - g_redwoodMetrics.levels[l-1].metric.eventReasons.addEventReason(events::pagerCacheLookup, r); + g_redwoodMetrics.level(l).metric.eventReasons.addEventReason(events::pagerCacheMiss, r); + g_redwoodMetrics.level(l).metric.eventReasons.addEventReason(events::pagerCacheLookup, r); } } // Finish initializing entry @@ -2314,7 +2327,9 @@ public: page->begin()); ++g_redwoodMetrics.metric.pagerDiskWrite; - if (l == 0){ g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerWrite, r);} + if (l == 0){ + g_redwoodMetrics.metric.eventReasons.addEventReason(events::pagerWrite, r); + } else{ auto& metrics = g_redwoodMetrics.level(l); metrics.metric.eventReasons.addEventReason(events::pagerWrite, r); From 8bf02943e4418ed78cf19dd5d6e92770ce86f4c0 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Fri, 25 Jun 2021 16:50:49 -0700 Subject: [PATCH 041/426] fix error in order of paramters in atomicUpdatePage function --- fdbserver/VersionedBTree.actor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 17c2eb9332..3270390baf 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -5057,7 +5057,7 @@ private: // LogicalPageIDs in previousID and try to update them atomically. if (pagesToBuild.size() == 1 && previousID.size() == pages.size()) { for (k = 0; k < pages.size(); ++k) { - LogicalPageID id = wait(self->m_pager->atomicUpdatePage(previousID[k], currLevel , pages[k], v)); + LogicalPageID id = wait(self->m_pager->atomicUpdatePage(currLevel, previousID[k], pages[k], v)); childPageID.push_back(records.arena(), id); } } else { @@ -5245,7 +5245,7 @@ private: if (oldID.size() == 1) { BTreePage* btPage = (BTreePage*)page->begin(); int currLevel = btPage->height; - LogicalPageID id = wait(self->m_pager->atomicUpdatePage(oldID.front(), currLevel, page, writeVersion)); + LogicalPageID id = wait(self->m_pager->atomicUpdatePage(currLevel, oldID.front(), page, writeVersion)); newID.front() = id; } else { state std::vector> pages; @@ -5266,7 +5266,7 @@ private: for (; i < pages.size(); ++i) { BTreePage* btPage = (BTreePage*)page->begin(); int currLevel = btPage->height; - LogicalPageID id = wait(self->m_pager->atomicUpdatePage(oldID[i], currLevel, pages[i], writeVersion)); + LogicalPageID id = wait(self->m_pager->atomicUpdatePage(currLevel, oldID[i], pages[i], writeVersion)); newID[i] = id; } } From 0fd39826c37d6e3239c2b7fd6e60517c47f2f5f7 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Fri, 25 Jun 2021 17:32:42 -0700 Subject: [PATCH 042/426] need to add detail information into log --- fdbserver/VersionedBTree.actor.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 3270390baf..4d3dca25d6 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1500,17 +1500,6 @@ struct RedwoodMetrics { *s += format("%-15s %-8u %8" PRId64 "/s ", m.first, m.second, int64_t(m.second / elapsed)); } } - const events eventsVector[] = {events::pagerCacheLookup, events::pagerCacheHit, events::pagerCacheMiss, events::pagerWrite}; - const vector reasonsVector = {pagerEventReasons::pointRead, pagerEventReasons::rangeRead, pagerEventReasons::rangePrefetch, pagerEventReasons::commit, pagerEventReasons::lazyClear, pagerEventReasons::metaData}; - for(events e : eventsVector){ - std::cout<<"\n"+getName(e)+": {"; - for(auto r = reasonsVector.begin() ; r != reasonsVector.end(); ++r){ - std::string temp = ""+getName(*r)+": "+std::to_string(metric.eventReasons.getEventReason(e,*r)); - temp += (std::next(r) != reasonsVector.end() ? ", " : "}"); - std::cout<drawHistogram(); - GetHistogramRegistry().logReport(); double elapsed_time_ms = std::chrono::duration(t_end-t_start).count(); std::cout<<"Time in millisecond: "< hCopy = + Histogram::getHistogram(LiteralStringRef("histogramTest"), LiteralStringRef("counts"), Histogram::Unit::bytes); + std::cout<drawHistogram(); + GetHistogramRegistry().logReport(); } { std::cout<<"Histogram Unit percentage: "< Date: Fri, 25 Jun 2021 22:33:26 -0700 Subject: [PATCH 043/426] Added /backup/containers/localdir/encrypted unit test --- fdbbackup/backup.actor.cpp | 2 +- fdbclient/BackupContainerFileSystem.actor.cpp | 43 +++++++++++++------ .../BackupContainerLocalDirectory.actor.cpp | 17 ++++---- fdbrpc/AsyncFileEncrypted.actor.cpp | 14 +++--- fdbrpc/AsyncFileEncrypted.h | 1 + fdbserver/workloads/UnitTests.actor.cpp | 16 +++++-- 6 files changed, 63 insertions(+), 30 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index f69b731260..0213bece1a 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -697,9 +697,9 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_DEVHELP, "--dev-help", SO_NONE }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, - { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, { OPT_RESTORE_BEGIN_VERSION, "--begin_version", SO_REQ_SEP }, { OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent_snapshot_only", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 87e4ffcbf0..5d5a7fa722 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -1466,7 +1466,7 @@ ACTOR Future writeAndVerifyFile(Reference c, Reference inputFile = wait(c->readFile(f->getFileName())); int64_t fileSize = wait(inputFile->size()); - ASSERT(size == fileSize); + ASSERT_EQ(size, fileSize); if (size > 0) { state Standalone> buf; buf.resize(buf.arena(), fileSize); @@ -1506,12 +1506,28 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key return Void(); } -ACTOR static Future testBackupContainer(std::string url) { +ACTOR Future createTestEncryptionKeyFile(std::string filename) { + state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( + filename, + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, + 0600)); + std::array testKey; + generateRandomData(testKey.data(), testKey.size()); + keyFile->write(testKey.data(), testKey.size(), 0); + wait(keyFile->sync()); + return Void(); +} + +ACTOR Future testBackupContainer(std::string url, Optional encryptionKeyFileName) { state FlowLock lock(100e6); + if (encryptionKeyFileName.present()) { + wait(createTestEncryptionKeyFile(encryptionKeyFileName.get())); + } + printf("BackupContainerTest URL %s\n", url.c_str()); - state Reference c = IBackupContainer::openContainer(url); + state Reference c = IBackupContainer::openContainer(url, encryptionKeyFileName); // Make sure container doesn't exist, then create it. try { @@ -1655,22 +1671,25 @@ ACTOR static Future testBackupContainer(std::string url) { return Void(); } -TEST_CASE("/backup/containers/localdir") { - if (g_network->isSimulated()) - wait(testBackupContainer(format("file://simfdb/backups/%llx", timer_int()))); - else - wait(testBackupContainer(format("file:///private/tmp/fdb_backups/%llx", timer_int()))); +TEST_CASE("/backup/containers/localdir/unencrypted") { + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {})); return Void(); -}; +} + +TEST_CASE("/backup/containers/localdir/encrypted") { + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), + format("%s/test_encryption_key", params.getDataDir().c_str()))); + return Void(); +} TEST_CASE("/backup/containers/url") { if (!g_network->isSimulated()) { const char* url = getenv("FDB_TEST_BACKUP_URL"); ASSERT(url != nullptr); - wait(testBackupContainer(url)); + wait(testBackupContainer(url, {})); } return Void(); -}; +} TEST_CASE("/backup/containers_list") { if (!g_network->isSimulated()) { @@ -1683,7 +1702,7 @@ TEST_CASE("/backup/containers_list") { } } return Void(); -}; +} TEST_CASE("/backup/time") { // test formatTime() diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp index d94c92441a..e15423df74 100644 --- a/fdbclient/BackupContainerLocalDirectory.actor.cpp +++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp @@ -134,13 +134,10 @@ std::string BackupContainerLocalDirectory::getURLFormat() { ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); - int64_t fileSize = wait(keyFile->size()); - // TODO: Use new error code and avoid hard-coding expected size - if (fileSize != 16) { - throw internal_error(); - } state std::array key; - wait(success(keyFile->read(key.data(), key.size(), 0))); + int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); + // TODO: Throw new error (fail gracefully) + ASSERT_EQ(bytesRead, key.size()); StreamCipher::Key::initializeKey(std::move(key)); return Void(); } @@ -216,6 +213,10 @@ Future> BackupContainerLocalDirectory::listURLs(const s } Future BackupContainerLocalDirectory::create() { + if (usesEncryption()) { + return encryptionSetupFuture; + } + // TODO: Update this comment: // Nothing should be done here because create() can be called by any process working with the container URL, // such as fdbbackup. Since "local directory" containers are by definition local to the machine they are // accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to @@ -284,8 +285,8 @@ Future> BackupContainerLocalDirectory::readFile(const std: } Future> BackupContainerLocalDirectory::writeFile(const std::string& path) { - int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | - IAsyncFile::OPEN_READWRITE; + int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; if (usesEncryption()) { flags |= IAsyncFile::OPEN_ENCRYPTED; } diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index ae37821f50..ffecfae3c1 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -30,7 +30,10 @@ public: // the filename. static auto getFirstBlockIV(const std::string& filename) { StreamCipher::IV iv; - auto hash = XXH3_128bits(filename.c_str(), filename.size()); + auto salt = basename(filename); + auto pos = salt.find('.'); + salt = salt.substr(0, pos); + auto hash = XXH3_128bits(salt.c_str(), salt.size()); auto high = reinterpret_cast(&hash.high64); auto low = reinterpret_cast(&hash.low64); std::copy(high, high + 8, &iv[0]); @@ -67,7 +70,6 @@ public: self->readBuffers.insert(block, _plaintext); plaintext = _plaintext; } - ASSERT(plaintext.size() == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); auto start = (block == firstBlock) ? plaintext.begin() + (offset % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) : plaintext.begin(); auto end = (block == lastBlock) @@ -86,7 +88,7 @@ public: ACTOR static Future write(AsyncFileEncrypted* self, void const* data, int length, int64_t offset) { ASSERT(self->canWrite); // All writes must append to the end of the file: - ASSERT(offset == self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock); + ASSERT_EQ(offset, self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock); state unsigned char const* input = reinterpret_cast(data); while (length > 0) { const auto chunkSize = std::min(length, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE - self->offsetInBlock); @@ -156,11 +158,13 @@ Future AsyncFileEncrypted::zeroRange(int64_t offset, int64_t length) { } Future AsyncFileEncrypted::truncate(int64_t size) { - ASSERT(false); // TODO: Not yet implemented + // FIXME: Not yet implemented + ASSERT(canWrite); return Void(); } Future AsyncFileEncrypted::sync() { + ASSERT(canWrite); return AsyncFileEncryptedImpl::sync(this); } @@ -169,7 +173,7 @@ Future AsyncFileEncrypted::flush() { } Future AsyncFileEncrypted::size() const { - return currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + offsetInBlock; + return file->size(); } std::string AsyncFileEncrypted::getFilename() const { diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index b474198e1c..af123f2650 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -57,6 +57,7 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted writeBuffer; + Future initialize(); public: AsyncFileEncrypted(Reference, bool canWrite); diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 285eb6fca5..a3e0dd7508 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -39,6 +39,7 @@ struct UnitTestWorkload : TestWorkload { std::string testPattern; int testRunLimit; UnitTestParameters testParams; + bool cleanupAfterTests; PerfIntCounter testsAvailable, testsExecuted, testsFailed; PerfDoubleCounter totalWallTime, totalSimTime; @@ -48,9 +49,14 @@ struct UnitTestWorkload : TestWorkload { testsFailed("Test Cases Failed"), totalWallTime("Total wall clock time (s)"), totalSimTime("Total flow time (s)") { enabled = !clientId; // only do this on the "first" client - testPattern = getOption(options, LiteralStringRef("testsMatching"), Value()).toString(); - testRunLimit = getOption(options, LiteralStringRef("maxTestCases"), -1); - testParams.setDataDir(getOption(options, LiteralStringRef("dataDir"), "simfdb/unittests/"_sr).toString()); + testPattern = getOption(options, "testsMatching"_sr, Value()).toString(); + testRunLimit = getOption(options, "maxTestCases"_sr, -1); + if (g_network->isSimulated()) { + testParams.setDataDir(getOption(options, "dataDir"_sr, "simfdb/unittests/"_sr).toString()); + } else { + testParams.setDataDir(getOption(options, "dataDir"_sr, "/private/tmp/"_sr).toString()); + } + cleanupAfterTests = getOption(options, "cleanupAfterTests"_sr, true); // Consume all remaining options as testParams which the unit test can access for (auto& kv : options) { @@ -121,7 +127,9 @@ struct UnitTestWorkload : TestWorkload { ++self->testsFailed; result = e; } - platform::eraseDirectoryRecursive(self->testParams.getDataDir()); + if (self->cleanupAfterTests) { + platform::eraseDirectoryRecursive(self->testParams.getDataDir()); + } ++self->testsExecuted; double wallTime = timer() - start_timer; double simTime = now() - start_now; From d88f8f76537ec09991daf222b2d566c4e0f444b3 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Fri, 25 Jun 2021 22:53:19 -0700 Subject: [PATCH 044/426] Make BackupContainer.actor.cpp less verbose --- fdbclient/BackupContainer.actor.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 56357a1467..29fef0486a 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -263,9 +263,9 @@ Reference IBackupContainer::openContainer(const std::string& u try { StringRef u(url); - if (u.startsWith(LiteralStringRef("file://"))) { + if (u.startsWith("file://"_sr)) { r = makeReference(url, encryptionKeyFileName); - } else if (u.startsWith(LiteralStringRef("blobstore://"))) { + } else if (u.startsWith("blobstore://"_sr)) { std::string resource; // The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options. @@ -278,15 +278,15 @@ Reference IBackupContainer::openContainer(const std::string& u for (auto c : resource) if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url(); - r = Reference(new BackupContainerS3BlobStore(bstore, resource, backupParams)); + r = makeReference(bstore, resource, backupParams); } #ifdef BUILD_AZURE_BACKUP - else if (u.startsWith(LiteralStringRef("azure://"))) { - u.eat(LiteralStringRef("azure://")); - auto address = NetworkAddress::parse(u.eat(LiteralStringRef("/")).toString()); - auto containerName = u.eat(LiteralStringRef("/")).toString(); - auto accountName = u.eat(LiteralStringRef("/")).toString(); - r = Reference(new BackupContainerAzureBlobStore(address, containerName, accountName)); + else if (u.startsWith("azure://"_sr)) { + u.eat("azure://"_sr); + auto address = NetworkAddress::parse(u.eat("/"_sr).toString()); + auto containerName = u.eat("/"_sr).toString(); + auto accountName = u.eat("/"_sr).toString(); + r = makeReference(address, containerName, accountName); } #endif else { @@ -316,10 +316,10 @@ Reference IBackupContainer::openContainer(const std::string& u ACTOR Future> listContainers_impl(std::string baseURL) { try { StringRef u(baseURL); - if (u.startsWith(LiteralStringRef("file://"))) { + if (u.startsWith("file://"_sr)) { std::vector results = wait(BackupContainerLocalDirectory::listURLs(baseURL)); return results; - } else if (u.startsWith(LiteralStringRef("blobstore://"))) { + } else if (u.startsWith("blobstore://"_sr)) { std::string resource; S3BlobStoreEndpoint::ParametersT backupParams; @@ -341,7 +341,7 @@ ACTOR Future> listContainers_impl(std::string baseURL) } // TODO: Enable this when Azure backups are ready /* - else if (u.startsWith(LiteralStringRef("azure://"))) { + else if (u.startsWith("azure://"_sr)) { std::vector results = wait(BackupContainerAzureBlobStore::listURLs(baseURL)); return results; } From 3d6515bd149b848fb46ba6b6a1feca5de391cbc7 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 00:07:27 -0700 Subject: [PATCH 045/426] Support encryption for blob store backups (not yet tested) --- fdbclient/AsyncFileS3BlobStore.actor.h | 2 +- fdbclient/BackupContainer.actor.cpp | 8 ++-- .../BackupContainerAzureBlobStore.actor.cpp | 39 +++++++++++++------ fdbclient/BackupContainerAzureBlobStore.h | 3 +- fdbclient/BackupContainerFileSystem.actor.cpp | 22 +++++++++++ fdbclient/BackupContainerFileSystem.h | 8 ++++ .../BackupContainerLocalDirectory.actor.cpp | 24 ++---------- fdbclient/BackupContainerLocalDirectory.h | 2 - .../BackupContainerS3BlobStore.actor.cpp | 33 +++++++++++----- fdbclient/BackupContainerS3BlobStore.h | 3 +- fdbrpc/AsyncFileEncrypted.h | 1 + 11 files changed, 94 insertions(+), 51 deletions(-) diff --git a/fdbclient/AsyncFileS3BlobStore.actor.h b/fdbclient/AsyncFileS3BlobStore.actor.h index bc520bda90..db436755b3 100644 --- a/fdbclient/AsyncFileS3BlobStore.actor.h +++ b/fdbclient/AsyncFileS3BlobStore.actor.h @@ -256,7 +256,7 @@ public: m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) { // Add first part - m_parts.push_back(Reference(new Part(1, m_bstore->knobs.multipart_min_part_size))); + m_parts.push_back(makeReference(1, m_bstore->knobs.multipart_min_part_size)); } }; diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 29fef0486a..fb7edbf753 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -58,6 +58,7 @@ ACTOR Future appendStringRefWithLen(Reference file, Standalon wait(file->append(s.begin(), s.size())); return Void(); } + } // namespace IBackupFile_impl Future IBackupFile::appendStringRefWithLen(Standalone s) { @@ -278,7 +279,7 @@ Reference IBackupContainer::openContainer(const std::string& u for (auto c : resource) if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url(); - r = makeReference(bstore, resource, backupParams); + r = makeReference(bstore, resource, backupParams, encryptionKeyFileName); } #ifdef BUILD_AZURE_BACKUP else if (u.startsWith("azure://"_sr)) { @@ -286,7 +287,8 @@ Reference IBackupContainer::openContainer(const std::string& u auto address = NetworkAddress::parse(u.eat("/"_sr).toString()); auto containerName = u.eat("/"_sr).toString(); auto accountName = u.eat("/"_sr).toString(); - r = makeReference(address, containerName, accountName); + r = makeReference( + address, containerName, accountName, encryptionKeyFileName); } #endif else { @@ -334,7 +336,7 @@ ACTOR Future> listContainers_impl(std::string baseURL) } // Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name - BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams); + BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {}); std::vector results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket())); return results; diff --git a/fdbclient/BackupContainerAzureBlobStore.actor.cpp b/fdbclient/BackupContainerAzureBlobStore.actor.cpp index dea07c382e..3af1a4a7cd 100644 --- a/fdbclient/BackupContainerAzureBlobStore.actor.cpp +++ b/fdbclient/BackupContainerAzureBlobStore.actor.cpp @@ -19,6 +19,7 @@ */ #include "fdbclient/BackupContainerAzureBlobStore.h" +#include "fdbrpc/AsyncFileEncrypted.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -167,8 +168,13 @@ public: if (!exists) { throw file_not_found(); } - return Reference( - new ReadFile(self->asyncTaskThread, self->containerName, fileName, self->client.get())); + Reference f = + makeReference(self->asyncTaskThread, self->containerName, fileName, self->client.get()); + if (self->usesEncryption()) { + f = makeReference(f, false); + } + f = makeReference(self->asyncTaskThread, self->containerName, fileName, self->client.get()); + return f; } ACTOR static Future> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) { @@ -177,10 +183,11 @@ public: auto outcome = client->create_append_blob(containerName, fileName).get(); return Void(); })); - return Reference( - new BackupFile(fileName, - Reference(new WriteFile( - self->asyncTaskThread, self->containerName, fileName, self->client.get())))); + auto f = makeReference(self->asyncTaskThread, self->containerName, fileName, self->client.get()); + if (self->usesEncryption()) { + f = makeReference(f, true); + } + return makeReference(fileName, f); } static void listFiles(AzureClient* client, @@ -213,6 +220,16 @@ public: } return Void(); } + + ACTOR static Future create(BackupContainerAzureBlobStore* self) { + state Future f1 = + self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] { + client->create_container(containerName).wait(); + return Void(); + }); + state Future f2 = self->usesEncryption() ? self->encryptionSetupComplete() : Void(); + return f1 && f2; + } }; Future BackupContainerAzureBlobStore::blobExists(const std::string& fileName) { @@ -225,10 +242,11 @@ Future BackupContainerAzureBlobStore::blobExists(const std::string& fileNa BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName, - const std::string& containerName) + const std::string& containerName, + const Optional& encryptionKeyFileName) : containerName(containerName) { + setEncryptionKey(encryptionKeyFileName); std::string accountKey = std::getenv("AZURE_KEY"); - auto credential = std::make_shared(accountName, accountKey); auto storageAccount = std::make_shared( accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str())); @@ -244,10 +262,7 @@ void BackupContainerAzureBlobStore::delref() { } Future BackupContainerAzureBlobStore::create() { - return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] { - client->create_container(containerName).wait(); - return Void(); - }); + return BackupContainerAzureBlobStoreImpl::create(this); } Future BackupContainerAzureBlobStore::exists() { return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] { diff --git a/fdbclient/BackupContainerAzureBlobStore.h b/fdbclient/BackupContainerAzureBlobStore.h index 193fe4a301..aae378fcf4 100644 --- a/fdbclient/BackupContainerAzureBlobStore.h +++ b/fdbclient/BackupContainerAzureBlobStore.h @@ -44,7 +44,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem, public: BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName, - const std::string& containerName); + const std::string& containerName, + const Optional& encryptionKeyFileName); void addref() override; void delref() override; diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 5d5a7fa722..94b2c10d99 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -23,6 +23,7 @@ #include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BackupContainerLocalDirectory.h" #include "fdbclient/JsonBuilder.h" +#include "flow/StreamCipher.h" #include "flow/UnitTest.h" #include @@ -1126,6 +1127,16 @@ public: return false; } + ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { + state Reference keyFile = + wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + state std::array key; + int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); + // TODO: Throw new error (fail gracefully) + ASSERT_EQ(bytesRead, key.size()); + StreamCipher::Key::initializeKey(std::move(key)); + return Void(); + } }; // class BackupContainerFileSystemImpl Future> BackupContainerFileSystem::writeLogFile(Version beginVersion, @@ -1432,6 +1443,17 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() { return { Reference::addRef(this), "mutation_log_type" }; } +bool BackupContainerFileSystem::usesEncryption() const { + return encryptionSetupFuture.isValid(); +} +Future BackupContainerFileSystem::encryptionSetupComplete() const { + return encryptionSetupFuture; +} +void BackupContainerFileSystem::setEncryptionKey(Optional const& encryptionKeyFileName) { + if (encryptionKeyFileName.present()) { + encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get()); + } +} namespace backup_test { diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h index cd0ddf4435..8c0967e701 100644 --- a/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/BackupContainerFileSystem.h @@ -186,6 +186,14 @@ private: Future> old_listRangeFiles(Version beginVersion, Version endVersion); friend class BackupContainerFileSystemImpl; + +protected: + bool usesEncryption() const; + void setEncryptionKey(Optional const& encryptionKeyFileName); + Future encryptionSetupComplete() const; + +private: + Future encryptionSetupFuture; }; #endif diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp index e15423df74..b89d085a64 100644 --- a/fdbclient/BackupContainerLocalDirectory.actor.cpp +++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp @@ -23,7 +23,6 @@ #include "fdbrpc/IAsyncFile.h" #include "flow/Platform.actor.h" #include "flow/Platform.h" -#include "flow/StreamCipher.h" #include "fdbrpc/simulator.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -132,25 +131,9 @@ std::string BackupContainerLocalDirectory::getURLFormat() { return "file://"; } -ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { - state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); - state std::array key; - int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); - // TODO: Throw new error (fail gracefully) - ASSERT_EQ(bytesRead, key.size()); - StreamCipher::Key::initializeKey(std::move(key)); - return Void(); -} - -bool BackupContainerLocalDirectory::usesEncryption() const { - return encryptionSetupFuture.isValid(); -} - BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url, const Optional& encryptionKeyFileName) { - if (encryptionKeyFileName.present()) { - encryptionSetupFuture = readEncryptionKey(encryptionKeyFileName.get()); - } + setEncryptionKey(encryptionKeyFileName); std::string path; if (url.find("file://") != 0) { @@ -214,10 +197,9 @@ Future> BackupContainerLocalDirectory::listURLs(const s Future BackupContainerLocalDirectory::create() { if (usesEncryption()) { - return encryptionSetupFuture; + return encryptionSetupComplete(); } - // TODO: Update this comment: - // Nothing should be done here because create() can be called by any process working with the container URL, + // No directory should be created here because create() can be called by any process working with the container URL, // such as fdbbackup. Since "local directory" containers are by definition local to the machine they are // accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to // every file creation, which is done in openFile(). Creating the directory here will result in unnecessary diff --git a/fdbclient/BackupContainerLocalDirectory.h b/fdbclient/BackupContainerLocalDirectory.h index 52cd810907..f7c77e4636 100644 --- a/fdbclient/BackupContainerLocalDirectory.h +++ b/fdbclient/BackupContainerLocalDirectory.h @@ -54,8 +54,6 @@ public: private: std::string m_path; - Future encryptionSetupFuture; - bool usesEncryption() const; }; #endif diff --git a/fdbclient/BackupContainerS3BlobStore.actor.cpp b/fdbclient/BackupContainerS3BlobStore.actor.cpp index 4e89402ae0..2144347cab 100644 --- a/fdbclient/BackupContainerS3BlobStore.actor.cpp +++ b/fdbclient/BackupContainerS3BlobStore.actor.cpp @@ -20,6 +20,7 @@ #include "fdbclient/AsyncFileS3BlobStore.actor.h" #include "fdbclient/BackupContainerS3BlobStore.h" +#include "fdbrpc/AsyncFileEncrypted.h" #include "fdbrpc/AsyncFileReadAhead.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -103,6 +104,10 @@ public: wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), "")); } + if (bc->usesEncryption()) { + wait(bc->encryptionSetupComplete()); + } + return Void(); } @@ -137,9 +142,10 @@ std::string BackupContainerS3BlobStore::indexEntry() { BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference bstore, const std::string& name, - const S3BlobStoreEndpoint::ParametersT& params) + const S3BlobStoreEndpoint::ParametersT& params, + const Optional& encryptionKeyFileName) : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") { - + setEncryptionKey(encryptionKeyFileName); // Currently only one parameter is supported, "bucket" for (const auto& [name, value] : params) { if (name == "bucket") { @@ -164,12 +170,16 @@ std::string BackupContainerS3BlobStore::getURLFormat() { } Future> BackupContainerS3BlobStore::readFile(const std::string& path) { - return Reference(new AsyncFileReadAheadCache( - Reference(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))), - m_bstore->knobs.read_block_size, - m_bstore->knobs.read_ahead_blocks, - m_bstore->knobs.concurrent_reads_per_file, - m_bstore->knobs.read_cache_blocks_per_file)); + Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); + if (usesEncryption()) { + f = makeReference(f, false); + } + f = makeReference(f, + m_bstore->knobs.read_block_size, + m_bstore->knobs.read_ahead_blocks, + m_bstore->knobs.concurrent_reads_per_file, + m_bstore->knobs.read_cache_blocks_per_file); + return f; } Future> BackupContainerS3BlobStore::listURLs(Reference bstore, @@ -178,8 +188,11 @@ Future> BackupContainerS3BlobStore::listURLs(Reference< } Future> BackupContainerS3BlobStore::writeFile(const std::string& path) { - return Reference(new BackupContainerS3BlobStoreImpl::BackupFile( - path, Reference(new AsyncFileS3BlobStoreWrite(m_bstore, m_bucket, dataPath(path))))); + Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); + if (usesEncryption()) { + f = makeReference(f, true); + } + return Future>(makeReference(path, f)); } Future BackupContainerS3BlobStore::deleteFile(const std::string& path) { diff --git a/fdbclient/BackupContainerS3BlobStore.h b/fdbclient/BackupContainerS3BlobStore.h index 57199fcb85..9e47483adf 100644 --- a/fdbclient/BackupContainerS3BlobStore.h +++ b/fdbclient/BackupContainerS3BlobStore.h @@ -43,7 +43,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem, public: BackupContainerS3BlobStore(Reference bstore, const std::string& name, - const S3BlobStoreEndpoint::ParametersT& params); + const S3BlobStoreEndpoint::ParametersT& params, + const Optional& encryptionKeyFileName); void addref() override; void delref() override; diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index af123f2650..6be26c8793 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -60,6 +60,7 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted initialize(); public: + // TODO: Remove boolean parameter here: AsyncFileEncrypted(Reference, bool canWrite); void addref() override; void delref() override; From c5b612510de65c39369a5bae1ba003d88f74b77e Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 11:14:23 -0700 Subject: [PATCH 046/426] Added invalid_encryption_key_file error --- fdbclient/BackupContainerFileSystem.actor.cpp | 3 +++ flow/error_definitions.h | 1 + 2 files changed, 4 insertions(+) diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 94b2c10d99..741ed55ce0 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -1454,6 +1454,9 @@ void BackupContainerFileSystem::setEncryptionKey(Optional const& en encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get()); } } +Future BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const &filename) { + return BackupContainerFileSystemImpl::createTestEncryptionKeyFile(filename); +} namespace backup_test { diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 0fd42b3ac0..8ffb54f290 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -227,6 +227,7 @@ ERROR( restore_destination_not_empty, 2370, "Attempted to restore into a non-emp ERROR( restore_duplicate_uid, 2371, "Attempted to restore using a UID that had been used for an aborted restore") ERROR( task_invalid_version, 2381, "Invalid task version") ERROR( task_interrupted, 2382, "Task execution stopped due to timeout, abort, or completion by another worker") +ERROR( invalid_encryption_key_file, 2383, "The provided encryption key file has invalid contents" ) ERROR( key_not_found, 2400, "Expected key is missing") ERROR( json_malformed, 2401, "JSON string was malformed") From 27e44c1bb9f9b7e6a3b1f40a026820791b51d066 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 11:15:12 -0700 Subject: [PATCH 047/426] Add support for encryption testing in BackupCorrectness workload --- fdbbackup/backup.actor.cpp | 7 +- fdbclient/AsyncTaskThread.actor.cpp | 2 +- fdbclient/BackupContainerFileSystem.actor.cpp | 79 +++++++++++-------- fdbclient/BackupContainerFileSystem.h | 13 +-- fdbrpc/AsyncFileEncrypted.actor.cpp | 6 +- fdbrpc/AsyncFileEncrypted.h | 1 - .../workloads/BackupCorrectness.actor.cpp | 77 ++++++++++++------ flow/StreamCipher.cpp | 4 +- flow/StreamCipher.h | 3 +- 9 files changed, 116 insertions(+), 76 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 0213bece1a..488ff04d76 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2490,9 +2490,8 @@ ACTOR Future runFastRestoreTool(Database db, ACTOR Future dumpBackupData(const char* name, std::string destinationContainer, Version beginVersion, - Version endVersion, - Optional encryptionKeyFile) { - state Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); + Version endVersion) { + state Reference c = openBackupContainer(name, destinationContainer); if (beginVersion < 0 || endVersion < 0) { BackupDescription desc = wait(c->describeBackup()); @@ -3994,7 +3993,7 @@ int main(int argc, char* argv[]) { case BackupType::DUMP: initTraceFile(); - f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd, encryptionKeyFile)); + f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd)); break; case BackupType::UNDEFINED: diff --git a/fdbclient/AsyncTaskThread.actor.cpp b/fdbclient/AsyncTaskThread.actor.cpp index 2e7c6e3596..050af68c29 100644 --- a/fdbclient/AsyncTaskThread.actor.cpp +++ b/fdbclient/AsyncTaskThread.actor.cpp @@ -83,6 +83,6 @@ TEST_CASE("/asynctaskthread/add") { clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100)); } wait(waitForAll(clients)); - ASSERT(sum == 1000); + ASSERT_EQ(sum, 1000); return Void(); } diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 741ed55ce0..f0a3e80e0f 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -291,13 +291,13 @@ public: std::map> tagIndices; // tagId -> indices in files for (int i = 0; i < logs.size(); i++) { - ASSERT(logs[i].tagId >= 0); - ASSERT(logs[i].tagId < logs[i].totalTags); + ASSERT_GE(logs[i].tagId, 0); + ASSERT_LT(logs[i].tagId, logs[i].totalTags); auto& indices = tagIndices[logs[i].tagId]; // filter out if indices.back() is subset of files[i] or vice versa if (!indices.empty()) { if (logs[indices.back()].isSubset(logs[i])) { - ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[indices.back()].fileSize, logs[i].fileSize); indices.back() = i; } else if (!logs[i].isSubset(logs[indices.back()])) { indices.push_back(i); @@ -865,7 +865,7 @@ public: int i = 0; for (int j = 1; j < logs.size(); j++) { if (logs[j].isSubset(logs[i])) { - ASSERT(logs[j].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[j].fileSize, logs[i].fileSize); continue; } @@ -1033,10 +1033,10 @@ public: } static std::string versionFolderString(Version v, int smallestBucket) { - ASSERT(smallestBucket < 14); + ASSERT_LT(smallestBucket, 14); // Get a 0-padded fixed size representation of v std::string vFixedPrecision = format("%019lld", v); - ASSERT(vFixedPrecision.size() == 19); + ASSERT_EQ(vFixedPrecision.size(), 19); // Truncate smallestBucket from the fixed length representation vFixedPrecision.resize(vFixedPrecision.size() - smallestBucket); @@ -1127,12 +1127,37 @@ public: return false; } + ACTOR static Future createTestEncryptionKeyFile(std::string filename) { + state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( + filename, + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, + 0600)); + StreamCipher::Key::RawKeyType testKey; + generateRandomData(testKey.data(), testKey.size()); + keyFile->write(testKey.data(), testKey.size(), 0); + wait(keyFile->sync()); + return Void(); + } + ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { - state Reference keyFile = - wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); - state std::array key; + state Reference keyFile; + try { + Reference _keyFile = + wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + } catch (Error& e) { + TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile") + .detail("FileName", encryptionKeyFileName) + .error(e); + throw e; + } + state StreamCipher::Key::RawKeyType key; int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); - // TODO: Throw new error (fail gracefully) + if (bytesRead != key.size()) { + TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize") + .detail("ExpectedSize", key.size()) + .detail("ActualSize", bytesRead); + throw invalid_encryption_key_file(); + } ASSERT_EQ(bytesRead, key.size()); StreamCipher::Key::initializeKey(std::move(key)); return Void(); @@ -1496,7 +1521,7 @@ ACTOR Future writeAndVerifyFile(Reference c, Reference> buf; buf.resize(buf.arena(), fileSize); int b = wait(inputFile->read(buf.begin(), buf.size(), 0)); - ASSERT(b == buf.size()); + ASSERT_EQ(b, buf.size()); ASSERT(buf == content); } return Void(); @@ -1510,7 +1535,7 @@ Version nextVersion(Version v) { // Write a snapshot file with only begin & end key ACTOR static Future testWriteSnapshotFile(Reference file, Key begin, Key end, uint32_t blockSize) { - ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size()); + ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size()); uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION; // write Header @@ -1531,23 +1556,11 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key return Void(); } -ACTOR Future createTestEncryptionKeyFile(std::string filename) { - state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( - filename, - IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, - 0600)); - std::array testKey; - generateRandomData(testKey.data(), testKey.size()); - keyFile->write(testKey.data(), testKey.size(), 0); - wait(keyFile->sync()); - return Void(); -} - ACTOR Future testBackupContainer(std::string url, Optional encryptionKeyFileName) { state FlowLock lock(100e6); if (encryptionKeyFileName.present()) { - wait(createTestEncryptionKeyFile(encryptionKeyFileName.get())); + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get())); } printf("BackupContainerTest URL %s\n", url.c_str()); @@ -1638,9 +1651,9 @@ ACTOR Future testBackupContainer(std::string url, Optional en wait(waitForAll(writes)); state BackupFileList listing = wait(c->dumpFileList()); - ASSERT(listing.ranges.size() == nRangeFiles); - ASSERT(listing.logs.size() == logs.size()); - ASSERT(listing.snapshots.size() == snapshots.size()); + ASSERT_EQ(listing.ranges.size(), nRangeFiles); + ASSERT_EQ(listing.logs.size(), logs.size()); + ASSERT_EQ(listing.snapshots.size(), snapshots.size()); state BackupDescription desc = wait(c->describeBackup()); printf("\n%s\n", desc.toString().c_str()); @@ -1670,8 +1683,8 @@ ACTOR Future testBackupContainer(std::string url, Optional en // If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot if (f.isError()) { - ASSERT(f.getError().code() == error_code_backup_cannot_expire); - ASSERT(i == listing.snapshots.size() - 1); + ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire); + ASSERT_EQ(i, listing.snapshots.size() - 1); wait(c->expireData(expireVersion, true)); } @@ -1687,9 +1700,9 @@ ACTOR Future testBackupContainer(std::string url, Optional en ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist); BackupFileList empty = wait(c->dumpFileList()); - ASSERT(empty.ranges.size() == 0); - ASSERT(empty.logs.size() == 0); - ASSERT(empty.snapshots.size() == 0); + ASSERT_EQ(empty.ranges.size(), 0); + ASSERT_EQ(empty.logs.size(), 0); + ASSERT_EQ(empty.snapshots.size(), 0); printf("BackupContainerTest URL=%s PASSED.\n", url.c_str()); diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h index 8c0967e701..6acf2d87a7 100644 --- a/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/BackupContainerFileSystem.h @@ -153,6 +153,13 @@ public: bool logsOnly, Version beginVersion) final; + static Future createTestEncryptionKeyFile(std::string const& filename); + +protected: + bool usesEncryption() const; + void setEncryptionKey(Optional const& encryptionKeyFileName); + Future encryptionSetupComplete() const; + private: struct VersionProperty { VersionProperty(Reference bc, const std::string& name) @@ -187,12 +194,6 @@ private: friend class BackupContainerFileSystemImpl; -protected: - bool usesEncryption() const; - void setEncryptionKey(Optional const& encryptionKeyFileName); - Future encryptionSetupComplete() const; - -private: Future encryptionSetupFuture; }; diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index ffecfae3c1..6e33243df3 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -102,7 +102,7 @@ public: if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) { wait(self->writeLastBlockToFile()); self->offsetInBlock = 0; - ASSERT(self->currentBlock < std::numeric_limits::max()); + ASSERT_LT(self->currentBlock, std::numeric_limits::max()); ++self->currentBlock; self->encryptor = std::make_unique(StreamCipher::Key::getKey(), self->getIV(self->currentBlock)); @@ -205,7 +205,7 @@ Future AsyncFileEncrypted::writeLastBlockToFile() { } size_t AsyncFileEncrypted::RandomCache::evict() { - ASSERT(vec.size() == maxSize); + ASSERT_EQ(vec.size(), maxSize); auto index = deterministicRandom()->randomInt(0, maxSize); hashMap.erase(vec[index]); return index; @@ -263,7 +263,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") { while (bytesRead < bytes) { chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead); int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead)); - ASSERT(bytesReadInChunk == chunkSize); + ASSERT_EQ(bytesReadInChunk, chunkSize); bytesRead += bytesReadInChunk; } ASSERT(writeBuffer == readBuffer); diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index 6be26c8793..af123f2650 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -60,7 +60,6 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted initialize(); public: - // TODO: Remove boolean parameter here: AsyncFileEncrypted(Reference, bool canWrite); void addref() override; void delref() override; diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp index abd3609325..070db26050 100644 --- a/fdbserver/workloads/BackupCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupCorrectness.actor.cpp @@ -21,6 +21,7 @@ #include "fdbrpc/simulator.h" #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" +#include "fdbclient/BackupContainerFileSystem.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/workloads/BulkSetup.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -41,35 +42,39 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { bool allowPauses; bool shareLogRange; bool shouldSkipRestoreRanges; + Optional encryptionKeyFileName; BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { locked = sharedRandomNumber % 2; - backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0); - restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0); - performRestore = getOption(options, LiteralStringRef("performRestore"), true); - backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag()); - backupRangesCount = getOption(options, LiteralStringRef("backupRangesCount"), 5); - backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1); + backupAfter = getOption(options, "backupAfter"_sr, 10.0); + restoreAfter = getOption(options, "restoreAfter"_sr, 35.0); + performRestore = getOption(options, "performRestore"_sr, true); + backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag()); + backupRangesCount = getOption(options, "backupRangesCount"_sr, 5); + backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1); abortAndRestartAfter = getOption(options, - LiteralStringRef("abortAndRestartAfter"), + "abortAndRestartAfter"_sr, deterministicRandom()->random01() < 0.5 ? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter : 0.0); - differentialBackup = getOption( - options, LiteralStringRef("differentialBackup"), deterministicRandom()->random01() < 0.5 ? true : false); + differentialBackup = + getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false); stopDifferentialAfter = getOption(options, - LiteralStringRef("stopDifferentialAfter"), + "stopDifferentialAfter"_sr, differentialBackup ? deterministicRandom()->random01() * (restoreAfter - std::max(abortAndRestartAfter, backupAfter)) + std::max(abortAndRestartAfter, backupAfter) : 0.0); - agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true); - allowPauses = getOption(options, LiteralStringRef("allowPauses"), true); - shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false); - restorePrefixesToInclude = getOption(options, LiteralStringRef("restorePrefixesToInclude"), std::vector()); + agentRequest = getOption(options, "simBackupAgents"_sr, true); + allowPauses = getOption(options, "allowPauses"_sr, true); + shareLogRange = getOption(options, "shareLogRange"_sr, false); + restorePrefixesToInclude = getOption(options, "restorePrefixesToInclude"_sr, std::vector()); shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false; + if (getOption(options, "encrypted"_sr, false)) { + encryptionKeyFileName = "simfdb/test_encryption_key_file"; + } TraceEvent("BARW_ClientId").detail("Id", wcx.clientId); UID randomID = nondeterministicRandom()->randomUniqueID(); @@ -77,11 +82,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { if (shareLogRange) { bool beforePrefix = sharedRandomNumber & 1; if (beforePrefix) - backupRanges.push_back_deep(backupRanges.arena(), - KeyRangeRef(normalKeys.begin, LiteralStringRef("\xfe\xff\xfe"))); + backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr)); else backupRanges.push_back_deep(backupRanges.arena(), - KeyRangeRef(strinc(LiteralStringRef("\x00\x00\x01")), normalKeys.end)); + KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end)); } else if (backupRangesCount <= 0) { backupRanges.push_back_deep(backupRanges.arena(), normalKeys); } else { @@ -265,7 +269,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { deterministicRandom()->randomInt(0, 100), tag.toString(), backupRanges, - stopDifferentialDelay ? false : true)); + stopDifferentialDelay ? false : true, + false, + false, + self->encryptionKeyFileName)); } catch (Error& e) { TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag)); if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) @@ -456,6 +463,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { BackupAndRestoreCorrectnessWorkload::backupAgentRequests++; } + if (self->encryptionKeyFileName.present()) { + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get())); + } + try { state Future startRestore = delay(self->restoreAfter); @@ -510,7 +521,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag)); try { extraBackup = backupAgent.submitBackup(cx, - LiteralStringRef("file://simfdb/backups/"), + "file://simfdb/backups/"_sr, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), self->backupTag.toString(), @@ -587,7 +598,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { range, Key(), Key(), - self->locked)); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName)); } } else { multipleRangesInOneTag = true; @@ -606,7 +621,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { true, Key(), Key(), - self->locked)); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName)); } // Sometimes kill and restart the restore @@ -632,7 +651,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { true, Key(), Key(), - self->locked); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName); } } else { for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) { @@ -657,7 +680,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { self->restoreRanges[restoreIndex], Key(), Key(), - self->locked); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName); } } } @@ -721,7 +748,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { .detail("TaskCount", taskCount) .detail("WaitCycles", waitCycles); printf("EndingNonZeroTasks: %ld\n", (long)taskCount); - wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); + wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef())); } loop { @@ -820,7 +847,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { } if (displaySystemKeys) { - wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); + wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef())); } TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag)); diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index e0066ae4a5..7a2c896564 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -44,7 +44,7 @@ void StreamCipher::cleanup() noexcept { } } -void StreamCipher::Key::initializeKey(std::array&& arr) { +void StreamCipher::Key::initializeKey(RawKeyType&& arr) { ASSERT(!globalKey); globalKey = std::make_unique(ConstructorTag{}); globalKey->arr = std::move(arr); @@ -180,7 +180,7 @@ TEST_CASE("flow/StreamCipher") { } const auto decrypted = decryptor.finish(arena); std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]); - ASSERT(decryptedOffset + decrypted.size() == plaintext.size()); + ASSERT_EQ(decryptedOffset + decrypted.size(), plaintext.size()); decryptedtext.resize(decryptedOffset + decrypted.size()); } diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index f91dc272ce..57c2e0e436 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -46,12 +46,13 @@ public: struct ConstructorTag {}; public: + using RawKeyType = decltype(arr); Key(ConstructorTag) {} Key(Key&&); Key& operator=(Key&&); ~Key(); unsigned char const* data() const { return arr.data(); } - static void initializeKey(decltype(arr)&&); + static void initializeKey(RawKeyType&&); static void initializeRandomTestKey(); static const Key& getKey(); static void cleanup() noexcept; From cbdf5bf6b7d5340fab5c47d9aaee1c1f9f04d392 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 17:38:57 -0700 Subject: [PATCH 048/426] Fix failing BackupCorrectness test with encryption --- fdbclient/BackupContainerFileSystem.actor.cpp | 3 ++- fdbserver/workloads/BackupCorrectness.actor.cpp | 1 + flow/StreamCipher.cpp | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index f0a3e80e0f..a4eb1f6e34 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -1141,16 +1141,17 @@ public: ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { state Reference keyFile; + state StreamCipher::Key::RawKeyType key; try { Reference _keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + keyFile = _keyFile; } catch (Error& e) { TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile") .detail("FileName", encryptionKeyFileName) .error(e); throw e; } - state StreamCipher::Key::RawKeyType key; int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); if (bytesRead != key.size()) { TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize") diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp index 070db26050..71c366481a 100644 --- a/fdbserver/workloads/BackupCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupCorrectness.actor.cpp @@ -172,6 +172,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { TraceEvent(SevInfo, "BARW_Param").detail("DifferentialBackup", differentialBackup); TraceEvent(SevInfo, "BARW_Param").detail("StopDifferentialAfter", stopDifferentialAfter); TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest); + TraceEvent(SevInfo, "BARW_Param").detail("Encrypted", encryptionKeyFileName.present()); return _start(cx, this); } diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index 7a2c896564..bd1f53f1e1 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -45,7 +45,9 @@ void StreamCipher::cleanup() noexcept { } void StreamCipher::Key::initializeKey(RawKeyType&& arr) { - ASSERT(!globalKey); + if (globalKey) { + ASSERT(globalKey->arr == arr); + } globalKey = std::make_unique(ConstructorTag{}); globalKey->arr = std::move(arr); memset(arr.data(), 0, arr.size()); From 9edfffb6a66f49c355322632725b64200f77b99a Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 17:40:40 -0700 Subject: [PATCH 049/426] Test backup encryption in 10% of backup tests --- fdbserver/workloads/BackupCorrectness.actor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp index 71c366481a..4ed98e7e34 100644 --- a/fdbserver/workloads/BackupCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupCorrectness.actor.cpp @@ -72,7 +72,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { shareLogRange = getOption(options, "shareLogRange"_sr, false); restorePrefixesToInclude = getOption(options, "restorePrefixesToInclude"_sr, std::vector()); shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false; - if (getOption(options, "encrypted"_sr, false)) { + if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) { encryptionKeyFileName = "simfdb/test_encryption_key_file"; } From 078d67d5fbc38c37cb92fe72d7dd9cec4aaf1be8 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 20:19:22 -0700 Subject: [PATCH 050/426] Fix flowbench build --- flowbench/BenchEncrypt.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flowbench/BenchEncrypt.cpp b/flowbench/BenchEncrypt.cpp index 91cf5fd978..54834bc829 100644 --- a/flowbench/BenchEncrypt.cpp +++ b/flowbench/BenchEncrypt.cpp @@ -41,7 +41,7 @@ static void bench_encrypt(benchmark::State& state) { auto bytes = state.range(0); auto chunks = state.range(1); auto chunkSize = bytes / chunks; - StreamCipher::Key::initializeRandomKey(); + StreamCipher::Key::initializeRandomTestKey(); const auto& key = StreamCipher::Key::getKey(); auto iv = getRandomIV(); auto data = getKey(bytes); @@ -57,7 +57,7 @@ static void bench_decrypt(benchmark::State& state) { auto bytes = state.range(0); auto chunks = state.range(1); auto chunkSize = bytes / chunks; - StreamCipher::Key::initializeRandomKey(); + StreamCipher::Key::initializeRandomTestKey(); const auto& key = StreamCipher::Key::getKey(); auto iv = getRandomIV(); auto data = getKey(bytes); From 8855c7ee8daa563b320b8d9f3a4c9dadc73e6146 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 27 Jun 2021 16:47:54 -0700 Subject: [PATCH 051/426] Set error kind to BugDetected for Crash trace events --- flow/Platform.actor.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 4ba8610b79..27cbb7d0dd 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -3419,10 +3419,13 @@ void crashHandler(int sig) { bool error = (sig != SIGUSR2); fflush(stdout); - TraceEvent(error ? SevError : SevInfo, error ? "Crash" : "ProcessTerminated") - .detail("Signal", sig) - .detail("Name", strsignal(sig)) - .detail("Trace", backtrace); + { + TraceEvent te(error ? SevError : SevInfo, error ? "Crash" : "ProcessTerminated"); + te.detail("Signal", sig).detail("Name", strsignal(sig)).detail("Trace", backtrace); + if (error) { + te.setErrorKind(ErrorKind::BugDetected); + } + } flushTraceFileVoid(); fprintf(stderr, "SIGNAL: %s (%d)\n", strsignal(sig), sig); From a5ecc11bbad7c97c0cea2cb67f8bb555c6a18ce1 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sun, 27 Jun 2021 18:55:57 -0700 Subject: [PATCH 052/426] Added AsyncFileEncrypted::mode field --- .../BackupContainerS3BlobStore.actor.cpp | 4 ++-- fdbrpc/AsyncFileEncrypted.actor.cpp | 23 +++++++++++-------- fdbrpc/AsyncFileEncrypted.h | 8 +++++-- fdbrpc/Net2FileSystem.cpp | 4 +++- fdbrpc/sim2.actor.cpp | 4 +++- 5 files changed, 27 insertions(+), 16 deletions(-) diff --git a/fdbclient/BackupContainerS3BlobStore.actor.cpp b/fdbclient/BackupContainerS3BlobStore.actor.cpp index 2144347cab..02112c2f58 100644 --- a/fdbclient/BackupContainerS3BlobStore.actor.cpp +++ b/fdbclient/BackupContainerS3BlobStore.actor.cpp @@ -172,7 +172,7 @@ std::string BackupContainerS3BlobStore::getURLFormat() { Future> BackupContainerS3BlobStore::readFile(const std::string& path) { Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); if (usesEncryption()) { - f = makeReference(f, false); + f = makeReference(f, AsyncFileEncrypted::Mode::READ_ONLY); } f = makeReference(f, m_bstore->knobs.read_block_size, @@ -190,7 +190,7 @@ Future> BackupContainerS3BlobStore::listURLs(Reference< Future> BackupContainerS3BlobStore::writeFile(const std::string& path) { Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); if (usesEncryption()) { - f = makeReference(f, true); + f = makeReference(f, AsyncFileEncrypted::Mode::APPEND_ONLY); } return Future>(makeReference(path, f)); } diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index 6e33243df3..588c6e5cc3 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -59,6 +59,7 @@ public: state uint16_t block; state unsigned char* output = reinterpret_cast(data); state int bytesRead = 0; + ASSERT(self->mode == AsyncFileEncrypted::Mode::READ_ONLY); for (block = firstBlock; block <= lastBlock; ++block) { state StringRef plaintext; @@ -86,7 +87,7 @@ public: } ACTOR static Future write(AsyncFileEncrypted* self, void const* data, int length, int64_t offset) { - ASSERT(self->canWrite); + ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY); // All writes must append to the end of the file: ASSERT_EQ(offset, self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock); state unsigned char const* input = reinterpret_cast(data); @@ -112,13 +113,14 @@ public: } ACTOR static Future sync(AsyncFileEncrypted* self) { - ASSERT(self->canWrite); + ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY); wait(self->writeLastBlockToFile()); wait(self->file->sync()); return Void(); } ACTOR static Future zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) { + ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY); // TODO: Could optimize this Arena arena; auto zeroes = new (arena) unsigned char[length]; @@ -128,10 +130,10 @@ public: } }; -AsyncFileEncrypted::AsyncFileEncrypted(Reference file, bool canWrite) - : file(file), canWrite(canWrite), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { +AsyncFileEncrypted::AsyncFileEncrypted(Reference file, Mode mode) + : file(file), mode(mode), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename()); - if (canWrite) { + if (mode == Mode::APPEND_ONLY) { encryptor = std::make_unique(StreamCipher::Key::getKey(), getIV(currentBlock)); writeBuffer = std::vector(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0); } @@ -158,21 +160,22 @@ Future AsyncFileEncrypted::zeroRange(int64_t offset, int64_t length) { } Future AsyncFileEncrypted::truncate(int64_t size) { - // FIXME: Not yet implemented - ASSERT(canWrite); - return Void(); + ASSERT(mode == Mode::APPEND_ONLY); + return file->truncate(size); } Future AsyncFileEncrypted::sync() { - ASSERT(canWrite); + ASSERT(mode == Mode::APPEND_ONLY); return AsyncFileEncryptedImpl::sync(this); } Future AsyncFileEncrypted::flush() { + ASSERT(mode == Mode::APPEND_ONLY); return Void(); } Future AsyncFileEncrypted::size() const { + ASSERT(mode == Mode::READ_ONLY); return file->size(); } @@ -190,7 +193,7 @@ void AsyncFileEncrypted::releaseZeroCopy(void* data, int length, int64_t offset) } int64_t AsyncFileEncrypted::debugFD() const { - return 0; + return file->debugFD(); } StreamCipher::IV AsyncFileEncrypted::getIV(uint16_t block) const { diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index af123f2650..ed5693de29 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -32,10 +32,14 @@ * Append-only file encrypted using AES-128-GCM. * */ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted { +public: + enum class Mode { APPEND_ONLY, READ_ONLY }; + +private: Reference file; StreamCipher::IV firstBlockIV; StreamCipher::IV getIV(uint16_t block) const; - bool canWrite; + Mode mode; Future writeLastBlockToFile(); friend class AsyncFileEncryptedImpl; @@ -60,7 +64,7 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted initialize(); public: - AsyncFileEncrypted(Reference, bool canWrite); + AsyncFileEncrypted(Reference, Mode); void addref() override; void delref() override; Future read(void* data, int length, int64_t offset) override; diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index 37b849525c..a2a8874bed 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -82,7 +82,9 @@ Future> Net2FileSystem::open(const std::string& file #if (!defined(TLS_DISABLED) && !defined(_WIN32)) if (flags & IAsyncFile::OPEN_ENCRYPTED) f = map(f, [flags](Reference r) { - return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); + auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY + : AsyncFileEncrypted::Mode::READ_ONLY; + return Reference(new AsyncFileEncrypted(r, mode)); }); #endif return f; diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 985bcece5e..9243bda4a1 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -2479,7 +2479,9 @@ Future> Sim2FileSystem::open(const std::string& file #if (!defined(TLS_DISABLED) && !defined(_WIN32)) if (flags & IAsyncFile::OPEN_ENCRYPTED) f = map(f, [flags](Reference r) { - return Reference(new AsyncFileEncrypted(r, flags & IAsyncFile::OPEN_READWRITE)); + auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY + : AsyncFileEncrypted::Mode::READ_ONLY; + return Reference(new AsyncFileEncrypted(r, mode)); }); #endif return f; From 9d3f17ab2c55caacf79030dcc9a68c9443530526 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Mon, 28 Jun 2021 09:32:10 -0700 Subject: [PATCH 053/426] fix error in mapping item to linear bucket, histogram class --- flow/Histogram.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/flow/Histogram.h b/flow/Histogram.h index f3ba327a71..1b8449b3f9 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -141,11 +141,10 @@ public: // This is used when the distance b/t upperBound and lowerBound are relativly small inline void sampleRecordCounter(uint32_t sample) { ASSERT(unit==Histogram::Unit::record_counter); - if(sample == upperBound){ - sample = upperBound - 1; - } - size_t idx = ( (sample - lowerBound) * 32.0 ) / (upperBound - lowerBound); - ASSERT(idx < 32); + + size_t idx = ( (sample - lowerBound) * 31.0 ) / (upperBound - lowerBound); + //ASSERT(idx < 32); + buckets[idx]++; } From c700feaa6ea25a9f3d6286814c39c0463c1ed87a Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Mon, 28 Jun 2021 11:08:20 -0700 Subject: [PATCH 054/426] Address Dan's comments --- fdbserver/CommitProxyServer.actor.cpp | 2 +- fdbserver/LogSystem.h | 24 ++++++++++++--------- fdbserver/TagPartitionedLogSystem.actor.cpp | 1 + 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index cd69a79f92..337a24e956 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -1186,7 +1186,7 @@ ACTOR Future postResolution(CommitBatchContext* self) { span.context, self->debugID); - float ratio = self->toCommit.getEmptyLocationRatio(); + float ratio = self->toCommit.getEmptyMessageRatio(); pProxyCommitData->stats.commitBatchingEmptyMessageRatio.addMeasurement(ratio); if (!self->forceRecovery) { diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 39d4a16871..60ce9ab733 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -29,6 +29,8 @@ #include "fdbserver/WorkerInterface.actor.h" #include "fdbclient/DatabaseConfiguration.h" #include "fdbserver/MutationTracking.h" +#include "flow/Arena.h" +#include "flow/Error.h" #include "flow/IndexedSet.h" #include "flow/Knobs.h" #include "fdbrpc/ReplicationPolicy.h" @@ -970,7 +972,7 @@ struct LogPushData : NonCopyable { } } } - written = std::vector(messagesWriter.size(), false); + isEmptyMessage = std::vector(messagesWriter.size(), false); } void addTxsTag() { @@ -1089,21 +1091,23 @@ struct LogPushData : NonCopyable { } Standalone getMessages(int loc) { - // Update written here because this is called less frequently. - Standalone value = messagesWriter[loc].toValue(); - if (!written[loc]) { + return messagesWriter[loc].toValue(); + } + + void recordEmptyMessage(int loc, const Standalone& value) { + if (!isEmptyMessage[loc]) { BinaryWriter w(AssumeVersion(g_network->protocolVersion())); Standalone v = w.toValue(); if (value.size() > v.size()) { - written[loc] = true; + isEmptyMessage[loc] = true; } } - return value; } - float getEmptyLocationRatio() const { - auto count = std::count(written.begin(), written.end(), false); - return 1.0 * count / written.size(); + float getEmptyMessageRatio() const { + auto count = std::count(isEmptyMessage.begin(), isEmptyMessage.end(), false); + ASSERT_WE_THINK(isEmptyMessage.size() > 0); + return 1.0 * count / isEmptyMessage.size(); } private: @@ -1111,7 +1115,7 @@ private: std::vector next_message_tags; std::vector prev_tags; std::vector messagesWriter; - std::vector written; // if messagesWriter has written anything + std::vector isEmptyMessage; // if messagesWriter has written anything std::vector msg_locations; // Stores message locations that have had span information written to them // for the current transaction. Adding transaction info will reset this diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp index 60063c8afd..0411bf2c86 100644 --- a/fdbserver/TagPartitionedLogSystem.actor.cpp +++ b/fdbserver/TagPartitionedLogSystem.actor.cpp @@ -566,6 +566,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted> tLogCommitResults; for (int loc = 0; loc < it->logServers.size(); loc++) { Standalone msg = data.getMessages(location); + data.recordEmptyMessage(location, msg); allReplies.push_back(recordPushMetrics( it->connectionResetTrackers[loc], it->logServers[loc]->get().interf().address(), From 22102641212f1f98a9234749cd354dfe9795616b Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Mon, 28 Jun 2021 11:38:17 -0700 Subject: [PATCH 055/426] Fix endpoint ordering by moving the new updateWorkerHealth to the end of ClusterControllerFullInterface --- fdbserver/WorkerInterface.actor.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index fe682611b9..eca8495092 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -149,9 +149,9 @@ struct ClusterControllerFullInterface { RequestStream registerWorker; RequestStream getWorkers; RequestStream registerMaster; - RequestStream updateWorkerHealth; RequestStream getServerDBInfo; // only used by testers; the cluster controller will send the serverDBInfo to workers + RequestStream updateWorkerHealth; UID id() const { return clientInterface.id(); } bool operator==(ClusterControllerFullInterface const& r) const { return id() == r.id(); } @@ -161,8 +161,8 @@ struct ClusterControllerFullInterface { return clientInterface.hasMessage() || recruitFromConfiguration.getFuture().isReady() || recruitRemoteFromConfiguration.getFuture().isReady() || recruitStorage.getFuture().isReady() || registerWorker.getFuture().isReady() || getWorkers.getFuture().isReady() || - registerMaster.getFuture().isReady() || updateWorkerHealth.getFuture().isReady() || - getServerDBInfo.getFuture().isReady(); + registerMaster.getFuture().isReady() || getServerDBInfo.getFuture().isReady() || + updateWorkerHealth.getFuture().isReady(); } void initEndpoints() { @@ -173,8 +173,8 @@ struct ClusterControllerFullInterface { registerWorker.getEndpoint(TaskPriority::ClusterControllerWorker); getWorkers.getEndpoint(TaskPriority::ClusterController); registerMaster.getEndpoint(TaskPriority::ClusterControllerRegister); - updateWorkerHealth.getEndpoint(TaskPriority::ClusterController); getServerDBInfo.getEndpoint(TaskPriority::ClusterController); + updateWorkerHealth.getEndpoint(TaskPriority::ClusterController); } template @@ -190,8 +190,8 @@ struct ClusterControllerFullInterface { registerWorker, getWorkers, registerMaster, - updateWorkerHealth, - getServerDBInfo); + getServerDBInfo, + updateWorkerHealth); } }; From 41ea722f978703f56d638c16354534fd2ff54e80 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Mon, 28 Jun 2021 12:23:09 -0700 Subject: [PATCH 056/426] extra check in getHeight function --- fdbserver/VersionedBTree.actor.cpp | 2 +- flow/Histogram.h | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 4d3dca25d6..817e50d470 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6422,7 +6422,7 @@ public: bool isValid() const { return valid; } int getHeight(){ - if(!path.empty()) return path.back().btPage()->height; + if(!path.empty() && path.back().btPage() != nullptr) return path.back().btPage()->height; return 0; } std::string toString() const { diff --git a/flow/Histogram.h b/flow/Histogram.h index 1b8449b3f9..75b749822d 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -68,11 +68,13 @@ private: Histogram(std::string const& group, std::string const& op, Unit unit, HistogramRegistry& registry, uint32_t lower, uint32_t upper) : UnitToStringMapper ( - { { Histogram::Unit::microseconds, "microseconds" },{ Histogram::Unit::bytes, "bytes" }, + { { Histogram::Unit::microseconds, "microseconds" }, + { Histogram::Unit::bytes, "bytes" }, { Histogram::Unit::bytes_per_second, "bytes_per_second" }, { Histogram::Unit::percentage, "percentage" }, { Histogram::Unit::record_counter, "record_counter" }, - }), group(group), op(op), unit(unit), registry(registry), lowerBound(lower), upperBound(upper), ReferenceCounted() { + }), + group(group), op(op), unit(unit), registry(registry), lowerBound(lower), upperBound(upper), ReferenceCounted() { ASSERT(UnitToStringMapper.find(unit) != UnitToStringMapper.end()); From 5137228d505d4252d66cfb2bb37a01fbc4e56f0c Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Mon, 28 Jun 2021 12:48:39 -0700 Subject: [PATCH 057/426] initialize sketches only once --- fdbserver/VersionedBTree.actor.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 817e50d470..54c3d7524b 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -1348,13 +1348,14 @@ struct RedwoodMetrics { void levelClear(unsigned int levelCounter = 0){ metric = {}; - - buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); - buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); - modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); + if(!buildFillPctSketch.isValid() || buildFillPctSketch->name() != ("buildFillPct:" + std::to_string(levelCounter))){ + buildFillPctSketch = Histogram::getHistogram(LiteralStringRef("buildFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + modifyFillPctSketch = Histogram::getHistogram(LiteralStringRef("modifyFillPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + buildStoredPctSketch = Histogram::getHistogram(LiteralStringRef("buildStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + modifyStoredPctSketch = Histogram::getHistogram(LiteralStringRef("modifyStoredPct"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::percentage); + buildItemCountSketch = Histogram::getHistogram(LiteralStringRef("buildItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); + modifyItemCountSketch = Histogram::getHistogram(LiteralStringRef("modifyItemCount"), LiteralStringRef(std::to_string(levelCounter).c_str()), Histogram::Unit::record_counter, 0, maxRecordCount); + } metric.eventReasons.clear(); buildFillPctSketch->clear(); @@ -1400,7 +1401,7 @@ struct RedwoodMetrics { void clear() { unsigned int levelCounter = 1; - for (auto& level : levels) { + for (RedwoodMetrics::Level& level : levels) { level.levelClear(levelCounter); ++levelCounter; } From 0bf035c3199160e9b84b25055214cf57b077563a Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Mon, 28 Jun 2021 13:55:22 -0700 Subject: [PATCH 058/426] add assert idx back --- flow/Histogram.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/Histogram.h b/flow/Histogram.h index 75b749822d..3863cf065b 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -145,7 +145,7 @@ public: ASSERT(unit==Histogram::Unit::record_counter); size_t idx = ( (sample - lowerBound) * 31.0 ) / (upperBound - lowerBound); - //ASSERT(idx < 32); + ASSERT(idx < 32); buckets[idx]++; } From e148b92430f4951b3f2c3e5c6329b57f7acb0807 Mon Sep 17 00:00:00 2001 From: Fuheng Zhao Date: Mon, 28 Jun 2021 14:16:41 -0700 Subject: [PATCH 059/426] check in idx --- fdbserver/IPager.h | 2 +- flow/Histogram.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fdbserver/IPager.h b/fdbserver/IPager.h index 99e21db861..e6f166cce3 100644 --- a/fdbserver/IPager.h +++ b/fdbserver/IPager.h @@ -43,7 +43,7 @@ typedef uint32_t QueueID; // Pager Events enum class events{ pagerCacheLookup = 0, pagerCacheHit, pagerCacheMiss, pagerWrite, MAXEVENTS}; // Reasons for page levle events. -enum class pagerEventReasons{ pointRead, rangeRead, rangePrefetch, commit, lazyClear, metaData, MAXEVENTREASONS}; +enum class pagerEventReasons{ pointRead = 0, rangeRead, rangePrefetch, commit, lazyClear, metaData, MAXEVENTREASONS}; // Represents a block of memory in a 4096-byte aligned location held by an Arena. class ArenaPage : public ReferenceCounted, public FastAllocated { diff --git a/flow/Histogram.h b/flow/Histogram.h index 3863cf065b..84bd785ad2 100644 --- a/flow/Histogram.h +++ b/flow/Histogram.h @@ -145,6 +145,7 @@ public: ASSERT(unit==Histogram::Unit::record_counter); size_t idx = ( (sample - lowerBound) * 31.0 ) / (upperBound - lowerBound); + if(idx >= 32){ idx = 31;} ASSERT(idx < 32); buckets[idx]++; From a33a0a7fff5fd398e40e0a069b731a033e2fc45b Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Mon, 28 Jun 2021 14:50:35 -0700 Subject: [PATCH 060/426] Add include to DeterministicRandom.cpp --- flow/DeterministicRandom.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flow/DeterministicRandom.cpp b/flow/DeterministicRandom.cpp index ab65c187da..c5043b4d62 100644 --- a/flow/DeterministicRandom.cpp +++ b/flow/DeterministicRandom.cpp @@ -20,6 +20,8 @@ #include "flow/DeterministicRandom.h" +#include + uint64_t DeterministicRandom::gen64() { uint64_t curr = next; next = (uint64_t(random()) << 32) ^ random(); From 9c82b22aa3e6b933d0ac487521a5475f1a8d688d Mon Sep 17 00:00:00 2001 From: Jingyu Zhou Date: Mon, 28 Jun 2021 16:09:07 -0700 Subject: [PATCH 061/426] Add comments for recordEmptyMessage and getEmptyMessageRatio. --- fdbserver/LogSystem.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 60ce9ab733..5aa14810de 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -1094,6 +1094,8 @@ struct LogPushData : NonCopyable { return messagesWriter[loc].toValue(); } + // Records if a tlog (specified by "loc") will receive an empty version batch message. + // "value" is the message returned by getMessages() call. void recordEmptyMessage(int loc, const Standalone& value) { if (!isEmptyMessage[loc]) { BinaryWriter w(AssumeVersion(g_network->protocolVersion())); @@ -1104,6 +1106,8 @@ struct LogPushData : NonCopyable { } } + // Returns the ratio of empty messages in this version batch. + // MUST be called after getMessages() and recordEmptyMessage(). float getEmptyMessageRatio() const { auto count = std::count(isEmptyMessage.begin(), isEmptyMessage.end(), false); ASSERT_WE_THINK(isEmptyMessage.size() > 0); From 9a87a65f47330af72d6009afa9e83fbfc456b0df Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Mon, 28 Jun 2021 16:19:55 -0700 Subject: [PATCH 062/426] Add tracing documentation --- .../sphinx/source/request-tracing.rst | 98 ++++++++++++++++++- documentation/sphinx/source/special-keys.rst | 2 + documentation/sphinx/source/visibility.rst | 2 +- fdbclient/vexillographer/fdb.options | 2 +- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/documentation/sphinx/source/request-tracing.rst b/documentation/sphinx/source/request-tracing.rst index cca170ad24..03d2d35c50 100644 --- a/documentation/sphinx/source/request-tracing.rst +++ b/documentation/sphinx/source/request-tracing.rst @@ -1,7 +1,97 @@ .. _request-tracing: -######################### -Request Tracing Framework -######################### +############### +Request Tracing +############### -.. include:: guide-common.rst.inc +The request tracing framework adds the ability to monitor transactions as they +move through FoundationDB. Tracing provides a detailed view into where +transactions spend time with data exported in near real-time, enabling fast +performance debugging. The FoundationDB tracing framework is based off the +`OpenTracing `_ specification. + +*Disambiguation:* :ref:`Trace files ` are +local log files containing debug and error output from a local ``fdbserver`` +binary. Request tracing produces similarly named *traces* which record the +amount of time a transaction spent in a part of the system. This document uses +the term tracing (or trace) to refer to these request traces, not local debug +information, unless otherwise specified. + +*Note*: Full request tracing capability requires at least ``TLogVersion::V6``. + +============== +Recording data +============== + +The request tracing framework produces no data by default. To enable collection +of traces, specify the collection type using the ``--tracer`` command line +option for ``fdbserver`` and the ``DISTRIBUTED_CLIENT_TRACER`` :ref:`network +option ` for clients. Both client +and server must have the same trace value set to perform correctly. + +========================= =============== +**Option** **Description** +------------------------- --------------- +none No tracing data is collected. +file, logfile, log_file Write tracing data to FDB trace files, specified with ``--logdir``. +network_lossy Send tracing data as UDP packets. Data is sent to ``localhost:8889``, but the default port can be changed by setting the ``TRACING_UDP_LISTENER_PORT`` knob. This option is useful if you have a log aggregation program to collect trace data. +========================= =============== + +----------- +Data format +----------- + +Spans are the building blocks of traces. A span represents an operation in the +life of a transaction, including the start and end timestamp and an operation. +A collection of spans make up a trace, representing a single transaction. The +tracing framework outputs individual spans, which can be reconstructed into +traces through their parent relationships. + +Trace data sent as UDP packets when using the ``network_lossy`` option is +serialized using `MessagePack `_. To save on the amount of +data sent, spans are serialized as an array of length 8 (if the span has one or +more parents), or length 7 (if the span has no parents). + +The fields of a span are specified below. The index at which the field appears +in the serialized msgpack array is also specified, for those using the UDP +collection format. + +================== ========= ======== =============== +**Field** **Index** **Type** **Description** +------------------ --------- -------- --------------- +Source IP:port 0 string The IP and port of the machine where the span originated. +Trace ID 1 uint64 The 64-bit identifier of the trace. All spans in a trace share the same trace ID. +Span ID 2 uint64 The 64-bit identifier of the span. All spans have a unique identifier. +Start timestamp 3 double The timestamp when the operation represented by the span began. +End timestamp 4 double The timestamp when the operation represented by the span ended. +Operation name 5 string The name of the operation the span represents. +Tags 6 map User defined tags, added manually to specify additional information. +Parent span IDs 7 vector (Optional) A list of span IDs representing parents of this span. +================== ========= ======== =============== + +^^^^^^^^^^^^^^^^^^^^^ +Multiple parent spans +^^^^^^^^^^^^^^^^^^^^^ + +Unlike traditional distributed tracing frameworks, FoundationDB spans can have +multiple parents. Because many FDB transactions are batched into a single +transaction, to continue tracing the request, the batched transaction must +treat all its component transactions as parents. + +--------------- +Control options +--------------- + +In addition to the command line parameter described above, tracing can be set +at a database and transaction level. + +Tracing can be globally disabled by setting the +``distributed_transaction_trace_disable`` database option. It can be enabled by +setting the ``distributed_transaction_trace_enable`` database option. If +neither option is specified but a tracer option is set as described above, +tracing will be enabled. + +Tracing can be enabled or disabled for individual transactions. The special key +space exposes an API to set a custom trace ID for a transaction, or to disable +tracing for the transaction. See the special key space :ref:`tracing module +documentation ` to learn more. diff --git a/documentation/sphinx/source/special-keys.rst b/documentation/sphinx/source/special-keys.rst index 4d8abdf177..2a6b0018b1 100644 --- a/documentation/sphinx/source/special-keys.rst +++ b/documentation/sphinx/source/special-keys.rst @@ -244,6 +244,8 @@ use the global configuration functions. #. ``\xff\xff/global_config/ := `` Read/write. Reading keys in the range will return a tuple decoded string representation of the value for the given key. Writing a value will update all processes in the cluster with the new key-value pair. Values must be written using the :ref:`api-python-tuple-layer`. +.. _special-key-space-tracing-module: + Tracing module -------------- diff --git a/documentation/sphinx/source/visibility.rst b/documentation/sphinx/source/visibility.rst index de16800ce0..200dea0447 100644 --- a/documentation/sphinx/source/visibility.rst +++ b/documentation/sphinx/source/visibility.rst @@ -6,7 +6,7 @@ Visibility Documents Curation of documents related to Visibility into FDB. -* :doc:`request-tracing` walks you through request-tracing framework. +* :doc:`request-tracing` provides fine-grained visibility into the flow of transactions through the system. .. toctree:: :maxdepth: 2 diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index 15ba1250ca..e0f908a69c 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -129,7 +129,7 @@ description is not currently required but encouraged. paramType="Int" paramDescription="probability expressed as a percentage between 0 and 100" description="Set the probability of an active CLIENT_BUGGIFY section being fired. A section will only fire if it was activated" />