diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index 18ae54aad6..71d04fc55b 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -536,3 +536,51 @@ sse2neon Authors (sse2neon) LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +rte_memcpy.h (from DPDK): + SPDX-License-Identifier: BSD-3-Clause + Copyright(c) 2010-2014 Intel Corporation + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +folly_memcpy: + + Copyright (c) Facebook, Inc. and its affiliates. + Author: Bin Liu + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e48afd84e..fa762949a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # limitations under the License. cmake_minimum_required(VERSION 3.13) project(foundationdb - VERSION 6.3.1 + VERSION 7.0.0 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) diff --git a/bindings/bindingtester/__init__.py b/bindings/bindingtester/__init__.py index 0adababb92..f8ad0030e2 100644 --- a/bindings/bindingtester/__init__.py +++ b/bindings/bindingtester/__init__.py @@ -26,7 +26,7 @@ sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', import util -FDB_API_VERSION = 630 +FDB_API_VERSION = 700 LOGGING = { 'version': 1, diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py index 6feed3b283..58db70f5db 100755 --- a/bindings/bindingtester/bindingtester.py +++ b/bindings/bindingtester/bindingtester.py @@ -157,7 +157,7 @@ def choose_api_version(selected_api_version, tester_min_version, tester_max_vers api_version = min_version elif random.random() < 0.9: api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430, - 440, 450, 460, 500, 510, 520, 600, 610, 620, 630] if v >= min_version and v <= max_version]) + 440, 450, 460, 500, 510, 520, 600, 610, 620, 630, 700] if v >= min_version and v <= max_version]) else: api_version = random.randint(min_version, max_version) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index ee82663411..e1522039db 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -20,7 +20,7 @@ import os -MAX_API_VERSION = 630 +MAX_API_VERSION = 700 COMMON_TYPES = ['null', 'bytes', 'string', 'int', 'uuid', 'bool', 'float', 'double', 'tuple'] ALL_TYPES = COMMON_TYPES + ['versionstamp'] diff --git a/bindings/bindingtester/tests/scripted.py b/bindings/bindingtester/tests/scripted.py index 60b1959864..c113ebc07f 100644 --- a/bindings/bindingtester/tests/scripted.py +++ b/bindings/bindingtester/tests/scripted.py @@ -34,7 +34,7 @@ fdb.api_version(FDB_API_VERSION) class ScriptedTest(Test): - TEST_API_VERSION = 630 + TEST_API_VERSION = 700 def __init__(self, subspace): super(ScriptedTest, self).__init__(subspace, ScriptedTest.TEST_API_VERSION, ScriptedTest.TEST_API_VERSION) diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index dae60c7ea8..ba56744dc7 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #define FDB_INCLUDE_LEGACY_TYPES #include "fdbclient/MultiVersionTransaction.h" diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index a930434819..b5dfa63d13 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -28,10 +28,10 @@ #endif #if !defined(FDB_API_VERSION) -#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 630) +#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 700) #elif FDB_API_VERSION < 13 #error API version no longer supported (upgrade to 13) -#elif FDB_API_VERSION > 630 +#elif FDB_API_VERSION > 700 #error Requested API version requires a newer version of this header #endif @@ -91,7 +91,7 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter); #pragma pack(push, 4) -#if FDB_API_VERSION >= 630 +#if FDB_API_VERSION >= 700 typedef struct keyvalue { const uint8_t* key; int key_length; diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h index 4f703e7271..792dd1d6dc 100644 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -3,7 +3,7 @@ #pragma once #ifndef FDB_API_VERSION -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #endif #include diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index 7a265e7d0f..319895554d 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -603,7 +603,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(630), "select API version", rs); + checkError(fdb_select_api_version(700), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index cbb7fcf304..4604dc41a8 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -244,7 +244,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(630), "select API version", rs); + checkError(fdb_select_api_version(700), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, keySize); diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index 5fb4268b78..7169689f76 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -29,7 +29,7 @@ #include #ifndef FDB_API_VERSION -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #endif #include diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c index 4f2744d199..7aa5a18576 100644 --- a/bindings/c/test/txn_size_test.c +++ b/bindings/c/test/txn_size_test.c @@ -97,7 +97,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(630), "select API version", rs); + checkError(fdb_select_api_version(700), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, KEY_SIZE); diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp index 35b18f71a3..7b19654874 100644 --- a/bindings/c/test/workloads/SimpleWorkload.cpp +++ b/bindings/c/test/workloads/SimpleWorkload.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include "foundationdb/fdb_c.h" #undef DLLEXPORT #include "workloads.h" @@ -258,7 +258,7 @@ struct SimpleWorkload : FDBWorkload { insertsPerTx = context->getOption("insertsPerTx", 100ul); opsPerTx = context->getOption("opsPerTx", 100ul); runFor = context->getOption("runFor", 10.0); - auto err = fdb_select_api_version(630); + auto err = fdb_select_api_version(700); if (err) { context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } }); diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 3ed3d93700..5fd56e653d 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -36,7 +36,7 @@ THREAD_FUNC networkThread(void* fdb) { } ACTOR Future _test() { - API *fdb = FDB::API::selectAPIVersion(630); + API *fdb = FDB::API::selectAPIVersion(700); auto db = fdb->createDatabase(); state Reference tr = db->createTransaction(); @@ -79,7 +79,7 @@ ACTOR Future _test() { } void fdb_flow_test() { - API *fdb = FDB::API::selectAPIVersion(630); + API *fdb = FDB::API::selectAPIVersion(700); fdb->setupNetwork(); startThread(networkThread, fdb); diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index e261052fae..66049cae0c 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -23,7 +23,7 @@ #include -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include #undef DLLEXPORT diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index a190299747..578f159f8c 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -1817,7 +1817,7 @@ ACTOR void _test_versionstamp() { try { g_network = newNet2(TLSConfig()); - API *fdb = FDB::API::selectAPIVersion(630); + API *fdb = FDB::API::selectAPIVersion(700); fdb->setupNetwork(); startThread(networkThread, fdb); diff --git a/bindings/go/README.md b/bindings/go/README.md index 7a03ea1d6f..8619e1692a 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -9,7 +9,7 @@ This package requires: - [Mono](http://www.mono-project.com/) (macOS or Linux) or [Visual Studio](https://www.visualstudio.com/) (Windows) (build-time only) - FoundationDB C API 2.0.x-6.1.x (part of the [FoundationDB client packages](https://apple.github.io/foundationdb/downloads.html#c)) -Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-630. +Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-700. To install this package, you can run the "fdb-go-install.sh" script (for versions 5.0.x and greater): diff --git a/bindings/go/src/fdb/cluster.go b/bindings/go/src/fdb/cluster.go index df895e9a51..5ab17b5273 100644 --- a/bindings/go/src/fdb/cluster.go +++ b/bindings/go/src/fdb/cluster.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go index 23cb4f19be..60f3f03d06 100644 --- a/bindings/go/src/fdb/database.go +++ b/bindings/go/src/fdb/database.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go index 5cfb157ad6..e1759701ff 100644 --- a/bindings/go/src/fdb/doc.go +++ b/bindings/go/src/fdb/doc.go @@ -46,7 +46,7 @@ A basic interaction with the FoundationDB API is demonstrated below: func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) // Open the default database from the system cluster db := fdb.MustOpenDefault() diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go index 94e699c89e..9c9f75b566 100644 --- a/bindings/go/src/fdb/errors.go +++ b/bindings/go/src/fdb/errors.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go index d0bfd5f699..bc05a05dba 100644 --- a/bindings/go/src/fdb/fdb.go +++ b/bindings/go/src/fdb/fdb.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include // #include import "C" @@ -108,7 +108,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // library, an error will be returned. APIVersion must be called prior to any // other functions in the fdb package. // -// Currently, this package supports API versions 200 through 630. +// Currently, this package supports API versions 200 through 700. // // Warning: When using the multi-version client API, setting an API version that // is not supported by a particular client library will prevent that client from @@ -116,7 +116,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // the API version of your application after upgrading your client until the // cluster has also been upgraded. func APIVersion(version int) error { - headerVersion := 630 + headerVersion := 700 networkMutex.Lock() defer networkMutex.Unlock() @@ -128,7 +128,7 @@ func APIVersion(version int) error { return errAPIVersionAlreadySet } - if version < 200 || version > 630 { + if version < 200 || version > 700 { return errAPIVersionNotSupported } diff --git a/bindings/go/src/fdb/fdb_test.go b/bindings/go/src/fdb/fdb_test.go index 7bcd588de8..e455dba473 100644 --- a/bindings/go/src/fdb/fdb_test.go +++ b/bindings/go/src/fdb/fdb_test.go @@ -32,7 +32,7 @@ import ( func ExampleOpenDefault() { var e error - e = fdb.APIVersion(630) + e = fdb.APIVersion(700) if e != nil { fmt.Printf("Unable to set API version: %v\n", e) return @@ -52,7 +52,7 @@ func ExampleOpenDefault() { } func TestVersionstamp(t *testing.T) { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() setVs := func(t fdb.Transactor, key fdb.Key) (fdb.FutureKey, error) { @@ -98,7 +98,7 @@ func TestVersionstamp(t *testing.T) { } func ExampleTransactor() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() setOne := func(t fdb.Transactor, key fdb.Key, value []byte) error { @@ -149,7 +149,7 @@ func ExampleTransactor() { } func ExampleReadTransactor() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() getOne := func(rt fdb.ReadTransactor, key fdb.Key) ([]byte, error) { @@ -202,7 +202,7 @@ func ExampleReadTransactor() { } func ExamplePrefixRange() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() tr, e := db.CreateTransaction() @@ -241,7 +241,7 @@ func ExamplePrefixRange() { } func ExampleRangeIterator() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() tr, e := db.CreateTransaction() diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index 17ae1d70a4..43718fe738 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -23,7 +23,7 @@ package fdb // #cgo LDFLAGS: -lfdb_c -lm -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include // #include // diff --git a/bindings/go/src/fdb/range.go b/bindings/go/src/fdb/range.go index 67a45c63b2..584f23cb2b 100644 --- a/bindings/go/src/fdb/range.go +++ b/bindings/go/src/fdb/range.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index 4102a0556b..6bd198b0da 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 630 +// #define FDB_API_VERSION 700 // #include import "C" diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 808485486b..e47208b6e6 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -19,7 +19,7 @@ */ #include -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include #include @@ -370,7 +370,7 @@ struct JVM { jmethodID selectMethod = env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(I)Lcom/apple/foundationdb/FDB;"); checkException(); - auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(630)); + auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(700)); checkException(); env->CallObjectMethod(fdbInstance, getMethod(fdbClass, "disableShutdownHook", "()V")); checkException(); diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 938ac498f3..a127a47864 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -21,7 +21,7 @@ #include #include -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include diff --git a/bindings/java/src/main/com/apple/foundationdb/FDB.java b/bindings/java/src/main/com/apple/foundationdb/FDB.java index ba96814cef..b945a5dc69 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDB.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDB.java @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This call is required before using any other part of the API. The call allows * an error to be thrown at this point to prevent client code from accessing a later library * with incorrect assumptions from the current version. The API version documented here is version - * {@code 630}.

+ * {@code 700}.

* FoundationDB encapsulates multiple versions of its interface by requiring * the client to explicitly specify the version of the API it uses. The purpose * of this design is to allow you to upgrade the server, client libraries, or @@ -181,8 +181,8 @@ public class FDB { } if(version < 510) throw new IllegalArgumentException("API version not supported (minimum 510)"); - if(version > 630) - throw new IllegalArgumentException("API version not supported (maximum 630)"); + if(version > 700) + throw new IllegalArgumentException("API version not supported (maximum 700)"); Select_API_version(version); singleton = new FDB(version); diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in index fd7c6ac80d..adaedd1a03 100644 --- a/bindings/java/src/main/overview.html.in +++ b/bindings/java/src/main/overview.html.in @@ -13,7 +13,7 @@ and then added to your classpath.

Getting started

To start using FoundationDB from Java, create an instance of the {@link com.apple.foundationdb.FDB FoundationDB API interface} with the version of the -API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 630}). +API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 700}). With this API object you can then open {@link com.apple.foundationdb.Cluster Cluster}s and {@link com.apple.foundationdb.Database Database}s and start using {@link com.apple.foundationdb.Transaction Transaction}s. @@ -29,7 +29,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java index 3a153e3582..e27e80b082 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java @@ -27,7 +27,7 @@ import com.apple.foundationdb.Database; import com.apple.foundationdb.FDB; public abstract class AbstractTester { - public static final int API_VERSION = 630; + public static final int API_VERSION = 700; protected static final int NUM_RUNS = 25; protected static final Charset ASCII = Charset.forName("ASCII"); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java index f21aabeb6a..68f7d74a95 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java @@ -33,7 +33,7 @@ public class BlockingBenchmark { private static final int PARALLEL = 100; public static void main(String[] args) throws InterruptedException { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); // The cluster file DOES NOT need to be valid, although it must exist. // This is because the database is never really contacted in this test. diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java index 53f13695c1..bddfd6f57d 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java @@ -48,7 +48,7 @@ public class ConcurrentGetSetGet { } public static void main(String[] args) { - try(Database database = FDB.selectAPIVersion(630).open()) { + try(Database database = FDB.selectAPIVersion(700).open()) { new ConcurrentGetSetGet().apply(database); } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java index c43dd71809..9f838d8eeb 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java @@ -33,7 +33,7 @@ import com.apple.foundationdb.directory.DirectorySubspace; public class DirectoryTest { public static void main(String[] args) throws Exception { try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { runTests(db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Example.java b/bindings/java/src/test/com/apple/foundationdb/test/Example.java index 74090eccc0..44e9087b3e 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/Example.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/Example.java @@ -26,7 +26,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java index aca9e918d2..ce1f623f4c 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java @@ -31,7 +31,7 @@ public class IterableTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java index 70f688e46a..29abab1471 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.tuple.ByteArrayUtil; public class LocalityTests { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database database = fdb.open(args[0])) { try(Transaction tr = database.createTransaction()) { String[] keyAddresses = LocalityUtil.getAddressesForKey(tr, "a".getBytes()).join(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java index 014f1f038d..624566964a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java @@ -43,7 +43,7 @@ public class ParallelRandomScan { private static final int PARALLELISM_STEP = 5; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(630); + FDB api = FDB.selectAPIVersion(700); try(Database database = api.open(args[0])) { for(int i = PARALLELISM_MIN; i <= PARALLELISM_MAX; i += PARALLELISM_STEP) { runTest(database, i, ROWS, DURATION_MS); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java index 3a99c68d56..81365ff44f 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; import com.apple.foundationdb.async.AsyncIterable; public class RangeTest { - private static final int API_VERSION = 630; + private static final int API_VERSION = 700; public static void main(String[] args) { System.out.println("About to use version " + API_VERSION); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java index f873e954e1..8ad91314c2 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java @@ -34,7 +34,7 @@ public class SerialInsertion { private static final int NODES = 1000000; public static void main(String[] args) { - FDB api = FDB.selectAPIVersion(630); + FDB api = FDB.selectAPIVersion(700); try(Database database = api.open()) { long start = System.currentTimeMillis(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java index cbcc2d713a..db63999daa 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java @@ -39,7 +39,7 @@ public class SerialIteration { private static final int THREAD_COUNT = 1; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(630); + FDB api = FDB.selectAPIVersion(700); try(Database database = api.open(args[0])) { for(int i = 1; i <= THREAD_COUNT; i++) { runThreadedTest(database, i); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java index 2aad1eb1bb..df084d564f 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java @@ -30,7 +30,7 @@ public class SerialTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java index d324463408..78de1ae3db 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java @@ -39,7 +39,7 @@ public class SnapshotTransactionTest { private static final Subspace SUBSPACE = new Subspace(Tuple.from("test", "conflict_ranges")); public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { snapshotReadShouldNotConflict(db); snapshotShouldNotAddConflictRange(db); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java index c7aa190ce7..066cf43383 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java @@ -50,7 +50,7 @@ public class TupleTest { public static void main(String[] args) throws NoSuchFieldException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); addMethods(); comparisons(); emptyTuple(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java index 12bef587d2..e50bc9c031 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java @@ -32,7 +32,7 @@ import com.apple.foundationdb.tuple.Versionstamp; public class VersionstampSmokeTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database db = fdb.open()) { db.run(tr -> { tr.clear(Tuple.from("prefix").range()); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java index b204e842e5..14c0aa1d43 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; public class WatchTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(630); + FDB fdb = FDB.selectAPIVersion(700); try(Database database = fdb.open(args[0])) { database.options().setLocationCacheSize(42); try(Transaction tr = database.createTransaction()) { diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index 0d54c96b5f..c969b6c70c 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -52,7 +52,7 @@ def get_api_version(): def api_version(ver): - header_version = 630 + header_version = 700 if '_version' in globals(): if globals()['_version'] != ver: diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index 3dd5e87077..91bdc2f3a0 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -253,7 +253,7 @@ def transactional(*tr_args, **tr_kwargs): @functools.wraps(func) def wrapper(*args, **kwargs): # We can't throw this from the decorator, as when a user runs - # >>> import fdb ; fdb.api_version(630) + # >>> import fdb ; fdb.api_version(700) # the code above uses @transactional before the API version is set if fdb.get_api_version() >= 630 and inspect.isgeneratorfunction(func): raise ValueError("Generators can not be wrapped with fdb.transactional") diff --git a/bindings/python/tests/size_limit_tests.py b/bindings/python/tests/size_limit_tests.py index 446f787bc1..756d9422e0 100644 --- a/bindings/python/tests/size_limit_tests.py +++ b/bindings/python/tests/size_limit_tests.py @@ -22,7 +22,7 @@ import fdb import sys if __name__ == '__main__': - fdb.api_version(630) + fdb.api_version(700) @fdb.transactional def setValue(tr, key, value): diff --git a/bindings/ruby/lib/fdb.rb b/bindings/ruby/lib/fdb.rb index b1b72d38d7..df8448ea0b 100644 --- a/bindings/ruby/lib/fdb.rb +++ b/bindings/ruby/lib/fdb.rb @@ -36,7 +36,7 @@ module FDB end end def self.api_version(version) - header_version = 630 + header_version = 700 if self.is_api_version_selected?() if @@chosen_version != version raise "FDB API already loaded at version #{@@chosen_version}." diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c index a15c1193e7..f26b2513c1 100644 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ b/build/cmake/package_tester/fdb_c_app/app.c @@ -1,7 +1,7 @@ -#define FDB_API_VERSION 630 +#define FDB_API_VERSION 700 #include int main(int argc, char* argv[]) { - fdb_select_api_version(630); + fdb_select_api_version(700); return 0; } diff --git a/build/cmake/package_tester/modules/tests.sh b/build/cmake/package_tester/modules/tests.sh index 88709a7953..35ff098a6f 100644 --- a/build/cmake/package_tester/modules/tests.sh +++ b/build/cmake/package_tester/modules/tests.sh @@ -65,7 +65,7 @@ then python setup.py install successOr "Installing python bindings failed" popd - python -c 'import fdb; fdb.api_version(630)' + python -c 'import fdb; fdb.api_version(700)' successOr "Loading python bindings failed" # Test cmake and pkg-config integration: https://github.com/apple/foundationdb/issues/1483 diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index ddb2f38792..f97c01a70a 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -209,6 +209,25 @@ else() # -mavx # -msse4.2) + # Tentatively re-enabling vector instructions + set(USE_AVX512F OFF CACHE BOOL "Enable AVX 512F instructions") + if (USE_AVX512F) + add_compile_options(-mavx512f) + endif() + set(USE_AVX ON CACHE BOOL "Enable AVX instructions") + if (USE_AVX) + add_compile_options(-mavx) + endif() + + # Intentionally using builtin memcpy. G++ does a good job on small memcpy's when the size is known at runtime. + # If the size is not known, then it falls back on the memcpy that's available at runtime (rte_memcpy, as of this + # writing; see flow.cpp). + # + # The downside of the builtin memcpy is that it's slower at large copies, so if we spend a lot of time on large + # copies of sizes that are known at compile time, this might not be a win. See the output of performance/memcpy + # for more information. + #add_compile_options(-fno-builtin-memcpy) + if (USE_VALGRIND) add_compile_options(-DVALGRIND -DUSE_VALGRIND) endif() @@ -241,7 +260,8 @@ else() -Wno-delete-non-virtual-dtor -Wno-undefined-var-template -Wno-tautological-pointer-compare - -Wno-format) + -Wno-format + -Woverloaded-virtual) if (USE_CCACHE) add_compile_options( -Wno-register @@ -253,7 +273,6 @@ else() endif() if (GCC) add_compile_options(-Wno-pragmas) - # Otherwise `state [[maybe_unused]] int x;` will issue a warning. # https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is add_compile_options(-Wno-attributes) @@ -267,6 +286,7 @@ else() -fvisibility=hidden -Wreturn-type -fPIC) + add_compile_options($<$:-Wclass-memaccess>) if (GPERFTOOLS_FOUND AND GCC) add_compile_options( -fno-builtin-malloc diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index 5374475bed..58e2cf2548 100755 --- a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -39,7 +39,9 @@ from json import JSONEncoder import logging import struct from bisect import bisect_left +from bisect import bisect_right import time +import datetime PROTOCOL_VERSION_5_2 = 0x0FDB00A552000001 PROTOCOL_VERSION_6_0 = 0x0FDB00A570010001 @@ -414,7 +416,7 @@ class TransactionInfoLoader(object): else: end_key = self.client_latency_end_key_selector - valid_transaction_infos = 0 + transaction_infos = 0 invalid_transaction_infos = 0 def build_client_transaction_info(v): @@ -446,11 +448,12 @@ class TransactionInfoLoader(object): info = build_client_transaction_info(v) if info.has_types(): buffer.append(info) - valid_transaction_infos += 1 except UnsupportedProtocolVersionError as e: invalid_transaction_infos += 1 except ValueError: invalid_transaction_infos += 1 + + transaction_infos += 1 else: if chunk_num == 1: # first chunk @@ -476,14 +479,15 @@ class TransactionInfoLoader(object): info = build_client_transaction_info(b''.join([chunk.value for chunk in c_list])) if info.has_types(): buffer.append(info) - valid_transaction_infos += 1 except UnsupportedProtocolVersionError as e: invalid_transaction_infos += 1 except ValueError: invalid_transaction_infos += 1 + + transaction_infos += 1 self._check_and_adjust_chunk_cache_size() - if (valid_transaction_infos + invalid_transaction_infos) % 1000 == 0: - print("Processed valid: %d, invalid: %d" % (valid_transaction_infos, invalid_transaction_infos)) + if transaction_infos % 1000 == 0: + print("Processed %d transactions, %d invalid" % (transaction_infos, invalid_transaction_infos)) if found == 0: more = False except fdb.FDBError as e: @@ -495,13 +499,15 @@ class TransactionInfoLoader(object): for item in buffer: yield item + print("Processed %d transactions, %d invalid\n" % (transaction_infos, invalid_transaction_infos)) + def has_sortedcontainers(): try: import sortedcontainers return True except ImportError: - logger.warn("Can't find sortedcontainers so disabling RangeCounter") + logger.warn("Can't find sortedcontainers so disabling ReadCounter") return False @@ -513,155 +519,197 @@ def has_dateparser(): logger.warn("Can't find dateparser so disabling human date parsing") return False - -class RangeCounter(object): - def __init__(self, k): - self.k = k +class ReadCounter(object): + def __init__(self): from sortedcontainers import SortedDict - self.ranges = SortedDict() + self.reads = SortedDict() + self.reads[b''] = [0, 0] + + self.read_counts = {} + self.hit_count=0 def process(self, transaction_info): + for get in transaction_info.gets: + self._insert_read(get.key, None) for get_range in transaction_info.get_ranges: - self._insert_range(get_range.key_range.start_key, get_range.key_range.end_key) + self._insert_read(get_range.key_range.start_key, get_range.key_range.end_key) - def _insert_range(self, start_key, end_key): - keys = self.ranges.keys() - if len(keys) == 0: - self.ranges[start_key] = end_key, 1 - return + def _insert_read(self, start_key, end_key): + self.read_counts.setdefault((start_key, end_key), 0) + self.read_counts[(start_key, end_key)] += 1 - start_pos = bisect_left(keys, start_key) - end_pos = bisect_left(keys, end_key) - #print("start_pos=%d, end_pos=%d" % (start_pos, end_pos)) + self.reads.setdefault(start_key, [0, 0])[0] += 1 + if end_key is not None: + self.reads.setdefault(end_key, [0, 0])[1] += 1 + else: + self.reads.setdefault(start_key+b'\x00', [0, 0])[1] += 1 - possible_intersection_keys = keys[max(0, start_pos - 1):min(len(keys), end_pos+1)] + def get_total_reads(self): + return sum([v for v in self.read_counts.values()]) + + def matches_filter(addresses, required_addresses): + for addr in required_addresses: + if addr not in addresses: + return False + return True - start_range_left = start_key + def get_top_k_reads(self, num, filter_addresses, shard_finder=None): + count_pairs = sorted([(v, k) for (k, v) in self.read_counts.items()], reverse=True, key=lambda item: item[0]) + if not filter_addresses: + count_pairs = count_pairs[0:num] - for key in possible_intersection_keys: - cur_end_key, cur_count = self.ranges[key] - #logger.debug("key=%s, cur_end_key=%s, cur_count=%d, start_range_left=%s" % (key, cur_end_key, cur_count, start_range_left)) - if start_range_left < key: - if end_key <= key: - self.ranges[start_range_left] = end_key, 1 - return - self.ranges[start_range_left] = key, 1 - start_range_left = key - assert start_range_left >= key - if start_range_left >= cur_end_key: - continue + if shard_finder: + results = [] + for (count, (start, end)) in count_pairs: + results.append((start, end, count, shard_finder.get_addresses_for_key(start))) - # [key, start_range_left) = cur_count - # if key == start_range_left this will get overwritten below - self.ranges[key] = start_range_left, cur_count + shard_finder.wait_for_shard_addresses(results, 0, 3) - if end_key <= cur_end_key: - # [start_range_left, end_key) = cur_count+1 - # [end_key, cur_end_key) = cur_count - self.ranges[start_range_left] = end_key, cur_count + 1 - if end_key != cur_end_key: - self.ranges[end_key] = cur_end_key, cur_count - start_range_left = end_key - break - else: - # [start_range_left, cur_end_key) = cur_count+1 - self.ranges[start_range_left] = cur_end_key, cur_count+1 - start_range_left = cur_end_key - assert start_range_left <= end_key + if filter_addresses: + filter_addresses = set(filter_addresses) + results = [r for r in results if filter_addresses.issubset(set(r[3]))][0:num] + else: + results = [(start, end, count) for (count, (start, end)) in count_pairs[0:num]] - # there may be some range left - if start_range_left < end_key: - self.ranges[start_range_left] = end_key, 1 + return results - def get_count_for_key(self, key): - if key in self.ranges: - return self.ranges[key][1] - - keys = self.ranges.keys() - index = bisect_left(keys, key) - if index == 0: - return 0 - - index_key = keys[index-1] - if index_key <= key < self.ranges[index_key][0]: - return self.ranges[index_key][1] - return 0 - - def get_range_boundaries(self, shard_finder=None): - total = sum([count for _, (_, count) in self.ranges.items()]) - range_size = total // self.k + def get_range_boundaries(self, num_buckets, shard_finder=None): + total = sum([start_count for (start_count, end_count) in self.reads.values()]) + range_size = total // num_buckets output_range_counts = [] - def add_boundary(start, end, count): + def add_boundary(start, end, started_count, total_count): if shard_finder: shard_count = shard_finder.get_shard_count(start, end) if shard_count == 1: addresses = shard_finder.get_addresses_for_key(start) else: addresses = None - output_range_counts.append((start, end, count, shard_count, addresses)) + output_range_counts.append((start, end, started_count, total_count, shard_count, addresses)) else: - output_range_counts.append((start, end, count, None, None)) + output_range_counts.append((start, end, started_count, total_count, None, None)) this_range_start_key = None + last_end = None + open_count = 0 + opened_this_range = 0 count_this_range = 0 - for (start_key, (end_key, count)) in self.ranges.items(): - if not this_range_start_key: - this_range_start_key = start_key - count_this_range += count - if count_this_range >= range_size: - add_boundary(this_range_start_key, end_key, count_this_range) - count_this_range = 0 - this_range_start_key = None - if count_this_range > 0: - add_boundary(this_range_start_key, end_key, count_this_range) + for (start_key, (start_count, end_count)) in self.reads.items(): + open_count -= end_count + + if opened_this_range >= range_size: + add_boundary(this_range_start_key, start_key, opened_this_range, count_this_range) + count_this_range = open_count + opened_this_range = 0 + this_range_start_key = None + + count_this_range += start_count + opened_this_range += start_count + open_count += start_count + + if count_this_range > 0 and this_range_start_key is None: + this_range_start_key = start_key + + if end_count > 0: + last_end = start_key + + if last_end is None: + last_end = b'\xff' + if count_this_range > 0: + add_boundary(this_range_start_key, last_end, opened_this_range, count_this_range) + + shard_finder.wait_for_shard_addresses(output_range_counts, 0, 5) return output_range_counts class ShardFinder(object): - def __init__(self, db): + def __init__(self, db, exclude_ports): self.db = db + self.exclude_ports = exclude_ports + + self.tr = db.create_transaction() + self.refresh_tr() + + self.outstanding = [] + self.boundary_keys = list(fdb.locality.get_boundary_keys(db, b'', b'\xff\xff')) + self.shard_cache = {} + + def _get_boundary_keys(self, begin, end): + start_pos = max(0, bisect_right(self.boundary_keys, begin)-1) + end_pos = max(0, bisect_right(self.boundary_keys, end)-1) + + return self.boundary_keys[start_pos:end_pos] + + def refresh_tr(self): + self.tr.options.set_read_lock_aware() + if not self.exclude_ports: + self.tr.options.set_include_port_in_address() @staticmethod - @fdb.transactional - def _get_boundary_keys(tr, begin, end): - tr.options.set_read_lock_aware() - return fdb.locality.get_boundary_keys(tr, begin, end) - - @staticmethod - @fdb.transactional def _get_addresses_for_key(tr, key): - tr.options.set_read_lock_aware() return fdb.locality.get_addresses_for_key(tr, key) def get_shard_count(self, start_key, end_key): - return len(list(self._get_boundary_keys(self.db, start_key, end_key))) + 1 + return len(self._get_boundary_keys(start_key, end_key)) + 1 def get_addresses_for_key(self, key): - return [a.decode('ascii') for a in self._get_addresses_for_key(self.db, key).wait()] + shard = self.boundary_keys[max(0, bisect_right(self.boundary_keys, key)-1)] + do_load = False + if not shard in self.shard_cache: + do_load = True + elif self.shard_cache[shard].is_ready(): + try: + self.shard_cache[shard].wait() + except fdb.FDBError as e: + self.tr.on_error(e).wait() + self.refresh_tr() + do_load = True + if do_load: + if len(self.outstanding) > 1000: + for f in self.outstanding: + try: + f.wait() + except fdb.FDBError as e: + pass -class TopKeysCounter(object): + self.outstanding = [] + self.tr.reset() + self.refresh_tr() + + self.outstanding.append(self._get_addresses_for_key(self.tr, shard)) + self.shard_cache[shard] = self.outstanding[-1] + + return self.shard_cache[shard] + + def wait_for_shard_addresses(self, ranges, key_idx, addr_idx): + for index in range(len(ranges)): + item = ranges[index] + if item[addr_idx] is not None: + while True: + try: + ranges[index] = item[0:addr_idx] + ([a.decode('ascii') for a in item[addr_idx].wait()],) + item[addr_idx+1:] + break + except fdb.FDBError as e: + ranges[index] = item[0:addr_idx] + (self.get_addresses_for_key(item[key_idx]),) + item[addr_idx+1:] + +class WriteCounter(object): mutation_types_to_consider = frozenset([MutationType.SET_VALUE, MutationType.ADD_VALUE]) - def __init__(self, k): - self.k = k - self.reads = defaultdict(lambda: 0) + def __init__(self): self.writes = defaultdict(lambda: 0) def process(self, transaction_info): - for get in transaction_info.gets: - self.reads[get.key] += 1 if transaction_info.commit: for mutation in transaction_info.commit.mutations: if mutation.code in self.mutation_types_to_consider: self.writes[mutation.param_one] += 1 - def _get_range_boundaries(self, counts, shard_finder=None): - total = sum([v for (k, v) in counts.items()]) - range_size = total // self.k - key_counts_sorted = sorted(counts.items()) + def get_range_boundaries(self, num_buckets, shard_finder=None): + total = sum([v for (k, v) in self.writes.items()]) + range_size = total // num_buckets + key_counts_sorted = sorted(self.writes.items()) output_range_counts = [] def add_boundary(start, end, count): @@ -671,9 +719,9 @@ class TopKeysCounter(object): addresses = shard_finder.get_addresses_for_key(start) else: addresses = None - output_range_counts.append((start, end, count, shard_count, addresses)) + output_range_counts.append((start, end, count, None, shard_count, addresses)) else: - output_range_counts.append((start, end, count, None, None)) + output_range_counts.append((start, end, count, None, None, None)) start_key = None count_this_range = 0 @@ -688,24 +736,31 @@ class TopKeysCounter(object): if count_this_range > 0: add_boundary(start_key, k, count_this_range) + shard_finder.wait_for_shard_addresses(output_range_counts, 0, 5) return output_range_counts - def _get_top_k(self, counts): - count_key_pairs = sorted([(v, k) for (k, v) in counts.items()], reverse=True) - return count_key_pairs[0:self.k] + def get_total_writes(self): + return sum([v for v in self.writes.values()]) - def get_top_k_reads(self): - return self._get_top_k(self.reads) + def get_top_k_writes(self, num, filter_addresses, shard_finder=None): + count_pairs = sorted([(v, k) for (k, v) in self.writes.items()], reverse=True) + if not filter_addresses: + count_pairs = count_pairs[0:num] - def get_top_k_writes(self): - return self._get_top_k(self.writes) + if shard_finder: + results = [] + for (count, key) in count_pairs: + results.append((key, None, count, shard_finder.get_addresses_for_key(key))) - def get_k_read_range_boundaries(self, shard_finder=None): - return self._get_range_boundaries(self.reads, shard_finder) + shard_finder.wait_for_shard_addresses(results, 0, 3) - def get_k_write_range_boundaries(self, shard_finder=None): - return self._get_range_boundaries(self.writes, shard_finder) + if filter_addresses: + filter_addresses = set(filter_addresses) + results = [r for r in results if filter_addresses.issubset(set(r[3]))][0:num] + else: + results = [(key, end, count) for (count, key) in count_pairs[0:num]] + return results def connect(cluster_file=None): db = fdb.open(cluster_file=cluster_file) @@ -722,6 +777,8 @@ def main(): help="Include get type. If no filter args are given all will be returned.") parser.add_argument("--filter-get-range", action="store_true", help="Include get_range type. If no filter args are given all will be returned.") + parser.add_argument("--filter-reads", action="store_true", + help="Include get and get_range type. If no filter args are given all will be returned.") parser.add_argument("--filter-commit", action="store_true", help="Include commit type. If no filter args are given all will be returned.") parser.add_argument("--filter-error-get", action="store_true", @@ -737,21 +794,34 @@ def main(): end_time_group = parser.add_mutually_exclusive_group() end_time_group.add_argument("--max-timestamp", type=int, help="Don't return events newer than this epoch time") end_time_group.add_argument("-e", "--end-time", type=str, help="Don't return events older than this parsed time") - parser.add_argument("--top-keys", type=int, help="If specified will output this many top keys for reads or writes", default=0) + parser.add_argument("--num-buckets", type=int, help="The number of buckets to partition the key-space into for operation counts", default=100) + parser.add_argument("--top-requests", type=int, help="If specified will output this many top keys for reads or writes", default=0) + parser.add_argument("--exclude-ports", action="store_true", help="Print addresses without the port number. Only works in versions older than 6.3, and is required in versions older than 6.2.") + parser.add_argument("--single-shard-ranges-only", action="store_true", help="Only print range boundaries that exist in a single shard") + parser.add_argument("-a", "--filter-address", action="append", help="Only print range boundaries that include the given address. This option can used multiple times to include more than one address in the filter, in which case all addresses must match.") + args = parser.parse_args() type_filter = set() if args.filter_get_version: type_filter.add("get_version") - if args.filter_get: type_filter.add("get") - if args.filter_get_range: type_filter.add("get_range") + if args.filter_get or args.filter_reads: type_filter.add("get") + if args.filter_get_range or args.filter_reads: type_filter.add("get_range") if args.filter_commit: type_filter.add("commit") if args.filter_error_get: type_filter.add("error_get") if args.filter_error_get_range: type_filter.add("error_get_range") if args.filter_error_commit: type_filter.add("error_commit") - top_keys = args.top_keys - key_counter = TopKeysCounter(top_keys) if top_keys else None - range_counter = RangeCounter(top_keys) if (has_sortedcontainers() and top_keys) else None - full_output = args.full_output or (top_keys is not None) + + if (not type_filter or "commit" in type_filter): + write_counter = WriteCounter() if args.num_buckets else None + else: + write_counter = None + + if (not type_filter or "get" in type_filter or "get_range" in type_filter): + read_counter = ReadCounter() if (has_sortedcontainers() and args.num_buckets) else None + else: + read_counter = None + + full_output = args.full_output or (args.num_buckets is not None) if args.min_timestamp: min_timestamp = args.min_timestamp @@ -784,48 +854,128 @@ def main(): db = connect(cluster_file=args.cluster_file) loader = TransactionInfoLoader(db, full_output=full_output, type_filter=type_filter, min_timestamp=min_timestamp, max_timestamp=max_timestamp) + for info in loader.fetch_transaction_info(): if info.has_types(): - if not key_counter and not range_counter: + if not write_counter and not read_counter: print(info.to_json()) else: - if key_counter: - key_counter.process(info) - if range_counter: - range_counter.process(info) + if write_counter: + write_counter.process(info) + if read_counter: + read_counter.process(info) - if key_counter: - def print_top(top): - for (count, key) in top: - print("%s %d" % (key, count)) - - def print_range_boundaries(range_boundaries): - for (start, end, count, shard_count, addresses) in range_boundaries: - if not shard_count: - print("[%s, %s] %d" % (start, end, count)) + def print_top(top, total, context): + if top: + running_count = 0 + for (idx, (start, end, count, addresses)) in enumerate(top): + running_count += count + if end is not None: + op_str = 'Range %r - %r' % (start, end) else: - addresses_string = "addresses=%s" % ','.join(addresses) if addresses else '' - print("[%s, %s] %d shards=%d %s" % (start, end, count, shard_count, addresses_string)) + op_str = 'Key %r' % start + + print(" %d. %s\n %d sampled %s (%.2f%%, %.2f%% cumulative)" % (idx+1, op_str, count, context, 100*count/total, 100*running_count/total)) + print(" shard addresses: %s\n" % ", ".join(addresses)) + + else: + print(" No %s found" % context) + + def print_range_boundaries(range_boundaries, context): + omit_start = None + for (idx, (start, end, start_count, total_count, shard_count, addresses)) in enumerate(range_boundaries): + omit = args.single_shard_ranges_only and shard_count is not None and shard_count > 1 + if args.filter_address: + if not addresses: + omit = True + else: + for addr in args.filter_address: + if addr not in addresses: + omit = True + break + + if not omit: + if omit_start is not None: + if omit_start == idx-1: + print(" %d. Omitted\n" % (idx)) + else: + print(" %d - %d. Omitted\n" % (omit_start+1, idx)) + omit_start = None + + if total_count is None: + count_str = '%d sampled %s' % (start_count, context) + else: + count_str = '%d sampled %s (%d intersecting)' % (start_count, context, total_count) + if not shard_count: + print(" %d. [%s, %s]\n %d sampled %s\n" % (idx+1, start, end, count, context)) + else: + addresses_string = "; addresses=%s" % ', '.join(addresses) if addresses else '' + print(" %d. [%s, %s]\n %s spanning %d shard(s)%s\n" % (idx+1, start, end, count_str, shard_count, addresses_string)) + elif omit_start is None: + omit_start = idx + + if omit_start is not None: + if omit_start == len(range_boundaries)-1: + print(" %d. Omitted\n" % len(range_boundaries)) + else: + print(" %d - %d. Omitted\n" % (omit_start+1, len(range_boundaries))) + + shard_finder = ShardFinder(db, args.exclude_ports) + + print("NOTE: shard locations are current and may not reflect where an operation was performed in the past\n") + + if write_counter: + if args.top_requests: + top_writes = write_counter.get_top_k_writes(args.top_requests, args.filter_address, shard_finder=shard_finder) + + range_boundaries = write_counter.get_range_boundaries(args.num_buckets, shard_finder=shard_finder) + num_writes = write_counter.get_total_writes() + + if args.top_requests or range_boundaries: + print("WRITES") + print("------\n") + print("Processed %d total writes\n" % num_writes) + + if args.top_requests: + suffix = "" + if args.filter_address: + suffix = " (%s)" % ", ".join(args.filter_address) + print("Top %d writes%s:\n" % (args.top_requests, suffix)) + + print_top(top_writes, write_counter.get_total_writes(), "writes") + print("") - shard_finder = ShardFinder(db) - top_reads = key_counter.get_top_k_reads() - if top_reads: - print("Top %d reads:" % min(top_keys, len(top_reads))) - print_top(top_reads) - print("Approx equal sized gets range boundaries:") - print_range_boundaries(key_counter.get_k_read_range_boundaries(shard_finder=shard_finder)) - top_writes = key_counter.get_top_k_writes() - if top_writes: - print("Top %d writes:" % min(top_keys, len(top_writes))) - print_top(top_writes) - print("Approx equal sized commits range boundaries:") - print_range_boundaries(key_counter.get_k_write_range_boundaries(shard_finder=shard_finder)) - if range_counter: - range_boundaries = range_counter.get_range_boundaries(shard_finder=shard_finder) if range_boundaries: - print("Approx equal sized get_ranges boundaries:") - print_range_boundaries(range_boundaries) + print("Key-space boundaries with approximately equal mutation counts:\n") + print_range_boundaries(range_boundaries, "writes") + if args.top_requests or range_boundaries: + print("") + + if read_counter: + if args.top_requests: + top_reads = read_counter.get_top_k_reads(args.top_requests, args.filter_address, shard_finder=shard_finder) + + range_boundaries = read_counter.get_range_boundaries(args.num_buckets, shard_finder=shard_finder) + num_reads = read_counter.get_total_reads() + + if args.top_requests or range_boundaries: + print("READS") + print("-----\n") + print("Processed %d total reads\n" % num_reads) + + if args.top_requests: + suffix = "" + if args.filter_address: + suffix = " (%s)" % ", ".join(args.filter_address) + print("Top %d reads%s:\n" % (args.top_requests, suffix)) + + print_top(top_reads, num_reads, "reads") + print("") + + if range_boundaries: + print("Key-space boundaries with approximately equal read counts:\n") + print_range_boundaries(range_boundaries, "reads") if __name__ == "__main__": main() diff --git a/documentation/sphinx/source/administration.rst b/documentation/sphinx/source/administration.rst index 935236e199..405a3b1368 100644 --- a/documentation/sphinx/source/administration.rst +++ b/documentation/sphinx/source/administration.rst @@ -493,7 +493,7 @@ If a process has had more than 10 TCP segments retransmitted in the last 5 secon 10.0.4.1:4500 ( 3% cpu; 2% machine; 0.004 Gbps; 0% disk; REXMIT! 2.5 GB / 4.1 GB RAM ) Machine-readable status --------------------------------- +----------------------- The status command can provide a complete summary of statistics about the cluster and the database with the ``json`` argument. Full documentation for ``status json`` output can be found :doc:`here `. From the output of ``status json``, operators can find useful health metrics to determine whether or not their cluster is hitting performance limits. @@ -505,6 +505,72 @@ Durable version lag ``cluster.qos.worst_durability_lag_storage_server`` cont Transaction log queue ``cluster.qos.worst_queue_bytes_log_server`` contains the maximum size in bytes of the mutations stored on a transaction log that have not yet been popped by storage servers. A large transaction log queue size can potentially cause the ratekeeper to increase throttling. ====================== ============================================================================================================== +Server-side latency band tracking +--------------------------------- + +As part of the status document, ``status json`` provides some sampled latency metrics obtained by running probe transactions internally. While this can often be useful, it does not necessarily reflect the distribution of latencies for requests originated by clients. + +FoundationDB additionally provides optional functionality to measure the latencies of all incoming get read version (GRV), read, and commit requests and report some basic details about those requests. The latencies are measured from the time the server receives the request to the point when it replies, and will therefore not include time spent in transit between the client and server or delays in the client process itself. + +The latency band tracking works by configuring various latency thresholds and counting the number of requests that occur in each band (i.e. between two consecutive thresholds). For example, if you wanted to define a service-level objective (SLO) for your cluster where 99.9% of read requests were answered within N seconds, you could set a read latency threshold at N. You could then count the number of requests below and above the threshold and determine whether the required percentage of requests are answered sufficiently quickly. + +Configuration of server-side latency bands is performed by setting the ``\xff\x02/latencyBandConfig`` key to a string encoding the following JSON document:: + + { + "get_read_version" : { + "bands" : [ 0.01, 0.1] + }, + "read" : { + "bands" : [ 0.01, 0.1], + "max_key_selector_offset" : 1000, + "max_read_bytes" : 1000000 + }, + "commit" : { + "bands" : [ 0.01, 0.1], + "max_commit_bytes" : 1000000 + } + } + +Every field in this configuration is optional, and any missing fields will be left unset (i.e. no bands will be tracked or limits will not apply). The configuration takes the following arguments: + +* ``bands`` - a list of thresholds (in seconds) to be measured for the given request type (``get_read_version``, ``read``, or ``commit``) +* ``max_key_selector_offset`` - an integer specifying the maximum key selector offset a read request can have and still be counted +* ``max_read_bytes`` - an integer specifying the maximum size in bytes of a read response that will be counted +* ``max_commit_bytes`` - an integer specifying the maximum size in bytes of a commit request that will be counted + +Setting this configuration key to a value that changes the configuration will result in the cluster controller server process logging a ``LatencyBandConfigChanged`` event. This event will indicate whether a configuration is present or not using its ``Present`` field. Specifying an invalid configuration will result in the latency band feature being unconfigured, and the server process running the cluster controller will log a ``InvalidLatencyBandConfiguration`` trace event. + +.. note:: GRV requests are counted only at default and immediate priority. Batch priority GRV requests are ignored for the purposes of latency band tracking. + +When configured, the ``status json`` output will include additional fields to report the number of requests in each latency band located at ``cluster.processes..roles[N].*_latency_bands``:: + + "grv_latency_bands" : { + 0.01: 10, + 0.1: 0, + inf: 1, + filtered: 0 + }, + "read_latency_bands" : { + 0.01: 12, + 0.1: 1, + inf: 0, + filtered: 0 + }, + "commit_latency_bands" : { + 0.01: 5, + 0.1: 5, + inf: 2, + filtered: 1 + } + +The ``grv_latency_bands`` and ``commit_latency_bands`` objects will only be logged for ``proxy`` roles, and ``read_latency_bands`` will only be logged for storage roles. Each threshold is represented as a key in the map, and its associated value will be the total number of requests in the lifetime of the process with a latency smaller than the threshold but larger than the next smaller threshold. + +For example, ``0.1: 1`` in ``read_latency_bands`` indicates that there has been 1 read request with a latency in the range ``[0.01, 0.1)``. For the smallest specified threshold, the lower bound is 0 (e.g. ``[0, 0.01)`` in the example above). Requests that took longer than any defined latency band will be reported in the ``inf`` (infinity) band. Requests that were filtered by the configuration (e.g. using ``max_read_bytes``) are reported in the ``filtered`` category. + +Because each threshold reports latencies strictly in the range between the next lower threshold and itself, it may be necessary to sum up the counts for multiple bands to determine the total number of requests below a certain threshold. + +.. note:: No history of request counts is recorded for processes that ran in the past. This includes the history prior to restart for a process that has been restarted, for which the counts get reset to 0. For this reason, it is recommended that you collect this information periodically if you need to be able to track requests from such processes. + .. _administration_fdbmonitor: ``fdbmonitor`` and ``fdbserver`` diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index 5c7cdd2c5d..40482d3b0b 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -133,7 +133,7 @@ API versioning Prior to including ``fdb_c.h``, you must define the ``FDB_API_VERSION`` macro. This, together with the :func:`fdb_select_api_version()` function, allows programs written against an older version of the API to compile and run with newer versions of the C library. The current version of the FoundationDB C API is |api-version|. :: - #define FDB_API_VERSION 630 + #define FDB_API_VERSION 700 #include .. function:: fdb_error_t fdb_select_api_version(int version) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index 6bce920a45..6ab190a052 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -147,7 +147,7 @@ .. |atomic-versionstamps-tuple-warning-value| replace:: At this time, versionstamped values are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped values may not be used with the Subspace and Directory layers except in those languages. -.. |api-version| replace:: 630 +.. |api-version| replace:: 700 .. |streaming-mode-blurb1| replace:: When using |get-range-func| and similar interfaces, API clients can request large ranges of the database to iterate over. Making such a request doesn't necessarily mean that the client will consume all of the data in the range - sometimes the client doesn't know how far it intends to iterate in advance. FoundationDB tries to balance latency and bandwidth by requesting data for iteration in batches. diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index fb75d3516d..69c6dab28a 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -108,7 +108,7 @@ Opening a database After importing the ``fdb`` module and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: import fdb - fdb.api_version(630) + fdb.api_version(700) db = fdb.open() .. function:: open( cluster_file=None, event_model=None ) diff --git a/documentation/sphinx/source/api-ruby.rst b/documentation/sphinx/source/api-ruby.rst index df73cf6dc4..84078b02d4 100644 --- a/documentation/sphinx/source/api-ruby.rst +++ b/documentation/sphinx/source/api-ruby.rst @@ -93,7 +93,7 @@ Opening a database After requiring the ``FDB`` gem and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: require 'fdb' - FDB.api_version 630 + FDB.api_version 700 db = FDB.open .. function:: open( cluster_file=nil ) -> Database diff --git a/documentation/sphinx/source/class-scheduling-go.rst b/documentation/sphinx/source/class-scheduling-go.rst index d8ea0a5b19..77d9c01e90 100644 --- a/documentation/sphinx/source/class-scheduling-go.rst +++ b/documentation/sphinx/source/class-scheduling-go.rst @@ -29,7 +29,7 @@ Before using the API, we need to specify the API version. This allows programs t .. code-block:: go - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. @@ -78,7 +78,7 @@ If this is all working, it looks like we are ready to start building a real appl func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) // Open the default database from the system cluster db := fdb.MustOpenDefault() @@ -666,7 +666,7 @@ Here's the code for the scheduling tutorial: } func main() { - fdb.MustAPIVersion(630) + fdb.MustAPIVersion(700) db := fdb.MustOpenDefault() db.Options().SetTransactionTimeout(60000) // 60,000 ms = 1 minute db.Options().SetTransactionRetryLimit(100) diff --git a/documentation/sphinx/source/class-scheduling-java.rst b/documentation/sphinx/source/class-scheduling-java.rst index c899c546dc..c5dda17d55 100644 --- a/documentation/sphinx/source/class-scheduling-java.rst +++ b/documentation/sphinx/source/class-scheduling-java.rst @@ -30,7 +30,7 @@ Before using the API, we need to specify the API version. This allows programs t private static final Database db; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); } @@ -66,7 +66,7 @@ If this is all working, it looks like we are ready to start building a real appl private static final Database db; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); } @@ -441,7 +441,7 @@ Here's the code for the scheduling tutorial: private static final Database db; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); db.options().setTransactionTimeout(60000); // 60,000 ms = 1 minute db.options().setTransactionRetryLimit(100); diff --git a/documentation/sphinx/source/class-scheduling-ruby.rst b/documentation/sphinx/source/class-scheduling-ruby.rst index d1f79c3725..c8d8483aad 100644 --- a/documentation/sphinx/source/class-scheduling-ruby.rst +++ b/documentation/sphinx/source/class-scheduling-ruby.rst @@ -23,7 +23,7 @@ Open a Ruby interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - > FDB.api_version 630 + > FDB.api_version 700 => nil Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -46,7 +46,7 @@ If this is all working, it looks like we are ready to start building a real appl .. code-block:: ruby require 'fdb' - FDB.api_version 630 + FDB.api_version 700 @db = FDB.open @db['hello'] = 'world' print 'hello ', @db['hello'] @@ -373,7 +373,7 @@ Here's the code for the scheduling tutorial: require 'fdb' - FDB.api_version 630 + FDB.api_version 700 #################################### ## Initialization ## diff --git a/documentation/sphinx/source/class-scheduling.rst b/documentation/sphinx/source/class-scheduling.rst index b516bc9f7c..23615a08a6 100644 --- a/documentation/sphinx/source/class-scheduling.rst +++ b/documentation/sphinx/source/class-scheduling.rst @@ -30,7 +30,7 @@ Open a Python interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - >>> fdb.api_version(630) + >>> fdb.api_version(700) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -48,7 +48,7 @@ When this command returns without exception, the modification is durably stored If this is all working, it looks like we are ready to start building a real application. For reference, here's the full code for "hello world":: import fdb - fdb.api_version(630) + fdb.api_version(700) db = fdb.open() db[b'hello'] = b'world' print 'hello', db[b'hello'] @@ -91,7 +91,7 @@ FoundationDB includes a few tools that make it easy to model data using this app opening a :ref:`directory ` in the database:: import fdb - fdb.api_version(630) + fdb.api_version(700) db = fdb.open() scheduling = fdb.directory.create_or_open(db, ('scheduling',)) @@ -337,7 +337,7 @@ Here's the code for the scheduling tutorial:: import fdb import fdb.tuple - fdb.api_version(630) + fdb.api_version(700) #################################### diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index b4d30eb629..6b81a98a82 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.3.0.pkg `_ +* `FoundationDB-6.3.1.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.3.0-1_amd64.deb `_ -* `foundationdb-server-6.3.0-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.3.1-1_amd64.deb `_ +* `foundationdb-server-6.3.1-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.3.0-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.3.0-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.1-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.3.1-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.3.0-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.3.0-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.1-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.3.1-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.3.0-x64.msi `_ +* `foundationdb-6.3.1-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package: -* `foundationdb-6.3.0.tar.gz `_ +* `foundationdb-6.3.1.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.3.0.gem `_ +* `fdb-6.3.1.gem `_ Java 8+ ------- -* `fdb-java-6.3.0.jar `_ -* `fdb-java-6.3.0-javadoc.jar `_ +* `fdb-java-6.3.1.jar `_ +* `fdb-java-6.3.1-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/hierarchical-documents-java.rst b/documentation/sphinx/source/hierarchical-documents-java.rst index c2631e5b36..db33abd4ef 100644 --- a/documentation/sphinx/source/hierarchical-documents-java.rst +++ b/documentation/sphinx/source/hierarchical-documents-java.rst @@ -69,7 +69,7 @@ Here’s a basic implementation of the recipe. private static final long EMPTY_ARRAY = -1; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); docSpace = new Subspace(Tuple.from("D")); } diff --git a/documentation/sphinx/source/multimaps-java.rst b/documentation/sphinx/source/multimaps-java.rst index 4ce8e1f3ba..3c9a46ad3c 100644 --- a/documentation/sphinx/source/multimaps-java.rst +++ b/documentation/sphinx/source/multimaps-java.rst @@ -74,7 +74,7 @@ Here’s a simple implementation of multimaps with multisets as described: private static final int N = 100; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); multi = new Subspace(Tuple.from("M")); } diff --git a/documentation/sphinx/source/old-release-notes/release-notes-620.rst b/documentation/sphinx/source/old-release-notes/release-notes-620.rst index 38b0c415bb..34ef860311 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-620.rst @@ -2,6 +2,15 @@ Release Notes ############# +6.2.22 +====== + +Fixes +----- + +* Coordinator class processes could be recruited as the cluster controller. `(PR #3282) `_ +* HTTPS requests made by backup would fail (introduced in 6.2.21). `(PR #3284) `_ + 6.2.21 ====== diff --git a/documentation/sphinx/source/old-release-notes/release-notes-630.rst b/documentation/sphinx/source/old-release-notes/release-notes-630.rst new file mode 100644 index 0000000000..1b1cc7a172 --- /dev/null +++ b/documentation/sphinx/source/old-release-notes/release-notes-630.rst @@ -0,0 +1,123 @@ +############# +Release Notes +############# + +6.3.1 +===== + +Features +-------- + +* Added the ability to set arbitrary tags on transactions. Tags can be specifically throttled using ``fdbcli``, and certain types of tags can be automatically throttled by ratekeeper. `(PR #2942) `_ +* Add an option for transactions to report conflicting keys by calling ``getRange`` with the special key prefix ``\xff\xff/transaction/conflicting_keys/``. `(PR 2257) `_ +* Added the ``exclude failed`` command to ``fdbcli``. This command designates that a process is dead and will never come back, so the transaction logs can forget about mutations sent to that process. `(PR #1955) `_ +* A new fast restore system that can restore a database to a point in time from backup files. It is a Spark-like parallel processing framework that processes backup data asynchronously, in parallel and in pipeline. `(Fast Restore Project) `_ +* Added backup workers for pulling mutations from transaction logs and uploading them to blob storage. Switching from the previous backup implementation will double a cluster's maximum write bandwidth. `(PR #1625) `_ `(PR #2588) `_ `(PR #2642) `_ +* Added a new API in all bindings that can be used to query the estimated byte size of a given range. `(PR #2537) `_ +* Added the ``lock`` and ``unlock`` commands to ``fdbcli`` which lock or unlock a cluster. `(PR #2890) `_ +* Add a framework which helps to add client functions using special keys (keys within ``[\xff\xff, \xff\xff\xff)``). `(PR #2662) `_ + +Performance +----------- + +* Improved the client's load balancing algorithm so that each proxy processes an equal number of requests. `(PR #2520) `_ +* Significantly reduced the amount of work done on the cluster controller by removing the centralized failure monitoring. `(PR #2518) `_ +* Improved master recovery speeds by more efficiently broadcasting the recovery state between processes. `(PR #2941) `_ +* Significantly reduced the number of network connections opened to the coordinators. `(PR #3069) `_ +* Improve GRV tail latencies, particularly as the transaction rate gets nearer the ratekeeper limit. `(PR #2735) `_ +* The proxies are now more responsive to changes in workload when unthrottling lower priority transactions. `(PR #2735) `_ +* Removed a lot of unnecessary copying across the codebase. `(PR #2986) `_ `(PR #2915) `_ `(PR #3024) `_ `(PR #2999) `_ +* Optimized the performance of the storage server. `(PR #1988) `_ `(PR #3103) `_ +* Optimized the performance of the resolver. `(PR #2648) `_ +* Replaced most uses of hashlittle2 with crc32 for better performance. `(PR #2538) `_ +* Significantly reduced the serialized size of conflict ranges and single key clears. `(PR #2513) `_ +* Improved range read performance when the reads overlap recently cleared key ranges. `(PR #2028) `_ +* Reduced the number of comparisons used by various map implementations. `(PR #2882) `_ +* Reduced the serialized size of empty strings. `(PR #3063) `_ +* Reduced the serialized size of various interfaces by 10x. `(PR #3068) `_ + +Reliability +----------- + +* Connections that disconnect frequently are not immediately marked available. `(PR #2932) `_ +* The data distributor will consider storage servers that are continually lagging behind as if they were failed. `(PR #2917) `_ +* Changing the storage engine type of a cluster will no longer cause the cluster to run out of memory. Instead, the cluster will gracefully migrate storage server processes to the new storage engine one by one. `(PR #1985) `_ +* Batch priority transactions which are being throttled by ratekeeper will get a ``batch_transaction_throttled`` error instead of hanging indefinitely. `(PR #1868) `_ +* Avoid using too much memory on the transaction logs when multiple types of transaction logs exist in the same process. `(PR #2213) `_ + +Fixes +----- + +* The ``SetVersionstampedKey`` atomic operation no longer conflicts with versions smaller than the current read version of the transaction. `(PR #2557) `_ +* Ratekeeper would measure durability lag a few seconds higher than reality. `(PR #2499) `_ +* In very rare scenarios, the data distributor process could get stuck in an infinite loop. `(PR #2228) `_ +* If the number of configured transaction logs were reduced at the exact same time a change to the system keyspace took place, it was possible for the transaction state store to become corrupted. `(PR #3051) `_ +* Fix multiple data races between threads on the client. `(PR #3026) `_ +* Transaction logs configured to spill by reference had an unintended delay between each spilled batch. `(PR #3153) `_ +* Added guards to honor ``DISABLE_POSIX_KERNEL_AIO``. `(PR #2888) `_ + +Status +------ + +* A process's ``memory.available_bytes`` can no longer exceed the memory limit of the process. For purposes of this statistic, processes on the same machine will be allocated memory proportionally based on the size of their memory limits. `(PR #3174) `_ +* Replaced ``cluster.database_locked`` status field with ``cluster.database_lock_state``, which contains two subfields: ``locked`` (boolean) and ``lock_uid`` (which contains the database lock uid if the database is locked). `(PR #2058) `_ +* Removed fields ``worst_version_lag_storage_server`` and ``limiting_version_lag_storage_server`` from the ``cluster.qos`` section. The ``worst_data_lag_storage_server`` and ``limiting_data_lag_storage_server`` objects can be used instead. `(PR #3196) `_ +* If a process is unable to flush trace logs to disk, the problem will now be reported via the output of ``status`` command inside ``fdbcli``. `(PR #2605) `_ `(PR #2820) `_ + +Bindings +-------- + +* API version updated to 630. See the :ref:`API version upgrade guide ` for upgrade details. +* Python: The ``@fdb.transactional`` decorator will now throw an error if the decorated function returns a generator. `(PR #1724) `_ +* Java: Add caching for various JNI objects to improve performance. `(PR #2809) `_ +* Java: Optimize byte array comparisons in ``ByteArrayUtil``. `(PR #2823) `_ +* Java: Add ``FDB.disableShutdownHook`` that can be used to prevent the default shutdown hook from running. Users of this new function should make sure to call ``stopNetwork`` before terminating a client process. `(PR #2635) `_ +* Java: Introduced ``keyAfter`` utility function that can be used to create the immediate next key for a given byte array. `(PR #2458) `_ +* Golang: The ``Transact`` function will unwrap errors that have been wrapped using ``xerrors`` to determine if a retryable FoundationDB error is in the error chain. `(PR #3131) `_ +* Golang: Added ``Subspace.PackWithVersionstamp`` that can be used to pack a ``Tuple`` that contains a versionstamp. `(PR #2243) `_ +* Golang: Implement ``Stringer`` interface for ``Tuple``, ``Subspace``, ``UUID``, and ``Versionstamp``. `(PR #3032) `_ +* C: The ``FDBKeyValue`` struct's ``key`` and ``value`` members have changed type from ``void*`` to ``uint8_t*``. `(PR #2622) `_ +* Deprecated ``enable_slow_task_profiling`` network option and replaced it with ``enable_run_loop_profiling``. `(PR #2608) `_ + +Other Changes +------------- + +* Small key ranges which are being heavily read will be reported in the logs using the trace event ``ReadHotRangeLog``. `(PR #2046) `_ `(PR #2378) `_ `(PR #2532) `_ +* Added the read version, commit version, and datacenter locality to the client transaction information. `(PR #3079) `_ `(PR #3205) `_ +* Added a network option ``TRACE_FILE_IDENTIFIER`` that can be used to provide a custom identifier string that will be part of the file name for all trace log files created on the client. `(PR #2869) `_ +* It is now possible to use the ``TRACE_LOG_GROUP`` option on a client process after the database has been created. `(PR #2862) `_ +* Added a network option ``TRACE_CLOCK_SOURCE`` that can be used to switch the trace event timestamps to use a realtime clock source. `(PR #2329) `_ +* The ``INCLUDE_PORT_IN_ADDRESS`` transaction option is now on by default. This means ``get_addresses_for_key`` will always return ports in the address strings. `(PR #2639) `_ +* Added the ``getversion`` command to ``fdbcli`` which returns the current read version of the cluster. `(PR #2882) `_ +* Added the ``advanceversion`` command to ``fdbcli`` which increases the current version of a cluster. `(PR #2965) `_ +* Improved the slow task profiler to also report backtraces for periods when the run loop is saturated. `(PR #2608) `_ +* Double the number of shard locations that the client will cache locally. `(PR #2198) `_ +* Replaced the ``-add_prefix`` and ``-remove_prefix`` options with ``--add_prefix`` and ``--remove_prefix`` in ``fdbrestore`` `(PR 3206) `_ +* Data distribution metrics can now be read using the special keyspace ``\xff\xff/metrics/data_distribution_stats``. `(PR #2547) `_ +* The ``\xff\xff/worker_interfaces/`` keyspace now begins at a key which includes a trailing ``/`` (previously ``\xff\xff/worker_interfaces``). Range reads to this range now respect the end key passed into the range and include the keyspace prefix in the resulting keys. `(PR #3095) `_ +* Added FreeBSD support. `(PR #2634) `_ +* Updated boost to 1.72. `(PR #2684) `_ + +Earlier release notes +--------------------- +* :doc:`6.2 (API Version 620) ` +* :doc:`6.1 (API Version 610) ` +* :doc:`6.0 (API Version 600) ` +* :doc:`5.2 (API Version 520) ` +* :doc:`5.1 (API Version 510) ` +* :doc:`5.0 (API Version 500) ` +* :doc:`4.6 (API Version 460) ` +* :doc:`4.5 (API Version 450) ` +* :doc:`4.4 (API Version 440) ` +* :doc:`4.3 (API Version 430) ` +* :doc:`4.2 (API Version 420) ` +* :doc:`4.1 (API Version 410) ` +* :doc:`4.0 (API Version 400) ` +* :doc:`3.0 (API Version 300) ` +* :doc:`2.0 (API Version 200) ` +* :doc:`1.0 (API Version 100) ` +* :doc:`Beta 3 (API Version 23) ` +* :doc:`Beta 2 (API Version 22) ` +* :doc:`Beta 1 (API Version 21) ` +* :doc:`Alpha 6 (API Version 16) ` +* :doc:`Alpha 5 (API Version 14) ` \ No newline at end of file diff --git a/documentation/sphinx/source/priority-queues-java.rst b/documentation/sphinx/source/priority-queues-java.rst index 068349d680..0fafb08b4b 100644 --- a/documentation/sphinx/source/priority-queues-java.rst +++ b/documentation/sphinx/source/priority-queues-java.rst @@ -74,7 +74,7 @@ Here's a basic implementation of the model: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); pq = new Subspace(Tuple.from("P")); diff --git a/documentation/sphinx/source/queues-java.rst b/documentation/sphinx/source/queues-java.rst index 1ed636146d..b4b60df48b 100644 --- a/documentation/sphinx/source/queues-java.rst +++ b/documentation/sphinx/source/queues-java.rst @@ -73,7 +73,7 @@ The following is a simple implementation of the basic pattern: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); queue = new Subspace(Tuple.from("Q")); randno = new Random(); diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index 2297176944..5e0b94afb6 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -2,7 +2,7 @@ Release Notes ############# -6.3.0 +6.3.1 ===== Features @@ -100,6 +100,7 @@ Other Changes Earlier release notes --------------------- +* :doc:`6.3 (API Version 630) ` * :doc:`6.2 (API Version 620) ` * :doc:`6.1 (API Version 610) ` * :doc:`6.0 (API Version 600) ` diff --git a/documentation/sphinx/source/simple-indexes-java.rst b/documentation/sphinx/source/simple-indexes-java.rst index 709bc4bc7c..c5edf02e71 100644 --- a/documentation/sphinx/source/simple-indexes-java.rst +++ b/documentation/sphinx/source/simple-indexes-java.rst @@ -87,7 +87,7 @@ In this example, we’re storing user data based on user ID but sometimes need t private static final Subspace index; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); main = new Subspace(Tuple.from("user")); index = new Subspace(Tuple.from("zipcode_index")); diff --git a/documentation/sphinx/source/tables-java.rst b/documentation/sphinx/source/tables-java.rst index 0f13cebd65..235dbd5b47 100644 --- a/documentation/sphinx/source/tables-java.rst +++ b/documentation/sphinx/source/tables-java.rst @@ -62,7 +62,7 @@ Here’s a simple implementation of the basic table pattern: private static final Subspace colIndex; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); table = new Subspace(Tuple.from("T")); rowIndex = table.subspace(Tuple.from("R")); diff --git a/documentation/sphinx/source/vector-java.rst b/documentation/sphinx/source/vector-java.rst index 254ca26cc2..17da6ebed8 100644 --- a/documentation/sphinx/source/vector-java.rst +++ b/documentation/sphinx/source/vector-java.rst @@ -77,7 +77,7 @@ Here’s the basic pattern: private static final Subspace vector; static { - fdb = FDB.selectAPIVersion(630); + fdb = FDB.selectAPIVersion(700); db = fdb.open(); vector = new Subspace(Tuple.from("V")); } diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 3b5518730d..128470674d 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -3465,16 +3465,6 @@ int main(int argc, char* argv[]) { std::set_new_handler( &platform::outOfMemory ); setMemoryQuota( memLimit ); - int total = 0; - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - total += i->second; - if (total) - printf("%d errors:\n", total); - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - if (i->second > 0) - printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what()); - - Reference ccf; Database db; Reference sourceCcf; diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h index 9670a2a6ac..f1c78806ef 100644 --- a/fdbclient/CommitTransaction.h +++ b/fdbclient/CommitTransaction.h @@ -132,6 +132,13 @@ struct MutationRef { }; }; +template<> +struct Traceable : std::true_type { + static std::string toString(MutationRef const& value) { + return value.toString(); + } +}; + static inline std::string getTypeString(MutationRef::Type type) { return type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset"; } @@ -206,7 +213,4 @@ struct CommitTransactionRef { } }; -bool debugMutation( const char* context, Version version, MutationRef const& m ); -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keyRange ); - #endif diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index c6aff2a804..b7d60d014b 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -26,6 +26,7 @@ #include #include +#include "flow/Arena.h" #include "flow/flow.h" #include "fdbclient/Knobs.h" @@ -77,6 +78,10 @@ struct Tag { serializer(ar, locality, id); } }; + +template <> +struct flow_ref : std::integral_constant {}; + #pragma pack(pop) template void load( Ar& ar, Tag& tag ) { tag.serialize_unversioned(ar); } @@ -108,6 +113,13 @@ struct struct_like_traits : std::true_type { } }; +template<> +struct Traceable : std::true_type { + static std::string toString(const Tag& value) { + return value.toString(); + } +}; + static const Tag invalidTag {tagLocalitySpecial, 0}; static const Tag txsTag {tagLocalitySpecial, 1}; static const Tag cacheTag {tagLocalitySpecial, 2}; @@ -222,11 +234,25 @@ std::string describe( std::vector const& items, int max_items = -1 ) { return describeList(items, max_items); } +template +struct Traceable> : std::true_type { + static std::string toString(const std::vector& value) { + return describe(value); + } +}; + template std::string describe( std::set const& items, int max_items = -1 ) { return describeList(items, max_items); } +template +struct Traceable> : std::true_type { + static std::string toString(const std::set& value) { + return describe(value); + } +}; + std::string printable( const StringRef& val ); std::string printable( const std::string& val ); std::string printable( const KeyRangeRef& range ); diff --git a/fdbclient/HTTP.actor.cpp b/fdbclient/HTTP.actor.cpp index 933dd15fff..3779b26ef5 100644 --- a/fdbclient/HTTP.actor.cpp +++ b/fdbclient/HTTP.actor.cpp @@ -352,6 +352,9 @@ namespace HTTP { send_start = timer(); loop { + wait(conn->onWritable()); + wait( delay( 0, TaskPriority::WriteSocket ) ); + // If we already got a response, before finishing sending the request, then close the connection, // set the Connection header to "close" as a hint to the caller that this connection can't be used // again, and break out of the send loop. @@ -372,11 +375,6 @@ namespace HTTP { pContent->sent(len); if(pContent->empty()) break; - - if(len == 0) { - wait(conn->onWritable()); - wait( delay( 0, TaskPriority::WriteSocket ) ); - } } wait(responseReading); diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h index 655bf25a14..4989bf28cc 100644 --- a/fdbrpc/fdbrpc.h +++ b/fdbrpc/fdbrpc.h @@ -246,20 +246,13 @@ public: // stream.send( request ) // Unreliable at most once delivery: Delivers request unless there is a connection failure (zero or one times) - void send(const T& value) const { + template + void send(U && value) const { if (queue->isRemoteEndpoint()) { - FlowTransport::transport().sendUnreliable(SerializeSource(value), getEndpoint(), true); + FlowTransport::transport().sendUnreliable(SerializeSource(std::forward(value)), getEndpoint(), true); } else - queue->send(value); - } - - void send(T&& value) const { - if (queue->isRemoteEndpoint()) { - FlowTransport::transport().sendUnreliable(SerializeSource(std::move(value)), getEndpoint(), true); - } - else - queue->send(std::move(value)); + queue->send(std::forward(value)); } /*void sendError(const Error& error) const { diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 0918f51a2d..f1fcb78729 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -1654,15 +1654,8 @@ public: this->currentProcess = t.machine; try { - //auto before = getCPUTicks(); t.action.send(Void()); ASSERT( this->currentProcess == t.machine ); - /*auto elapsed = getCPUTicks() - before; - currentProcess->cpuTicks += elapsed; - if (deterministicRandom()->random01() < 0.01){ - TraceEvent("TaskDuration").detail("CpuTicks", currentProcess->cpuTicks); - currentProcess->cpuTicks = 0; - }*/ } catch (Error& e) { TraceEvent(SevError, "UnhandledSimulationEventError").error(e, true); killProcess(t.machine, KillInstantly); diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index d81e5763a1..6f8164a0f4 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -58,7 +58,6 @@ public: bool failed; bool excluded; bool cleared; - int64_t cpuTicks; bool rebooting; std::vector globals; @@ -68,12 +67,11 @@ public: double fault_injection_p1, fault_injection_p2; ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddressList addresses, - INetworkConnections *net, const char* dataFolder, const char* coordinationFolder ) - : name(name), locality(locality), startingClass(startingClass), - addresses(addresses), address(addresses.address), dataFolder(dataFolder), - network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0), - rebooting(false), fault_injection_p1(0), fault_injection_p2(0), - fault_injection_r(0), machine(0), cleared(false) {} + INetworkConnections* net, const char* dataFolder, const char* coordinationFolder) + : name(name), locality(locality), startingClass(startingClass), addresses(addresses), + address(addresses.address), dataFolder(dataFolder), network(net), coordinationFolder(coordinationFolder), + failed(false), excluded(false), rebooting(false), fault_injection_p1(0), fault_injection_p2(0), + fault_injection_r(0), machine(0), cleared(false) {} Future onShutdown() { return shutdownSignal.getFuture(); } diff --git a/fdbserver/BackupProgress.actor.cpp b/fdbserver/BackupProgress.actor.cpp index 898ce31b70..037c0bb6e3 100644 --- a/fdbserver/BackupProgress.actor.cpp +++ b/fdbserver/BackupProgress.actor.cpp @@ -115,7 +115,7 @@ std::map, std::map> BackupProgr // ASSERT(info.logRouterTags == epochTags[rit->first]); updateTagVersions(&tagVersions, &tags, rit->second, info.epochEnd, adjustedBeginVersion, epoch); - break; + if (tags.empty()) break; } rit++; } diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index d9c341e629..2c38671eae 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -734,13 +734,11 @@ ACTOR Future saveMutationsToFile(BackupData* self, Version popVersion, int MutationRef m; if (!message.isBackupMessage(&m)) continue; - if (debugMutation("addMutation", message.version.version, m)) { - TraceEvent("BackupWorkerDebug", self->myId) + DEBUG_MUTATION("addMutation", message.version.version, m) .detail("Version", message.version.toString()) - .detail("Mutation", m.toString()) + .detail("Mutation", m) .detail("KCV", self->minKnownCommittedVersion) .detail("SavedVersion", self->savedVersion); - } std::vector> adds; if (m.type != MutationRef::Type::ClearRange) { diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 8472ab0a1b..f30ef11d1a 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -46,8 +46,10 @@ set(FDBSERVER_SRCS MasterInterface.h MasterProxyServer.actor.cpp masterserver.actor.cpp - MoveKeys.actor.cpp + MutationTracking.h + MutationTracking.cpp MoveKeys.actor.h + MoveKeys.actor.cpp networktest.actor.cpp NetworkTest.h OldTLogServer_4_6.actor.cpp diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp index 1f38dfb8ee..9ec422ad7c 100644 --- a/fdbserver/DiskQueue.actor.cpp +++ b/fdbserver/DiskQueue.actor.cpp @@ -1013,7 +1013,7 @@ private: ASSERT( nextPageSeq%sizeof(Page)==0 ); auto& p = backPage(); - memset(&p, 0, sizeof(Page)); // FIXME: unnecessary? + memset(static_cast(&p), 0, sizeof(Page)); // FIXME: unnecessary? p.magic = 0xFDB; switch (diskQueueVersion) { case DiskQueueVersion::V0: diff --git a/fdbserver/LogSystem.h b/fdbserver/LogSystem.h index 7b24b09dda..f2ee81bc5f 100644 --- a/fdbserver/LogSystem.h +++ b/fdbserver/LogSystem.h @@ -27,6 +27,7 @@ #include "fdbserver/TLogInterface.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbclient/DatabaseConfiguration.h" +#include "fdbserver/MutationTracking.h" #include "flow/IndexedSet.h" #include "fdbrpc/ReplicationPolicy.h" #include "fdbrpc/Locality.h" @@ -877,17 +878,27 @@ struct LogPushData : NonCopyable { msg_locations.clear(); logSystem->getPushLocations(prev_tags, msg_locations, allLocations); + BinaryWriter bw(AssumeVersion(currentProtocolVersion)); uint32_t subseq = this->subsequence++; + bool first = true; + int firstOffset=-1, firstLength=-1; for(int loc : msg_locations) { - // FIXME: memcpy after the first time - BinaryWriter& wr = messagesWriter[loc]; - int offset = wr.getLength(); - wr << uint32_t(0) << subseq << uint16_t(prev_tags.size()); - for(auto& tag : prev_tags) { - wr << tag; + if (first) { + BinaryWriter& wr = messagesWriter[loc]; + firstOffset = wr.getLength(); + wr << uint32_t(0) << subseq << uint16_t(prev_tags.size()); + for(auto& tag : prev_tags) + wr << tag; + wr << item; + firstLength = wr.getLength() - firstOffset; + *(uint32_t*)((uint8_t*)wr.getData() + firstOffset) = firstLength - sizeof(uint32_t); + DEBUG_TAGS_AND_MESSAGE("ProxyPushLocations", invalidVersion, StringRef(((uint8_t*)wr.getData() + firstOffset), firstLength)).detail("PushLocations", msg_locations); + first = false; + } else { + BinaryWriter& wr = messagesWriter[loc]; + BinaryWriter& from = messagesWriter[msg_locations[0]]; + wr.serializeBytes( (uint8_t*)from.getData() + firstOffset, firstLength ); } - wr << item; - *(uint32_t*)((uint8_t*)wr.getData() + offset) = wr.getLength() - offset - sizeof(uint32_t); } next_message_tags.clear(); } diff --git a/fdbserver/LogSystemPeekCursor.actor.cpp b/fdbserver/LogSystemPeekCursor.actor.cpp index 733c8bceba..4bcd6ccafe 100644 --- a/fdbserver/LogSystemPeekCursor.actor.cpp +++ b/fdbserver/LogSystemPeekCursor.actor.cpp @@ -21,6 +21,7 @@ #include "fdbserver/LogSystem.h" #include "fdbrpc/FailureMonitor.h" #include "fdbserver/Knobs.h" +#include "fdbserver/MutationTracking.h" #include "fdbrpc/ReplicationUtils.h" #include "flow/actorcompiler.h" // has to be last include @@ -90,6 +91,7 @@ void ILogSystem::ServerPeekCursor::nextMessage() { } messageAndTags.loadFromArena(&rd, &messageVersion.sub); + DEBUG_TAGS_AND_MESSAGE("ServerPeekCursor", messageVersion.version, messageAndTags.getRawMessage()).detail("CursorID", this->randomID); // Rewind and consume the header so that reader() starts from the message. rd.rewind(); rd.readBytes(messageAndTags.getHeaderSize()); diff --git a/fdbserver/MasterProxyServer.actor.cpp b/fdbserver/MasterProxyServer.actor.cpp index 7d50d077e6..f29428d598 100644 --- a/fdbserver/MasterProxyServer.actor.cpp +++ b/fdbserver/MasterProxyServer.actor.cpp @@ -38,6 +38,7 @@ #include "fdbserver/LogSystem.h" #include "fdbserver/LogSystemDiskQueueAdapter.h" #include "fdbserver/MasterInterface.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/RecoveryState.h" #include "fdbserver/ServerDBInfo.h" #include "fdbserver/WaitFailure.h" @@ -759,7 +760,7 @@ ACTOR Future addBackupMutations(ProxyCommitData* self, std::mapaddTags(tags); toCommit->addTypedMessage(backupMutation); -// if (debugMutation("BackupProxyCommit", commitVersion, backupMutation)) { +// if (DEBUG_MUTATION("BackupProxyCommit", commitVersion, backupMutation)) { // TraceEvent("BackupProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("BackupMutation", backupMutation.toString()) // .detail("BackupMutationSize", val.size()).detail("Version", commitVersion).detail("DestPath", logRangeMutation.first) // .detail("PartIndex", part).detail("PartIndexEndian", bigEndian32(part)).detail("PartData", backupMutation.param1); @@ -1079,8 +1080,7 @@ ACTOR Future commitBatch( self->singleKeyMutationEvent->log(); } - if (debugMutation("ProxyCommit", commitVersion, m)) - TraceEvent("ProxyCommitTo", self->dbgid).detail("To", describe(tags)).detail("Mutation", m.toString()).detail("Version", commitVersion); + DEBUG_MUTATION("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", tags).detail("Mutation", m); toCommit.addTags(tags); if(self->cacheInfo[m.param1]) { @@ -1095,8 +1095,7 @@ ACTOR Future commitBatch( ++firstRange; if (firstRange == ranges.end()) { // Fast path - if (debugMutation("ProxyCommit", commitVersion, m)) - TraceEvent("ProxyCommitTo", self->dbgid).detail("To", describe(ranges.begin().value().tags)).detail("Mutation", m.toString()).detail("Version", commitVersion); + DEBUG_MUTATION("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", ranges.begin().value().tags).detail("Mutation", m); ranges.begin().value().populateTags(); toCommit.addTags(ranges.begin().value().tags); @@ -1108,8 +1107,7 @@ ACTOR Future commitBatch( r.value().populateTags(); allSources.insert(r.value().tags.begin(), r.value().tags.end()); } - if (debugMutation("ProxyCommit", commitVersion, m)) - TraceEvent("ProxyCommitTo", self->dbgid).detail("To", describe(allSources)).detail("Mutation", m.toString()).detail("Version", commitVersion); + DEBUG_MUTATION("ProxyCommit", commitVersion, m).detail("Dbgid", self->dbgid).detail("To", allSources).detail("Mutation", m); toCommit.addTags(allSources); } diff --git a/fdbserver/MutationTracking.cpp b/fdbserver/MutationTracking.cpp new file mode 100644 index 0000000000..ddd17437a3 --- /dev/null +++ b/fdbserver/MutationTracking.cpp @@ -0,0 +1,101 @@ +/* + * MutationTracking.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "fdbserver/MutationTracking.h" +#include "fdbserver/LogProtocolMessage.h" + +#if defined(FDB_CLEAN_BUILD) && MUTATION_TRACKING_ENABLED +#error "You cannot use mutation tracking in a clean/release build." +#endif + +// Track up to 2 keys in simulation via enabling MUTATION_TRACKING_ENABLED and setting the keys here. +StringRef debugKey = LiteralStringRef( "" ); +StringRef debugKey2 = LiteralStringRef( "\xff\xff\xff\xff" ); + +TraceEvent debugMutationEnabled( const char* context, Version version, MutationRef const& mutation ) { + if ((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) && + ((mutation.param1<=debugKey && mutation.param2>debugKey) || (mutation.param1<=debugKey2 && mutation.param2>debugKey2))) { + return std::move(TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", typeString[mutation.type]).detail("KeyBegin", mutation.param1).detail("KeyEnd", mutation.param2)); + } else if (mutation.param1 == debugKey || mutation.param1 == debugKey2) { + return std::move(TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", typeString[mutation.type]).detail("Key", mutation.param1).detail("Value", mutation.param2)); + } else { + return std::move(TraceEvent()); + } +} + +TraceEvent debugKeyRangeEnabled( const char* context, Version version, KeyRangeRef const& keys ) { + if (keys.contains(debugKey) || keys.contains(debugKey2)) { + return std::move(debugMutation(context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) )); + } else { + return std::move(TraceEvent()); + } +} + +TraceEvent debugTagsAndMessageEnabled( const char* context, Version version, StringRef commitBlob ) { + BinaryReader rdr(commitBlob, AssumeVersion(currentProtocolVersion)); + while (!rdr.empty()) { + if (*(int32_t*)rdr.peekBytes(4) == VERSION_HEADER) { + int32_t dummy; + rdr >> dummy >> version; + continue; + } + TagsAndMessage msg; + msg.loadFromArena(&rdr, nullptr); + bool logAdapterMessage = std::any_of( + msg.tags.begin(), msg.tags.end(), [](const Tag& t) { return t == txsTag || t.locality == tagLocalityTxs; }); + StringRef mutationData = msg.getMessageWithoutTags(); + uint8_t mutationType = *mutationData.begin(); + if (logAdapterMessage) { + // Skip the message, as there will always be an idential non-logAdapterMessage mutation + // that we can match against in the same commit. + } else if (LogProtocolMessage::startsLogProtocolMessage(mutationType)) { + BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion())); + LogProtocolMessage lpm; + br >> lpm; + rdr.setProtocolVersion(br.protocolVersion()); + } else { + MutationRef m; + BinaryReader br(mutationData, AssumeVersion(rdr.protocolVersion())); + br >> m; + TraceEvent&& event = debugMutation(context, version, m); + if (event.isEnabled()) { + return std::move(event.detail("MessageTags", msg.tags)); + } + } + } + return std::move(TraceEvent()); +} + +#if MUTATION_TRACKING_ENABLED +TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { + return debugMutationEnabled( context, version, mutation ); +} +TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { + return debugKeyRangeEnabled( context, version, keys ); +} +TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ) { + return debugTagsAndMessageEnabled( context, version, commitBlob ); +} +#else +TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ) { return std::move(TraceEvent()); } +TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return std::move(TraceEvent()); } +TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ) { return std::move(TraceEvent()); } +#endif diff --git a/fdbserver/MutationTracking.h b/fdbserver/MutationTracking.h new file mode 100644 index 0000000000..978ddca6a3 --- /dev/null +++ b/fdbserver/MutationTracking.h @@ -0,0 +1,49 @@ +/* + * MutationTracking.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _FDBSERVER_MUTATIONTRACKING_H_ +#define _FDBSERVER_MUTATIONTRACKING_H_ +#pragma once + +#include "fdbclient/FDBTypes.h" +#include "fdbclient/CommitTransaction.h" + +#define MUTATION_TRACKING_ENABLED 0 +// The keys to track are defined in the .cpp file to limit recompilation. + + +#define DEBUG_MUTATION(context, version, mutation) MUTATION_TRACKING_ENABLED && debugMutation(context, version, mutation) +TraceEvent debugMutation( const char* context, Version version, MutationRef const& mutation ); + +// debugKeyRange and debugTagsAndMessage only log the *first* occurrence of a key in their range/commit. +// TODO: Create a TraceEventGroup that forwards all calls to each element of a vector, +// to allow "multiple" TraceEvents to be returned. + +#define DEBUG_KEY_RANGE(context, version, keys) MUTATION_TRACKING_ENABLED && debugKeyRange(context, version, keys) +TraceEvent debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ); + +#define DEBUG_TAGS_AND_MESSAGE(context, version, commitBlob) MUTATION_TRACKING_ENABLED && debugTagsAndMessage(context, version, commitBlob) +TraceEvent debugTagsAndMessage( const char* context, Version version, StringRef commitBlob ); + + +// TODO: Version Tracking. If the bug is in handling a version rather than a key, then it'd be good to be able to log each time +// that version is handled within simulation. A similar set of functions should be implemented. + +#endif diff --git a/fdbserver/RestoreApplier.actor.cpp b/fdbserver/RestoreApplier.actor.cpp index 42d1c6511b..7df2b61a57 100644 --- a/fdbserver/RestoreApplier.actor.cpp +++ b/fdbserver/RestoreApplier.actor.cpp @@ -83,6 +83,7 @@ ACTOR Future restoreApplierCore(RestoreApplierInterface applierInterf, int updateProcessStats(self); updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL); } + when(wait(actors.getResult())) {} when(wait(exitRole)) { TraceEvent("RestoreApplierCoreExitRole", self->id()); break; @@ -92,6 +93,7 @@ ACTOR Future restoreApplierCore(RestoreApplierInterface applierInterf, int TraceEvent(SevWarn, "FastRestoreApplierError", self->id()) .detail("RequestType", requestTypeStr) .error(e, true); + actors.clear(false); break; } } @@ -211,52 +213,75 @@ ACTOR static Future applyClearRangeMutations(Standalone> getValue(Reference tr, Key key, int i, + std::set* keysNotFound) { + try { + Optional v = wait(tr->get(key)); + return v; + } catch (Error& e) { + if (e.code() == error_code_key_not_found) { + keysNotFound->insert(i); + return Optional(); + } else { + throw; + } + } +} + // Get keys in incompleteStagingKeys and precompute the stagingKey which is stored in batchData->stagingKeys ACTOR static Future getAndComputeStagingKeys( std::map::iterator> incompleteStagingKeys, double delayTime, Database cx, UID applierID, int batchIndex) { state Reference tr(new ReadYourWritesTransaction(cx)); - state std::vector>> fValues; + state std::vector>> fValues(incompleteStagingKeys.size(), Never()); state int retries = 0; + state UID randomID = deterministicRandom()->randomUniqueID(); wait(delay(delayTime + deterministicRandom()->random01() * delayTime)); TraceEvent("FastRestoreApplierGetAndComputeStagingKeysStart", applierID) + .detail("RandomUID", randomID) .detail("BatchIndex", batchIndex) .detail("GetKeys", incompleteStagingKeys.size()) .detail("DelayTime", delayTime); + state std::set keysNotFound; + + state int i = 0; loop { try { - tr->reset(); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); + i = 0; for (auto& key : incompleteStagingKeys) { - fValues.push_back(tr->get(key.first)); + if (!keysNotFound.count(i)) { // only get exist-keys + fValues[i] = getValue(tr, key.first, i, &keysNotFound); + } + ++i; } wait(waitForAll(fValues)); break; } catch (Error& e) { - if (retries++ > 10) { // TODO: Can we stop retry at the first error? - TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysGetKeysStuck", applierID) + bool ok = (e.code() != error_code_key_not_found); + if (!ok || retries++ > incompleteStagingKeys.size()) { + TraceEvent(!ok ? SevError : SevWarnAlways, "GetAndComputeStagingKeys", applierID) .detail("BatchIndex", batchIndex) - .detail("GetKeys", incompleteStagingKeys.size()) + .detail("KeyIndex", i) .error(e); - break; } wait(tr->onError(e)); - fValues.clear(); } } ASSERT(fValues.size() == incompleteStagingKeys.size()); int i = 0; for (auto& key : incompleteStagingKeys) { - if (!fValues[i].get().present()) { // Debug info to understand which key does not exist in DB + if (keysNotFound.count(i) || (!fValues[i].get().present())) { // Key not exist in DB + // if condition: fValues[i].Valid() && fValues[i].isReady() && !fValues[i].isError() && TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB", applierID) .detail("BatchIndex", batchIndex) .detail("Key", key.first) - .detail("Reason", "Not found in DB") + .detail("IsReady", fValues[i].isReady()) .detail("PendingMutations", key.second->second.pendingMutations.size()) - .detail("StagingKeyType", (int)key.second->second.type); + .detail("StagingKeyType", getTypeString(key.second->second.type)); for (auto& vm : key.second->second.pendingMutations) { TraceEvent(SevWarn, "FastRestoreApplierGetAndComputeStagingKeysNoBaseValueInDB") .detail("PendingMutationVersion", vm.first.toString()) @@ -274,8 +299,10 @@ ACTOR static Future getAndComputeStagingKeys( } TraceEvent("FastRestoreApplierGetAndComputeStagingKeysDone", applierID) + .detail("RandomUID", randomID) .detail("BatchIndex", batchIndex) - .detail("GetKeys", incompleteStagingKeys.size()); + .detail("GetKeys", incompleteStagingKeys.size()) + .detail("DelayTime", delayTime); return Void(); } @@ -502,6 +529,7 @@ ACTOR static Future handleApplyToDBRequest(RestoreVersionBatchRequest req, .detail("FinishedBatch", self->finishedBatch.get()); // Ensure batch (i-1) is applied before batch i + // TODO: Add a counter to warn when too many requests are waiting on the actor wait(self->finishedBatch.whenAtLeast(req.batchIndex - 1)); state bool isDuplicated = true; @@ -523,6 +551,8 @@ ACTOR static Future handleApplyToDBRequest(RestoreVersionBatchRequest req, } ASSERT(batchData->dbApplier.present()); + ASSERT(!batchData->dbApplier.get().isError()); // writeMutationsToDB actor cannot have error. + // We cannot blindly retry because it is not idempodent wait(batchData->dbApplier.get()); @@ -578,4 +608,4 @@ Value applyAtomicOp(Optional existingValue, Value value, MutationRef: ASSERT(false); } return Value(); -} +} \ No newline at end of file diff --git a/fdbserver/RestoreApplier.actor.h b/fdbserver/RestoreApplier.actor.h index 06cff36b75..ff0561a7c3 100644 --- a/fdbserver/RestoreApplier.actor.h +++ b/fdbserver/RestoreApplier.actor.h @@ -36,6 +36,7 @@ #include "fdbrpc/Locality.h" #include "fdbserver/CoordinationInterface.h" #include "fdbclient/RestoreWorkerInterface.actor.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/RestoreUtil.h" #include "fdbserver/RestoreRoleCommon.actor.h" @@ -60,19 +61,17 @@ struct StagingKey { // Assume: SetVersionstampedKey and SetVersionstampedValue have been converted to set void add(const MutationRef& m, LogMessageVersion newVersion) { ASSERT(m.type != MutationRef::SetVersionstampedKey && m.type != MutationRef::SetVersionstampedValue); - if (debugMutation("StagingKeyAdd", newVersion.version, m)) { - TraceEvent("StagingKeyAdd") - .detail("Version", version.toString()) - .detail("NewVersion", newVersion.toString()) - .detail("Mutation", m.toString()); - } + DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m) + .detail("Version", version.toString()) + .detail("NewVersion", newVersion.toString()) + .detail("Mutation", m); if (version == newVersion) { // This could happen because the same mutation can be present in // overlapping mutation logs, because new TLogs can copy mutations // from old generation TLogs (or backup worker is recruited without // knowning previously saved progress). ASSERT(type == m.type && key == m.param1 && val == m.param2); - TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m.toString()); + TraceEvent("SameVersion").detail("Version", version.toString()).detail("Mutation", m); return; } @@ -84,15 +83,13 @@ struct StagingKey { ASSERT(m.param1 == m.param2); } if (version < newVersion) { - if (debugMutation("StagingKeyAdd", newVersion.version, m)) { - TraceEvent("StagingKeyAdd") + DEBUG_MUTATION("StagingKeyAdd", newVersion.version, m) .detail("Version", version.toString()) .detail("NewVersion", newVersion.toString()) .detail("MType", getTypeString(type)) .detail("Key", key) .detail("Val", val) .detail("NewMutation", m.toString()); - } key = m.param1; val = m.param2; type = (MutationRef::Type)m.type; @@ -108,8 +105,8 @@ struct StagingKey { TraceEvent("SameVersion") .detail("Version", version.toString()) .detail("NewVersion", newVersion.toString()) - .detail("OldMutation", it->second.toString()) - .detail("NewMutation", m.toString()); + .detail("OldMutation", it->second) + .detail("NewMutation", m); ASSERT(it->second.type == m.type && it->second.param1 == m.param1 && it->second.param2 == m.param2); } } @@ -126,7 +123,8 @@ struct StagingKey { .detail("Value", val) .detail("MType", type < MutationRef::MAX_ATOMIC_OP ? getTypeString(type) : "[Unset]") .detail("LargestPendingVersion", - (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString())); + (pendingMutations.empty() ? "[none]" : pendingMutations.rbegin()->first.toString())) + .detail("PendingMutations", pendingMutations.size()); std::map>::iterator lb = pendingMutations.lower_bound(version); if (lb == pendingMutations.end()) { return; diff --git a/fdbserver/RestoreLoader.actor.cpp b/fdbserver/RestoreLoader.actor.cpp index c919e77778..9e6c4b3c91 100644 --- a/fdbserver/RestoreLoader.actor.cpp +++ b/fdbserver/RestoreLoader.actor.cpp @@ -26,6 +26,7 @@ #include "fdbclient/BackupAgent.actor.h" #include "fdbserver/RestoreLoader.actor.h" #include "fdbserver/RestoreRoleCommon.actor.h" +#include "fdbserver/MutationTracking.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -110,13 +111,17 @@ ACTOR Future restoreLoaderCore(RestoreLoaderInterface loaderInterf, int no updateProcessStats(self); updateProcessStatsTimer = delay(SERVER_KNOBS->FASTRESTORE_UPDATE_PROCESS_STATS_INTERVAL); } + when(wait(actors.getResult())) {} when(wait(exitRole)) { TraceEvent("FastRestoreLoaderCoreExitRole", self->id()); break; } } } catch (Error& e) { - TraceEvent(SevWarn, "FastRestoreLoader", self->id()).detail("RequestType", requestTypeStr).error(e, true); + TraceEvent(SevWarn, "FastRestoreLoaderError", self->id()) + .detail("RequestType", requestTypeStr) + .error(e, true); + actors.clear(false); break; } } @@ -508,23 +513,23 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat nodeIDs.contents()); ASSERT(mvector.size() == nodeIDs.size()); - if (debugMutation("RestoreLoader", commitVersion.version, kvm)) { - TraceEvent e("DebugSplit"); - int i = 0; - for (auto& [key, uid] : *pRangeToApplier) { - e.detail(format("Range%d", i).c_str(), printable(key)) - .detail(format("UID%d", i).c_str(), uid.toString()); - i++; + if (MUTATION_TRACKING_ENABLED) { + TraceEvent&& e = debugMutation("RestoreLoaderDebugSplit", commitVersion.version, kvm); + if (e.isEnabled()) { + int i = 0; + for (auto& [key, uid] : *pRangeToApplier) { + e.detail(format("Range%d", i).c_str(), printable(key)) + .detail(format("UID%d", i).c_str(), uid.toString()); + i++; + } } } for (splitMutationIndex = 0; splitMutationIndex < mvector.size(); splitMutationIndex++) { MutationRef mutation = mvector[splitMutationIndex]; UID applierID = nodeIDs[splitMutationIndex]; - if (debugMutation("RestoreLoader", commitVersion.version, mutation)) { - TraceEvent("SplittedMutation") - .detail("Version", commitVersion.toString()) - .detail("Mutation", mutation.toString()); - } + DEBUG_MUTATION("RestoreLoaderSplittedMutation", commitVersion.version, mutation) + .detail("Version", commitVersion.toString()) + .detail("Mutation", mutation); // CAREFUL: The splitted mutations' lifetime is shorter than the for-loop // Must use deep copy for splitted mutations applierVersionedMutationsBuffer[applierID].push_back_deep( @@ -540,12 +545,10 @@ ACTOR Future sendMutationsToApplier(VersionedMutationsMap* pkvOps, int bat UID applierID = itlow->second; kvCount++; - if (debugMutation("RestoreLoader", commitVersion.version, kvm)) { - TraceEvent("SendMutation") - .detail("Applier", applierID) - .detail("Version", commitVersion.toString()) - .detail("Mutation", kvm.toString()); - } + DEBUG_MUTATION("RestoreLoaderSendMutation", commitVersion.version, kvm) + .detail("Applier", applierID) + .detail("Version", commitVersion.toString()) + .detail("Mutation", kvm); // kvm data is saved in pkvOps in batchData, so shallow copy is ok here. applierVersionedMutationsBuffer[applierID].push_back(applierVersionedMutationsBuffer[applierID].arena(), VersionedMutation(kvm, commitVersion)); @@ -1057,4 +1060,4 @@ TEST_CASE("/FastRestore/RestoreLoader/splitMutation") { } return Void(); -} \ No newline at end of file +} diff --git a/fdbserver/RestoreMaster.actor.cpp b/fdbserver/RestoreMaster.actor.cpp index de1fc909f2..6e7ceb961d 100644 --- a/fdbserver/RestoreMaster.actor.cpp +++ b/fdbserver/RestoreMaster.actor.cpp @@ -866,6 +866,7 @@ ACTOR static Future notifyApplierToApplyMutations(ReferenceapplyToDB.present()); + ASSERT(!batchData->applyToDB.get().isError()); wait(batchData->applyToDB.get()); // Sanity check all appliers have applied data to destination DB diff --git a/fdbserver/RestoreUtil.actor.cpp b/fdbserver/RestoreUtil.actor.cpp index 7965ab60e4..7451f16570 100644 --- a/fdbserver/RestoreUtil.actor.cpp +++ b/fdbserver/RestoreUtil.actor.cpp @@ -76,4 +76,4 @@ bool isRangeMutation(MutationRef m) { ASSERT(m.type == MutationRef::Type::SetValue || isAtomicOp((MutationRef::Type)m.type)); return false; } -} \ No newline at end of file +} diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp index df56901490..4d5b68b3d5 100644 --- a/fdbserver/StorageCache.actor.cpp +++ b/fdbserver/StorageCache.actor.cpp @@ -31,6 +31,7 @@ #include "fdbclient/Notified.h" #include "fdbserver/LogProtocolMessage.h" #include "fdbserver/LogSystem.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbclient/DatabaseContext.h" @@ -488,9 +489,8 @@ ACTOR Future getValueQ( StorageCacheData* data, GetValueRequest req ) { data->checkChangeCounter(changeCounter, req.key); } - // FIXME: enable when debugMutation is active - //debugMutation("CacheGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); - //debugMutation("CacheGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); + //DEBUG_MUTATION("CacheGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); + //DEBUG_MUTATION("CacheGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); if (v.present()) { ++data->counters.rowsQueried; @@ -1022,23 +1022,11 @@ void StorageCacheData::addMutation(KeyRangeRef const& cachedKeyRange, Version ve return; } expanded = addMutationToMutationLog(mLog, expanded); - // FIXME: enable when debugMutation is active - if (false && debugMutation("expandedMutation", version, expanded)) { - const char* type = - mutation.type == MutationRef::SetValue ? "SetValue" : - mutation.type == MutationRef::ClearRange ? "ClearRange" : - mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" : - mutation.type == MutationRef::DebugKey ? "DebugKey" : - "UnknownMutation"; - printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%s\t%s\t%s\n", - now(), g_network->getLocalAddress().toString().c_str(), "originalMutation", - type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str()); - printf(" Cached Key-range: %s - %s\n", printable(cachedKeyRange.begin).c_str(), printable(cachedKeyRange.end).c_str()); - } - applyMutation( expanded, mLog.arena(), mutableData() ); + + DEBUG_MUTATION("expandedMutation", version, expanded).detail("Begin", cachedKeyRange.begin).detail("End", cachedKeyRange.end); + applyMutation( this, expanded, mLog.arena(), mutableData() ); //printf("\nSCUpdate: Printing versioned tree after applying mutation\n"); //mutableData().printTree(version); - } void removeDataRange( StorageCacheData *sc, Standalone &mLV, KeyRangeMap>& cacheRanges, KeyRangeRef range ) { @@ -1596,15 +1584,11 @@ public: data->mutableData().createNewVersion(ver); } + DEBUG_MUTATION("SCUpdateMutation", ver, m); if (m.param1.startsWith( systemKeys.end )) { //TraceEvent("SCPrivateData", data->thisServerID).detail("Mutation", m.toString()).detail("Version", ver); applyPrivateCacheData( data, m ); } else { - // FIXME: enable when debugMutation is active - //for(auto m = changes[c].mutations.begin(); m; ++m) { - // debugMutation("SCUpdateMutation", changes[c].version, *m); - //} - splitMutation(data, data->cachedRangeMap, m, ver); } @@ -1623,7 +1607,8 @@ private: // that this cache server is responsible for // TODO Revisit during failure handling. Might we loose some private mutations? void applyPrivateCacheData( StorageCacheData* data, MutationRef const& m ) { - //TraceEvent(SevDebug, "SCPrivateCacheMutation", data->thisServerID).detail("Mutation", m.toString()); + + //TraceEvent(SevDebug, "SCPrivateCacheMutation", data->thisServerID).detail("Mutation", m); if (processedCacheStartKey) { // we expect changes in pairs, [begin,end). This mutation is for end key of the range @@ -1896,9 +1881,8 @@ ACTOR Future pullAsyncData( StorageCacheData *data ) { data->debug_inApplyUpdate = false; - if(ver != invalidVersion && ver > data->version.get()) { - // FIXME: enable when debugKeyRange is active - //debugKeyRange("SCUpdate", ver, allKeys); + if(ver != invalidVersion && ver > data->version.get()) { + DEBUG_KEY_RANGE("SCUpdate", ver, allKeys); data->mutableData().createNewVersion(ver); diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index 6bc99cfb9d..42bdfe61c1 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -32,6 +32,7 @@ #include "fdbserver/TLogInterface.h" #include "fdbserver/Knobs.h" #include "fdbserver/IKeyValueStore.h" +#include "fdbserver/MutationTracking.h" #include "flow/ActorCollection.h" #include "fdbrpc/FailureMonitor.h" #include "fdbserver/IDiskQueue.h" @@ -1261,6 +1262,7 @@ void commitMessages( TLogData* self, Reference logData, Version version block.reserve(block.arena(), std::max(SERVER_KNOBS->TLOG_MESSAGE_BLOCK_BYTES, msgSize)); } + DEBUG_TAGS_AND_MESSAGE("TLogCommitMessages", version, msg.getRawMessage()).detail("UID", self->dbgid).detail("LogId", logData->logId); block.append(block.arena(), msg.message.begin(), msg.message.size()); for(auto tag : msg.tags) { if(logData->locality == tagLocalitySatellite) { @@ -1375,7 +1377,12 @@ void peekMessagesFromMemory( Reference self, TLogPeekRequest const& req messages << VERSION_HEADER << currentVersion; } + // We need the 4 byte length prefix to be a TagsAndMessage format, but that prefix is added as part of StringRef serialization. + int offset = messages.getLength(); messages << it->second.toStringRef(); + void* data = messages.getData(); + DEBUG_TAGS_AND_MESSAGE("TLogPeek", currentVersion, StringRef((uint8_t*)data+offset, messages.getLength()-offset)) + .detail("LogId", self->logId).detail("PeekTag", req.tag); } } @@ -1637,6 +1644,7 @@ ACTOR Future tLogPeekMessages( TLogData* self, TLogPeekRequest req, Refere wait(parseMessagesForTag(entry.messages, req.tag, logData->logRouterTags)); for (const StringRef& msg : rawMessages) { messages.serializeBytes(msg); + DEBUG_TAGS_AND_MESSAGE("TLogPeekFromDisk", entry.version, msg).detail("UID", self->dbgid).detail("LogId", logData->logId).detail("PeekTag", req.tag); } lastRefMessageVersion = entry.version; diff --git a/fdbserver/VFSAsync.cpp b/fdbserver/VFSAsync.cpp index 3d53aaccfb..0a1feff976 100644 --- a/fdbserver/VFSAsync.cpp +++ b/fdbserver/VFSAsync.cpp @@ -531,7 +531,7 @@ static int asyncOpen( if (flags & SQLITE_OPEN_WAL) oflags |= IAsyncFile::OPEN_LARGE_PAGES; oflags |= IAsyncFile::OPEN_LOCK; - memset(p, 0, sizeof(VFSAsyncFile)); + memset(static_cast(p), 0, sizeof(VFSAsyncFile)); new (p) VFSAsyncFile(zName, flags); try { // Note that SQLiteDB::open also opens the db file, so its flags and modes are important, too diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 2959c06b2a..5311c0f5b9 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -2204,6 +2204,7 @@ struct SplitStringRef { // A BTree "page id" is actually a list of LogicalPageID's whose contents should be concatenated together. // NOTE: Uses host byte order typedef VectorRef BTreePageIDRef; +constexpr LogicalPageID maxPageID = (LogicalPageID)-1; std::string toString(BTreePageIDRef id) { return std::string("BTreePageID") + toString(id.begin(), id.end()); @@ -2246,6 +2247,10 @@ struct RedwoodRecordRef { inline RedwoodRecordRef withoutValue() const { return RedwoodRecordRef(key, version); } + inline RedwoodRecordRef withMaxPageID() const { + return RedwoodRecordRef(key, version, StringRef((uint8_t *)&maxPageID, sizeof(maxPageID))); + } + // Truncate (key, version, part) tuple to len bytes. void truncate(int len) { ASSERT(len <= key.size()); @@ -3872,7 +3877,7 @@ private: // If the decode upper boundary is the subtree upper boundary the pointers will be the same // For the lower boundary, if the pointers are not the same there is still a possibility // that the keys are the same. This happens for the first remaining subtree of an internal page - // after the previous first subtree was cleared. + // after the prior subtree(s) were cleared. return (decodeUpperBound == subtreeUpperBound) && (decodeLowerBound == subtreeLowerBound || decodeLowerBound->sameExceptValue(*subtreeLowerBound)); } @@ -4984,6 +4989,246 @@ public: Future moveLast() { return move_end(this, false); } }; + // Cursor designed for short lifespans. + // Holds references to all pages touched. + // All record references returned from it are valid until the cursor is destroyed. + class BTreeCursor { + Arena arena; + Reference pager; + std::unordered_map> pages; + VersionedBTree* btree; + bool valid; + + struct PathEntry { + BTreePage* btPage; + BTreePage::BinaryTree::Cursor cursor; + }; + VectorRef path; + + public: + BTreeCursor() {} + + bool isValid() const { return valid; } + + std::string toString() const { + std::string r; + for (int i = 0; i < path.size(); ++i) { + r += format("[%d/%d: %s] ", i + 1, path.size(), + path[i].cursor.valid() ? path[i].cursor.get().toString(path[i].btPage->isLeaf()).c_str() + : ""); + } + if (!valid) { + r += " (invalid) "; + } + return r; + } + + const RedwoodRecordRef& get() { return path.back().cursor.get(); } + + bool inRoot() const { return path.size() == 1; } + + // Pop and return the page cursor at the end of the path. + // This is meant to enable range scans to consume the contents of a leaf page more efficiently. + // Can only be used when inRoot() is true. + BTreePage::BinaryTree::Cursor popPath() { + BTreePage::BinaryTree::Cursor c = path.back().cursor; + path.pop_back(); + return c; + } + + Future pushPage(BTreePageIDRef id, const RedwoodRecordRef& lowerBound, + const RedwoodRecordRef& upperBound) { + Reference& page = pages[id.front()]; + if (page.isValid()) { + path.push_back(arena, { (BTreePage*)page->begin(), getCursor(page) }); + return Void(); + } + + return map(readPage(pager, id, &lowerBound, &upperBound), [this, &page, id](Reference p) { + page = p; + path.push_back(arena, { (BTreePage*)p->begin(), getCursor(p) }); + return Void(); + }); + } + + Future pushPage(BTreePage::BinaryTree::Cursor c) { + const RedwoodRecordRef& rec = c.get(); + auto next = c; + next.moveNext(); + BTreePageIDRef id = rec.getChildPage(); + return pushPage(id, rec, next.getOrUpperBound()); + } + + Future init(VersionedBTree* btree_in, Reference pager_in, BTreePageIDRef root) { + btree = btree_in; + pager = pager_in; + path.reserve(arena, 6); + valid = false; + return pushPage(root, dbBegin, dbEnd); + } + + // Seeks cursor to query if it exists, the record before or after it, or an undefined and invalid + // position between those records + // If 0 is returned, then + // If the cursor is valid then it points to query + // If the cursor is not valid then the cursor points to some place in the btree such that + // If there is a record in the tree < query then movePrev() will move to it, and + // If there is a record in the tree > query then moveNext() will move to it. + // If non-zero is returned then the cursor is valid and the return value is logically equivalent + // to query.compare(cursor.get()) + ACTOR Future seek_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + state RedwoodRecordRef internalPageQuery = query.withMaxPageID(); + self->path = self->path.slice(0, 1); + debug_printf("seek(%s, %d) start cursor = %s\n", query.toString().c_str(), prefetchBytes, + self->toString().c_str()); + + loop { + auto& entry = self->path.back(); + if (entry.btPage->isLeaf()) { + int cmp = entry.cursor.seek(query); + self->valid = entry.cursor.valid() && !entry.cursor.node->isDeleted(); + debug_printf("seek(%s, %d) loop exit cmp=%d cursor=%s\n", query.toString().c_str(), prefetchBytes, + cmp, self->toString().c_str()); + return self->valid ? cmp : 0; + } + + // Internal page, so seek to the branch where query must be + // Currently, after a subtree deletion internal page boundaries are still strictly adhered + // to and will be updated if anything is inserted into the cleared range, so if the seek fails + // or it finds an entry with a null child page then query does not exist in the BTree. + if (entry.cursor.seekLessThan(internalPageQuery) && entry.cursor.get().value.present()) { + debug_printf("seek(%s, %d) loop seek success cursor=%s\n", query.toString().c_str(), prefetchBytes, + self->toString().c_str()); + Future f = self->pushPage(entry.cursor); + + // Prefetch siblings, at least prefetchBytes, at level 2 but without jumping to another level 2 + // sibling + if (prefetchBytes != 0 && entry.btPage->height == 2) { + auto c = entry.cursor; + bool fwd = prefetchBytes > 0; + prefetchBytes = abs(prefetchBytes); + // While we should still preload more bytes and a move in the target direction is successful + while (prefetchBytes > 0 && (fwd ? c.moveNext() : c.movePrev())) { + // If there is a page link, preload it. + if (c.get().value.present()) { + BTreePageIDRef childPage = c.get().getChildPage(); + preLoadPage(self->pager.getPtr(), childPage); + prefetchBytes -= self->btree->m_blockSize * childPage.size(); + } + } + } + + wait(f); + } else { + self->valid = false; + debug_printf("seek(%s, %d) loop exit cmp=0 cursor=%s\n", query.toString().c_str(), prefetchBytes, + self->toString().c_str()); + return 0; + } + } + } + + Future seek(RedwoodRecordRef query, int prefetchBytes) { return seek_impl(this, query, prefetchBytes); } + + ACTOR Future seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + debug_printf("seekGTE(%s, %d) start\n", query.toString().c_str(), prefetchBytes); + int cmp = wait(self->seek(query, prefetchBytes)); + if (cmp > 0 || (cmp == 0 && !self->isValid())) { + wait(self->moveNext()); + } + return Void(); + } + + Future seekGTE(RedwoodRecordRef query, int prefetchBytes) { + return seekGTE_impl(this, query, prefetchBytes); + } + + ACTOR Future seekLT_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + debug_printf("seekLT(%s, %d) start\n", query.toString().c_str(), prefetchBytes); + int cmp = wait(self->seek(query, prefetchBytes)); + if (cmp <= 0) { + wait(self->movePrev()); + } + return Void(); + } + + Future seekLT(RedwoodRecordRef query, int prefetchBytes) { + return seekLT_impl(this, query, -prefetchBytes); + } + + ACTOR Future move_impl(BTreeCursor* self, bool forward) { + // Try to the move cursor at the end of the path in the correct direction + debug_printf("move%s() start cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + while (1) { + debug_printf("move%s() first loop cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + auto& entry = self->path.back(); + bool success; + if(entry.cursor.valid()) { + success = forward ? entry.cursor.moveNext() : entry.cursor.movePrev(); + } else { + success = forward ? entry.cursor.moveFirst() : false; + } + + // Skip over internal page entries that do not link to child pages. There should never be two in a row. + if (success && !entry.btPage->isLeaf() && !entry.cursor.get().value.present()) { + success = forward ? entry.cursor.moveNext() : entry.cursor.movePrev(); + ASSERT(!success || entry.cursor.get().value.present()); + } + + // Stop if successful + if (success) { + break; + } + + if (self->path.size() == 1) { + self->valid = false; + return Void(); + } + + // Move to parent + self->path = self->path.slice(0, self->path.size() - 1); + } + + // While not on a leaf page, move down to get to one. + while (1) { + debug_printf("move%s() second loop cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + auto& entry = self->path.back(); + if (entry.btPage->isLeaf()) { + break; + } + + // The last entry in an internal page could be a null link, if so move back + if (!forward && !entry.cursor.get().value.present()) { + ASSERT(entry.cursor.movePrev()); + ASSERT(entry.cursor.get().value.present()); + } + + wait(self->pushPage(entry.cursor)); + auto& newEntry = self->path.back(); + ASSERT(forward ? newEntry.cursor.moveFirst() : newEntry.cursor.moveLast()); + } + + self->valid = true; + + debug_printf("move%s() exit cursor=%s\n", forward ? "Next" : "Prev", self->toString().c_str()); + return Void(); + } + + Future moveNext() { return move_impl(this, true); } + Future movePrev() { return move_impl(this, false); } + }; + + Future initBTreeCursor(BTreeCursor* cursor, Version snapshotVersion) { + // Only committed versions can be read. + ASSERT(snapshotVersion <= m_lastCommittedVersion); + Reference snapshot = m_pager->getReadSnapshot(snapshotVersion); + + // This is a ref because snapshot will continue to hold the metakey value memory + KeyRef m = snapshot->getMetaKey(); + + return cursor->init(this, snapshot, ((MetaKey*)m.begin())->root.get()); + } + // Cursor is for reading and interating over user visible KV pairs at a specific version // KeyValueRefs returned become invalid once the cursor is moved class Cursor : public IStoreCursor, public ReferenceCounted, public FastAllocated, NonCopyable { @@ -5264,10 +5509,13 @@ public: ACTOR static Future> readRange_impl(KeyValueStoreRedwoodUnversioned* self, KeyRange keys, int rowLimit, int byteLimit) { + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - ++g_redwoodMetrics.opGetRange; + state Standalone result; state int accumulatedBytes = 0; ASSERT(byteLimit > 0); @@ -5276,33 +5524,58 @@ public: return result; } - state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); - // Prefetch is currently only done in the forward direction - state int prefetchBytes = rowLimit > 1 ? byteLimit : 0; + // Prefetch is disabled for now pending some decent logic for deciding how much to fetch + state int prefetchBytes = 0; if (rowLimit > 0) { - wait(cur->findFirstEqualOrGreater(keys.begin, prefetchBytes)); - while (cur->isValid() && cur->getKey() < keys.end) { - KeyValueRef kv(KeyRef(result.arena(), cur->getKey()), ValueRef(result.arena(), cur->getValue())); - accumulatedBytes += kv.expectedSize(); - result.push_back(result.arena(), kv); - if (--rowLimit == 0 || accumulatedBytes >= byteLimit) { + wait(cur.seekGTE(keys.begin, prefetchBytes)); + while (cur.isValid()) { + // Read page contents without using waits + bool isRoot = cur.inRoot(); + BTreePage::BinaryTree::Cursor leafCursor = cur.popPath(); + while(leafCursor.valid()) { + KeyValueRef kv = leafCursor.get().toKeyValueRef(); + if(kv.key >= keys.end) { + break; + } + accumulatedBytes += kv.expectedSize(); + result.push_back_deep(result.arena(), kv); + if (--rowLimit == 0 || accumulatedBytes >= byteLimit) { + break; + } + leafCursor.moveNext(); + } + // Stop if the leaf cursor is still valid which means we hit a key or size limit or + // if we started in the root page + if(leafCursor.valid() || isRoot) { break; } - wait(cur->next()); + wait(cur.moveNext()); } } else { - wait(cur->findLastLessOrEqual(keys.end)); - if (cur->isValid() && cur->getKey() == keys.end) wait(cur->prev()); - - while (cur->isValid() && cur->getKey() >= keys.begin) { - KeyValueRef kv(KeyRef(result.arena(), cur->getKey()), ValueRef(result.arena(), cur->getValue())); - accumulatedBytes += kv.expectedSize(); - result.push_back(result.arena(), kv); - if (++rowLimit == 0 || accumulatedBytes >= byteLimit) { + wait(cur.seekLT(keys.end, prefetchBytes)); + while (cur.isValid()) { + // Read page contents without using waits + bool isRoot = cur.inRoot(); + BTreePage::BinaryTree::Cursor leafCursor = cur.popPath(); + while(leafCursor.valid()) { + KeyValueRef kv = leafCursor.get().toKeyValueRef(); + if(kv.key < keys.begin) { + break; + } + accumulatedBytes += kv.expectedSize(); + result.push_back_deep(result.arena(), kv); + if (++rowLimit == 0 || accumulatedBytes >= byteLimit) { + break; + } + leafCursor.movePrev(); + } + // Stop if the leaf cursor is still valid which means we hit a key or size limit or + // if we started in the root page + if(leafCursor.valid() || isRoot) { break; } - wait(cur->prev()); + wait(cur.movePrev()); } } @@ -5316,15 +5589,16 @@ public: ACTOR static Future> readValue_impl(KeyValueStoreRedwoodUnversioned* self, Key key, Optional debugID) { + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - ++g_redwoodMetrics.opGet; - state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); - wait(cur->findEqual(key)); - if (cur->isValid()) { - return cur->getValue(); + wait(cur.seekGTE(key, 0)); + if (cur.isValid() && cur.get().key == key) { + return cur.get().value.get(); } return Optional(); } @@ -5335,18 +5609,20 @@ public: ACTOR static Future> readValuePrefix_impl(KeyValueStoreRedwoodUnversioned* self, Key key, int maxLength, Optional debugID) { + state VersionedBTree::BTreeCursor cur; + wait(self->m_tree->initBTreeCursor(&cur, self->m_tree->getLastCommittedVersion())); + wait(self->m_concurrentReads.take()); state FlowLock::Releaser releaser(self->m_concurrentReads); - ++g_redwoodMetrics.opGet; - state Reference cur = self->m_tree->readAtVersion(self->m_tree->getLastCommittedVersion()); - wait(cur->findEqual(key)); - if (cur->isValid()) { - Value v = cur->getValue(); + wait(cur.seekGTE(key, 0)); + if (cur.isValid() && cur.get().key == key) { + Value v = cur.get().value.get(); int len = std::min(v.size(), maxLength); - return Value(cur->getValue().substr(0, len)); + return Value(v.substr(0, len)); } + return Optional(); } @@ -5411,6 +5687,157 @@ KeyValue randomKV(int maxKeySize = 10, int maxValueSize = 5) { return kv; } +// Verify a range using a BTreeCursor. +// Assumes that the BTree holds a single data version and the version is 0. +ACTOR Future verifyRangeBTreeCursor(VersionedBTree* btree, Key start, Key end, Version v, + std::map, Optional>* written, + int* pErrorCount) { + state int errors = 0; + if (end <= start) end = keyAfter(start); + + state std::map, Optional>::const_iterator i = + written->lower_bound(std::make_pair(start.toString(), 0)); + state std::map, Optional>::const_iterator iEnd = + written->upper_bound(std::make_pair(end.toString(), 0)); + state std::map, Optional>::const_iterator iLast; + + state VersionedBTree::BTreeCursor cur; + wait(btree->initBTreeCursor(&cur, v)); + debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Start\n", v, start.printable().c_str(), end.printable().c_str()); + + // Randomly use the cursor for something else first. + if (deterministicRandom()->coinflip()) { + state Key randomKey = randomKV().key; + debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Dummy seek to '%s'\n", v, start.printable().c_str(), + end.printable().c_str(), randomKey.toString().c_str()); + wait(success(cur.seek(randomKey, 0))); + } + + debug_printf("VerifyRange(@%" PRId64 ", %s, %s): Actual seek\n", v, start.printable().c_str(), + end.printable().c_str()); + wait(cur.seekGTE(start, 0)); + + state std::vector results; + + while (cur.isValid() && cur.get().key < end) { + // Find the next written kv pair that would be present at this version + while (1) { + iLast = i; + if (i == iEnd) break; + ++i; + + if (iLast->first.second <= v && iLast->second.present() && + (i == iEnd || i->first.first != iLast->first.first || i->first.second > v)) { + debug_printf("VerifyRange(@%" PRId64 ", %s, %s) Found key in written map: %s\n", v, + start.printable().c_str(), end.printable().c_str(), iLast->first.first.c_str()); + break; + } + } + + if (iLast == iEnd) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' vs nothing in written map.\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str()); + break; + } + + if (cur.get().key != iLast->first.first) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' but expected '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + iLast->first.first.c_str()); + break; + } + if (cur.get().value.get() != iLast->second.get()) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' has tree value '%s' but expected '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + cur.get().value.get().toString().c_str(), iLast->second.get().c_str()); + break; + } + + ASSERT(errors == 0); + + results.push_back(KeyValue(KeyValueRef(cur.get().key, cur.get().value.get()))); + wait(cur.moveNext()); + } + + // Make sure there are no further written kv pairs that would be present at this version. + while (1) { + iLast = i; + if (i == iEnd) break; + ++i; + if (iLast->first.second <= v && iLast->second.present() && + (i == iEnd || i->first.first != iLast->first.first || i->first.second > v)) + break; + } + + if (iLast != iEnd) { + ++errors; + ++*pErrorCount; + printf("VerifyRange(@%" PRId64 ", %s, %s) ERROR: Tree range ended but written has @%" PRId64 " '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), iLast->first.second, iLast->first.first.c_str()); + } + + debug_printf("VerifyRangeReverse(@%" PRId64 ", %s, %s): start\n", v, start.printable().c_str(), + end.printable().c_str()); + + // Randomly use a new cursor at the same version for the reverse range read, if the version is still available for + // opening new cursors + if (v >= btree->getOldestVersion() && deterministicRandom()->coinflip()) { + cur = VersionedBTree::BTreeCursor(); + wait(btree->initBTreeCursor(&cur, v)); + } + + // Now read the range from the tree in reverse order and compare to the saved results + wait(cur.seekLT(end, 0)); + + state std::vector::const_reverse_iterator r = results.rbegin(); + + while (cur.isValid() && cur.get().key >= start) { + if (r == results.rend()) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' vs nothing in written map.\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str()); + break; + } + + if (cur.get().key != r->key) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 ", %s, %s) ERROR: Tree key '%s' but expected '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + r->key.toString().c_str()); + break; + } + if (cur.get().value.get() != r->value) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 + ", %s, %s) ERROR: Tree key '%s' has tree value '%s' but expected '%s'\n", + v, start.printable().c_str(), end.printable().c_str(), cur.get().key.toString().c_str(), + cur.get().value.get().toString().c_str(), r->value.toString().c_str()); + break; + } + + ++r; + wait(cur.movePrev()); + } + + if (r != results.rend()) { + ++errors; + ++*pErrorCount; + printf("VerifyRangeReverse(@%" PRId64 ", %s, %s) ERROR: Tree range ended but written has '%s'\n", v, + start.printable().c_str(), end.printable().c_str(), r->key.toString().c_str()); + } + + return errors; +} + ACTOR Future verifyRange(VersionedBTree* btree, Key start, Key end, Version v, std::map, Optional>* written, int* pErrorCount) { @@ -5607,6 +6034,58 @@ ACTOR Future seekAll(VersionedBTree* btree, Version v, return errors; } +// Verify the result of point reads for every set or cleared key at the given version +ACTOR Future seekAllBTreeCursor(VersionedBTree* btree, Version v, + std::map, Optional>* written, int* pErrorCount) { + state std::map, Optional>::const_iterator i = written->cbegin(); + state std::map, Optional>::const_iterator iEnd = written->cend(); + state int errors = 0; + state VersionedBTree::BTreeCursor cur; + + wait(btree->initBTreeCursor(&cur, v)); + + while (i != iEnd) { + state std::string key = i->first.first; + state Version ver = i->first.second; + if (ver == v) { + state Optional val = i->second; + debug_printf("Verifying @%" PRId64 " '%s'\n", ver, key.c_str()); + state Arena arena; + wait(cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key), 0), 0)); + bool foundKey = cur.isValid() && cur.get().key == key; + bool hasValue = foundKey && cur.get().value.present(); + + if (val.present()) { + bool valueMatch = hasValue && cur.get().value.get() == val.get(); + if (!foundKey || !hasValue || !valueMatch) { + ++errors; + ++*pErrorCount; + if (!foundKey) { + printf("Verify ERROR: key_not_found: '%s' -> '%s' @%" PRId64 "\n", key.c_str(), + val.get().c_str(), ver); + } + else if (!hasValue) { + printf("Verify ERROR: value_not_found: '%s' -> '%s' @%" PRId64 "\n", key.c_str(), + val.get().c_str(), ver); + } + else if (!valueMatch) { + printf("Verify ERROR: value_incorrect: for '%s' found '%s' expected '%s' @%" PRId64 "\n", + key.c_str(), cur.get().value.get().toString().c_str(), val.get().c_str(), + ver); + } + } + } else if (foundKey && hasValue) { + ++errors; + ++*pErrorCount; + printf("Verify ERROR: cleared_key_found: '%s' -> '%s' @%" PRId64 "\n", key.c_str(), + cur.get().value.get().toString().c_str(), ver); + } + } + ++i; + } + return errors; +} + ACTOR Future verify(VersionedBTree* btree, FutureStream vStream, std::map, Optional>* written, int* pErrorCount, bool serial) { @@ -5637,7 +6116,13 @@ ACTOR Future verify(VersionedBTree* btree, FutureStream vStream, state Reference cur = btree->readAtVersion(v); debug_printf("Verifying entire key range at version %" PRId64 "\n", v); - fRangeAll = verifyRange(btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, pErrorCount); + if(deterministicRandom()->coinflip()) { + fRangeAll = verifyRange(btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, + pErrorCount); + } else { + fRangeAll = verifyRangeBTreeCursor(btree, LiteralStringRef(""), LiteralStringRef("\xff\xff"), v, written, + pErrorCount); + } if (serial) { wait(success(fRangeAll)); } @@ -5646,13 +6131,21 @@ ACTOR Future verify(VersionedBTree* btree, FutureStream vStream, Key end = randomKV().key; debug_printf("Verifying range (%s, %s) at version %" PRId64 "\n", toString(begin).c_str(), toString(end).c_str(), v); - fRangeRandom = verifyRange(btree, begin, end, v, written, pErrorCount); + if(deterministicRandom()->coinflip()) { + fRangeRandom = verifyRange(btree, begin, end, v, written, pErrorCount); + } else { + fRangeRandom = verifyRangeBTreeCursor(btree, begin, end, v, written, pErrorCount); + } if (serial) { wait(success(fRangeRandom)); } debug_printf("Verifying seeks to each changed key at version %" PRId64 "\n", v); - fSeekAll = seekAll(btree, v, written, pErrorCount); + if(deterministicRandom()->coinflip()) { + fSeekAll = seekAll(btree, v, written, pErrorCount); + } else { + fSeekAll = seekAllBTreeCursor(btree, v, written, pErrorCount); + } if (serial) { wait(success(fSeekAll)); } @@ -6485,11 +6978,11 @@ TEST_CASE("!/redwood/correctness/btree") { state int maxKeySize = deterministicRandom()->randomInt(1, pageSize * 2); state int maxValueSize = randomSize(pageSize * 25); state int maxCommitSize = shortTest ? 1000 : randomSize(std::min((maxKeySize + maxValueSize) * 20000, 10e6)); - state int mutationBytesTarget = shortTest ? 100000 : randomSize(std::min(maxCommitSize * 100, 100e6)); + state int mutationBytesTarget = shortTest ? 100000 : randomSize(std::min(maxCommitSize * 100, pageSize * 100000)); state double clearProbability = deterministicRandom()->random01() * .1; state double clearSingleKeyProbability = deterministicRandom()->random01(); state double clearPostSetProbability = deterministicRandom()->random01() * .1; - state double coldStartProbability = pagerMemoryOnly ? 0 : deterministicRandom()->random01(); + state double coldStartProbability = pagerMemoryOnly ? 0 : (deterministicRandom()->random01() * 0.3); state double advanceOldVersionProbability = deterministicRandom()->random01(); state double maxDuration = 60; state int64_t cacheSizeBytes = diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index 54d4ebe3f7..265b4560e7 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -617,26 +617,6 @@ struct EventLogRequest { } }; -struct DebugEntryRef { - double time; - NetworkAddress address; - StringRef context; - Version version; - MutationRef mutation; - DebugEntryRef() {} - DebugEntryRef( const char* c, Version v, MutationRef const& m ) : context((const uint8_t*)c,strlen(c)), version(v), mutation(m), time(now()), address( g_network->getLocalAddress() ) {} - DebugEntryRef( Arena& a, DebugEntryRef const& d ) : time(d.time), address(d.address), context(d.context), version(d.version), mutation(a, d.mutation) {} - - size_t expectedSize() const { - return context.expectedSize() + mutation.expectedSize(); - } - - template - void serialize(Ar& ar) { - serializer(ar, time, address, context, version, mutation); - } -}; - struct DiskStoreRequest { constexpr static FileIdentifier file_identifier = 1986262; bool includePartialStores; diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 3c1ec52643..b43ddf11b3 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -196,63 +196,6 @@ bool enableFailures = true; #define test_assert(x) if (!(x)) { cout << "Test failed: " #x << endl; return false; } -vector< Standalone> > debugEntries; -int64_t totalDebugEntriesSize = 0; - -#if CENABLED(0, NOT_IN_CLEAN) -StringRef debugKey = LiteralStringRef(""); -StringRef debugKey2 = LiteralStringRef("\xff\xff\xff\xff"); - -bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { - if ((mutation.type == mutation.SetValue || mutation.type == mutation.AddValue || mutation.type==mutation.DebugKey) && (mutation.param1 == debugKey || mutation.param1 == debugKey2)) - ;//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", "SetValue").detail("Key", mutation.param1).detail("Value", mutation.param2); - else if ((mutation.type == mutation.ClearRange || mutation.type == mutation.DebugKeyRange) && ((mutation.param1<=debugKey && mutation.param2>debugKey) || (mutation.param1<=debugKey2 && mutation.param2>debugKey2))) - ;//TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("MutationType", "ClearRange").detail("KeyBegin", mutation.param1).detail("KeyEnd", mutation.param2); - else - return false; - const char* type = - mutation.type == MutationRef::SetValue ? "SetValue" : - mutation.type == MutationRef::ClearRange ? "ClearRange" : - mutation.type == MutationRef::AddValue ? "AddValue" : - mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" : - mutation.type == MutationRef::DebugKey ? "DebugKey" : - "UnknownMutation"; - printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%lld\t%s\t%s\t%s\n", now(), g_network->getLocalAddress().toString().c_str(), context, version, type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str()); - - return true; -} - -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { - if (keys.contains(debugKey) || keys.contains(debugKey2)) { - debugMutation(context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) ); - //TraceEvent("MutationTracking").detail("At", context).detail("Version", version).detail("KeyBegin", keys.begin).detail("KeyEnd", keys.end); - return true; - } else - return false; -} - -#elif CENABLED(0, NOT_IN_CLEAN) -bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { - if (!debugEntries.size() || debugEntries.back().size() >= 1000) { - if (debugEntries.size()) totalDebugEntriesSize += debugEntries.back().arena().getSize() + sizeof(debugEntries.back()); - debugEntries.push_back(Standalone>()); - TraceEvent("DebugMutationBuffer").detail("Bytes", totalDebugEntriesSize); - } - auto& v = debugEntries.back(); - v.push_back_deep( v.arena(), DebugEntryRef(context, version, mutation) ); - - return false; // No auxiliary logging -} - -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { - return debugMutation( context, version, MutationRef(MutationRef::DebugKeyRange, keys.begin, keys.end) ); -} - -#else // Default implementation. -bool debugMutation( const char* context, Version version, MutationRef const& mutation ) { return false; } -bool debugKeyRange( const char* context, Version version, KeyRangeRef const& keys ) { return false; } -#endif - #ifdef _WIN32 #include @@ -1978,20 +1921,6 @@ int main(int argc, char* argv[]) { cout << " " << i->second << " " << i->first << endl;*/ // cout << " " << Actor::allActors[i]->getName() << endl; - int total = 0; - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - total += i->second; - if (total) - printf("%d errors:\n", total); - for(auto i = Error::errorCounts().begin(); i != Error::errorCounts().end(); ++i) - if (i->second > 0) - printf(" %d: %d %s\n", i->second, i->first, Error::fromCode(i->first).what()); - - if (&g_simulator == g_network) { - auto processes = g_simulator.getAllProcesses(); - for(auto i = processes.begin(); i != processes.end(); ++i) - printf("%s %s: %0.3f Mclocks\n", (*i)->name, (*i)->address.toString().c_str(), (*i)->cpuTicks / 1e6); - } if (role == Simulation) { unsigned long sevErrorEventsLogged = TraceEvent::CountEventsLoggedAt(SevError); if (sevErrorEventsLogged > 0) { diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 0c4a9db4f9..9cd62ca710 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -42,6 +42,7 @@ #include "fdbserver/LogProtocolMessage.h" #include "fdbserver/LogSystem.h" #include "fdbserver/MoveKeys.actor.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/RecoveryState.h" #include "fdbserver/StorageMetrics.h" #include "fdbserver/ServerDBInfo.h" @@ -951,8 +952,8 @@ ACTOR Future getValueQ( StorageServer* data, GetValueRequest req ) { v = vv; } - debugMutation("ShardGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); - debugMutation("ShardGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); + DEBUG_MUTATION("ShardGetValue", version, MutationRef(MutationRef::DebugKey, req.key, v.present()?v.get():LiteralStringRef(""))); + DEBUG_MUTATION("ShardGetPath", version, MutationRef(MutationRef::DebugKey, req.key, path==0?LiteralStringRef("0"):path==1?LiteralStringRef("1"):LiteralStringRef("2"))); /* StorageMetrics m; @@ -1031,7 +1032,7 @@ ACTOR Future watchValue_impl( StorageServer* data, WatchValueRequest req ) throw reply.error.get(); } - debugMutation("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("") ) ); + DEBUG_MUTATION("ShardWatchValue", latest, MutationRef(MutationRef::DebugKey, req.key, reply.value.present() ? StringRef( reply.value.get() ) : LiteralStringRef("") ) ); if( req.debugID.present() ) g_traceBatch.addEvent("WatchValueDebug", req.debugID.get().first(), "watchValueQ.AfterRead"); //.detail("TaskID", g_network->getCurrentTask()); @@ -2096,7 +2097,7 @@ ACTOR Future fetchKeys( StorageServer *data, AddingShard* shard ) { wait( data->coreStarted.getFuture() && delay( 0 ) ); try { - debugKeyRange("fetchKeysBegin", data->version.get(), shard->keys); + DEBUG_KEY_RANGE("fetchKeysBegin", data->version.get(), shard->keys); TraceEvent(SevDebug, interval.begin(), data->thisServerID) .detail("KeyBegin", shard->keys.begin) @@ -2164,8 +2165,8 @@ ACTOR Future fetchKeys( StorageServer *data, AddingShard* shard ) { .detail("KeyBegin", keys.begin).detail("KeyEnd", keys.end) .detail("Last", this_block.size() ? this_block.end()[-1].key : std::string()) .detail("Version", fetchVersion).detail("More", this_block.more); - debugKeyRange("fetchRange", fetchVersion, keys); - for(auto k = this_block.begin(); k != this_block.end(); ++k) debugMutation("fetch", fetchVersion, MutationRef(MutationRef::SetValue, k->key, k->value)); + DEBUG_KEY_RANGE("fetchRange", fetchVersion, keys); + for(auto k = this_block.begin(); k != this_block.end(); ++k) DEBUG_MUTATION("fetch", fetchVersion, MutationRef(MutationRef::SetValue, k->key, k->value)); data->counters.bytesFetched += expectedSize; if( fetchBlockBytes > expectedSize ) { @@ -2312,7 +2313,7 @@ ACTOR Future fetchKeys( StorageServer *data, AddingShard* shard ) { ASSERT( b->version >= checkv ); checkv = b->version; for(auto& m : b->mutations) - debugMutation("fetchKeysFinalCommitInject", batch->changes[0].version, m); + DEBUG_MUTATION("fetchKeysFinalCommitInject", batch->changes[0].version, m); } shard->updates.clear(); @@ -2421,7 +2422,8 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss // .detail("Context", changeServerKeysContextName[(int)context]); validate(data); - debugKeyRange( nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys ); + // TODO(alexmiller): Figure out how to selectively enable spammy data distribution events. + //DEBUG_KEY_RANGE( nowAssigned ? "KeysAssigned" : "KeysUnassigned", version, keys ); bool isDifferent = false; auto existingShards = data->shards.intersectingRanges(keys); @@ -2526,7 +2528,7 @@ void changeServerKeys( StorageServer* data, const KeyRangeRef& keys, bool nowAss void rollback( StorageServer* data, Version rollbackVersion, Version nextVersion ) { TEST(true); // call to shard rollback - debugKeyRange("Rollback", rollbackVersion, allKeys); + DEBUG_KEY_RANGE("Rollback", rollbackVersion, allKeys); // We used to do a complicated dance to roll back in MVCC history. It's much simpler, and more testable, // to simply restart the storage server actor and restore from the persistent disk state, and then roll @@ -2547,18 +2549,7 @@ void StorageServer::addMutation(Version version, MutationRef const& mutation, Ke return; } expanded = addMutationToMutationLog(mLog, expanded); - if (debugMutation("expandedMutation", version, expanded)) { - const char* type = - mutation.type == MutationRef::SetValue ? "SetValue" : - mutation.type == MutationRef::ClearRange ? "ClearRange" : - mutation.type == MutationRef::DebugKeyRange ? "DebugKeyRange" : - mutation.type == MutationRef::DebugKey ? "DebugKey" : - "UnknownMutation"; - printf("DEBUGMUTATION:\t%.6f\t%s\t%s\t%" PRId64 "\t%s\t%s\t%s\n", now(), g_network->getLocalAddress().toString().c_str(), "originalMutation", version, type, printable(mutation.param1).c_str(), printable(mutation.param2).c_str()); - printf(" shard: %s - %s\n", printable(shard.begin).c_str(), printable(shard.end).c_str()); - if (mutation.type == MutationRef::ClearRange && mutation.param2 != shard.end) - printf(" eager: %s\n", printable( eagerReads->getKeyEnd( mutation.param2 ) ).c_str() ); - } + DEBUG_MUTATION("applyMutation", version, expanded).detail("UID", thisServerID).detail("ShardBegin", shard.begin).detail("ShardEnd", shard.end); applyMutation( this, expanded, mLog.arena(), mutableData() ); //printf("\nSSUpdate: Printing versioned tree after applying mutation\n"); //mutableData().printTree(version); @@ -2610,9 +2601,9 @@ public: applyPrivateData( data, m ); } } else { - // FIXME: enable when debugMutation is active + // FIXME: enable when DEBUG_MUTATION is active //for(auto m = changes[c].mutations.begin(); m; ++m) { - // debugMutation("SSUpdateMutation", changes[c].version, *m); + // DEBUG_MUTATION("SSUpdateMutation", changes[c].version, *m); //} splitMutation(data, data->shards, m, ver); @@ -2897,7 +2888,8 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) rd >> msg; if (ver != invalidVersion) { // This change belongs to a version < minVersion - if (debugMutation("SSPeek", ver, msg) || ver == 1) { + DEBUG_MUTATION("SSPeek", ver, msg).detail("ServerID", data->thisServerID); + if (ver == 1) { TraceEvent("SSPeekMutation", data->thisServerID); // The following trace event may produce a value with special characters //TraceEvent("SSPeekMutation", data->thisServerID).detail("Mutation", msg.toString()).detail("Version", cloneCursor2->version().toString()); @@ -2950,7 +2942,8 @@ ACTOR Future update( StorageServer* data, bool* pReceivedUpdate ) } if(ver != invalidVersion && ver > data->version.get()) { - debugKeyRange("SSUpdate", ver, allKeys); + // TODO(alexmiller): Update to version tracking. + DEBUG_KEY_RANGE("SSUpdate", ver, KeyRangeRef()); data->mutableData().createNewVersion(ver); if (data->otherError.getFuture().isReady()) data->otherError.getFuture().get(); @@ -3153,7 +3146,7 @@ void StorageServerDisk::writeKeyValue( KeyValueRef kv ) { } void StorageServerDisk::writeMutation( MutationRef mutation ) { - // FIXME: debugMutation(debugContext, debugVersion, *m); + // FIXME: DEBUG_MUTATION(debugContext, debugVersion, *m); if (mutation.type == MutationRef::SetValue) { storage->set( KeyValueRef(mutation.param1, mutation.param2) ); } else if (mutation.type == MutationRef::ClearRange) { @@ -3164,7 +3157,7 @@ void StorageServerDisk::writeMutation( MutationRef mutation ) { void StorageServerDisk::writeMutations( MutationListRef mutations, Version debugVersion, const char* debugContext ) { for(auto m = mutations.begin(); m; ++m) { - debugMutation(debugContext, debugVersion, *m); + DEBUG_MUTATION(debugContext, debugVersion, *m).detail("UID", data->thisServerID); if (m->type == MutationRef::SetValue) { storage->set( KeyValueRef(m->param1, m->param2) ); } else if (m->type == MutationRef::ClearRange) { @@ -3181,7 +3174,8 @@ bool StorageServerDisk::makeVersionMutationsDurable( Version& prevStorageVersion if (u != data->getMutationLog().end() && u->first <= newStorageVersion) { VersionUpdateRef const& v = u->second; ASSERT( v.version > prevStorageVersion && v.version <= newStorageVersion ); - debugKeyRange("makeVersionMutationsDurable", v.version, allKeys); + // TODO(alexmiller): Update to version tracking. + DEBUG_KEY_RANGE("makeVersionMutationsDurable", v.version, KeyRangeRef()); writeMutations(v.mutations, v.version, "makeVersionDurable"); for(auto m=v.mutations.begin(); m; ++m) bytesLeft -= mvccStorageBytes(*m); @@ -3367,7 +3361,8 @@ ACTOR Future restoreDurableState( StorageServer* data, IKeyValueStore* sto for(auto it = data->newestAvailableVersion.ranges().begin(); it != data->newestAvailableVersion.ranges().end(); ++it) { if (it->value() == invalidVersion) { KeyRangeRef clearRange(it->begin(), it->end()); - debugKeyRange("clearInvalidVersion", invalidVersion, clearRange); + // TODO(alexmiller): Figure out how to selectively enable spammy data distribution events. + //DEBUG_KEY_RANGE("clearInvalidVersion", invalidVersion, clearRange); storage->clear( clearRange ); data->byteSampleApplyClear( clearRange, invalidVersion ); } diff --git a/fdbserver/workloads/ApiCorrectness.actor.cpp b/fdbserver/workloads/ApiCorrectness.actor.cpp index c122c7c550..afd04d6f97 100644 --- a/fdbserver/workloads/ApiCorrectness.actor.cpp +++ b/fdbserver/workloads/ApiCorrectness.actor.cpp @@ -20,6 +20,7 @@ #include "fdbserver/QuietDatabase.h" +#include "fdbserver/MutationTracking.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/workloads/ApiWorkload.h" #include "fdbserver/workloads/MemoryKeyValueStore.h" @@ -328,7 +329,7 @@ public: wait(transaction->commit()); for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, data.size()); i++) - debugMutation("ApiCorrectnessSet", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, data[i].key, data[i].value)); + DEBUG_MUTATION("ApiCorrectnessSet", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, data[i].key, data[i].value)); currentIndex += self->maxKeysPerTransaction; break; @@ -660,7 +661,7 @@ public: wait(transaction->commit()); for(int i = currentIndex; i < std::min(currentIndex + self->maxKeysPerTransaction, keys.size()); i++) - debugMutation("ApiCorrectnessClear", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, keys[i], StringRef())); + DEBUG_MUTATION("ApiCorrectnessClear", transaction->getCommittedVersion(), MutationRef(MutationRef::DebugKey, keys[i], StringRef())); currentIndex += self->maxKeysPerTransaction; break; @@ -711,7 +712,7 @@ public: } transaction->clear(range); wait(transaction->commit()); - debugKeyRange("ApiCorrectnessClear", transaction->getCommittedVersion(), range); + DEBUG_KEY_RANGE("ApiCorrectnessClear", transaction->getCommittedVersion(), range); break; } catch(Error &e) { diff --git a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp index 61010cbaa1..15cc2d3fb2 100644 --- a/fdbserver/workloads/FuzzApiCorrectness.actor.cpp +++ b/fdbserver/workloads/FuzzApiCorrectness.actor.cpp @@ -59,7 +59,9 @@ struct ExceptionContract { e.code() == error_code_transaction_cancelled || e.code() == error_code_key_too_large || e.code() == error_code_value_too_large || - e.code() == error_code_process_behind) + e.code() == error_code_process_behind || + e.code() == error_code_batch_transaction_throttled || + e.code() == error_code_tag_throttled) { return; } diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 91692fd6eb..479ac8c7cc 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -26,6 +26,8 @@ void forceLinkIndexedSetTests(); void forceLinkDequeTests(); void forceLinkFlowTests(); void forceLinkVersionedMapTests(); +void forceLinkMemcpyTests(); +void forceLinkMemcpyPerfTests(); struct UnitTestWorkload : TestWorkload { bool enabled; @@ -45,6 +47,8 @@ struct UnitTestWorkload : TestWorkload { forceLinkDequeTests(); forceLinkFlowTests(); forceLinkVersionedMapTests(); + forceLinkMemcpyTests(); + forceLinkMemcpyPerfTests(); } virtual std::string description() { return "UnitTests"; } diff --git a/fdbservice/FDBService.cpp b/fdbservice/FDBService.cpp index 8efee25593..fe761a0109 100644 --- a/fdbservice/FDBService.cpp +++ b/fdbservice/FDBService.cpp @@ -28,8 +28,8 @@ #include #include -#include "..\flow\SimpleOpt.h" -#include "..\fdbmonitor\SimpleIni.h" +#include "flow/SimpleOpt.h" +#include "fdbmonitor/SimpleIni.h" #include "fdbclient/versions.h" // For PathFileExists diff --git a/flow/Arena.h b/flow/Arena.h index 5de81ee3bb..ae4ea967ae 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -380,12 +380,11 @@ public: } #else Standalone( const T& t, const Arena& arena ) : Arena( arena ), T( t ) {} - Standalone( const Standalone & t ) : Arena((Arena const&)t), T((T const&)t) {} - Standalone& operator=( const Standalone & t ) { - *(Arena*)this = (Arena const&)t; - *(T*)this = (T const&)t; - return *this; - } + Standalone(const Standalone&) = default; + Standalone& operator=(const Standalone&) = default; + Standalone(Standalone&&) = default; + Standalone& operator=(Standalone&&) = default; + ~Standalone() = default; #endif template Standalone castTo() const { @@ -710,15 +709,20 @@ inline bool operator != (const StringRef& lhs, const StringRef& rhs ) { return ! inline bool operator <= ( const StringRef& lhs, const StringRef& rhs ) { return !(lhs>rhs); } inline bool operator >= ( const StringRef& lhs, const StringRef& rhs ) { return !(lhs -struct memcpy_able : std::is_trivial {}; +struct flow_ref : std::integral_constant> {}; template <> -struct memcpy_able : std::integral_constant {}; +struct flow_ref : std::integral_constant {}; + +template +struct flow_ref> : std::integral_constant {}; template struct string_serialized_traits : std::false_type { @@ -794,7 +798,7 @@ public: using value_type = T; static_assert(SerStrategy == VecSerStrategy::FlatBuffers || string_serialized_traits::value); - // T must be trivially destructible (and copyable)! + // T must be trivially destructible! VectorRef() : data(0), m_size(0), m_capacity(0) {} template @@ -809,19 +813,19 @@ public: return *this; } - // Arena constructor for non-Ref types, identified by memcpy_able + // Arena constructor for non-Ref types, identified by !flow_ref template - VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) + VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) : VPS(toCopy), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) { if (m_size > 0) { - memcpy(data, toCopy.data, m_size * sizeof(T)); + std::copy(toCopy.data, toCopy.data + m_size, data); } } // Arena constructor for Ref types, which must have an Arena constructor template - VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) + VectorRef(Arena& p, const VectorRef& toCopy, typename std::enable_if::value, int>::type = 0) : VPS(), data((T*)new (p) uint8_t[sizeof(T) * toCopy.size()]), m_size(toCopy.size()), m_capacity(toCopy.size()) { for (int i = 0; i < m_size; i++) { auto ptr = new (&data[i]) T(p, toCopy[i]); @@ -917,7 +921,7 @@ public: if (m_size + count > m_capacity) reallocate(p, m_size + count); VPS::invalidate(); if (count > 0) { - memcpy(data + m_size, begin, sizeof(T) * count); + std::copy(begin, begin + count, data + m_size); } m_size += count; } @@ -957,15 +961,15 @@ public: if (size > m_capacity) reallocate(p, size); } - // expectedSize() for non-Ref types, identified by memcpy_able + // expectedSize() for non-Ref types, identified by !flow_ref template - typename std::enable_if::value, size_t>::type expectedSize() const { + typename std::enable_if::value, size_t>::type expectedSize() const { return sizeof(T) * m_size; } // expectedSize() for Ref types, which must in turn have expectedSize() implemented. template - typename std::enable_if::value, size_t>::type expectedSize() const { + typename std::enable_if::value, size_t>::type expectedSize() const { size_t t = sizeof(T) * m_size; for (int i = 0; i < m_size; i++) t += data[i].expectedSize(); return t; @@ -982,9 +986,9 @@ private: void reallocate(Arena& p, int requiredCapacity) { requiredCapacity = std::max(m_capacity * 2, requiredCapacity); // SOMEDAY: Maybe we are right at the end of the arena and can expand cheaply - T* newData = (T*)new (p) uint8_t[requiredCapacity * sizeof(T)]; + T* newData = new (p) T[requiredCapacity]; if (m_size > 0) { - memcpy(newData, data, m_size * sizeof(T)); + std::move(data, data + m_size, newData); } data = newData; m_capacity = requiredCapacity; diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index 6c3b8eca24..61ce9ed17b 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -67,7 +67,7 @@ set(FLOW_SRCS XmlTraceLogFormatter.cpp XmlTraceLogFormatter.h actorcompiler.h - crc32c.h + crc32c.h crc32c.cpp error_definitions.h ${CMAKE_CURRENT_BINARY_DIR}/SourceVersion.h @@ -75,14 +75,18 @@ set(FLOW_SRCS flat_buffers.h flow.cpp flow.h + folly_memcpy.S genericactors.actor.cpp genericactors.actor.h network.cpp network.h + rte_memcpy.h serialize.cpp serialize.h stacktrace.amalgamation.cpp stacktrace.h + test_memcpy.cpp + test_memcpy_perf.cpp version.cpp) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/SourceVersion.h.cmake ${CMAKE_CURRENT_BINARY_DIR}/SourceVersion.h) diff --git a/flow/Error.cpp b/flow/Error.cpp index 3edb81adf9..cf2fa34b63 100644 --- a/flow/Error.cpp +++ b/flow/Error.cpp @@ -28,11 +28,6 @@ using std::make_pair; bool g_crashOnError = false; -std::map& Error::errorCounts() { - static std::map counts; - return counts; -} - #include Error Error::fromUnvalidatedCode(int code) { @@ -70,8 +65,6 @@ Error::Error(int error_code) crashAndDie(); } } - /*if (error_code) - errorCounts()[error_code]++;*/ } ErrorCodeTable& Error::errorCodeTable() { diff --git a/flow/Error.h b/flow/Error.h index 0afe4d0d99..98cb35c576 100644 --- a/flow/Error.h +++ b/flow/Error.h @@ -58,9 +58,12 @@ public: explicit Error(int error_code); static void init(); - static std::map& errorCounts(); static ErrorCodeTable& errorCodeTable(); - static Error fromCode(int error_code) { Error e; e.error_code = error_code; return e; } // Doesn't change errorCounts + static Error fromCode(int error_code) { + Error e; + e.error_code = error_code; + return e; + } static Error fromUnvalidatedCode(int error_code); // Converts codes that are outside the legal range (but not necessarily individually unknown error codes) to unknown_error() Error asInjectedFault() const; // Returns an error with the same code() as this but isInjectedFault() is true diff --git a/flow/IKeyValueContainer.h b/flow/IKeyValueContainer.h index 64a5752c2e..2167f32801 100644 --- a/flow/IKeyValueContainer.h +++ b/flow/IKeyValueContainer.h @@ -69,23 +69,33 @@ bool operator<(CompatibleWithKey const& l, KeyValueMapPair const& r) { class IKeyValueContainer { public: - typedef typename IndexedSet::iterator iterator; + using const_iterator = IndexedSet::const_iterator; + using iterator = IndexedSet::iterator; IKeyValueContainer() = default; ~IKeyValueContainer() = default; - bool empty() { return data.empty(); } + bool empty() const { return data.empty(); } void clear() { return data.clear(); } - std::tuple size() { return std::make_tuple(0, 0, 0); } + std::tuple size() const { return std::make_tuple(0, 0, 0); } + const_iterator find(const StringRef& key) const { return data.find(key); } iterator find(const StringRef& key) { return data.find(key); } + const_iterator begin() const { return data.begin(); } iterator begin() { return data.begin(); } + const_iterator cbegin() const { return begin(); } + const_iterator end() const { return data.end(); } iterator end() { return data.end(); } + const_iterator cend() const { return end(); } + const_iterator lower_bound(const StringRef& key) const { return data.lower_bound(key); } iterator lower_bound(const StringRef& key) { return data.lower_bound(key); } + const_iterator upper_bound(const StringRef& key) const { return data.upper_bound(key); } iterator upper_bound(const StringRef& key) { return data.upper_bound(key); } - iterator previous(iterator i) const { return data.previous(i); } + const_iterator previous(const_iterator i) const { return data.previous(i); } + const_iterator previous(iterator i) const { return data.previous(const_iterator{ i }); } + iterator previous(iterator i) { return data.previous(i); } void erase(iterator begin, iterator end) { data.erase(begin, end); } iterator insert(const StringRef& key, const StringRef& val, bool replaceExisting = true) { @@ -96,7 +106,8 @@ public: return data.insert(pairs, replaceExisting); } - uint64_t sumTo(iterator to) { return data.sumTo(to); } + uint64_t sumTo(const_iterator to) const { return data.sumTo(to); } + uint64_t sumTo(iterator to) const { return data.sumTo(const_iterator{ to }); } static int getElementBytes() { return IndexedSet::getElementBytes(); } diff --git a/flow/IndexedSet.cpp b/flow/IndexedSet.cpp index 2065fabac0..16e9ce12ca 100644 --- a/flow/IndexedSet.cpp +++ b/flow/IndexedSet.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include "flow/TreeBenchmark.h" #include "flow/UnitTest.h" template @@ -204,18 +205,25 @@ TEST_CASE("/flow/IndexedSet/strings") { template struct IndexedSetHarness { using map = IndexedSet; + using const_result = typename map::const_iterator; using result = typename map::iterator; using key_type = K; map s; void insert(K const& k) { s.insert(K(k), 1); } - result find(K const& k) const { return s.find(k); } - result not_found() const { return s.end(); } - result begin() const { return s.begin(); } - result end() const { return s.end(); } - result lower_bound(K const& k) const { return s.lower_bound(k); } - result upper_bound(K const& k) const { return s.upper_bound(k); } + const_result find(K const& k) const { return s.find(k); } + result find(K const& k) { return s.find(k); } + const_result not_found() const { return s.end(); } + result not_found() { return s.end(); } + const_result begin() const { return s.begin(); } + result begin() { return s.begin(); } + const_result end() const { return s.end(); } + result end() { return s.end(); } + const_result lower_bound(K const& k) const { return s.lower_bound(k); } + result lower_bound(K const& k) { return s.lower_bound(k); } + const_result upper_bound(K const& k) const { return s.upper_bound(k); } + result upper_bound(K const& k) { return s.upper_bound(k); } void erase(K const& k) { s.erase(k); } }; @@ -494,4 +502,60 @@ TEST_CASE("/flow/IndexedSet/all numbers") { return Void(); } +template +static constexpr bool is_const_ref_v = std::is_const_v>; + +TEST_CASE("/flow/IndexedSet/const_iterator") { + struct Key { + int key; + explicit Key(int key) : key(key) {} + }; + struct Metric { + int metric; + explicit Metric(int metric) : metric(metric) {} + }; + + IndexedSet is; + for (int i = 0; i < 10; ++i) is.insert(i, 1); + + IndexedSet& ncis = is; + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + static_assert(!is_const_ref_v); + + const IndexedSet& cis = is; + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + static_assert(is_const_ref_v); + + for (auto& val : ncis) { + static_assert(!is_const_ref_v); + } + for (const auto& val : ncis) { + static_assert(is_const_ref_v); + } + for (auto& val : cis) { + static_assert(is_const_ref_v); + } + + return Void(); +} void forceLinkIndexedSetTests() {} diff --git a/flow/IndexedSet.h b/flow/IndexedSet.h index 2e22e71e64..44c6d87be3 100644 --- a/flow/IndexedSet.h +++ b/flow/IndexedSet.h @@ -29,6 +29,7 @@ #include "flow/Error.h" #include +#include #include // IndexedSet is similar to a std::set, with the following additional features: @@ -39,7 +40,6 @@ // - Search functions (find(), lower_bound(), etc) can accept a type comparable to T instead of T // (e.g. StringRef when T is std::string or Standalone). This can save a lot of needless // copying at query time for read-mostly sets with string keys. -// - iterators are not const; the responsibility of not changing the order lies with the caller // - the size() function is missing; if the metric being used is a count sumTo(end()) will do instead // A number of STL compatibility features are missing and should be added as needed. // T must define operator <, which must define a total order. Unlike std::set, @@ -70,8 +70,10 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou // combinations, but still take advantage of move constructors when available (or required). template Node(T_&& data, Metric_&& m, Node* parent=0) : data(std::forward(data)), total(std::forward(m)), parent(parent), balance(0) { - child[0] = child[1] = NULL; + child[0] = child[1] = nullptr; } + Node(Node const&) = delete; + Node& operator=(Node const&) = delete; ~Node(){ delete child[0]; delete child[1]; @@ -84,35 +86,93 @@ private: // Forward-declare IndexedSet::Node because Clang is much stricter abou Node *parent; }; -public: - struct iterator{ - typename IndexedSet::Node *i; - iterator() : i(0) {}; - iterator(typename IndexedSet::Node *n) : i(n) {}; - T& operator*() { return i->data; }; - T* operator->() { return &i->data; } + template + struct IteratorImpl { + typename std::conditional_t* node; + + explicit IteratorImpl(const IteratorImpl& nonConstIter) : node(nonConstIter.node) { + static_assert(isConst); + } + + explicit IteratorImpl(decltype(node) n = nullptr) : node(n){}; + + typename std::conditional_t& operator*() const { return node->data; } + + typename std::conditional_t* operator->() const { return &node->data; } + void operator++(); void decrementNonEnd(); - bool operator == ( const iterator& r ) const { return i == r.i; } - bool operator != ( const iterator& r ) const { return i != r.i; } + bool operator==(const IteratorImpl& r) const { return node == r.node; } + bool operator!=(const IteratorImpl& r) const { return node != r.node; } // following two methods are for memory storage engine(KeyValueStoreMemory class) use only // in order to have same interface as radixtree - StringRef& getKey(uint8_t* dummyContent) const { return i->data.key; } - StringRef& getValue() const { return i->data.value; } + typename std::conditional_t& getKey(uint8_t* dummyContent) const { + return node->data.key; + } + typename std::conditional_t& getValue() const { return node->data.value; } }; - IndexedSet() : root(NULL) {}; + template + struct Impl { + using NodeT = std::conditional_t; + using IteratorT = IteratorImpl; + using SetT = std::conditional_t, IndexedSet>; + + static IteratorT begin(SetT&); + + template + static IteratorImpl previous(SetT&, IteratorImpl); + + template + static IteratorT index(SetT&, const M&); + + template + static IteratorT find(SetT&, const Key&); + + template + static IteratorT upper_bound(SetT&, const Key&); + + template + static IteratorT lower_bound(SetT&, const Key&); + + template + static IteratorT lastLessOrEqual(SetT&, const Key&); + + static IteratorT lastItem(SetT&); + }; + + using ConstImpl = Impl; + using NonConstImpl = Impl; + +public: + using iterator = IteratorImpl; + using const_iterator = IteratorImpl; + + IndexedSet() : root(nullptr){}; ~IndexedSet() { delete root; } - IndexedSet(IndexedSet&& r) BOOST_NOEXCEPT : root(r.root) { r.root = NULL; } + IndexedSet(IndexedSet&& r) BOOST_NOEXCEPT : root(r.root) { r.root = nullptr; } IndexedSet& operator=(IndexedSet&& r) BOOST_NOEXCEPT { delete root; root = r.root; r.root = 0; return *this; } - iterator begin() const; - iterator end() const { return iterator(); } - iterator previous(iterator i) const; - iterator lastItem() const; + const_iterator begin() const { return ConstImpl::begin(*this); }; + iterator begin() { return NonConstImpl::begin(*this); }; + const_iterator cbegin() const { return begin(); } + + const_iterator end() const { return const_iterator{}; } + iterator end() { return iterator{}; } + const_iterator cend() const { return end(); } + + const_iterator previous(const_iterator i) const { return ConstImpl::previous(*this, i); } + const_iterator previous(iterator i) const { return ConstImpl::previous(*this, const_iterator{ i }); } + iterator previous(iterator i) { return NonConstImpl::previous(*this, i); } + + const_iterator lastItem() const { return ConstImpl::lastItem(*this); } + iterator lastItem() { return NonConstImpl::lastItem(*this); } bool empty() const { return !root; } - void clear() { delete root; root = NULL; } + void clear() { + delete root; + root = nullptr; + } void swap( IndexedSet& r ) { std::swap( root, r.root ); } // Place data in the set with the given metric. If an item equal to data is already in the set and, @@ -159,36 +219,78 @@ public: // Returns x such that key==*x, or end() template - iterator find(const Key &key) const; + const_iterator find(const Key& key) const { + return ConstImpl::find(*this, key); + } + + template + iterator find(const Key& key) { + return NonConstImpl::find(*this, key); + } // Returns the smallest x such that *x>=key, or end() template - iterator lower_bound(const Key &key) const; + const_iterator lower_bound(const Key& key) const { + return ConstImpl::lower_bound(*this, key); + } + + template + iterator lower_bound(const Key& key) { + return NonConstImpl::lower_bound(*this, key); + }; // Returns the smallest x such that *x>key, or end() template - iterator upper_bound(const Key &key) const; + const_iterator upper_bound(const Key& key) const { + return ConstImpl::upper_bound(*this, key); + } + + template + iterator upper_bound(const Key& key) { + return NonConstImpl::upper_bound(*this, key); + }; // Returns the largest x such that *x<=key, or end() template - iterator lastLessOrEqual( const Key &key ) const; + const_iterator lastLessOrEqual(const Key& key) const { + return ConstImpl::lastLessOrEqual(*this, key); + }; + + template + iterator lastLessOrEqual(const Key& key) { + return NonConstImpl::lastLessOrEqual(*this, key); + } // Returns smallest x such that sumTo(x+1) > metric, or end() template - iterator index( M const& metric ) const; + const_iterator index(M const& metric) const { + return ConstImpl::index(*this, metric); + }; + + template + iterator index(M const& metric) { + return NonConstImpl::index(*this, metric); + } // Return the metric inserted with item x - Metric getMetric(iterator x) const; + Metric getMetric(const_iterator x) const; + Metric getMetric(iterator x) const { return getMetric(const_iterator{ x }); } // Return the sum of getMetric(x) for begin()<=x - Metric sumRange(const Key& begin, const Key& end) const { return sumRange(lower_bound(begin), lower_bound(end)); } + template + Metric sumRange(const Key& begin, const Key& end) const { + return sumRange(lower_bound(begin), lower_bound(end)); + } // Return the amount of memory used by an entry in the IndexedSet static int getElementBytes() { return sizeof(Node); } @@ -212,18 +314,25 @@ private: newNode->parent = oldNode->parent; } + template + static void moveIteratorImpl(std::conditional_t*& node) { + if (node->child[0 ^ direction]) { + node = node->child[0 ^ direction]; + while (node->child[1 ^ direction]) node = node->child[1 ^ direction]; + } else { + while (node->parent && node->parent->child[0 ^ direction] == node) node = node->parent; + node = node->parent; + } + } + // direction 0 = left, 1 = right template - static void moveIterator(Node* &i){ - if (i->child[0^direction]) { - i = i->child[0^direction]; - while (i->child[1^direction]) - i = i->child[1^direction]; - } else { - while (i->parent && i->parent->child[0^direction] == i) - i = i->parent; - i = i->parent; - } + static void moveIterator(Node const*& node) { + moveIteratorImpl(node); + } + template + static void moveIterator(Node*& node) { + moveIteratorImpl(node); } public: // but testonly @@ -284,12 +393,19 @@ template , class Metric= class Map { public: typedef typename IndexedSet::iterator iterator; + typedef typename IndexedSet::const_iterator const_iterator; Map() {} - iterator begin() const { return set.begin(); } - iterator end() const { return set.end(); } - iterator lastItem() const { return set.lastItem(); } - iterator previous(iterator i) const { return set.previous(i); } + const_iterator begin() const { return set.begin(); } + iterator begin() { return set.begin(); } + const_iterator cbegin() const { return begin(); } + const_iterator end() const { return set.end(); } + iterator end() { return set.end(); } + const_iterator cend() const { return end(); } + const_iterator lastItem() const { return set.lastItem(); } + iterator lastItem() { return set.lastItem(); } + const_iterator previous(const_iterator i) const { return set.previous(i); } + iterator previous(iterator i) { return set.previous(i); } bool empty() const { return set.empty(); } Value& operator[]( const Key& key ) { @@ -317,18 +433,58 @@ public: } template - iterator find( KeyCompatible const& k ) const { return set.find(k); } + const_iterator find(KeyCompatible const& k) const { + return set.find(k); + } template - iterator lower_bound( KeyCompatible const& k ) const { return set.lower_bound(k); } + iterator find(KeyCompatible const& k) { + return set.find(k); + } + template - iterator upper_bound( KeyCompatible const& k ) const { return set.upper_bound(k); } + const_iterator lower_bound(KeyCompatible const& k) const { + return set.lower_bound(k); + } template - iterator lastLessOrEqual( KeyCompatible const& k ) const { return set.lastLessOrEqual(k); } - template - iterator index( M const& metric ) const { return set.index(metric); } - Metric getMetric(iterator x) const { return set.getMetric(x); } - Metric sumTo(iterator to) const { return set.sumTo(to); } - Metric sumRange(iterator begin, iterator end) const { return set.sumRange(begin,end); } + iterator lower_bound(KeyCompatible const& k) { + return set.lower_bound(k); + } + + template + const_iterator upper_bound(KeyCompatible const& k) const { + return set.upper_bound(k); + } + template + iterator upper_bound(KeyCompatible const& k) { + return set.upper_bound(k); + } + + template + const_iterator lastLessOrEqual(KeyCompatible const& k) const { + return set.lastLessOrEqual(k); + } + template + iterator lastLessOrEqual(KeyCompatible const& k) { + return set.lastLessOrEqual(k); + } + + template + const_iterator index(M const& metric) const { + return set.index(metric); + } + template + iterator index(M const& metric) { + return set.index(metric); + } + + Metric getMetric(const_iterator x) const { return set.getMetric(x); } + Metric getMetric(iterator x) const { return getMetric(const_iterator{ x }); } + + Metric sumTo(const_iterator to) const { return set.sumTo(to); } + Metric sumTo(iterator to) const { return sumTo(const_iterator{ to }); } + + Metric sumRange(const_iterator begin, const_iterator end) const { return set.sumRange(begin, end); } + Metric sumRange(iterator begin, iterator end) const { return set.sumRange(begin, end); } template Metric sumRange(const KeyCompatible& begin, const KeyCompatible& end) const { return set.sumRange(begin,end); } @@ -347,13 +503,15 @@ private: /////////////////////// implementation ////////////////////////// template -void IndexedSet::iterator::operator++(){ - moveIterator<1>(i); +template +void IndexedSet::IteratorImpl::operator++() { + moveIterator<1>(node); } template -void IndexedSet::iterator::decrementNonEnd(){ - moveIterator<0>(i); +template +void IndexedSet::IteratorImpl::decrementNonEnd() { + moveIterator<0>(node); } template @@ -578,28 +736,33 @@ Node* ISCommonSubtreeRoot(Node* first, Node* last) { } template -typename IndexedSet::iterator IndexedSet::begin() const { - Node *x = root; - while (x && x->child[0]) - x = x->child[0]; - return x; +template +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::begin( + IndexedSet::Impl::SetT& self) { + NodeT* x = self.root; + while (x && x->child[0]) x = x->child[0]; + return IteratorT{ x }; } template -typename IndexedSet::iterator IndexedSet::previous(typename IndexedSet::iterator i) const { - if (i==end()) - return lastItem(); +template +template +typename IndexedSet::template IteratorImpl +IndexedSet::Impl::previous(IndexedSet::Impl::SetT& self, + IndexedSet::IteratorImpl iter) { + if (iter == self.end()) return self.lastItem(); - moveIterator<0>(i.i); - return i; + moveIterator<0>(iter.node); + return iter; } template -typename IndexedSet::iterator IndexedSet::lastItem() const { - Node *x = root; - while (x && x->child[1]) - x = x->child[1]; - return x; +template +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lastItem( + IndexedSet::Impl::SetT& self) { + NodeT* x = self.root; + while (x && x->child[1]) x = x->child[1]; + return IteratorT{ x }; } template template @@ -617,9 +780,9 @@ Metric IndexedSet::addMetric(T_&& data, Metric_&& metric){ template template typename IndexedSet::iterator IndexedSet::insert(T_&& data, Metric_&& metric, bool replaceExisting){ - if (root == NULL){ + if (root == nullptr) { root = new Node(std::forward(data), std::forward(metric)); - return root; + return iterator{ root }; } Node *t = root; int d; // direction @@ -642,7 +805,7 @@ typename IndexedSet::iterator IndexedSet::insert(T_&& data, } } - return returnNode; + return iterator{ returnNode }; } d = cmp > 0; Node *nextT = t->child[d]; @@ -685,23 +848,23 @@ typename IndexedSet::iterator IndexedSet::insert(T_&& data, t->total = t->total + metric; } - return newNode; + return iterator{ newNode }; } template int IndexedSet::insert(const std::vector>& dataVector, bool replaceExisting) { int num_inserted = 0; - Node *blockStart = NULL; - Node *blockEnd = NULL; + Node* blockStart = nullptr; + Node* blockEnd = nullptr; for(int i = 0; i < dataVector.size(); ++i) { Metric metric = dataVector[i].second; T data = std::move(dataVector[i].first); int d = 1; // direction - if(blockStart == NULL || (blockEnd != NULL && data >= blockEnd->data)) { - blockEnd = NULL; - if (root == NULL) { + if (blockStart == nullptr || (blockEnd != nullptr && data >= blockEnd->data)) { + blockEnd = nullptr; + if (root == nullptr) { root = new Node(std::move(data), metric); num_inserted++; blockStart = root; @@ -842,8 +1005,8 @@ Metric IndexedSet::eraseHalf(Node* start, Node* end, int eraseDir, in metricDelta = metricDelta - n->total; n->parent = start->parent; } - - start->child[fromDir] = NULL; + + start->child[fromDir] = nullptr; toFree.push_back( start ); } @@ -874,13 +1037,13 @@ void IndexedSet::erase( typename IndexedSet::iterator begin, // Removes all nodes in the set between first and last, inclusive. // toFree is extended with the roots of completely removed subtrees. - ASSERT(!end.i || (begin.i && (::compare(*begin, *end) <= 0))); + ASSERT(!end.node || (begin.node && (::compare(*begin, *end) <= 0))); if(begin == end) return; - - IndexedSet::Node* first = begin.i; - IndexedSet::Node* last = previous(end).i; + + IndexedSet::Node* first = begin.node; + IndexedSet::Node* last = previous(end).node; IndexedSet::Node* subRoot = ISCommonSubtreeRoot(first, last); @@ -897,7 +1060,7 @@ void IndexedSet::erase( typename IndexedSet::iterator begin, int heightDelta = leftHeightDelta + rightHeightDelta; // Rebalance and update metrics for all nodes from subRoot up to the root - for(auto p = subRoot; p != NULL; p = p->parent) { + for (auto p = subRoot; p != nullptr; p = p->parent) { p->total = p->total - metricDelta; auto& pc = p->parent ? p->parent->child[p->parent->child[1]==p] : root; @@ -925,7 +1088,7 @@ void IndexedSet::erase(iterator toErase) { { // Find the node to erase - Node* t = toErase.i; + Node* t = toErase.node; if (!t) return; if (!t->child[0] || !t->child[1]) { @@ -1005,101 +1168,106 @@ void IndexedSet::erase(iterator toErase) { // Returns x such that key==*x, or end() template +template template -typename IndexedSet::iterator IndexedSet::find(const Key &key) const { - Node* t = root; +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::find( + IndexedSet::Impl::SetT& self, const Key& key) { + NodeT* t = self.root; while (t){ int cmp = compare(key, t->data); - if (cmp == 0) return iterator(t); + if (cmp == 0) return IteratorT{ t }; t = t->child[cmp > 0]; } - return end(); + return self.end(); } // Returns the smallest x such that *x>=key, or end() template +template template -typename IndexedSet::iterator IndexedSet::lower_bound(const Key &key) const { - Node* t = root; - if (!t) return iterator(); +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lower_bound( + IndexedSet::Impl::SetT& self, const Key& key) { + NodeT* t = self.root; + if (!t) return self.end(); bool less; while (true) { less = t->data < key; - Node* n = t->child[less]; + NodeT* n = t->child[less]; if (!n) break; t = n; } if (less) moveIterator<1>(t); - return iterator(t); + return IteratorT{ t }; } // Returns the smallest x such that *x>key, or end() template +template template -typename IndexedSet::iterator IndexedSet::upper_bound(const Key &key) const { - Node* t = root; - if (!t) return iterator(); +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::upper_bound( + IndexedSet::Impl::SetT& self, const Key& key) { + NodeT* t = self.root; + if (!t) return self.end(); bool not_less; while (true) { not_less = !(key < t->data); - Node* n = t->child[not_less]; + NodeT* n = t->child[not_less]; if (!n) break; t = n; } if (not_less) moveIterator<1>(t); - return iterator(t); + return IteratorT{ t }; } template +template template -typename IndexedSet::iterator IndexedSet::lastLessOrEqual(const Key &key) const { - iterator i = upper_bound(key); - if (i == begin()) return end(); - return previous(i); +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::lastLessOrEqual( + IndexedSet::Impl::SetT& self, const Key& key) { + auto i = self.upper_bound(key); + if (i == self.begin()) return self.end(); + return self.previous(i); } // Returns first x such that metric < sum(begin(), x+1), or end() template +template template -typename IndexedSet::iterator IndexedSet::index( M const& metric ) const -{ +typename IndexedSet::template Impl::IteratorT IndexedSet::Impl::index( + IndexedSet::Impl::SetT& self, const M& metric) { M m = metric; - Node* t = root; + NodeT* t = self.root; while (t) { if (t->child[0] && m < t->child[0]->total) t = t->child[0]; else { m = m - t->total; - if (t->child[1]) - m = m + t->child[1]->total; - if (m < M()) - return iterator(t); + if (t->child[1]) m = m + t->child[1]->total; + if (m < M()) return IteratorT{ t }; t = t->child[1]; } } - return end(); + return self.end(); } template -Metric IndexedSet::getMetric(typename IndexedSet::iterator x) const { - Metric m = x.i->total; +Metric IndexedSet::getMetric(typename IndexedSet::const_iterator x) const { + Metric m = x.node->total; for(int i=0; i<2; i++) - if (x.i->child[i]) - m = m - x.i->child[i]->total; + if (x.node->child[i]) m = m - x.node->child[i]->total; return m; } template -Metric IndexedSet::sumTo(typename IndexedSet::iterator end) const { - if (!end.i) - return root ? root->total : Metric(); +Metric IndexedSet::sumTo(typename IndexedSet::const_iterator end) const { + if (!end.node) return root ? root->total : Metric(); - Metric m = end.i->child[0] ? end.i->child[0]->total : Metric(); - for(Node* p = end.i; p->parent; p=p->parent) { + Metric m = end.node->child[0] ? end.node->child[0]->total : Metric(); + for (const Node* p = end.node; p->parent; p = p->parent) { if (p->parent->child[1] == p) { m = m - p->total; m = m + p->parent->total; diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index 9c78cff5c3..7da3b85840 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -296,6 +296,35 @@ public: } }; +struct SendBufferIterator { + typedef boost::asio::const_buffer value_type; + typedef std::forward_iterator_tag iterator_category; + typedef size_t difference_type; + typedef boost::asio::const_buffer* pointer; + typedef boost::asio::const_buffer& reference; + + SendBuffer const* p; + int limit; + + SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits::max()) : p(p), limit(limit) { + ASSERT(limit > 0); + } + + bool operator == (SendBufferIterator const& r) const { return p == r.p; } + bool operator != (SendBufferIterator const& r) const { return p != r.p; } + void operator++() { + limit -= p->bytes_written - p->bytes_sent; + if(limit > 0) + p = p->next; + else + p = NULL; + } + + boost::asio::const_buffer operator*() const { + return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) ); + } +}; + class Connection : public IConnection, ReferenceCounted { public: virtual void addref() { ReferenceCounted::addref(); } @@ -420,35 +449,6 @@ private: tcp::socket socket; NetworkAddress peer_address; - struct SendBufferIterator { - typedef boost::asio::const_buffer value_type; - typedef std::forward_iterator_tag iterator_category; - typedef size_t difference_type; - typedef boost::asio::const_buffer* pointer; - typedef boost::asio::const_buffer& reference; - - SendBuffer const* p; - int limit; - - SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits::max()) : p(p), limit(limit) { - ASSERT(limit > 0); - } - - bool operator == (SendBufferIterator const& r) const { return p == r.p; } - bool operator != (SendBufferIterator const& r) const { return p != r.p; } - void operator++() { - limit -= p->bytes_written - p->bytes_sent; - if(limit > 0) - p = p->next; - else - p = NULL; - } - - boost::asio::const_buffer operator*() const { - return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) ); - } - }; - void init() { // Socket settings that have to be set after connect or accept succeeds socket.non_blocking(true); @@ -721,6 +721,10 @@ public: // Writes as many bytes as possible from the given SendBuffer chain into the write buffer and returns the number of bytes written (might be 0) virtual int write( SendBuffer const* data, int limit ) { +#ifdef __APPLE__ + // For some reason, writing ssl_sock with more than 2016 bytes when socket is writeable sometimes results in a broken pipe error. + limit = std::min(limit, 2016); +#endif boost::system::error_code err; ++g_net2->countWrites; @@ -763,35 +767,6 @@ private: NetworkAddress peer_address; Reference> sslContext; - struct SendBufferIterator { - typedef boost::asio::const_buffer value_type; - typedef std::forward_iterator_tag iterator_category; - typedef size_t difference_type; - typedef boost::asio::const_buffer* pointer; - typedef boost::asio::const_buffer& reference; - - SendBuffer const* p; - int limit; - - SendBufferIterator(SendBuffer const* p=0, int limit = std::numeric_limits::max()) : p(p), limit(limit) { - ASSERT(limit > 0); - } - - bool operator == (SendBufferIterator const& r) const { return p == r.p; } - bool operator != (SendBufferIterator const& r) const { return p != r.p; } - void operator++() { - limit -= p->bytes_written - p->bytes_sent; - if(limit > 0) - p = p->next; - else - p = NULL; - } - - boost::asio::const_buffer operator*() const { - return boost::asio::const_buffer( p->data + p->bytes_sent, std::min(limit, p->bytes_written - p->bytes_sent) ); - } - }; - void init() { // Socket settings that have to be set after connect or accept succeeds socket.non_blocking(true); diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index cd78f6f56b..f1864bcd0c 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -132,7 +132,7 @@ public: // introduced features // // xyzdev // vvvv -constexpr ProtocolVersion currentProtocolVersion(0x0FDB00B063010001LL); +constexpr ProtocolVersion currentProtocolVersion(0x0FDB00B070010001LL); // This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to // change when we reach version 10. static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version"); diff --git a/flow/Trace.cpp b/flow/Trace.cpp index adf1921fe3..2f15483b30 100644 --- a/flow/Trace.cpp +++ b/flow/Trace.cpp @@ -783,6 +783,7 @@ TraceEvent::TraceEvent(TraceEvent &&ev) { tmpEventMetric = ev.tmpEventMetric; trackingKey = ev.trackingKey; type = ev.type; + timeIndex = ev.timeIndex; ev.initialized = true; ev.enabled = false; @@ -803,6 +804,7 @@ TraceEvent& TraceEvent::operator=(TraceEvent &&ev) { tmpEventMetric = ev.tmpEventMetric; trackingKey = ev.trackingKey; type = ev.type; + timeIndex = ev.timeIndex; ev.initialized = true; ev.enabled = false; diff --git a/flow/Trace.h b/flow/Trace.h index 3aeb5a9a8d..ae32302a74 100644 --- a/flow/Trace.h +++ b/flow/Trace.h @@ -479,6 +479,10 @@ public: return enabled; } + explicit operator bool() const { + return enabled; + } + void log(); ~TraceEvent(); // Actually logs the event diff --git a/flow/flow.cpp b/flow/flow.cpp index fb1cf81f60..5c5450f065 100644 --- a/flow/flow.cpp +++ b/flow/flow.cpp @@ -21,9 +21,28 @@ #include "flow/flow.h" #include "flow/DeterministicRandom.h" #include "flow/UnitTest.h" +#include "flow/rte_memcpy.h" +#include "flow/folly_memcpy.h" #include #include +#if (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) +// For benchmarking; need a version of rte_memcpy that doesn't live in the same compilation unit as the test. +void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { + return rte_memcpy(__dest, __src, __n); +} + +// This compilation unit will be linked in to the main binary, so this should override glibc memcpy +__attribute__((visibility ("default"))) void *memcpy (void *__restrict __dest, const void *__restrict __src, size_t __n) { + // folly_memcpy is faster for small copies, but rte seems to win out in most other circumstances + return rte_memcpy(__dest, __src, __n); +} +#else +void * rte_memcpy_noinline(void *__restrict __dest, const void *__restrict __src, size_t __n) { + return memcpy(__dest, __src, __n); +} +#endif // (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) + INetwork *g_network = 0; FILE* randLog = 0; diff --git a/flow/flow.h b/flow/flow.h index 8890ad13b8..b58916ea44 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -560,10 +560,8 @@ public: cb->insertChain(this); } - virtual void unwait() { - delFutureRef(); - } - virtual void fire() { ASSERT(false); } + virtual void unwait() override { delFutureRef(); } + virtual void fire(T const&) override { ASSERT(false); } }; template @@ -644,10 +642,9 @@ struct NotifiedQueue : private SingleCallback, FastAllocated ASSERT(SingleCallback::next == this); cb->insert(this); } - virtual void unwait() { - delFutureRef(); - } - virtual void fire() { ASSERT(false); } + virtual void unwait() override { delFutureRef(); } + virtual void fire(T const&) override { ASSERT(false); } + virtual void fire(T&&) override { ASSERT(false); } }; @@ -1006,12 +1003,8 @@ struct Actor { template struct ActorCallback : Callback { - virtual void fire(ValueType const& value) { - static_cast(this)->a_callback_fire(this, value); - } - virtual void error(Error e) { - static_cast(this)->a_callback_error(this, e); - } + virtual void fire(ValueType const& value) override { static_cast(this)->a_callback_fire(this, value); } + virtual void error(Error e) override { static_cast(this)->a_callback_error(this, e); } }; template diff --git a/flow/folly_memcpy.S b/flow/folly_memcpy.S new file mode 100644 index 0000000000..66361a774a --- /dev/null +++ b/flow/folly_memcpy.S @@ -0,0 +1,178 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * memcpy: An optimized memcpy implementation for x86_64. It uses AVX when + * __AVX__ is defined, and uses SSE2 otherwise. + * + * @author Bin Liu + */ + +#if defined(__x86_64__) && defined(__linux__) && !defined(__CYGWIN__) + + .file "memcpy.S" + .text + +/* + * _memcpy_short is a local helper used when length < 8. It cannot be called + * from outside, because it expects a non-standard calling convention: + * + * %rax: destination buffer address. + * %rsi: source buffer address. + * %edx: length, in the range of [0, 7] + */ + .type _memcpy_short, @function +_memcpy_short: +.LSHORT: + .cfi_startproc + // if (length == 0) return; + test %edx, %edx + jz .LEND + + movzbl (%rsi), %ecx + // if (length - 4 < 0) goto LS4; + sub $4, %edx + jb .LS4 + + mov (%rsi), %ecx + mov (%rsi, %rdx), %edi + mov %ecx, (%rax) + mov %edi, (%rax, %rdx) +.LEND: + rep + ret + nop + +.LS4: + // At this point, length can be 1 or 2 or 3, and $cl contains + // the first byte. + mov %cl, (%rax) + // if (length - 4 + 2 < 0) return; + add $2, %edx + jnc .LEND + + // length is 2 or 3 here. In either case, just copy the last + // two bytes. + movzwl (%rsi, %rdx), %ecx + mov %cx, (%rax, %rdx) + ret + + .cfi_endproc + .size _memcpy_short, .-_memcpy_short + + +/* + * void* memcpy(void* dst, void* src, uint32_t length); + * + */ + .align 16 + .globl folly_memcpy + .type folly_memcpy, @function +folly_memcpy: + .cfi_startproc + + mov %rdx, %rcx + mov %rdi, %rax + cmp $8, %rdx + jb .LSHORT + + mov -8(%rsi, %rdx), %r8 + mov (%rsi), %r9 + mov %r8, -8(%rdi, %rdx) + and $24, %rcx + jz .L32 + + mov %r9, (%rdi) + mov %rcx, %r8 + sub $16, %rcx + jb .LT32 +#ifndef __AVX__ + movdqu (%rsi, %rcx), %xmm1 + movdqu %xmm1, (%rdi, %rcx) +#else + vmovdqu (%rsi, %rcx), %xmm1 + vmovdqu %xmm1, (%rdi, %rcx) +#endif + // Test if there are 32-byte groups +.LT32: + add %r8, %rsi + and $-32, %rdx + jnz .L32_adjDI + ret + + .align 16 +.L32_adjDI: + add %r8, %rdi +.L32: +#ifndef __AVX__ + movdqu (%rsi), %xmm0 + movdqu 16(%rsi), %xmm1 +#else + vmovdqu (%rsi), %ymm0 +#endif + shr $6, %rdx + jnc .L64_32read +#ifndef __AVX__ + movdqu %xmm0, (%rdi) + movdqu %xmm1, 16(%rdi) +#else + vmovdqu %ymm0, (%rdi) +#endif + lea 32(%rsi), %rsi + jnz .L64_adjDI +#ifdef __AVX__ + vzeroupper +#endif + ret + +.L64_adjDI: + add $32, %rdi + +.L64: +#ifndef __AVX__ + movdqu (%rsi), %xmm0 + movdqu 16(%rsi), %xmm1 +#else + vmovdqu (%rsi), %ymm0 +#endif + +.L64_32read: +#ifndef __AVX__ + movdqu 32(%rsi), %xmm2 + movdqu 48(%rsi), %xmm3 + add $64, %rsi + movdqu %xmm0, (%rdi) + movdqu %xmm1, 16(%rdi) + movdqu %xmm2, 32(%rdi) + movdqu %xmm3, 48(%rdi) +#else + vmovdqu 32(%rsi), %ymm1 + add $64, %rsi + vmovdqu %ymm0, (%rdi) + vmovdqu %ymm1, 32(%rdi) +#endif + add $64, %rdi + dec %rdx + jnz .L64 +#ifdef __AVX__ + vzeroupper +#endif + ret + + .cfi_endproc + .size folly_memcpy, .-folly_memcpy + +#endif diff --git a/flow/folly_memcpy.h b/flow/folly_memcpy.h new file mode 100644 index 0000000000..9b74507a0d --- /dev/null +++ b/flow/folly_memcpy.h @@ -0,0 +1,33 @@ +/* + * flow.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLOW_FOLLY_MEMCPY_H +#define FLOW_FOLLY_MEMCPY_H +#pragma once + +#if (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) + +extern "C" { + void* folly_memcpy(void* dst, const void* src, uint32_t length); +} + +#endif // linux or bsd and avx + +#endif \ No newline at end of file diff --git a/flow/rte_memcpy.h b/flow/rte_memcpy.h new file mode 100644 index 0000000000..e5986e6500 --- /dev/null +++ b/flow/rte_memcpy.h @@ -0,0 +1,913 @@ +/* +SPDX-License-Identifier: BSD-3-Clause +Copyright(c) 2010-2014 Intel Corporation + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _RTE_MEMCPY_X86_64_H_ +#define _RTE_MEMCPY_X86_64_H_ + +/** + * @file + * + * Functions for SSE/AVX/AVX2/AVX512 implementation of memcpy(). + */ + +#include +#include +#include + +#include + +#if (defined (__linux__) || defined (__FreeBSD__)) && defined(__AVX__) + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Copy bytes from one location to another. The locations must not overlap. + * + * @note This is implemented as a macro, so it's address should not be taken + * and care is needed as parameter expressions may be evaluated multiple times. + * + * @param dst + * Pointer to the destination of the data. + * @param src + * Pointer to the source data. + * @param n + * Number of bytes to copy. + * @return + * Pointer to the destination data. + */ +static force_inline void * +rte_memcpy(void *dst, const void *src, size_t n); + +#ifdef __AVX512F__ +#define RTE_MACHINE_CPUFLAG_AVX512F +#elif defined(__AVX__) +#define RTE_MACHINE_CPUFLAG_AVX2 +#endif + +#ifdef RTE_MACHINE_CPUFLAG_AVX512F + +#define ALIGNMENT_MASK 0x3F + +/** + * AVX512 implementation below + */ + +/** + * Copy 16 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i xmm0; + + xmm0 = _mm_loadu_si128((const __m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm0); +} + +/** + * Copy 32 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + __m256i ymm0; + + ymm0 = _mm256_loadu_si256((const __m256i *)src); + _mm256_storeu_si256((__m256i *)dst, ymm0); +} + +/** + * Copy 64 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + __m512i zmm0; + + zmm0 = _mm512_loadu_si512((const void *)src); + _mm512_storeu_si512((void *)dst, zmm0); +} + +/** + * Copy 128 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + rte_mov64(dst + 0 * 64, src + 0 * 64); + rte_mov64(dst + 1 * 64, src + 1 * 64); +} + +/** + * Copy 256 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov256(uint8_t *dst, const uint8_t *src) +{ + rte_mov64(dst + 0 * 64, src + 0 * 64); + rte_mov64(dst + 1 * 64, src + 1 * 64); + rte_mov64(dst + 2 * 64, src + 2 * 64); + rte_mov64(dst + 3 * 64, src + 3 * 64); +} + +/** + * Copy 128-byte blocks from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + __m512i zmm0, zmm1; + + while (n >= 128) { + zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64)); + n -= 128; + zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64)); + src = src + 128; + _mm512_storeu_si512((void *)(dst + 0 * 64), zmm0); + _mm512_storeu_si512((void *)(dst + 1 * 64), zmm1); + dst = dst + 128; + } +} + +/** + * Copy 512-byte blocks from one location to another, + * locations should not overlap. + */ +static inline void +rte_mov512blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + __m512i zmm0, zmm1, zmm2, zmm3, zmm4, zmm5, zmm6, zmm7; + + while (n >= 512) { + zmm0 = _mm512_loadu_si512((const void *)(src + 0 * 64)); + n -= 512; + zmm1 = _mm512_loadu_si512((const void *)(src + 1 * 64)); + zmm2 = _mm512_loadu_si512((const void *)(src + 2 * 64)); + zmm3 = _mm512_loadu_si512((const void *)(src + 3 * 64)); + zmm4 = _mm512_loadu_si512((const void *)(src + 4 * 64)); + zmm5 = _mm512_loadu_si512((const void *)(src + 5 * 64)); + zmm6 = _mm512_loadu_si512((const void *)(src + 6 * 64)); + zmm7 = _mm512_loadu_si512((const void *)(src + 7 * 64)); + src = src + 512; + _mm512_storeu_si512((void *)(dst + 0 * 64), zmm0); + _mm512_storeu_si512((void *)(dst + 1 * 64), zmm1); + _mm512_storeu_si512((void *)(dst + 2 * 64), zmm2); + _mm512_storeu_si512((void *)(dst + 3 * 64), zmm3); + _mm512_storeu_si512((void *)(dst + 4 * 64), zmm4); + _mm512_storeu_si512((void *)(dst + 5 * 64), zmm5); + _mm512_storeu_si512((void *)(dst + 6 * 64), zmm6); + _mm512_storeu_si512((void *)(dst + 7 * 64), zmm7); + dst = dst + 512; + } +} + +static force_inline void * +rte_memcpy_generic(void *dst, const void *src, size_t n) +{ + uintptr_t dstu = (uintptr_t)dst; + uintptr_t srcu = (uintptr_t)src; + void *ret = dst; + size_t dstofss; + size_t bits; + + /** + * Copy less than 16 bytes + */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dstu = *(const uint8_t *)srcu; + srcu = (uintptr_t)((const uint8_t *)srcu + 1); + dstu = (uintptr_t)((uint8_t *)dstu + 1); + } + if (n & 0x02) { + *(uint16_t *)dstu = *(const uint16_t *)srcu; + srcu = (uintptr_t)((const uint16_t *)srcu + 1); + dstu = (uintptr_t)((uint16_t *)dstu + 1); + } + if (n & 0x04) { + *(uint32_t *)dstu = *(const uint32_t *)srcu; + srcu = (uintptr_t)((const uint32_t *)srcu + 1); + dstu = (uintptr_t)((uint32_t *)dstu + 1); + } + if (n & 0x08) + *(uint64_t *)dstu = *(const uint64_t *)srcu; + return ret; + } + + /** + * Fast way when copy size doesn't exceed 512 bytes + */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + if (n <= 512) { + if (n >= 256) { + n -= 256; + rte_mov256((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 256; + dst = (uint8_t *)dst + 256; + } + if (n >= 128) { + n -= 128; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } +COPY_BLOCK_128_BACK63: + if (n > 64) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + return ret; + } + if (n > 0) + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + return ret; + } + + /** + * Make store aligned when copy size exceeds 512 bytes + */ + dstofss = ((uintptr_t)dst & 0x3F); + if (dstofss > 0) { + dstofss = 64 - dstofss; + n -= dstofss; + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + + /** + * Copy 512-byte blocks. + * Use copy block function for better instruction order control, + * which is important when load is unaligned. + */ + rte_mov512blocks((uint8_t *)dst, (const uint8_t *)src, n); + bits = n; + n = n & 511; + bits -= n; + src = (const uint8_t *)src + bits; + dst = (uint8_t *)dst + bits; + + /** + * Copy 128-byte blocks. + * Use copy block function for better instruction order control, + * which is important when load is unaligned. + */ + if (n >= 128) { + rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n); + bits = n; + n = n & 127; + bits -= n; + src = (const uint8_t *)src + bits; + dst = (uint8_t *)dst + bits; + } + + /** + * Copy whatever left + */ + goto COPY_BLOCK_128_BACK63; +} + +#elif defined RTE_MACHINE_CPUFLAG_AVX2 + +#define ALIGNMENT_MASK 0x1F + +/** + * AVX2 implementation below + */ + +/** + * Copy 16 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i xmm0; + + xmm0 = _mm_loadu_si128((const __m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm0); +} + +/** + * Copy 32 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + __m256i ymm0; + + ymm0 = _mm256_loadu_si256((const __m256i *)src); + _mm256_storeu_si256((__m256i *)dst, ymm0); +} + +/** + * Copy 64 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); + rte_mov32((uint8_t *)dst + 1 * 32, (const uint8_t *)src + 1 * 32); +} + +/** + * Copy 128 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + rte_mov32((uint8_t *)dst + 0 * 32, (const uint8_t *)src + 0 * 32); + rte_mov32((uint8_t *)dst + 1 * 32, (const uint8_t *)src + 1 * 32); + rte_mov32((uint8_t *)dst + 2 * 32, (const uint8_t *)src + 2 * 32); + rte_mov32((uint8_t *)dst + 3 * 32, (const uint8_t *)src + 3 * 32); +} + +/** + * Copy 128-byte blocks from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov128blocks(uint8_t *dst, const uint8_t *src, size_t n) +{ + __m256i ymm0, ymm1, ymm2, ymm3; + + while (n >= 128) { + ymm0 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 0 * 32)); + n -= 128; + ymm1 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 1 * 32)); + ymm2 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 2 * 32)); + ymm3 = _mm256_loadu_si256((const __m256i *)((const uint8_t *)src + 3 * 32)); + src = (const uint8_t *)src + 128; + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 0 * 32), ymm0); + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 1 * 32), ymm1); + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 2 * 32), ymm2); + _mm256_storeu_si256((__m256i *)((uint8_t *)dst + 3 * 32), ymm3); + dst = (uint8_t *)dst + 128; + } +} + +static force_inline void * +rte_memcpy_generic(void *dst, const void *src, size_t n) +{ + uintptr_t dstu = (uintptr_t)dst; + uintptr_t srcu = (uintptr_t)src; + void *ret = dst; + size_t dstofss; + size_t bits; + + /** + * Copy less than 16 bytes + */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dstu = *(const uint8_t *)srcu; + srcu = (uintptr_t)((const uint8_t *)srcu + 1); + dstu = (uintptr_t)((uint8_t *)dstu + 1); + } + if (n & 0x02) { + *(uint16_t *)dstu = *(const uint16_t *)srcu; + srcu = (uintptr_t)((const uint16_t *)srcu + 1); + dstu = (uintptr_t)((uint16_t *)dstu + 1); + } + if (n & 0x04) { + *(uint32_t *)dstu = *(const uint32_t *)srcu; + srcu = (uintptr_t)((const uint32_t *)srcu + 1); + dstu = (uintptr_t)((uint32_t *)dstu + 1); + } + if (n & 0x08) { + *(uint64_t *)dstu = *(const uint64_t *)srcu; + } + return ret; + } + + /** + * Fast way when copy size doesn't exceed 256 bytes + */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 48) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst + 16, (const uint8_t *)src + 16); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + if (n <= 256) { + if (n >= 128) { + n -= 128; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } +COPY_BLOCK_128_BACK31: + if (n >= 64) { + n -= 64; + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 64; + dst = (uint8_t *)dst + 64; + } + if (n > 32) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + return ret; + } + if (n > 0) { + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + } + return ret; + } + + /** + * Make store aligned when copy size exceeds 256 bytes + */ + dstofss = (uintptr_t)dst & 0x1F; + if (dstofss > 0) { + dstofss = 32 - dstofss; + n -= dstofss; + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + + /** + * Copy 128-byte blocks + */ + rte_mov128blocks((uint8_t *)dst, (const uint8_t *)src, n); + bits = n; + n = n & 127; + bits -= n; + src = (const uint8_t *)src + bits; + dst = (uint8_t *)dst + bits; + + /** + * Copy whatever left + */ + goto COPY_BLOCK_128_BACK31; +} + +#else /* RTE_MACHINE_CPUFLAG */ + +#define ALIGNMENT_MASK 0x0F + +/** + * SSE & AVX implementation below + */ + +/** + * Copy 16 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov16(uint8_t *dst, const uint8_t *src) +{ + __m128i xmm0; + + xmm0 = _mm_loadu_si128((const __m128i *)(const __m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm0); +} + +/** + * Copy 32 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov32(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); +} + +/** + * Copy 64 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov64(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); + rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16); + rte_mov16((uint8_t *)dst + 3 * 16, (const uint8_t *)src + 3 * 16); +} + +/** + * Copy 128 bytes from one location to another, + * locations should not overlap. + */ +static force_inline void +rte_mov128(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); + rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16); + rte_mov16((uint8_t *)dst + 3 * 16, (const uint8_t *)src + 3 * 16); + rte_mov16((uint8_t *)dst + 4 * 16, (const uint8_t *)src + 4 * 16); + rte_mov16((uint8_t *)dst + 5 * 16, (const uint8_t *)src + 5 * 16); + rte_mov16((uint8_t *)dst + 6 * 16, (const uint8_t *)src + 6 * 16); + rte_mov16((uint8_t *)dst + 7 * 16, (const uint8_t *)src + 7 * 16); +} + +/** + * Copy 256 bytes from one location to another, + * locations should not overlap. + */ +static inline void +rte_mov256(uint8_t *dst, const uint8_t *src) +{ + rte_mov16((uint8_t *)dst + 0 * 16, (const uint8_t *)src + 0 * 16); + rte_mov16((uint8_t *)dst + 1 * 16, (const uint8_t *)src + 1 * 16); + rte_mov16((uint8_t *)dst + 2 * 16, (const uint8_t *)src + 2 * 16); + rte_mov16((uint8_t *)dst + 3 * 16, (const uint8_t *)src + 3 * 16); + rte_mov16((uint8_t *)dst + 4 * 16, (const uint8_t *)src + 4 * 16); + rte_mov16((uint8_t *)dst + 5 * 16, (const uint8_t *)src + 5 * 16); + rte_mov16((uint8_t *)dst + 6 * 16, (const uint8_t *)src + 6 * 16); + rte_mov16((uint8_t *)dst + 7 * 16, (const uint8_t *)src + 7 * 16); + rte_mov16((uint8_t *)dst + 8 * 16, (const uint8_t *)src + 8 * 16); + rte_mov16((uint8_t *)dst + 9 * 16, (const uint8_t *)src + 9 * 16); + rte_mov16((uint8_t *)dst + 10 * 16, (const uint8_t *)src + 10 * 16); + rte_mov16((uint8_t *)dst + 11 * 16, (const uint8_t *)src + 11 * 16); + rte_mov16((uint8_t *)dst + 12 * 16, (const uint8_t *)src + 12 * 16); + rte_mov16((uint8_t *)dst + 13 * 16, (const uint8_t *)src + 13 * 16); + rte_mov16((uint8_t *)dst + 14 * 16, (const uint8_t *)src + 14 * 16); + rte_mov16((uint8_t *)dst + 15 * 16, (const uint8_t *)src + 15 * 16); +} + +/** + * Macro for copying unaligned block from one location to another with constant load offset, + * 47 bytes leftover maximum, + * locations should not overlap. + * Requirements: + * - Store is aligned + * - Load offset is , which must be immediate value within [1, 15] + * - For , make sure bit backwards & <16 - offset> bit forwards are available for loading + * - , , must be variables + * - __m128i ~ must be pre-defined + */ +#define MOVEUNALIGNED_LEFT47_IMM(dst, src, len, offset) \ +__extension__ ({ \ + size_t tmp; \ + while (len >= 128 + 16 - offset) { \ + xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16)); \ + len -= 128; \ + xmm1 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 1 * 16)); \ + xmm2 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 2 * 16)); \ + xmm3 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 3 * 16)); \ + xmm4 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 4 * 16)); \ + xmm5 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 5 * 16)); \ + xmm6 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 6 * 16)); \ + xmm7 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 7 * 16)); \ + xmm8 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 8 * 16)); \ + src = (const uint8_t *)src + 128; \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 0 * 16), _mm_alignr_epi8(xmm1, xmm0, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 1 * 16), _mm_alignr_epi8(xmm2, xmm1, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 2 * 16), _mm_alignr_epi8(xmm3, xmm2, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 3 * 16), _mm_alignr_epi8(xmm4, xmm3, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 4 * 16), _mm_alignr_epi8(xmm5, xmm4, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 5 * 16), _mm_alignr_epi8(xmm6, xmm5, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 6 * 16), _mm_alignr_epi8(xmm7, xmm6, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 7 * 16), _mm_alignr_epi8(xmm8, xmm7, offset)); \ + dst = (uint8_t *)dst + 128; \ + } \ + tmp = len; \ + len = ((len - 16 + offset) & 127) + 16 - offset; \ + tmp -= len; \ + src = (const uint8_t *)src + tmp; \ + dst = (uint8_t *)dst + tmp; \ + if (len >= 32 + 16 - offset) { \ + while (len >= 32 + 16 - offset) { \ + xmm0 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 0 * 16)); \ + len -= 32; \ + xmm1 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 1 * 16)); \ + xmm2 = _mm_loadu_si128((const __m128i *)((const uint8_t *)src - offset + 2 * 16)); \ + src = (const uint8_t *)src + 32; \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 0 * 16), _mm_alignr_epi8(xmm1, xmm0, offset)); \ + _mm_storeu_si128((__m128i *)((uint8_t *)dst + 1 * 16), _mm_alignr_epi8(xmm2, xmm1, offset)); \ + dst = (uint8_t *)dst + 32; \ + } \ + tmp = len; \ + len = ((len - 16 + offset) & 31) + 16 - offset; \ + tmp -= len; \ + src = (const uint8_t *)src + tmp; \ + dst = (uint8_t *)dst + tmp; \ + } \ +}) + +/** + * Macro for copying unaligned block from one location to another, + * 47 bytes leftover maximum, + * locations should not overlap. + * Use switch here because the aligning instruction requires immediate value for shift count. + * Requirements: + * - Store is aligned + * - Load offset is , which must be within [1, 15] + * - For , make sure bit backwards & <16 - offset> bit forwards are available for loading + * - , , must be variables + * - __m128i ~ used in MOVEUNALIGNED_LEFT47_IMM must be pre-defined + */ +#define MOVEUNALIGNED_LEFT47(dst, src, len, offset) \ +__extension__ ({ \ + switch (offset) { \ + case 0x01: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x01); break; \ + case 0x02: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x02); break; \ + case 0x03: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x03); break; \ + case 0x04: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x04); break; \ + case 0x05: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x05); break; \ + case 0x06: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x06); break; \ + case 0x07: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x07); break; \ + case 0x08: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x08); break; \ + case 0x09: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x09); break; \ + case 0x0A: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0A); break; \ + case 0x0B: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0B); break; \ + case 0x0C: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0C); break; \ + case 0x0D: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0D); break; \ + case 0x0E: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0E); break; \ + case 0x0F: MOVEUNALIGNED_LEFT47_IMM(dst, src, n, 0x0F); break; \ + default:; \ + } \ +}) + +static force_inline void * +rte_memcpy_generic(void *dst, const void *src, size_t n) +{ + __m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8; + uintptr_t dstu = (uintptr_t)dst; + uintptr_t srcu = (uintptr_t)src; + void *ret = dst; + size_t dstofss; + size_t srcofs; + + /** + * Copy less than 16 bytes + */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dstu = *(const uint8_t *)srcu; + srcu = (uintptr_t)((const uint8_t *)srcu + 1); + dstu = (uintptr_t)((uint8_t *)dstu + 1); + } + if (n & 0x02) { + *(uint16_t *)dstu = *(const uint16_t *)srcu; + srcu = (uintptr_t)((const uint16_t *)srcu + 1); + dstu = (uintptr_t)((uint16_t *)dstu + 1); + } + if (n & 0x04) { + *(uint32_t *)dstu = *(const uint32_t *)srcu; + srcu = (uintptr_t)((const uint32_t *)srcu + 1); + dstu = (uintptr_t)((uint32_t *)dstu + 1); + } + if (n & 0x08) { + *(uint64_t *)dstu = *(const uint64_t *)srcu; + } + return ret; + } + + /** + * Fast way when copy size doesn't exceed 512 bytes + */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 48) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst + 32, (const uint8_t *)src + 32); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n <= 128) { + goto COPY_BLOCK_128_BACK15; + } + if (n <= 512) { + if (n >= 256) { + n -= 256; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + rte_mov128((uint8_t *)dst + 128, (const uint8_t *)src + 128); + src = (const uint8_t *)src + 256; + dst = (uint8_t *)dst + 256; + } +COPY_BLOCK_255_BACK15: + if (n >= 128) { + n -= 128; + rte_mov128((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 128; + dst = (uint8_t *)dst + 128; + } +COPY_BLOCK_128_BACK15: + if (n >= 64) { + n -= 64; + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 64; + dst = (uint8_t *)dst + 64; + } +COPY_BLOCK_64_BACK15: + if (n >= 32) { + n -= 32; + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + 32; + dst = (uint8_t *)dst + 32; + } + if (n > 16) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + return ret; + } + if (n > 0) { + rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n); + } + return ret; + } + + /** + * Make store aligned when copy size exceeds 512 bytes, + * and make sure the first 15 bytes are copied, because + * unaligned copy functions require up to 15 bytes + * backwards access. + */ + dstofss = (uintptr_t)dst & 0x0F; + if (dstofss > 0) { + dstofss = 16 - dstofss + 16; + n -= dstofss; + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + src = (const uint8_t *)src + dstofss; + dst = (uint8_t *)dst + dstofss; + } + srcofs = ((uintptr_t)src & 0x0F); + + /** + * For aligned copy + */ + if (srcofs == 0) { + /** + * Copy 256-byte blocks + */ + for (; n >= 256; n -= 256) { + rte_mov256((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 256; + src = (const uint8_t *)src + 256; + } + + /** + * Copy whatever left + */ + goto COPY_BLOCK_255_BACK15; + } + + /** + * For copy with unaligned load + */ + MOVEUNALIGNED_LEFT47(dst, src, n, srcofs); + + /** + * Copy whatever left + */ + goto COPY_BLOCK_64_BACK15; +} + +#endif /* RTE_MACHINE_CPUFLAG */ + +static force_inline void * +rte_memcpy_aligned(void *dst, const void *src, size_t n) +{ + void *ret = dst; + + /* Copy size <= 16 bytes */ + if (n < 16) { + if (n & 0x01) { + *(uint8_t *)dst = *(const uint8_t *)src; + src = (const uint8_t *)src + 1; + dst = (uint8_t *)dst + 1; + } + if (n & 0x02) { + *(uint16_t *)dst = *(const uint16_t *)src; + src = (const uint16_t *)src + 1; + dst = (uint16_t *)dst + 1; + } + if (n & 0x04) { + *(uint32_t *)dst = *(const uint32_t *)src; + src = (const uint32_t *)src + 1; + dst = (uint32_t *)dst + 1; + } + if (n & 0x08) + *(uint64_t *)dst = *(const uint64_t *)src; + + return ret; + } + + /* Copy 16 <= size <= 32 bytes */ + if (n <= 32) { + rte_mov16((uint8_t *)dst, (const uint8_t *)src); + rte_mov16((uint8_t *)dst - 16 + n, + (const uint8_t *)src - 16 + n); + + return ret; + } + + /* Copy 32 < size <= 64 bytes */ + if (n <= 64) { + rte_mov32((uint8_t *)dst, (const uint8_t *)src); + rte_mov32((uint8_t *)dst - 32 + n, + (const uint8_t *)src - 32 + n); + + return ret; + } + + /* Copy 64 bytes blocks */ + for (; n >= 64; n -= 64) { + rte_mov64((uint8_t *)dst, (const uint8_t *)src); + dst = (uint8_t *)dst + 64; + src = (const uint8_t *)src + 64; + } + + /* Copy whatever left */ + rte_mov64((uint8_t *)dst - 64 + n, + (const uint8_t *)src - 64 + n); + + return ret; +} + +static force_inline void * +rte_memcpy(void *dst, const void *src, size_t n) +{ + if (!(((uintptr_t)dst | (uintptr_t)src) & ALIGNMENT_MASK)) + return rte_memcpy_aligned(dst, src, n); + else + return rte_memcpy_generic(dst, src, n); +} + +static inline uint64_t +rte_rdtsc(void) +{ + union { + uint64_t tsc_64; + struct { + uint32_t lo_32; + uint32_t hi_32; + }; + } tsc; + + asm volatile("rdtsc" : + "=a" (tsc.lo_32), + "=d" (tsc.hi_32)); + return tsc.tsc_64; +} + +#ifdef __cplusplus +} +#endif + +#endif /* defined (__linux__) || defined (__FreeBSD__) */ + +#endif /* _RTE_MEMCPY_X86_64_H_ */ diff --git a/flow/test_memcpy.cpp b/flow/test_memcpy.cpp new file mode 100644 index 0000000000..3d8c408205 --- /dev/null +++ b/flow/test_memcpy.cpp @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation + */ + +#include +#include +#include +#include + +#include "flow/folly_memcpy.h" +#include "flow/rte_memcpy.h" +#include "flow/IRandom.h" + +#include "flow/UnitTest.h" + +/* + * Set this to the maximum buffer size you want to test. If it is 0, then the + * values in the buf_sizes[] array below will be used. + */ +#define TEST_VALUE_RANGE 0 + +/* List of buffer sizes to test */ +#if TEST_VALUE_RANGE == 0 +static size_t buf_sizes[] = { + 0, 1, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, + 256, 257, 320, 384, 511, 512, 513, 1023, 1024, 1025, 1518, 1522, 1600, + 2048, 3072, 4096, 5120, 6144, 7168, 8192 +}; +/* MUST be as large as largest packet size above */ +#define SMALL_BUFFER_SIZE 8192 +#else /* TEST_VALUE_RANGE != 0 */ +static size_t buf_sizes[TEST_VALUE_RANGE]; +#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE +#endif /* TEST_VALUE_RANGE == 0 */ + +/* Data is aligned on this many bytes (power of 2) */ +#define ALIGNMENT_UNIT 32 + + +/* + * Create two buffers, and initialise one with random values. These are copied + * to the second buffer and then compared to see if the copy was successful. + * The bytes outside the copied area are also checked to make sure they were not + * changed. + */ +static int +test_single_memcpy(unsigned int off_src, unsigned int off_dst, size_t size) +{ + unsigned int i; + uint8_t dest[SMALL_BUFFER_SIZE + ALIGNMENT_UNIT]; + uint8_t src[SMALL_BUFFER_SIZE + ALIGNMENT_UNIT]; + void * ret; + + /* Setup buffers */ + for (i = 0; i < SMALL_BUFFER_SIZE + ALIGNMENT_UNIT; i++) { + dest[i] = 0; + src[i] = (uint8_t) deterministicRandom()->randomUInt32(); + } + + /* Do the copy */ + ret = memcpy(dest + off_dst, src + off_src, size); + if (ret != (dest + off_dst)) { + printf("memcpy() returned %p, not %p\n", + ret, dest + off_dst); + } + + /* Check nothing before offset is affected */ + for (i = 0; i < off_dst; i++) { + if (dest[i] != 0) { + printf("memcpy() failed for %u bytes (offsets=%u,%u): " + "[modified before start of dst].\n", + (unsigned)size, off_src, off_dst); + return -1; + } + } + + /* Check everything was copied */ + for (i = 0; i < size; i++) { + if (dest[i + off_dst] != src[i + off_src]) { + printf("memcpy() failed for %u bytes (offsets=%u,%u): " + "[didn't copy byte %u].\n", + (unsigned)size, off_src, off_dst, i); + return -1; + } + } + + /* Check nothing after copy was affected */ + for (i = size; i < SMALL_BUFFER_SIZE; i++) { + if (dest[i + off_dst] != 0) { + printf("memcpy() failed for %u bytes (offsets=%u,%u): " + "[copied too many].\n", + (unsigned)size, off_src, off_dst); + return -1; + } + } + return 0; +} + +/* + * Check functionality for various buffer sizes and data offsets/alignments. + */ +TEST_CASE("/rte/memcpy") { + unsigned int off_src, off_dst, i; + unsigned int num_buf_sizes = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + int ret; + + for (off_src = 0; off_src < ALIGNMENT_UNIT; off_src++) { + for (off_dst = 0; off_dst < ALIGNMENT_UNIT; off_dst++) { + for (i = 0; i < num_buf_sizes; i++) { + ret = test_single_memcpy(off_src, off_dst, + buf_sizes[i]); + ASSERT(ret == 0); + } + } + } + return Void(); +} + +void forceLinkMemcpyTests() { } \ No newline at end of file diff --git a/flow/test_memcpy_perf.cpp b/flow/test_memcpy_perf.cpp new file mode 100644 index 0000000000..b51463534e --- /dev/null +++ b/flow/test_memcpy_perf.cpp @@ -0,0 +1,357 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation + */ + +#include +#include +#include +#include + +#include "flow/rte_memcpy.h" +#include "flow/IRandom.h" +#include "flow/UnitTest.h" +#include "flow/flow.h" + +#if (defined (__linux__) || defined (__FreeBSD__)) && defined (__AVX__) +extern "C" { + void* folly_memcpy(void* dst, const void* src, uint32_t length); +} + + +void * rte_memcpy_noinline(void* dst, const void* src, size_t length); // for performance comparisons + +/* + * Set this to the maximum buffer size you want to test. If it is 0, then the + * values in the buf_sizes[] array below will be used. + */ +#define TEST_VALUE_RANGE 0 + +/* List of buffer sizes to test */ +#if TEST_VALUE_RANGE == 0 +static size_t buf_sizes[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128, + 129, 191, 192, 193, 255, 256, 257, 319, 320, 321, 383, 384, 385, 447, 448, + 449, 511, 512, 513, 767, 768, 769, 1023, 1024, 1025, 1518, 1522, 1536, 1600, + 2048, 2560, 3072, 3584, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 +}; +/* MUST be as large as largest packet size above */ +#define SMALL_BUFFER_SIZE 8192 +#else /* TEST_VALUE_RANGE != 0 */ +static size_t buf_sizes[TEST_VALUE_RANGE]; +#define SMALL_BUFFER_SIZE TEST_VALUE_RANGE +#endif /* TEST_VALUE_RANGE == 0 */ + + +/* + * Arrays of this size are used for measuring uncached memory accesses by + * picking a random location within the buffer. Make this smaller if there are + * memory allocation errors. + */ +#define LARGE_BUFFER_SIZE (100 * 1024 * 1024) + +/* How many times to run timing loop for performance tests */ +#define TEST_ITERATIONS 1000000 +#define TEST_BATCH_SIZE 100 + +/* Data is aligned on this many bytes (power of 2) */ +// #ifdef RTE_MACHINE_CPUFLAG_AVX512F +#define ALIGNMENT_UNIT 64 +// #elif defined RTE_MACHINE_CPUFLAG_AVX2 +// #define ALIGNMENT_UNIT 32 +// #else /* RTE_MACHINE_CPUFLAG */ +// #define ALIGNMENT_UNIT 16 +// #endif /* RTE_MACHINE_CPUFLAG */ + +/* + * Pointers used in performance tests. The two large buffers are for uncached + * access where random addresses within the buffer are used for each + * memcpy. The two small buffers are for cached access. + */ +static uint8_t *large_buf_read, *large_buf_write; +static uint8_t *small_buf_read, *small_buf_write; + +static size_t round_up(size_t sz, size_t alignment) { + return (((sz - 1) / alignment) + 1) * alignment; +} + +static uint8_t * rte_malloc(char const * ignored, size_t sz, size_t align) { + return (uint8_t*) aligned_alloc(align, round_up(sz, align)); +} + +static void rte_free(void * ptr) { + if (!!ptr) { + free(ptr); + } +} + +/* Initialise data buffers. */ +static int +init_buffers(void) +{ + unsigned i; + + large_buf_read = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (large_buf_read == NULL) + goto error_large_buf_read; + + large_buf_write = rte_malloc("memcpy", LARGE_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (large_buf_write == NULL) + goto error_large_buf_write; + + small_buf_read = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (small_buf_read == NULL) + goto error_small_buf_read; + + small_buf_write = rte_malloc("memcpy", SMALL_BUFFER_SIZE + ALIGNMENT_UNIT, ALIGNMENT_UNIT); + if (small_buf_write == NULL) + goto error_small_buf_write; + + for (i = 0; i < LARGE_BUFFER_SIZE; i++) + large_buf_read[i] = deterministicRandom()->randomUInt32(); + for (i = 0; i < SMALL_BUFFER_SIZE; i++) + small_buf_read[i] = deterministicRandom()->randomUInt32(); + + return 0; + +error_small_buf_write: + rte_free(small_buf_read); +error_small_buf_read: + rte_free(large_buf_write); +error_large_buf_write: + rte_free(large_buf_read); +error_large_buf_read: + printf("ERROR: not enough memory\n"); + return -1; +} + +/* Cleanup data buffers */ +static void +free_buffers(void) +{ + rte_free(large_buf_read); + rte_free(large_buf_write); + rte_free(small_buf_read); + rte_free(small_buf_write); +} + +/* + * Get a random offset into large array, with enough space needed to perform + * max copy size. Offset is aligned, uoffset is used for unalignment setting. + */ +static inline size_t +get_rand_offset(size_t uoffset) +{ + return ((deterministicRandom()->randomUInt32() % (LARGE_BUFFER_SIZE - SMALL_BUFFER_SIZE)) & + ~(ALIGNMENT_UNIT - 1)) + uoffset; +} + +/* Fill in source and destination addresses. */ +static inline void +fill_addr_arrays(size_t *dst_addr, int is_dst_cached, size_t dst_uoffset, + size_t *src_addr, int is_src_cached, size_t src_uoffset) +{ + unsigned int i; + + for (i = 0; i < TEST_BATCH_SIZE; i++) { + dst_addr[i] = (is_dst_cached) ? dst_uoffset : get_rand_offset(dst_uoffset); + src_addr[i] = (is_src_cached) ? src_uoffset : get_rand_offset(src_uoffset); + } +} + +/* + * WORKAROUND: For some reason the first test doing an uncached write + * takes a very long time (~25 times longer than is expected). So we do + * it once without timing. + */ +static void +do_uncached_write(uint8_t *dst, int is_dst_cached, + const uint8_t *src, int is_src_cached, size_t size) +{ + unsigned i, j; + size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE]; + + for (i = 0; i < (TEST_ITERATIONS / TEST_BATCH_SIZE); i++) { + fill_addr_arrays(dst_addrs, is_dst_cached, 0, + src_addrs, is_src_cached, 0); + for (j = 0; j < TEST_BATCH_SIZE; j++) { + memcpy(dst+dst_addrs[j], src+src_addrs[j], size); + } + } +} + +/* + * Run a single memcpy performance test. This is a macro to ensure that if + * the "size" parameter is a constant it won't be converted to a variable. + */ +#define SINGLE_PERF_TEST(dst, is_dst_cached, dst_uoffset, \ + src, is_src_cached, src_uoffset, size) \ +do { \ + unsigned int iter, t; \ + size_t dst_addrs[TEST_BATCH_SIZE], src_addrs[TEST_BATCH_SIZE]; \ + uint64_t start_time, total_time = 0; \ + uint64_t total_time2 = 0; \ + for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ + fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, \ + src_addrs, is_src_cached, src_uoffset); \ + start_time = rte_rdtsc(); \ + for (t = 0; t < TEST_BATCH_SIZE; t++) \ + rte_memcpy_noinline(dst+dst_addrs[t], src+src_addrs[t], size); \ + total_time += rte_rdtsc() - start_time; \ + } \ + for (iter = 0; iter < (TEST_ITERATIONS / TEST_BATCH_SIZE); iter++) { \ + fill_addr_arrays(dst_addrs, is_dst_cached, dst_uoffset, \ + src_addrs, is_src_cached, src_uoffset); \ + start_time = rte_rdtsc(); \ + for (t = 0; t < TEST_BATCH_SIZE; t++) \ + memcpy(dst+dst_addrs[t], src+src_addrs[t], size); \ + total_time2 += rte_rdtsc() - start_time; \ + } \ + printf("%3.0f -", (double)total_time / TEST_ITERATIONS); \ + printf("%3.0f", (double)total_time2 / TEST_ITERATIONS); \ + printf("(%6.2f%%) ", ((double)total_time - total_time2)*100/total_time2); \ +} while (0) + +/* Run aligned memcpy tests for each cached/uncached permutation */ +#define ALL_PERF_TESTS_FOR_SIZE(n) \ +do { \ + if (__builtin_constant_p(n)) \ + printf("\nC%6u", (unsigned)n); \ + else \ + printf("\n%7u", (unsigned)n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 0, small_buf_read, 1, 0, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 0, small_buf_read, 1, 0, n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 0, large_buf_read, 0, 0, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 0, large_buf_read, 0, 0, n); \ +} while (0) + +/* Run unaligned memcpy tests for each cached/uncached permutation */ +#define ALL_PERF_TESTS_FOR_SIZE_UNALIGNED(n) \ +do { \ + if (__builtin_constant_p(n)) \ + printf("\nC%6u", (unsigned)n); \ + else \ + printf("\n%7u", (unsigned)n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 1, small_buf_read, 1, 5, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 1, small_buf_read, 1, 5, n); \ + SINGLE_PERF_TEST(small_buf_write, 1, 1, large_buf_read, 0, 5, n); \ + SINGLE_PERF_TEST(large_buf_write, 0, 1, large_buf_read, 0, 5, n); \ +} while (0) + +/* Run memcpy tests for constant length */ +#define ALL_PERF_TEST_FOR_CONSTANT \ +do { \ + TEST_CONSTANT(6U); TEST_CONSTANT(64U); TEST_CONSTANT(128U); \ + TEST_CONSTANT(192U); TEST_CONSTANT(256U); TEST_CONSTANT(512U); \ + TEST_CONSTANT(768U); TEST_CONSTANT(1024U); TEST_CONSTANT(1536U); \ +} while (0) + +/* Run all memcpy tests for aligned constant cases */ +static inline void +perf_test_constant_aligned(void) +{ +#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE + ALL_PERF_TEST_FOR_CONSTANT; +#undef TEST_CONSTANT +} + +/* Run all memcpy tests for unaligned constant cases */ +static inline void +perf_test_constant_unaligned(void) +{ +#define TEST_CONSTANT ALL_PERF_TESTS_FOR_SIZE_UNALIGNED + ALL_PERF_TEST_FOR_CONSTANT; +#undef TEST_CONSTANT +} + +/* Run all memcpy tests for aligned variable cases */ +static inline void +perf_test_variable_aligned(void) +{ + unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + unsigned i; + for (i = 0; i < n; i++) { + ALL_PERF_TESTS_FOR_SIZE((size_t)buf_sizes[i]); + } +} + +/* Run all memcpy tests for unaligned variable cases */ +static inline void +perf_test_variable_unaligned(void) +{ + unsigned n = sizeof(buf_sizes) / sizeof(buf_sizes[0]); + unsigned i; + for (i = 0; i < n; i++) { + ALL_PERF_TESTS_FOR_SIZE_UNALIGNED((size_t)buf_sizes[i]); + } +} + +/* Run all memcpy tests */ +TEST_CASE("performance/memcpy/rte") { + int ret; + struct timeval tv_begin, tv_end; + double time_aligned, time_unaligned; + double time_aligned_const, time_unaligned_const; + + ret = init_buffers(); + ASSERT(ret == 0); + +#if TEST_VALUE_RANGE != 0 + /* Set up buf_sizes array, if required */ + unsigned i; + for (i = 0; i < TEST_VALUE_RANGE; i++) + buf_sizes[i] = i; +#endif + + /* See function comment */ + do_uncached_write(large_buf_write, 0, small_buf_read, 1, SMALL_BUFFER_SIZE); + + printf("\n** rte_memcpy() - memcpy perf. tests (C = compile-time constant) **\n" + "======= ================= ================= ================= =================\n" + " Size Cache to cache Cache to mem Mem to cache Mem to mem\n" + "(bytes) (ticks) (ticks) (ticks) (ticks)\n" + "------- ----------------- ----------------- ----------------- -----------------"); + + printf("\n================================= %2dB aligned =================================", + ALIGNMENT_UNIT); + /* Do aligned tests where size is a variable */ + gettimeofday(&tv_begin, NULL); + perf_test_variable_aligned(); + gettimeofday(&tv_end, NULL); + time_aligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n------- ----------------- ----------------- ----------------- -----------------"); + /* Do aligned tests where size is a compile-time constant */ + gettimeofday(&tv_begin, NULL); + perf_test_constant_aligned(); + gettimeofday(&tv_end, NULL); + time_aligned_const = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n================================== Unaligned =================================="); + /* Do unaligned tests where size is a variable */ + gettimeofday(&tv_begin, NULL); + perf_test_variable_unaligned(); + gettimeofday(&tv_end, NULL); + time_unaligned = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n------- ----------------- ----------------- ----------------- -----------------"); + /* Do unaligned tests where size is a compile-time constant */ + gettimeofday(&tv_begin, NULL); + perf_test_constant_unaligned(); + gettimeofday(&tv_end, NULL); + time_unaligned_const = (double)(tv_end.tv_sec - tv_begin.tv_sec) + + ((double)tv_end.tv_usec - tv_begin.tv_usec)/1000000; + printf("\n======= ================= ================= ================= =================\n\n"); + + printf("Test Execution Time (seconds):\n"); + printf("Aligned variable copy size = %8.3f\n", time_aligned); + printf("Aligned constant copy size = %8.3f\n", time_aligned_const); + printf("Unaligned variable copy size = %8.3f\n", time_unaligned); + printf("Unaligned constant copy size = %8.3f\n", time_unaligned_const); + free_buffers(); + + return Void(); +} + +#endif // defined (__linux__) || defined (__FreeBSD__) + +void forceLinkMemcpyPerfTests() {} diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs index 2d2109e696..92aa3fa86e 100644 --- a/packaging/msi/FDBInstaller.wxs +++ b/packaging/msi/FDBInstaller.wxs @@ -32,7 +32,7 @@