diff --git a/CMakeLists.txt b/CMakeLists.txt index 08df8edfe0..8ff0c2c704 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.13) +if(WIN32) + cmake_minimum_required(VERSION 3.15) +else() + cmake_minimum_required(VERSION 3.13) +endif() + project(foundationdb VERSION 7.1.0 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." @@ -196,9 +201,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CUR if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json - COMMAND $ ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py + COMMAND $ ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json COMMENT "Build compile commands for IDE" ) add_custom_target(processed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json) diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index eb89a9de25..d8696705ef 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -308,7 +308,7 @@ if(NOT OPEN_FOR_IDE) if(RUN_JUNIT_TESTS) # Sets up the JUnit testing structure to run through ctest # - # To add a new junit test, add the class to the JAVA_JUNIT_TESTS variable in `src/tests.cmake`. Note that if you run a Suite, + # To add a new junit test, add the class to the JAVA_JUNIT_TESTS variable in `src/tests.cmake`. Note that if you run a Suite, # ctest will NOT display underlying details of the suite itself, so it's best to avoid junit suites in general. Also, # if you need a different runner other than JUnitCore, you'll have to modify this so be aware. # @@ -316,8 +316,8 @@ if(NOT OPEN_FOR_IDE) # # ctest . # - # from the ${BUILD_DIR}/bindings/java subdirectory. - # + # from the ${BUILD_DIR}/bindings/java subdirectory. + # # Note: if you are running from ${BUILD_DIR}, additional tests of the native logic will be run. To avoid these, use # # ctest . -R java-unit @@ -325,15 +325,15 @@ if(NOT OPEN_FOR_IDE) # ctest has lots of flexible command options, so be sure to refer to its documentation if you want to do something specific(documentation # can be found at https://cmake.org/cmake/help/v3.19/manual/ctest.1.html) - add_jar(fdb-junit SOURCES ${JAVA_JUNIT_TESTS} ${JUNIT_RESOURCES} INCLUDE_JARS fdb-java - ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar + add_jar(fdb-junit SOURCES ${JAVA_JUNIT_TESTS} ${JUNIT_RESOURCES} INCLUDE_JARS fdb-java + ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-engine-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-params-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar ${CMAKE_BINARY_DIR}/packages/apiguardian-api-1.1.1.jar ) get_property(junit_jar_path TARGET fdb-junit PROPERTY JAR_FILE) - + add_test(NAME java-unit COMMAND ${Java_JAVA_EXECUTABLE} -classpath "${target_jar}:${junit_jar_path}:${JUNIT_CLASSPATH}" @@ -346,12 +346,12 @@ if(NOT OPEN_FOR_IDE) if(RUN_JAVA_INTEGRATION_TESTS) # Set up the integration tests. These tests generally require a running database server to function properly. Most tests # should be written such that they can be run in parallel with other integration tests (e.g. try to use a unique key range for each test - # whenever possible), because it's a reasonable assumption that a single server will be shared among multiple tests, and might do so + # whenever possible), because it's a reasonable assumption that a single server will be shared among multiple tests, and might do so # concurrently. # # Integration tests are run through ctest the same way as unit tests, but their label is prefixed with the entry 'integration-'. - # Note that most java integration tests will fail if they can't quickly connect to a running FDB instance(depending on how the test is written, anyway). - # However, if you want to explicitly skip them, you can run + # Note that most java integration tests will fail if they can't quickly connect to a running FDB instance(depending on how the test is written, anyway). + # However, if you want to explicitly skip them, you can run # # `ctest -E integration` # @@ -368,8 +368,8 @@ if(NOT OPEN_FOR_IDE) # empty, consider generating a random prefix for the keys you write, use # the directory layer with a unique path, etc.) # - add_jar(fdb-integration SOURCES ${JAVA_INTEGRATION_TESTS} ${JAVA_INTEGRATION_RESOURCES} INCLUDE_JARS fdb-java - ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar + add_jar(fdb-integration SOURCES ${JAVA_INTEGRATION_TESTS} ${JAVA_INTEGRATION_RESOURCES} INCLUDE_JARS fdb-java + ${CMAKE_BINARY_DIR}/packages/junit-jupiter-api-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-engine-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/junit-jupiter-params-5.7.1.jar ${CMAKE_BINARY_DIR}/packages/opentest4j-1.2.0.jar @@ -382,7 +382,14 @@ if(NOT OPEN_FOR_IDE) COMMAND ${Java_JAVA_EXECUTABLE} -classpath "${target_jar}:${integration_jar_path}:${JUNIT_CLASSPATH}" -Djava.library.path=${CMAKE_BINARY_DIR}/lib - org.junit.platform.console.ConsoleLauncher "--details=summary" "--class-path=${integration_jar_path}" "--scan-classpath" "--disable-banner" + org.junit.platform.console.ConsoleLauncher "--details=summary" "--class-path=${integration_jar_path}" "--scan-classpath" "--disable-banner" "-T MultiClient" + ) + + add_multi_fdbclient_test(NAME java-multi-integration + COMMAND ${Java_JAVA_EXECUTABLE} + -classpath "${target_jar}:${integration_jar_path}:${JUNIT_CLASSPATH}" + -Djava.library.path=${CMAKE_BINARY_DIR}/lib + org.junit.platform.console.ConsoleLauncher "--details=summary" "--class-path=${integration_jar_path}" "--scan-classpath" "--disable-banner" "-t MultiClient" ) endif() diff --git a/bindings/java/src/README.md b/bindings/java/src/README.md index 6fedba6368..6bd377b85a 100644 --- a/bindings/java/src/README.md +++ b/bindings/java/src/README.md @@ -22,4 +22,19 @@ To skip integration tests, execute `ctest -E integration` from `${BUILD_DIR}/bin To run _only_ integration tests, run `ctest -R integration` from `${BUILD_DIR}/bindings/java`. There are lots of other useful `ctest` commands, which we don't need to get into here. For more information, -see the [https://cmake.org/cmake/help/v3.19/manual/ctest.1.html](ctest documentation). \ No newline at end of file +see the [https://cmake.org/cmake/help/v3.19/manual/ctest.1.html](ctest documentation). + +### Multi-Client tests +Multi-Client tests are integration tests that can only be executed when multiple clusters are running. To write a multi-client +test, do the following: + +1. Tag all tests that require multiple clients with `@Tag("MultiClient")` +2. Ensure that your tests have the `MultiClientHelper` extension present, and Registered as an extension +3. Ensure that your test class is in the the JAVA_INTEGRATION_TESTS list in `test.cmake` + +( see `BasicMultiClientIntegrationTest` for a good reference example) + +It is important to note that it requires significant time to start and stop 3 separate clusters; if the underying test takes a long time to run, +ctest will time out and kill the test. When that happens, there is no guarantee that the FDB clusters will be properly stopped! It is thus +in your best interest to ensure that all tests run in a relatively small amount of time, or have a longer timeout attached. + diff --git a/bindings/java/src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java new file mode 100644 index 0000000000..2b02a3656f --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java @@ -0,0 +1,69 @@ +/* + * BasicMultiClientIntegrationTest + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.Collection; +import java.util.Random; + +import com.apple.foundationdb.tuple.Tuple; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Simple class to test multi-client logic. + * + * Note that all Multi-client-only tests _must_ be tagged with "MultiClient", which will ensure that they are excluded + * from non-multi-threaded tests. + */ +public class BasicMultiClientIntegrationTest { + @RegisterExtension public static final MultiClientHelper clientHelper = new MultiClientHelper(); + + @Test + @Tag("MultiClient") + void testMultiClientWritesAndReadsData() throws Exception { + FDB fdb = FDB.selectAPIVersion(630); + fdb.options().setKnob("min_trace_severity=5"); + + Collection dbs = clientHelper.openDatabases(fdb); // the clientHelper will close the databases for us + System.out.print("Starting tests."); + Random rand = new Random(); + for (int counter = 0; counter < 25; ++counter) { + for (Database db : dbs) { + String key = Integer.toString(rand.nextInt(100000000)); + String val = Integer.toString(rand.nextInt(100000000)); + + db.run(tr -> { + tr.set(Tuple.from(key).pack(), Tuple.from(val).pack()); + return null; + }); + + String fetchedVal = db.run(tr -> { + byte[] result = tr.get(Tuple.from(key).pack()).join(); + return Tuple.fromBytes(result).getString(0); + }); + Assertions.assertEquals(val, fetchedVal, "Wrong result!"); + } + Thread.sleep(200); + } + } +} diff --git a/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java b/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java index 5634e7d741..cf8c1bde1d 100644 --- a/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java +++ b/bindings/java/src/integration/com/apple/foundationdb/DirectoryTest.java @@ -19,8 +19,6 @@ */ package com.apple.foundationdb; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/bindings/java/src/integration/com/apple/foundationdb/MultiClientHelper.java b/bindings/java/src/integration/com/apple/foundationdb/MultiClientHelper.java new file mode 100644 index 0000000000..671163955f --- /dev/null +++ b/bindings/java/src/integration/com/apple/foundationdb/MultiClientHelper.java @@ -0,0 +1,82 @@ +/* + * MultiClientHelper.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb; + +import java.util.ArrayList; +import java.util.Collection; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * Callback to help define a multi-client scenario and ensure that + * the clients can be configured properly. + */ +public class MultiClientHelper implements BeforeAllCallback,AfterEachCallback{ + private String[] clusterFiles; + private Collection openDatabases; + + public static String[] readClusterFromEnv() { + /* + * Reads the cluster file lists from the ENV variable + * FDB_CLUSTERS. + */ + String clusterFilesProp = System.getenv("FDB_CLUSTERS"); + if (clusterFilesProp == null) { + throw new IllegalStateException("Missing FDB cluster connection file names"); + } + + return clusterFilesProp.split(";"); + } + + Collection openDatabases(FDB fdb){ + if(openDatabases!=null){ + return openDatabases; + } + if(clusterFiles==null){ + clusterFiles = readClusterFromEnv(); + } + Collection dbs = new ArrayList(); + for (String arg : clusterFiles) { + System.out.printf("Opening Cluster: %s\n", arg); + dbs.add(fdb.open(arg)); + } + + this.openDatabases = dbs; + return dbs; + } + + @Override + public void beforeAll(ExtensionContext arg0) throws Exception { + clusterFiles = readClusterFromEnv(); + } + + @Override + public void afterEach(ExtensionContext arg0) throws Exception { + //close any databases that have been opened + if(openDatabases!=null){ + for(Database db : openDatabases){ + db.close(); + } + } + openDatabases = null; + } + +} diff --git a/bindings/java/src/tests.cmake b/bindings/java/src/tests.cmake index 8bed62ecb8..bf3131eeb3 100644 --- a/bindings/java/src/tests.cmake +++ b/bindings/java/src/tests.cmake @@ -48,12 +48,14 @@ set(JUNIT_RESOURCES set(JAVA_INTEGRATION_TESTS src/integration/com/apple/foundationdb/DirectoryTest.java src/integration/com/apple/foundationdb/RangeQueryIntegrationTest.java + src/integration/com/apple/foundationdb/BasicMultiClientIntegrationTest.java ) # Resources that are used in integration testing, but are not explicitly test files (JUnit rules, # utility classes, and so forth) set(JAVA_INTEGRATION_RESOURCES src/integration/com/apple/foundationdb/RequiresDatabase.java + src/integration/com/apple/foundationdb/MultiClientHelper.java ) diff --git a/build/artifacts/.gitkeep b/build/artifacts/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/build/cmake/Dockerfile b/build/cmake/Dockerfile deleted file mode 100644 index 0452606a1f..0000000000 --- a/build/cmake/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM centos:6 -LABEL version=0.0.4 - -RUN yum install -y yum-utils -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN yum -y install centos-release-scl -RUN yum install -y devtoolset-7 - -# install cmake -RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz > /tmp/cmake.tar.gz &&\ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\ - sha256sum -c /tmp/cmake-sha.txt &&\ - cd /tmp && tar xf cmake.tar.gz && cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ - -# install boost -RUN curl -L https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 > /tmp/boost.tar.bz2 &&\ - cd /tmp && echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost.tar.bz2" > boost-sha.txt &&\ - sha256sum -c boost-sha.txt && tar xf boost.tar.bz2 && cp -r boost_1_72_0/boost /usr/local/include/ &&\ - rm -rf boost.tar.bz2 boost_1_72_0 - -# install mono (for actorcompiler) -RUN yum install -y epel-release -RUN yum install -y mono-core - -# install Java -RUN yum install -y java-1.8.0-openjdk-devel - -# install LibreSSL -RUN curl https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz > /tmp/libressl.tar.gz &&\ - cd /tmp && echo "b8cb31e59f1294557bfc80f2a662969bc064e83006ceef0574e2553a1c254fd5 libressl.tar.gz" > libressl-sha.txt &&\ - sha256sum -c libressl-sha.txt && tar xf libressl.tar.gz &&\ - cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ - cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- make -j`nproc` install &&\ - rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz - - -# install dependencies for bindings and documentation -# python 2.7 is required for the documentation -RUN yum install -y rh-python36-python-devel rh-ruby24 golang python27 - -# install packaging tools -RUN yum install -y rpm-build debbuild - -CMD scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash diff --git a/build/cmake/build.sh b/build/cmake/build.sh deleted file mode 100644 index ff02e78080..0000000000 --- a/build/cmake/build.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash - -arguments_usage() { - cat </dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/deb.sh -source ${source_dir}/modules/tests.sh -source ${source_dir}/modules/test_args.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - test_args_parse "$@" - __res=$? - if [ ${__res} -eq 2 ] - then - __res=0 - break - elif [ ${__res} -ne 0 ] - then - break - fi - tests_main - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/cmake/package_tester/fdb_c_app/CMakeLists.txt b/build/cmake/package_tester/fdb_c_app/CMakeLists.txt deleted file mode 100644 index 60ed25f0ca..0000000000 --- a/build/cmake/package_tester/fdb_c_app/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required(VERSION 2.8.0) -project(fdb_c_app C) -find_package(FoundationDB-Client REQUIRED) -add_executable(app app.c) -target_link_libraries(app PRIVATE fdb_c) diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c deleted file mode 100644 index 6fe24068f9..0000000000 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ /dev/null @@ -1,7 +0,0 @@ -#define FDB_API_VERSION 710 -#include - -int main(int argc, char* argv[]) { - fdb_select_api_version(710); - return 0; -} diff --git a/build/cmake/package_tester/modules/arguments.sh b/build/cmake/package_tester/modules/arguments.sh deleted file mode 100644 index afe46ee035..0000000000 --- a/build/cmake/package_tester/modules/arguments.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${arguments_sh_included+x} ] -then - arguments_sh_included=1 - - source ${source_dir}/modules/util.sh - - arguments_usage() { - cat <&2 - __res=1 - break - elif [ $docker_parallelism -lt 1 ] - then - echo -e "${RED}Error: -j ${OPTARG} makes no sense" >&2 - __res=1 - break - fi - ;; - P ) - pruning_strategy="${OPTARG}" - if ! [[ "${pruning_strategy}" =~ ^(ALL|FAILED|SUCCEEDED|NONE)$ ]] - then - fail "Unknown pruning strategy ${pruning_strategy}" - fi - ;; - \? ) - curr_index="$((OPTIND-1))" - echo "Unknown option ${@:${curr_index}:1}" - arguments_usage - __res=1 - break - ;; - esac - done - shift $((OPTIND -1)) - commands=("$@") - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/config.sh b/build/cmake/package_tester/modules/config.sh deleted file mode 100644 index 70fc35692c..0000000000 --- a/build/cmake/package_tester/modules/config.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${config_sh_included+x} ] -then - config_sh_included=1 - - source ${source_dir}/modules/util.sh - - config_load_vms() { - local __res=0 - enterfun - for _ in 1 - do - if [ -z "${docker_ini+x}"] - then - docker_file="${source_dir}/../docker.ini" - fi - # parse the ini file and read it into an - # associative array - eval "$(awk -F ' *= *' '{ if ($1 ~ /^\[/) section=$1; else if ($1 !~ /^$/) printf "ini_%s%s=\47%s\47\n", $1, section, $2 }' ${docker_file})" - vms=( "${!ini_name[@]}" ) - if [ $? -ne 0 ] - then - echo "ERROR: Could not parse config-file ${docker_file}" - __res=1 - break - fi - done - exitfun - return ${__res} - } - - config_find_packages() { - local __res=0 - enterfun - for _ in 1 - do - cd ${fdb_build} - while read f - do - if [[ "${f}" =~ .*"clients".* || "${f}" =~ .*"server".* ]] - then - if [ -z ${fdb_packages+x} ] - then - fdb_packages="${f}" - else - fdb_packages="${fdb_packages}:${f}" - fi - fi - done <<< "$(ls *.deb *.rpm)" - if [ $? -ne 0 ] - then - __res=1 - break - fi - done - exitfun - return ${__res} - } - - get_fdb_source() { - local __res=0 - enterfun - cd ${source_dir} - while true - do - if [ -d .git ] - then - # we found the root - pwd - break - fi - if [ `pwd` = "/" ] - then - __res=1 - break - fi - cd .. - done - exitfun - return ${__res} - } - - fdb_build=0 - - config_verify() { - local __res=0 - enterfun - for _ in 1 - do - if [ -z ${fdb_source+x} ] - then - fdb_source=`get_fdb_source` - fi - if [ ! -d "${fdb_build}" ] - then - __res=1 - echo "ERROR: Could not find fdb build dir: ${fdb_build}" - echo " Either set the environment variable fdb_build or" - echo " pass it with -b " - fi - if [ ! -f "${fdb_source}/flow/Net2.actor.cpp" ] - then - __res=1 - echo "ERROR: ${fdb_source} does not appear to be a fdb source" - echo " directory. Either pass it with -s or set" - echo " the environment variable fdb_source." - fi - if [ ${__res} -ne 0 ] - then - break - fi - config_load_vms - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/deb.sh b/build/cmake/package_tester/modules/deb.sh deleted file mode 100644 index fcef96805a..0000000000 --- a/build/cmake/package_tester/modules/deb.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${deb_sh_included}" ] -then - deb_sh_included=1 - - source ${source_dir}/modules/util.sh - - install_build_tools() { - apt-get -y install cmake gcc - } - - install() { - local __res=0 - enterfun - echo "Install FoundationDB" - cd /build/packages - package_names=() - for f in "${package_files[@]}" - do - package_name="$(dpkg -I ${f} | grep Package | sed 's/.*://')" - package_names+=( "${package_name}" ) - done - dpkg -i ${package_files[@]} - apt-get -yf -o Dpkg::Options::="--force-confold" install - __res=$? - sleep 5 - exitfun - return ${__res} - } - - uninstall() { - local __res=0 - enterfun - apt-get -y remove ${package_names[@]} - __res=$? - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/docker.sh b/build/cmake/package_tester/modules/docker.sh deleted file mode 100644 index d38b939957..0000000000 --- a/build/cmake/package_tester/modules/docker.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${docker_sh_included+x}" ] -then - docker_sh_included=1 - source ${source_dir}/modules/util.sh - source ${source_dir}/modules/config.sh - source ${source_dir}/modules/tests.sh - - failed_tests=() - - docker_ids=() - docker_threads=() - docker_logs=() - docker_error_logs=() - - docker_wait_any() { - local __res=0 - enterfun - while [ "${#docker_threads[@]}" -gt 0 ] - do - IFS=";" read -ra res <${pipe_file} - docker_id=${res[0]} - result=${res[1]} - i=0 - for (( idx=0; idx<${#docker_ids[@]}; idx++ )) - do - if [ "${docker_id}" = "${docker_ids[idx]}" ] - then - i=idx - break - fi - done - if [ "${result}" -eq 0 ] - then - echo -e "${GREEN}Test succeeded: ${docker_threads[$i]}" - echo -e "\tDocker-ID: ${docker_ids[$i]} " - echo -e "\tLog-File: ${docker_logs[$i]}" - echo -e "\tErr-File: ${docker_error_logs[$i]} ${NC}" - else - echo -e "${RED}Test FAILED: ${docker_threads[$i]}" - echo -e "\tDocker-ID: ${docker_ids[$i]} " - echo -e "\tLog-File: ${docker_logs[$i]}" - echo -e "\tErr-File: ${docker_error_logs[$i]} ${NC}" - failed_tests+=( "${docker_threads[$i]}" ) - fi - n=$((i+1)) - docker_ids=( "${docker_ids[@]:0:$i}" "${docker_ids[@]:$n}" ) - docker_threads=( "${docker_threads[@]:0:$i}" "${docker_threads[@]:$n}" ) - docker_logs=( "${docker_logs[@]:0:$i}" "${docker_logs[@]:$n}" ) - docker_error_logs=( "${docker_error_logs[@]:0:$i}" "${docker_error_logs[@]:$n}" ) - break - done - exitfun - return "${__res}" - } - - docker_wait_all() { - local __res=0 - while [ "${#docker_threads[@]}" -gt 0 ] - do - docker_wait_any - if [ "$?" -ne 0 ] - then - __res=1 - fi - done - return ${__res} - } - - docker_run() { - local __res=0 - enterfun - for _ in 1 - do - echo "Testing the following:" - echo "======================" - for K in "${vms[@]}" - do - curr_packages=( $(cd ${fdb_build}/packages; ls | grep -P ${ini_packages[${K}]} ) ) - echo "Will test the following ${#curr_packages[@]} packages in docker-image ${K}:" - for p in "${curr_packages[@]}" - do - echo " ${p}" - done - echo - done - log_dir="${fdb_build}/pkg_tester" - pipe_file="${fdb_build}/pkg_tester.pipe" - lock_file="${fdb_build}/pkg_tester.lock" - if [ -p "${pipe_file}" ] - then - rm "${pipe_file}" - successOr "Could not delete old pipe file" - fi - if [ -f "${lock_file}" ] - then - rm "${lock_file}" - successOr "Could not delete old pipe file" - fi - touch "${lock_file}" - successOr "Could not create lock file" - mkfifo "${pipe_file}" - successOr "Could not create pipe file" - mkdir -p "${log_dir}" - # setup the containers - # TODO: shall we make this parallel as well? - for vm in "${vms[@]}" - do - curr_name="${ini_name[$vm]}" - curr_location="${ini_location[$vm]}" - if [[ "$curr_location" = /* ]] - then - cd "${curr_location}" - else - cd ${source_dir}/../${curr_location} - fi - docker_buid_logs="${log_dir}/docker_build_${curr_name}" - docker build . -t ${curr_name} 1> "${docker_buid_logs}.log" 2> "${docker_buid_logs}.err" - successOr "Building Docker image ${curr_name} failed - see ${docker_buid_logs}.log and ${docker_buid_logs}.err" - done - if [ ! -z "${tests_to_run+x}"] - then - tests=() - IFS=';' read -ra tests <<< "${tests_to_run}" - fi - for vm in "${vms[@]}" - do - curr_name="${ini_name[$vm]}" - curr_format="${ini_format[$vm]}" - curr_packages=( $(cd ${fdb_build}/packages; ls | grep -P ${ini_packages[${vm}]} ) ) - for curr_test in "${tests[@]}" - do - if [ "${#docker_ids[@]}" -ge "${docker_parallelism}" ] - then - docker_wait_any - fi - log_file="${log_dir}/${curr_name}_${curr_test}.log" - err_file="${log_dir}/${curr_name}_${curr_test}.err" - docker_id=$( docker run -d -v "${fdb_source}:/foundationdb"\ - -v "${fdb_build}:/build"\ - ${curr_name} /sbin/init ) - echo "Starting Test ${curr_name}/${curr_test} Docker-ID: ${docker_id}" - { - docker exec "${docker_id}" bash \ - /foundationdb/build/cmake/package_tester/${curr_format}_tests.sh -n ${curr_test} ${curr_packages[@]}\ - 2> ${err_file} 1> ${log_file} - res=$? - if [ "${pruning_strategy}" = "ALL" ] - then - docker kill "${docker_id}" > /dev/null - elif [ "${res}" -eq 0 ] && [ "${pruning_strategy}" = "SUCCEEDED" ] - then - docker kill "${docker_id}" > /dev/null - elif [ "${res}" -ne 0 ] && [ "${pruning_strategy}" = "FAILED" ] - then - docker kill "${docker_id}" > /dev/null - fi - flock "${lock_file}" echo "${docker_id};${res}" >> "${pipe_file}" - } & - docker_ids+=( "${docker_id}" ) - docker_threads+=( "${curr_name}/${curr_test}" ) - docker_logs+=( "${log_file}" ) - docker_error_logs+=( "${err_file}" ) - done - done - docker_wait_all - rm ${pipe_file} - if [ "${#failed_tests[@]}" -eq 0 ] - then - echo -e "${GREEN}SUCCESS${NC}" - else - echo -e "${RED}FAILURE" - echo "The following tests failed:" - for t in "${failed_tests[@]}" - do - echo " - ${t}" - done - echo -e "${NC}" - __res=1 - fi - done - exitfun - return "${__res}" - } -fi diff --git a/build/cmake/package_tester/modules/globals.sh b/build/cmake/package_tester/modules/globals.sh deleted file mode 100644 index 795a4adc66..0000000000 --- a/build/cmake/package_tester/modules/globals.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# This module has to be included first and only once. -# This is because of a limitation of older bash versions -# that doesn't allow us to declare associative arrays -# globally. - -if [ -z "${global_sh_included+x}"] -then - global_sh_included=1 -else - echo "global.sh can only be included once" - exit 1 -fi - -declare -A ini_name -declare -A ini_location -declare -A ini_packages -declare -A ini_format -declare -A test_start_state -declare -A test_exit_state -declare -a tests -declare -a vms diff --git a/build/cmake/package_tester/modules/rpm.sh b/build/cmake/package_tester/modules/rpm.sh deleted file mode 100644 index 866bde558e..0000000000 --- a/build/cmake/package_tester/modules/rpm.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${rpm_sh_included}" ] -then - rpm_sh_included=1 - - source ${source_dir}/modules/util.sh - - conf_save_extension=".rpmsave" - - install_build_tools() { - yum -y install cmake gcc - } - - install() { - local __res=0 - enterfun - cd /build/packages - package_names=() - for f in "${package_files[@]}" - do - package_names+=( "$(rpm -qp ${f})" ) - done - yum install -y ${package_files[@]} - __res=$? - # give the server some time to come up - sleep 5 - exitfun - return ${__res} - } - - uninstall() { - local __res=0 - enterfun - if [ "$1" == "purge" ] - then - yum remove --purge -y ${package_names[@]} - else - yum remove -y ${package_names[@]} - fi - __res=$? - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/test_args.sh b/build/cmake/package_tester/modules/test_args.sh deleted file mode 100644 index bb88da945f..0000000000 --- a/build/cmake/package_tester/modules/test_args.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${test_args_sh_included+x} ] -then - test_args_sh_included=1 - - source ${source_dir}/modules/util.sh - - test_args_usage() { - me=`basename "$0"` - echo "usage: ${me} [-h] files..." - cat < /tmp/fdb.cluster - successOr "Could not create fdb.cluster file" - sed '/\[fdbserver.4500\]/a \[fdbserver.4501\]' /foundationdb/packaging/foundationdb.conf > /tmp/foundationdb.conf - successOr "Could not change foundationdb.conf file" - # we need to keep these files around for testing that the install didn't change them - cp /tmp/fdb.cluster /etc/foundationdb/fdb.cluster - cp /tmp/foundationdb.conf /etc/foundationdb/foundationdb.conf - - install - successOr "FoundationDB install failed" - # make sure we are not in build directory as there is a fdbc.cluster file there - echo "Configure new database - Install isn't supposed to do this for us" - echo "as there was an existing configuration" - cd / - timeout 2 fdbcli --exec 'configure new single ssd' - successOr "Couldn't configure new database" - tests_healthy - num_processes="$(timeout 2 fdbcli --exec 'status' | grep "FoundationDB processes" | sed -e 's/.*- //')" - if [ "${num_processes}" -ne 2 ] - then - fail Number of processes incorrect after config change - fi - - differences="$(diff /tmp/fdb.cluster /etc/foundationdb/fdb.cluster)" - if [ -n "${differences}" ] - then - fail Install changed configuration files - fi - differences="$(diff /tmp/foundationdb.conf /etc/foundationdb/foundationdb.conf)" - if [ -n "${differences}" ] - then - fail Install changed configuration files - fi - - uninstall - # make sure config didn't get deleted - # RPM, however, renames the file on remove, so we need to check for this - conffile="/etc/foundationdb/foundationdb.conf${conf_save_extension}" - if [ ! -f /etc/foundationdb/fdb.cluster ] || [ ! -f "${conffile}" ] - then - fail "Uninstall removed configuration" - fi - differences="$(diff /tmp/foundationdb.conf ${conffile})" - if [ -n "${differences}" ] - then - fail "${conffile} changed during remove" - fi - differences="$(diff /tmp/fdb.cluster /etc/foundationdb/fdb.cluster)" - if [ -n "${differences}" ] - then - fail "/etc/foundationdb/fdb.cluster changed during remove" - fi - - return 0 - } -fi diff --git a/build/cmake/package_tester/modules/util.sh b/build/cmake/package_tester/modules/util.sh deleted file mode 100644 index c3d643bdfc..0000000000 --- a/build/cmake/package_tester/modules/util.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${util_sh_included+x} ] -then - util_sh_included=1 - - # for colored output - RED='\033[0;31m' - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - NC='\033[0m' # No Color - - - enterfun() { - pushd . > /dev/null - } - - exitfun() { - popd > /dev/null - } - - fail() { - false - successOr ${@:1} - } - - successOr() { - local __res=$? - if [ ${__res} -ne 0 ] - then - if [ "$#" -gt 1 ] - then - >&2 echo -e "${RED}${@:1} ${NC}" - fi - exit ${__res} - fi - return 0 - } - -fi diff --git a/build/cmake/package_tester/rpm_tests.sh b/build/cmake/package_tester/rpm_tests.sh deleted file mode 100755 index a88bfb4f15..0000000000 --- a/build/cmake/package_tester/rpm_tests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/rpm.sh -source ${source_dir}/modules/tests.sh -source ${source_dir}/modules/test_args.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - test_args_parse "$@" - __res=$? - if [ ${__res} -eq 2 ] - then - __res=0 - break - elif [ ${__res} -ne 0 ] - then - break - fi - tests_main - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/cmake/package_tester/test_packages.sh b/build/cmake/package_tester/test_packages.sh deleted file mode 100755 index 05642073d8..0000000000 --- a/build/cmake/package_tester/test_packages.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/config.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/arguments.sh -source ${source_dir}/modules/docker.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - arguments_parse "$@" - if [ $? -ne 0 ] - then - __res=1 - break - fi - config_verify - if [ $? -ne 0 ] - then - __res=1 - break - fi - docker_run - __res=$? - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml deleted file mode 100644 index 4dcc30c683..0000000000 --- a/build/docker-compose.yaml +++ /dev/null @@ -1,105 +0,0 @@ -version: "3" - -services: - common: &common - image: foundationdb/foundationdb-build:0.1.24 - - build-setup: &build-setup - <<: *common - depends_on: [common] - volumes: - - ..:/__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb - working_dir: /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb - environment: - - MAKEJOBS=1 - - USE_CCACHE=1 - - BUILD_DIR=./work - - release-setup: &release-setup - <<: *build-setup - environment: - - MAKEJOBS=1 - - USE_CCACHE=1 - - RELEASE=true - - BUILD_DIR=./work - - snapshot-setup: &snapshot-setup - <<: *build-setup - - build-docs: - <<: *build-setup - volumes: - - ..:/foundationdb - working_dir: /foundationdb - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage' - - - release-packages: &release-packages - <<: *release-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' - - snapshot-packages: &snapshot-packages - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' - - prb-packages: - <<: *snapshot-packages - - - release-bindings: &release-bindings - <<: *release-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' - - snapshot-bindings: &snapshot-bindings - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' - - prb-bindings: - <<: *snapshot-bindings - - - snapshot-cmake: &snapshot-cmake - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' - - prb-cmake: - <<: *snapshot-cmake - - - snapshot-bindings-cmake: &snapshot-bindings-cmake - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "bindings/all"' - - prb-bindings-cmake: - <<: *snapshot-bindings-cmake - - - snapshot-cmake: &snapshot-testpackages - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}"' - - prb-testpackages: - <<: *snapshot-testpackages - - - snapshot-ctest: &snapshot-ctest - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' - - prb-ctest: - <<: *snapshot-ctest - - - snapshot-correctness: &snapshot-correctness - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' - - prb-correctness: - <<: *snapshot-correctness - - - shell: - <<: *build-setup - volumes: - - ..:/foundationdb - entrypoint: /bin/bash diff --git a/build/docker/centos6/build/Dockerfile b/build/docker/centos6/build/Dockerfile deleted file mode 100644 index 0a1fbbd70a..0000000000 --- a/build/docker/centos6/build/Dockerfile +++ /dev/null @@ -1,290 +0,0 @@ -FROM centos:6 - -WORKDIR /tmp - -RUN sed -i -e '/enabled/d' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -e '/gpgcheck=1/a enabled=0' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo && \ - sed -i -e "s/6\.0/$(cut -d\ -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo && \ - sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo && \ - yum install -y \ - centos-release-scl-rh \ - epel-release \ - scl-utils \ - yum-utils && \ - yum-config-manager --enable rhel-server-rhscl-7-rpms && \ - sed -i -e 's/#baseurl=/baseurl=/g' \ - -e 's/mirror.centos.org/vault.centos.org/g' \ - -e 's/mirrorlist=/#mirrorlist=/g' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo && \ - yum install -y \ - binutils-devel \ - curl \ - debbuild \ - devtoolset-8 \ - devtoolset-8-libasan-devel \ - devtoolset-8-libtsan-devel \ - devtoolset-8-libubsan-devel \ - devtoolset-8-valgrind-devel \ - dos2unix \ - dpkg \ - gettext-devel \ - git \ - golang \ - java-1.8.0-openjdk-devel \ - libcurl-devel \ - libuuid-devel \ - libxslt \ - lz4 \ - lz4-devel \ - lz4-static \ - mono-devel \ - redhat-lsb-core \ - rpm-build \ - tcl-devel \ - unzip \ - wget \ - rh-python36 \ - rh-python36-python-devel \ - rh-ruby24 && \ - yum clean all && \ - rm -rf /var/cache/yum - -# build/install autoconf -- same version installed by yum in centos7 -RUN curl -Ls http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz -o autoconf.tar.gz && \ - echo "954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 autoconf.tar.gz" > autoconf-sha.txt && \ - sha256sum -c autoconf-sha.txt && \ - mkdir autoconf && \ - tar --strip-components 1 --no-same-owner --directory autoconf -xf autoconf.tar.gz && \ - cd autoconf && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install automake -- same version installed by yum in centos7 -RUN curl -Ls http://ftp.gnu.org/gnu/automake/automake-1.13.4.tar.gz -o automake.tar.gz && \ - echo "4c93abc0bff54b296f41f92dd3aa1e73e554265a6f719df465574983ef6f878c automake.tar.gz" > automake-sha.txt && \ - sha256sum -c automake-sha.txt && \ - mkdir automake && \ - tar --strip-components 1 --no-same-owner --directory automake -xf automake.tar.gz && \ - cd automake && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install git -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/git/git/archive/v2.30.0.tar.gz -o git.tar.gz && \ - echo "8db4edd1a0a74ebf4b78aed3f9e25c8f2a7db3c00b1aaee94d1e9834fae24e61 git.tar.gz" > git-sha.txt && \ - sha256sum -c git-sha.txt && \ - mkdir git && \ - tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \ - cd git && \ - make configure && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install ninja -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip && \ - echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > ninja-sha.txt && \ - sha256sum -c ninja-sha.txt && \ - unzip ninja.zip && \ - cd ninja-1.9.0 && \ - ./configure.py --bootstrap && \ - cp ninja /usr/bin && \ - cd .. && \ - rm -rf /tmp/* - -# install cmake -RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz && \ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt && \ - sha256sum -c cmake-sha.txt && \ - mkdir cmake && \ - tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \ - cp -r cmake/* /usr/local/ && \ - rm -rf /tmp/* - -# build/install LLVM -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - curl -Ls https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/llvm-project-10.0.0.tar.xz -o llvm.tar.xz && \ - echo "6287a85f4a6aeb07dbffe27847117fe311ada48005f2b00241b523fe7b60716e llvm.tar.xz" > llvm-sha.txt && \ - sha256sum -c llvm-sha.txt && \ - mkdir llvm-project && \ - tar --strip-components 1 --no-same-owner --directory llvm-project -xf llvm.tar.xz && \ - mkdir -p llvm-project/build && \ - cd llvm-project/build && \ - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb" \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - ../llvm && \ - cmake --build . && \ - cmake --build . --target install && \ - cd ../.. && \ - rm -rf /tmp/* - -# build/install openssl -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz && \ - echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt && \ - sha256sum -c openssl-sha.txt && \ - mkdir openssl && \ - tar --strip-components 1 --no-same-owner --directory openssl -xf openssl.tar.gz && \ - cd openssl && \ - ./config CFLAGS="-fPIC -O3" --prefix=/usr/local && \ - make -j`nproc` && \ - make -j1 install && \ - ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ && \ - cd .. && \ - rm -rf /tmp/* - -# install rocksdb to /opt -RUN curl -Ls https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz && \ - echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt && \ - sha256sum -c rocksdb-sha.txt && \ - tar --directory /opt -xf rocksdb.tar.gz && \ - rm -rf /tmp/* - -# install boost 1.67 to /opt -RUN curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 && \ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt && \ - sha256sum -c boost-sha-67.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_67_0.tar.bz2 && \ - rm -rf /opt/boost_1_67_0/libs && \ - rm -rf /tmp/* - -# install boost 1.72 to /opt -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 && \ - echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt && \ - sha256sum -c boost-sha-72.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_72_0.tar.bz2 && \ - cd /opt/boost_1_72_0 &&\ - ./bootstrap.sh --with-libraries=context &&\ - ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ - rm -rf /opt/boost_1_72_0/libs && \ - rm -rf /tmp/* - -# jemalloc (needed for FDB after 6.3) -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 && \ - echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt && \ - sha256sum -c jemalloc-sha.txt && \ - mkdir jemalloc && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory jemalloc -xjf jemalloc-5.2.1.tar.bz2 && \ - cd jemalloc && \ - ./configure --enable-static --disable-cxx && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# Install CCACHE -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz -o ccache.tar.gz && \ - echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt && \ - sha256sum -c ccache-sha256.txt && \ - mkdir ccache &&\ - tar --strip-components 1 --no-same-owner --directory ccache -xf ccache.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# build/install toml -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz -o toml.tar.gz && \ - echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt && \ - sha256sum -c toml-sha256.txt && \ - mkdir toml && \ - tar --strip-components 1 --no-same-owner --directory toml -xf toml.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# download old fdbserver binaries -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /opt/foundationdb/old && \ - curl -Ls https://www.foundationdb.org/downloads/misc/fdbservers-${FDB_VERSION}.tar.gz | \ - tar --no-same-owner --directory /opt/foundationdb/old -xz && \ - chmod +x /opt/foundationdb/old/* && \ - ln -sf /opt/foundationdb/old/fdbserver-${FDB_VERSION} /opt/foundationdb/old/fdbserver - -# build/install distcc -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \ - echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \ - sha256sum -c distcc-sha256.txt && \ - mkdir distcc && \ - tar --strip-components 1 --no-same-owner --directory distcc -xf distcc.tar.gz && \ - cd distcc && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -RUN curl -Ls https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o /opt/ctest2junit.xsl - -# # Setting this environment variable switches from OpenSSL to BoringSSL -# ENV OPENSSL_ROOT_DIR=/opt/boringssl -# -# # install BoringSSL: TODO: They don't seem to have releases(?) I picked today's master SHA. -# RUN cd /opt &&\ -# git clone https://boringssl.googlesource.com/boringssl &&\ -# cd boringssl &&\ -# git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\ -# mkdir build -# -# # ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS -# # also, enable -fPIC. -# # this is moderately uglier than creating a patchfile, but easier to maintain. -# RUN cd /opt/boringssl &&\ -# for f in crypto/fipsmodule/rand/fork_detect_test.cc \ -# include/openssl/bn.h \ -# ssl/test/bssl_shim.cc ; do \ -# perl -p -i -e 's/#include /#define __STDC_FORMAT_MACROS 1\n#include /g;' $f ; \ -# done &&\ -# perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\ -# git diff -# -# RUN cd /opt/boringssl/build &&\ -# scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\ -# scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\ -# ./ssl/ssl_test &&\ -# mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib -# -# # Localize time zone -# ARG TIMEZONEINFO=America/Los_Angeles -# RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime -# -# LABEL version=${IMAGE_TAG} -# ENV DOCKER_IMAGEVER=${IMAGE_TAG} -# ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 -# ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc -# ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ -# -# ENV CCACHE_NOHASHDIR=true -# ENV CCACHE_UMASK=0000 -# ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches" -# -# CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash diff --git a/build/docker/centos6/devel/Dockerfile b/build/docker/centos6/devel/Dockerfile deleted file mode 100644 index c5c9db2914..0000000000 --- a/build/docker/centos6/devel/Dockerfile +++ /dev/null @@ -1,84 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos6-latest -FROM ${REPOSITORY}:${VERSION} - -# add vscode server -RUN yum repolist && \ - yum -y install \ - bash-completion \ - byobu \ - cgdb \ - emacs-nox \ - jq \ - the_silver_searcher \ - tmux \ - tree \ - vim \ - zsh && \ - yum clean all && \ - rm -rf /var/cache/yum - -WORKDIR /tmp -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - pip3 install \ - lxml \ - psutil \ - python-dateutil \ - subprocess32 && \ - mkdir fdb-joshua && \ - cd fdb-joshua && \ - git clone https://github.com/FoundationDB/fdb-joshua . && \ - pip3 install /tmp/fdb-joshua && \ - cd /tmp && \ - curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \ - echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \ - echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /usr/lib/foundationdb/plugins && \ - curl -Ls https://www.foundationdb.org/downloads/misc/joshua_tls_library.tar.gz | \ - tar --strip-components=1 --no-same-owner --directory /usr/lib/foundationdb/plugins -xz && \ - ln -sf /usr/lib/foundationdb/plugins/FDBGnuTLS.so /usr/lib/foundationdb/plugins/fdb-libressl-plugin.so && \ - curl -Ls https://www.foundationdb.org/downloads/${FDB_VERSION}/linux/libfdb_c_${FDB_VERSION}.so -o /usr/lib64/libfdb_c_${FDB_VERSION}.so && \ - ln -sf /usr/lib64/libfdb_c_${FDB_VERSION}.so /usr/lib64/libfdb_c.so - -WORKDIR /root -RUN rm -f /root/anaconda-ks.cfg && \ - printf '%s\n' \ - 'source /opt/rh/devtoolset-8/enable' \ - 'source /opt/rh/rh-python36/enable' \ - 'source /opt/rh/rh-ruby26/enable' \ - '' \ - 'function cmk_ci() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -v -C ${HOME}/build_output -j 84 all packages strip_targets' \ - '}' \ - 'function cmk() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -C ${HOME}/build_output -j 84' \ - '}' \ - 'function ct() {' \ - ' cd ${HOME}/build_output && ctest -j 32 --no-compress-output -T test --output-on-failure' \ - '}' \ - 'function j() {' \ - ' python3 -m joshua.joshua "${@}"' \ - '}' \ - 'function jsd() {' \ - ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ - '}' \ - '' \ - 'USER_BASHRC="$HOME/src/.bashrc.local"' \ - 'if test -f "$USER_BASHRC"; then' \ - ' source $USER_BASHRC' \ - 'fi' \ - '' \ - >> .bashrc diff --git a/build/docker/centos6/distcc/Dockerfile b/build/docker/centos6/distcc/Dockerfile deleted file mode 100644 index a96e67dff2..0000000000 --- a/build/docker/centos6/distcc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos6-latest -FROM ${REPOSITORY}:${VERSION} - -RUN useradd distcc && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - update-distcc-symlinks - -EXPOSE 3632 -EXPOSE 3633 -USER distcc -ENV ALLOW 0.0.0.0/0 - -ENTRYPOINT distccd \ - --daemon \ - --enable-tcp-insecure \ - --no-detach \ - --port 3632 \ - --log-stderr \ - --log-level info \ - --listen 0.0.0.0 \ - --allow ${ALLOW} \ - --jobs `nproc` \ No newline at end of file diff --git a/build/docker/centos7/build/Dockerfile b/build/docker/centos7/build/Dockerfile deleted file mode 100644 index de376d2557..0000000000 --- a/build/docker/centos7/build/Dockerfile +++ /dev/null @@ -1,247 +0,0 @@ -FROM centos:7 - -WORKDIR /tmp -COPY mono-project.com.rpmkey.pgp ./ -RUN rpmkeys --import mono-project.com.rpmkey.pgp && \ - curl -Ls https://download.mono-project.com/repo/centos7-stable.repo -o /etc/yum.repos.d/mono-centos7-stable.repo && \ - yum repolist && \ - yum install -y \ - centos-release-scl-rh \ - epel-release \ - scl-utils \ - yum-utils && \ - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ - yum install -y \ - autoconf \ - automake \ - binutils-devel \ - curl \ - debbuild \ - devtoolset-8 \ - devtoolset-8-libasan-devel \ - devtoolset-8-libtsan-devel \ - devtoolset-8-libubsan-devel \ - devtoolset-8-systemtap-sdt-devel \ - docker-ce \ - dos2unix \ - dpkg \ - gettext-devel \ - git \ - golang \ - java-11-openjdk-devel \ - libcurl-devel \ - libuuid-devel \ - libxslt \ - lz4 \ - lz4-devel \ - lz4-static \ - mono-devel \ - redhat-lsb-core \ - rpm-build \ - tcl-devel \ - unzip \ - wget && \ - if [ "$(uname -p)" == "aarch64" ]; then \ - yum install -y \ - rh-python38 \ - rh-python38-python-devel \ - rh-ruby27; \ - else \ - yum install -y \ - rh-python36 \ - rh-python36-python-devel \ - rh-ruby26; \ - fi && \ - yum clean all && \ - rm -rf /var/cache/yum - -# build/install git -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/git/git/archive/v2.30.0.tar.gz -o git.tar.gz && \ - echo "8db4edd1a0a74ebf4b78aed3f9e25c8f2a7db3c00b1aaee94d1e9834fae24e61 git.tar.gz" > git-sha.txt && \ - sha256sum -c git-sha.txt && \ - mkdir git && \ - tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \ - cd git && \ - make configure && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install ninja -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip && \ - echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > ninja-sha.txt && \ - sha256sum -c ninja-sha.txt && \ - unzip ninja.zip && \ - cd ninja-1.9.0 && \ - ./configure.py --bootstrap && \ - cp ninja /usr/bin && \ - cd .. && \ - rm -rf /tmp/* - -# install cmake -RUN if [ "$(uname -p)" == "aarch64" ]; then \ - curl -Ls https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-aarch64.tar.gz -o cmake.tar.gz; \ - echo "69ec045c6993907a4f4a77349d0a0668f1bd3ce8bc5f6fbab6dc7a7e2ffc4f80 cmake.tar.gz" > cmake-sha.txt; \ - else \ - curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz; \ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt; \ - fi && \ - sha256sum -c cmake-sha.txt && \ - mkdir cmake && \ - tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \ - cp -r cmake/* /usr/local/ && \ - rm -rf /tmp/* - -# build/install LLVM -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz -o llvm.tar.xz && \ - echo "b7b639fc675fa1c86dd6d0bc32267be9eb34451748d2efd03f674b773000e92b llvm.tar.xz" > llvm-sha.txt && \ - sha256sum -c llvm-sha.txt && \ - mkdir llvm-project && \ - tar --strip-components 1 --no-same-owner --directory llvm-project -xf llvm.tar.xz && \ - mkdir -p llvm-project/build && \ - cd llvm-project/build && \ - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb" \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - ../llvm && \ - cmake --build . && \ - cmake --build . --target install && \ - cd ../.. && \ - rm -rf /tmp/* - -# build/install openssl -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz && \ - echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt && \ - sha256sum -c openssl-sha.txt && \ - mkdir openssl && \ - tar --strip-components 1 --no-same-owner --directory openssl -xf openssl.tar.gz && \ - cd openssl && \ - ./config CFLAGS="-fPIC -O3" --prefix=/usr/local && \ - make -j`nproc` && \ - make -j1 install && \ - ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ && \ - cd .. && \ - rm -rf /tmp/* - -# install rocksdb to /opt -RUN curl -Ls https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz && \ - echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt && \ - sha256sum -c rocksdb-sha.txt && \ - tar --directory /opt -xf rocksdb.tar.gz && \ - rm -rf /tmp/* - -# install boost 1.67 to /opt -RUN curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 && \ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt && \ - sha256sum -c boost-sha-67.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_67_0.tar.bz2 && \ - rm -rf /opt/boost_1_67_0/libs && \ - rm -rf /tmp/* - -# install boost 1.72 to /opt -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 && \ - echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt && \ - sha256sum -c boost-sha-72.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_72_0.tar.bz2 && \ - cd /opt/boost_1_72_0 &&\ - ./bootstrap.sh --with-libraries=context &&\ - ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ - rm -rf /opt/boost_1_72_0/libs && \ - rm -rf /tmp/* - -# jemalloc (needed for FDB after 6.3) -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 && \ - echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt && \ - sha256sum -c jemalloc-sha.txt && \ - mkdir jemalloc && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory jemalloc -xjf jemalloc-5.2.1.tar.bz2 && \ - cd jemalloc && \ - ./configure --enable-static --disable-cxx && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# Install CCACHE -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz -o ccache.tar.gz && \ - echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt && \ - sha256sum -c ccache-sha256.txt && \ - mkdir ccache &&\ - tar --strip-components 1 --no-same-owner --directory ccache -xf ccache.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# build/install toml -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz -o toml.tar.gz && \ - echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt && \ - sha256sum -c toml-sha256.txt && \ - mkdir toml && \ - tar --strip-components 1 --no-same-owner --directory toml -xf toml.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# download old fdbserver binaries -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /opt/foundationdb/old && \ - curl -Ls https://www.foundationdb.org/downloads/misc/fdbservers-${FDB_VERSION}.tar.gz | \ - tar --no-same-owner --directory /opt/foundationdb/old -xz && \ - chmod +x /opt/foundationdb/old/* && \ - ln -sf /opt/foundationdb/old/fdbserver-${FDB_VERSION} /opt/foundationdb/old/fdbserver - -# build/install distcc -RUN source /opt/rh/devtoolset-8/enable && \ - if [ "$(uname -p)" == "aarch64" ]; then \ - source /opt/rh/rh-python38/enable; \ - else \ - source /opt/rh/rh-python36/enable; \ - fi && \ - curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \ - echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \ - sha256sum -c distcc-sha256.txt && \ - mkdir distcc && \ - tar --strip-components 1 --no-same-owner --directory distcc -xf distcc.tar.gz && \ - cd distcc && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# valgrind -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://sourceware.org/pub/valgrind/valgrind-3.17.0.tar.bz2 -o valgrind-3.17.0.tar.bz2 && \ - echo "ad3aec668e813e40f238995f60796d9590eee64a16dff88421430630e69285a2 valgrind-3.17.0.tar.bz2" > valgrind-sha.txt && \ - sha256sum -c valgrind-sha.txt && \ - mkdir valgrind && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory valgrind -xjf valgrind-3.17.0.tar.bz2 && \ - cd valgrind && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -RUN curl -Ls https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o /opt/ctest2junit.xsl diff --git a/build/docker/centos7/build/mono-project.com.rpmkey.pgp b/build/docker/centos7/build/mono-project.com.rpmkey.pgp deleted file mode 100644 index 4d7f8726d4..0000000000 --- a/build/docker/centos7/build/mono-project.com.rpmkey.pgp +++ /dev/null @@ -1,40 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: SKS 1.1.6 -Comment: Hostname: sks.pod01.fleetstreetops.com - -mQENBFPfqCcBCADctOzyTxfWvf40Nlb+AMkcJyb505WSbzhWU8yPmBNAJOnbwueMsTkNMHEO -u8fGRNxRWj5o/Db1N7EoSQtK3OgFnBef8xquUyrzA1nJ2aPfUWX+bhTG1TwyrtLaOssFRz6z -/h/ChUIFvt2VZCw+Yx4BiKi+tvgwrHTYB/Yf2J9+R/1O6949n6veFFRBfgPOL0djhvRqXzhv -FjJkh4xhTaGVeOnRR3+YQkblmti2n6KYl0n2kNB40ujSqpTloSfnR5tmJpz00WoOA9MJBdvH -txTTn8l6rVzXbm4mW9ZmB1kht/BgWaNLaIisW5AZSkQKer35wOWf0G7Gw+cWHq+I7W9pABEB -AAG0OlhhbWFyaW4gUHVibGljIEplbmtpbnMgKGF1dG8tc2lnbmluZykgPHJlbGVuZ0B4YW1h -cmluLmNvbT6JARwEEAECAAYFAlQIhKQACgkQyQ+cuQ4frQyc1wf+MCusJK4ANLWikbgiSSx1 -qMBveBlLKLEdCxYY+B9rc/pRDw448iBdd+nuSVdbRoqLgoN8gHbClboP+i22yw+mga0KASD7 -b1mpdYB0npR3H73zbYArn3qTV8s/yUXkIAEFUtj0yoEuv8KjO8P7nZJh8OuqqAupUVN0s3Kj -ONqXqi6Ro3fvVEZWOUFZl/FmY5KmXlpcw+YwE5CaNhJ2WunrjFTDqynRU/LeoPEKuwyYvfo9 -37zJFCrpAUMTr/9QpEKmV61H7fEHA9oHq97FBwWfjOU0l2mrXt1zJ97xVd2DXxrZodlkiY6B -76rhaT4ZhltY1E7WB2Z9WPfTe1Y6jz4fZ4kBHAQQAQgABgUCWEyoiAAKCRABFQplW72BAn/P -CAC0GkRBR3JTmG8WGeQMLb/o6Gon9cxpLnKv1GgFbHSM7XYMe7ySh5zxORwFuECuJ5+qcA6c -Ve/kJAV8rewLULL9yvHK3oK7R8zoVGbFVm+lyoxiaXpkkWg21Mb8IubiO+tA/dJc7hKQSpoI -0+dmJNaNrTVwqj0tQ8e0OL9KvBOYwFbSe06bocSNPVmKCt0EOvpGcQfzFw5UEjJVkqFn/moU -rSxj0YsJpwRXB1pOsBaQC6r9oCgUvxPf4H77U07+ImXzxRWInVPYFSXSiBA7p+hzvsikmZEl -iIAia8mTteUF1GeK4kafUk6iZZUfBlCIb9sV4O9Vvv8W0VjK4Vg6O2UAiQE4BBMBAgAiBQJT -36gnAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCmoZs409gx75DoB/9h5p8u1cUS -y6Mp2PjjW398LJZaqWwaa2W/lcLEKN7oWTC5Yf5BEuVsO9270pVln9Cv7hiqcbC8kywk+sZv -RsYO3uoTRwsmImc/7uaK382hey1A2hvkH5fYHmY/5Z/Z0bm/A0k0chhG2ycjWjZXYLZ96I0V -U3ZBQBHoh3qRtgWq4yWTsCJBX+FKPBdmkIpgcPXQw+hak0mj2sILqjScRZT1Oe+WJsMNMaLa -8dSdw+pPm8NM/VGLmO9iTTDApuAsRixpCYLdJY+ThGNrKe6xDswQo8gr3gbBkJi0wLRDP2Rz -q7rD0TC2PxOaWOZ7hmyz+EhjLcjZhHNJTaa+NV0k8YAwuQENBFPfqCcBCACtc7HssC9S3PxJ -m1youvGfYLhm+KzMO+gIoy7R32VXIZNxrkMYzaeerqSsMwxdhEjyOscT+rJbRGZ+9iPOGeh4 -AqZlzzOuxQ/Lg5h+2mGVXe0Avb+A2zC56mLSQCL3W8NjABUZdknnc1YIf9Dz05fy4jPEttNS -y+Rzte0ITLH1Hy/PKBrlF5n+G1/86f3L5n1ZZXmV3vi+rXT/OyEh9xRS4usmR6kVh4o2XGlI -zUrUjhZvb4lxrHfWgzKlWFoUSydaZDk7eikTKF692RiSSpLbDLW2sNOdzT2eqv2B8CJRF5sL -bD6BB3dAbH7KfqKiCT3xcCZhNEZw+M+GcRO/HNbnABEBAAGJAR8EGAECAAkFAlPfqCcCGwwA -CgkQpqGbONPYMe+sNQgAwjm9PJ45t7NBNTXn1zadoQQbPqz9qAlWiII0k+zzJCTTVqgyIXJY -I6zdNiB/Oh1Xajs/T9z9tL54+LLqgtZKa0lzDOmcxn6Iujf3a1MFdYxKgaQtT2ADxAimuBoz -3Y1ohxXgAs2+VISWYoPBI+UWhYqg11zq3uwpFIYQBRgkVydCxefCxY19okNp9FPC7KJPpJkO -NgDAK693Y9mOZXSq+XeGhjy3Sxesl0PYLIfV33z+vCpc2o1dDA5wuycgfqupNQITkQm6gPOH -1jLu8Vttm4fdEtVMcqkn8dJFomo3JW3qxI7IWwjbVRg10G8LGAuBbD6CA0dGSf8PkHFYv2Xs -dQ== -=MWcF ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/build/docker/centos7/devel/Dockerfile b/build/docker/centos7/devel/Dockerfile deleted file mode 100644 index 98f1923c17..0000000000 --- a/build/docker/centos7/devel/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -# add vscode server -RUN yum-config-manager --add-repo=https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/repo/epel-7/carlwgeorge-ripgrep-epel-7.repo && \ - yum repolist && \ - yum -y install \ - bash-completion \ - byobu \ - cgdb \ - emacs-nox \ - fish \ - jq \ - ripgrep \ - the_silver_searcher \ - tmux \ - tree \ - vim \ - zsh && \ - yum clean all && \ - rm -rf /var/cache/yum - -WORKDIR /tmp -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - pip3 install \ - lxml \ - psutil \ - python-dateutil \ - subprocess32 && \ - mkdir fdb-joshua && \ - cd fdb-joshua && \ - git clone https://github.com/FoundationDB/fdb-joshua . && \ - pip3 install /tmp/fdb-joshua && \ - cd /tmp && \ - curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \ - echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \ - echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /usr/lib/foundationdb/plugins && \ - curl -Ls https://www.foundationdb.org/downloads/misc/joshua_tls_library.tar.gz | \ - tar --strip-components=1 --no-same-owner --directory /usr/lib/foundationdb/plugins -xz && \ - ln -sf /usr/lib/foundationdb/plugins/FDBGnuTLS.so /usr/lib/foundationdb/plugins/fdb-libressl-plugin.so && \ - curl -Ls https://www.foundationdb.org/downloads/${FDB_VERSION}/linux/libfdb_c_${FDB_VERSION}.so -o /usr/lib64/libfdb_c_${FDB_VERSION}.so && \ - ln -sf /usr/lib64/libfdb_c_${FDB_VERSION}.so /usr/lib64/libfdb_c.so - -WORKDIR /root -RUN curl -Ls https://update.code.visualstudio.com/latest/server-linux-x64/stable -o /tmp/vscode-server-linux-x64.tar.gz && \ - mkdir -p .vscode-server/bin/latest && \ - tar --strip-components 1 --no-same-owner --directory .vscode-server/bin/latest -xf /tmp/vscode-server-linux-x64.tar.gz && \ - touch .vscode-server/bin/latest/0 && \ - rm -rf /tmp/* -RUN rm -f /root/anaconda-ks.cfg && \ - printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -Eeuo pipefail' \ - '' \ - 'mkdir -p ~/.docker' \ - 'cat > ~/.docker/config.json << EOF' \ - '{' \ - ' "proxies":' \ - ' {' \ - ' "default":' \ - ' {' \ - ' "httpProxy": "${HTTP_PROXY}",' \ - ' "httpsProxy": "${HTTPS_PROXY}",' \ - ' "noProxy": "${NO_PROXY}"' \ - ' }' \ - ' }' \ - '}' \ - 'EOF' \ - > docker_proxy.sh && \ - chmod 755 docker_proxy.sh && \ - printf '%s\n' \ - 'source /opt/rh/devtoolset-8/enable' \ - 'source /opt/rh/rh-python36/enable' \ - 'source /opt/rh/rh-ruby26/enable' \ - '' \ - 'function cmk_ci() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -v -C ${HOME}/build_output -j 84 all packages strip_targets' \ - '}' \ - 'function cmk() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -C ${HOME}/build_output -j 84' \ - '}' \ - 'function ct() {' \ - ' cd ${HOME}/build_output && ctest -j 32 --no-compress-output -T test --output-on-failure' \ - '}' \ - 'function j() {' \ - ' python3 -m joshua.joshua "${@}"' \ - '}' \ - 'function jsd() {' \ - ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ - '}' \ - '' \ - 'USER_BASHRC="$HOME/src/.bashrc.local"' \ - 'if test -f "$USER_BASHRC"; then' \ - ' source $USER_BASHRC' \ - 'fi' \ - '' \ - 'bash ${HOME}/docker_proxy.sh' \ - >> .bashrc diff --git a/build/docker/centos7/distcc/Dockerfile b/build/docker/centos7/distcc/Dockerfile deleted file mode 100644 index 785e6bee93..0000000000 --- a/build/docker/centos7/distcc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -RUN useradd distcc && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - update-distcc-symlinks - -EXPOSE 3632 -EXPOSE 3633 -USER distcc -ENV ALLOW 0.0.0.0/0 - -ENTRYPOINT distccd \ - --daemon \ - --enable-tcp-insecure \ - --no-detach \ - --port 3632 \ - --log-stderr \ - --log-level info \ - --listen 0.0.0.0 \ - --allow ${ALLOW} \ - --jobs `nproc` \ No newline at end of file diff --git a/build/docker/centos7/ycsb/Dockerfile b/build/docker/centos7/ycsb/Dockerfile deleted file mode 100644 index a8b60230b3..0000000000 --- a/build/docker/centos7/ycsb/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -ENV YCSB_VERSION=ycsb-foundationdb-binding-0.17.0 \ - PATH=${PATH}:/usr/bin - -RUN cd /opt \ - && eval curl "-Ls https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-foundationdb-binding-0.17.0.tar.gz" \ - | tar -xzvf - - -RUN rm -Rf /opt/${YCSB_VERSION}/lib/fdb-java-5.2.5.jar - -# COPY The Appropriate fdb-java-.jar Aaron from packages -# COPY binary RPM for foundationd-db -# Install Binary - -WORKDIR "/opt/${YCSB_VERSION}" - -ENTRYPOINT ["bin/ycsb.sh"] \ No newline at end of file diff --git a/build/gen_dev_docker.sh b/build/gen_dev_docker.sh deleted file mode 100755 index 89129d5a86..0000000000 --- a/build/gen_dev_docker.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# we first check whether the user is in the group docker -user=$(id -un) -DIR_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) -group=$(id -gn) -uid=$(id -u) -gid=$(id -g) -gids=( $(id -G) ) -groups=( $(id -Gn) ) -tmpdir="/tmp/fdb-docker-${DIR_UUID}" -image=fdb-dev - -pushd . -mkdir ${tmpdir} -cd ${tmpdir} - -echo - -cat <> Dockerfile -FROM foundationdb/foundationdb-dev:0.11.1 -RUN yum install -y sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -RUN groupadd -g 1100 sudo -EOF - -num_groups=${#gids[@]} -additional_groups="-G sudo" -for ((i=0;i> Dockerfile - if [ ${gids[i]} -ne ${gid} ] - then - additional_groups="${additional_groups},${gids[$i]}" - fi -done - -cat <> Dockerfile -RUN useradd -u ${uid} -g ${gid} ${additional_groups} -m ${user} - -USER ${user} -CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash - -EOF - -echo "Created ${tmpdir}" -echo "Buidling Docker container ${image}" -sudo docker build -t ${image} . - -popd - -echo "Writing startup script" -mkdir -p $HOME/bin -cat < $HOME/bin/fdb-dev -#!/usr/bin/bash - -if [ -d "\${CCACHE_DIR}" ] -then - args="-v \${CCACHE_DIR}:\${CCACHE_DIR}" - args="\${args} -e CCACHE_DIR=\${CCACHE_DIR}" - args="\${args} -e CCACHE_UMASK=\${CCACHE_UMASK}" - ccache_args=\$args -fi - -if [ -t 1 ] ; then - TERMINAL_ARGS=-it `# Run in interactive mode and simulate a TTY` -else - TERMINAL_ARGS=-i `# Run in interactive mode` -fi - -sudo docker run --rm `# delete (temporary) image after return` \\ - \${TERMINAL_ARGS} \\ - --privileged=true `# Run in privileged mode ` \\ - --cap-add=SYS_PTRACE \\ - --security-opt seccomp=unconfined \\ - -v "${HOME}:${HOME}" `# Mount home directory` \\ - -w="\$(pwd)" \\ - \${ccache_args} \\ - ${image} "\$@" -EOF - -cat < $HOME/bin/clangd -#!/usr/bin/bash - -fdb-dev scl enable devtoolset-8 rh-python36 rh-ruby24 -- clangd -EOF - -if [[ ":$PATH:" != *":$HOME/bin:"* ]] -then - echo "WARNING: $HOME/bin is not in your PATH!" - echo -e "\tThis can cause problems with some scripts (like fdb-clangd)" -fi -chmod +x $HOME/bin/fdb-dev -chmod +x $HOME/bin/clangd -echo "To start the dev docker image run $HOME/bin/fdb-dev" -echo "$HOME/bin/clangd can be used for IDE integration" -echo "You can edit these files but be aware that this script will overwrite your changes if you rerun it" diff --git a/build/get_package_name.sh b/build/get_package_name.sh deleted file mode 100755 index c2c94d126b..0000000000 --- a/build/get_package_name.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -cat $1 | grep '' | sed -e 's,^[^>]*>,,' -e 's,<.*,,' diff --git a/build/get_version.sh b/build/get_version.sh deleted file mode 100755 index a7a2a179f2..0000000000 --- a/build/get_version.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -cat $1 | grep '' | sed -e 's,^[^>]*>,,' -e 's,<.*,,' - diff --git a/build/txt-to-toml.py b/build/txt-to-toml.py deleted file mode 100755 index 68d1dcdbb5..0000000000 --- a/build/txt-to-toml.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -import sys - - -def main(): - if len(sys.argv) != 2: - print("Usage: txt-to-toml.py [src.txt]") - return 1 - - filename = sys.argv[1] - - indent = " " - in_workload = False - first_test = False - keys_before_test = False - - for line in open(filename): - k = "" - v = "" - - if line.strip().startswith(";"): - print((indent if in_workload else "") + line.strip().replace(";", "#")) - continue - - if "=" in line: - (k, v) = line.strip().split("=") - (k, v) = (k.strip(), v.strip()) - - if k == "testTitle": - first_test = True - if in_workload: - print("") - in_workload = False - if keys_before_test: - print("") - keys_before_test = False - print("[[test]]") - - if k == "testName": - in_workload = True - print("") - print(indent + "[[test.workload]]") - - if not first_test: - keys_before_test = True - - if v.startswith("."): - v = "0" + v - - if any(c.isalpha() or c in ["/", "!"] for c in v): - if v != "true" and v != "false": - v = "'" + v + "'" - - if k == "buggify": - print("buggify = " + ("true" if v == "'on'" else "false")) - elif k: - print((indent if in_workload else "") + k + " = " + v) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index f8a364f753..1ecbb70b8b 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -394,6 +394,7 @@ function(package_bindingtester) add_dependencies(bindingtester copy_bindingtester_binaries) endfunction() +# Creates a single cluster before running the specified command (usually a ctest test) function(add_fdbclient_test) set(options DISABLED ENABLED) set(oneValueArgs NAME) @@ -417,7 +418,37 @@ function(add_fdbclient_test) --build-dir ${CMAKE_BINARY_DIR} -- ${T_COMMAND}) - set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) +endfunction() + +# Creates 3 distinct clusters before running the specified command. +# This is useful for testing features that require multiple clusters (like the +# multi-cluster FDB client) +function(add_multi_fdbclient_test) + set(options DISABLED ENABLED) + set(oneValueArgs NAME) + set(multiValueArgs COMMAND) + cmake_parse_arguments(T "${options}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}") + if(OPEN_FOR_IDE) + return() + endif() + if(NOT T_ENABLED AND T_DISABLED) + return() + endif() + if(NOT T_NAME) + message(FATAL_ERROR "NAME is a required argument for add_multi_fdbclient_test") + endif() + if(NOT T_COMMAND) + message(FATAL_ERROR "COMMAND is a required argument for add_multi_fdbclient_test") + endif() + message(STATUS "Adding Client test ${T_NAME}") + add_test(NAME "${T_NAME}" + COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py + --build-dir ${CMAKE_BINARY_DIR} + --clusters 3 + -- + ${T_COMMAND}) + set_tests_properties("${T_NAME}" PROPERTIES TIMEOUT 60) endfunction() function(add_java_test) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index c14c5011c5..a1c231ec06 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -100,8 +100,7 @@ if(WIN32) endif() add_compile_options(/W0 /EHsc /bigobj $<$:/Zi> /MP /FC /Gm-) add_compile_definitions(NOMINMAX) - set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") else() set(GCC NO) set(CLANG NO) diff --git a/build/gen_compile_db.py b/contrib/gen_compile_db.py similarity index 100% rename from build/gen_compile_db.py rename to contrib/gen_compile_db.py diff --git a/design/tlog-spilling.md.html b/design/tlog-spilling.md.html index 58bbde503e..da72f8eccf 100644 --- a/design/tlog-spilling.md.html +++ b/design/tlog-spilling.md.html @@ -352,7 +352,7 @@ API for random reads to the DiskQueue. That ability is now required for peeking, and thus, `IDiskQueue`'s API has been enhanced correspondingly: ``` CPP -enum class CheckHashes { NO, YES }; +BOOLEAN_PARAM(CheckHashes); class IDiskQueue { // ... @@ -369,9 +369,9 @@ and not `(start, length)`. Spilled data, when using spill-by-value, was resistant to bitrot via data being checksummed interally within SQLite's B-tree. Now that reads can be done directly, the responsibility for verifying data integrity falls upon the -DiskQueue. `CheckHashes::YES` will cause the DiskQueue to use the checksum in +DiskQueue. `CheckHashes::TRUE` will cause the DiskQueue to use the checksum in each DiskQueue page to verify data integrity. If an externally maintained -checksums exists to verify the returned data, then `CheckHashes::NO` can be +checksums exists to verify the returned data, then `CheckHashes::FALSE` can be used to elide the checksumming. A page failing its checksum will cause the transaction log to die with an `io_error()`. diff --git a/documentation/CMakeLists.txt b/documentation/CMakeLists.txt index ccd60a2bbd..e734e28e91 100644 --- a/documentation/CMakeLists.txt +++ b/documentation/CMakeLists.txt @@ -1,4 +1,8 @@ add_subdirectory(tutorial) +if(WIN32) + return() +endif() + # build a virtualenv set(sphinx_dir ${CMAKE_CURRENT_SOURCE_DIR}/sphinx) set(venv_dir ${CMAKE_CURRENT_BINARY_DIR}/venv) diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp index 4f102a31de..d875b2345c 100644 --- a/fdbbackup/FileConverter.actor.cpp +++ b/fdbbackup/FileConverter.actor.cpp @@ -598,7 +598,7 @@ int main(int argc, char** argv) { Error::init(); StringRef url(param.container_url); - setupNetwork(0, true); + setupNetwork(0, UseMetrics::TRUE); TraceEvent::setNetworkThread(); openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "convert", param.trace_log_group); diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp index 193564d905..ecf8963950 100644 --- a/fdbbackup/FileDecoder.actor.cpp +++ b/fdbbackup/FileDecoder.actor.cpp @@ -579,7 +579,7 @@ int main(int argc, char** argv) { Error::init(); StringRef url(param.container_url); - setupNetwork(0, true); + setupNetwork(0, UseMetrics::TRUE); TraceEvent::setNetworkThread(); openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "decode", param.trace_log_group); diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 04fc42cded..62ddd24179 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -133,6 +133,7 @@ enum { OPT_WAITFORDONE, OPT_BACKUPKEYS_FILTER, OPT_INCREMENTALONLY, + OPT_ENCRYPTION_KEY_FILE, // Backup Modify OPT_MOD_ACTIVE_INTERVAL, @@ -259,6 +260,7 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = { { OPT_KNOB, "--knob_", SO_REQ_SEP }, { OPT_BLOB_CREDENTIALS, "--blob_credentials", SO_REQ_SEP }, { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -697,6 +699,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = { { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, { OPT_RESTORE_BEGIN_VERSION, "--begin_version", SO_REQ_SEP }, { OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent_snapshot_only", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption_key_file", SO_REQ_SEP }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif @@ -1089,6 +1092,8 @@ static void printBackupUsage(bool devhelp) { " Performs incremental backup without the base backup.\n" " This option indicates to the backup agent that it will only need to record the log files, " "and ignore the range files.\n"); + printf(" --encryption_key_file" + " The AES-128-GCM key in the provided file is used for encrypting backup files.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1162,6 +1167,8 @@ static void printRestoreUsage(bool devhelp) { " To be used in conjunction with incremental restore.\n" " Indicates to the backup agent to only begin replaying log files from a certain version, " "instead of the entire set.\n"); + printf(" --encryption_key_file" + " The AES-128-GCM key in the provided file is used for decrypting backup files.\n"); #ifndef TLS_DISABLED printf(TLS_HELP); #endif @@ -1463,7 +1470,7 @@ ACTOR Future getLayerStatus(Reference tr std::string id, ProgramExe exe, Database dest, - bool snapshot = false) { + Snapshot snapshot = Snapshot::FALSE) { // This process will write a document that looks like this: // { backup : { $expires : {}, version: } // so that the value under 'backup' will eventually expire to null and thus be ignored by @@ -1639,7 +1646,7 @@ ACTOR Future cleanupStatus(Reference tr, std::string name, std::string id, int limit = 1) { - state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); + state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::TRUE)); state bool readMore = false; state int i; for (i = 0; i < docs.size(); ++i) { @@ -1668,7 +1675,7 @@ ACTOR Future cleanupStatus(Reference tr, } if (readMore) { limit = 10000; - RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, true)); + RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::TRUE)); docs = std::move(docs2); readMore = false; } @@ -1705,7 +1712,10 @@ ACTOR Future getLayerStatus(Database src, std::string root // Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay // based on that and BACKUP_AGGREGATE_POLL_RATE -ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::string name, double* pollDelay) { +ACTOR Future updateAgentPollRate(Database src, + std::string rootKey, + std::string name, + std::shared_ptr pollDelay) { loop { try { json_spirit::mObject status = wait(getLayerStatus(src, rootKey)); @@ -1727,7 +1737,7 @@ ACTOR Future updateAgentPollRate(Database src, std::string rootKey, std::s ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name, ProgramExe exe, - double* pollDelay, + std::shared_ptr pollDelay, Database taskDest = Database(), std::string id = nondeterministicRandom()->randomUniqueID().toString()) { state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name; @@ -1757,7 +1767,8 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state Future futureStatusDoc = getLayerStatus(tr, name, id, exe, taskDest, true); + state Future futureStatusDoc = + getLayerStatus(tr, name, id, exe, taskDest, Snapshot::TRUE); wait(cleanupStatus(tr, rootKey, name, id)); std::string statusdoc = wait(futureStatusDoc); tr->set(instanceKey, statusdoc); @@ -1774,7 +1785,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, // Now that status was written at least once by this process (and hopefully others), start the poll rate // control updater if it wasn't started yet - if (!pollRateUpdater.isValid() && pollDelay != nullptr) + if (!pollRateUpdater.isValid()) pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay); } catch (Error& e) { TraceEvent(SevWarnAlways, "UnableToWriteStatus").error(e); @@ -1784,17 +1795,17 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, } ACTOR Future runDBAgent(Database src, Database dest) { - state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; + state std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); std::string id = nondeterministicRandom()->randomUniqueID().toString(); - state Future status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, &pollDelay, dest, id); + state Future status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id); state Future status_other = - statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, &pollDelay, dest, id); + statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id); state DatabaseBackupAgent backupAgent(src); loop { try { - wait(backupAgent.run(dest, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); + wait(backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; } catch (Error& e) { if (e.code() == error_code_operation_cancelled) @@ -1811,14 +1822,14 @@ ACTOR Future runDBAgent(Database src, Database dest) { } ACTOR Future runAgent(Database db) { - state double pollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; - state Future status = statusUpdateActor(db, "backup", ProgramExe::AGENT, &pollDelay); + state std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); + state Future status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay); state FileBackupAgent backupAgent; loop { try { - wait(backupAgent.run(db, &pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); + wait(backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); break; } catch (Error& e) { if (e.code() == error_code_operation_cancelled) @@ -1846,7 +1857,8 @@ ACTOR Future submitDBBackup(Database src, backupRanges.push_back_deep(backupRanges.arena(), normalKeys); } - wait(backupAgent.submitBackup(dest, KeyRef(tagName), backupRanges, false, StringRef(), StringRef(), true)); + wait(backupAgent.submitBackup( + dest, KeyRef(tagName), backupRanges, StopWhenDone::FALSE, StringRef(), StringRef(), LockDB::TRUE)); // Check if a backup agent is running bool agentRunning = wait(backupAgent.checkActive(dest)); @@ -1890,10 +1902,10 @@ ACTOR Future submitBackup(Database db, Standalone> backupRanges, std::string tagName, bool dryRun, - bool waitForCompletion, - bool stopWhenDone, - bool usePartitionedLog, - bool incrementalBackupOnly) { + WaitForComplete waitForCompletion, + StopWhenDone stopWhenDone, + UsePartitionedLog usePartitionedLog, + IncrementalBackupOnly incrementalBackupOnly) { try { state FileBackupAgent backupAgent; @@ -1996,7 +2008,7 @@ ACTOR Future switchDBBackup(Database src, Database dest, Standalone> backupRanges, std::string tagName, - bool forceAction) { + ForceAction forceAction) { try { state DatabaseBackupAgent backupAgent(src); @@ -2046,7 +2058,7 @@ ACTOR Future statusDBBackup(Database src, Database dest, std::string tagNa return Void(); } -ACTOR Future statusBackup(Database db, std::string tagName, bool showErrors, bool json) { +ACTOR Future statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) { try { state FileBackupAgent backupAgent; @@ -2063,11 +2075,15 @@ ACTOR Future statusBackup(Database db, std::string tagName, bool showError return Void(); } -ACTOR Future abortDBBackup(Database src, Database dest, std::string tagName, bool partial, bool dstOnly) { +ACTOR Future abortDBBackup(Database src, + Database dest, + std::string tagName, + PartialBackup partial, + DstOnly dstOnly) { try { state DatabaseBackupAgent backupAgent(src); - wait(backupAgent.abortBackup(dest, Key(tagName), partial, false, dstOnly)); + wait(backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::FALSE, dstOnly)); wait(backupAgent.unlockBackup(dest, Key(tagName))); printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); @@ -2118,7 +2134,7 @@ ACTOR Future abortBackup(Database db, std::string tagName) { return Void(); } -ACTOR Future cleanupMutations(Database db, bool deleteData) { +ACTOR Future cleanupMutations(Database db, DeleteData deleteData) { try { wait(cleanupBackup(db, deleteData)); } catch (Error& e) { @@ -2131,7 +2147,7 @@ ACTOR Future cleanupMutations(Database db, bool deleteData) { return Void(); } -ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDone) { +ACTOR Future waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) { try { state FileBackupAgent backupAgent; @@ -2150,7 +2166,7 @@ ACTOR Future waitBackup(Database db, std::string tagName, bool stopWhenDon return Void(); } -ACTOR Future discontinueBackup(Database db, std::string tagName, bool waitForCompletion) { +ACTOR Future discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) { try { state FileBackupAgent backupAgent; @@ -2220,7 +2236,9 @@ ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause return Void(); } -Reference openBackupContainer(const char* name, std::string destinationContainer) { +Reference openBackupContainer(const char* name, + std::string destinationContainer, + Optional const& encryptionKeyFile = {}) { // Error, if no dest container was specified if (destinationContainer.empty()) { fprintf(stderr, "ERROR: No backup destination was specified.\n"); @@ -2230,7 +2248,7 @@ Reference openBackupContainer(const char* name, std::string de Reference c; try { - c = IBackupContainer::openContainer(destinationContainer); + c = IBackupContainer::openContainer(destinationContainer, encryptionKeyFile); } catch (Error& e) { std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { @@ -2255,12 +2273,13 @@ ACTOR Future runRestore(Database db, Version targetVersion, std::string targetTimestamp, bool performRestore, - bool verbose, - bool waitForDone, + Verbose verbose, + WaitForComplete waitForDone, std::string addPrefix, std::string removePrefix, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly) { + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, + Optional encryptionKeyFile) { if (ranges.empty()) { ranges.push_back_deep(ranges.arena(), normalKeys); } @@ -2296,7 +2315,8 @@ ACTOR Future runRestore(Database db, try { state FileBackupAgent backupAgent; - state Reference bc = openBackupContainer(exeRestore.toString().c_str(), container); + state Reference bc = + openBackupContainer(exeRestore.toString().c_str(), container, encryptionKeyFile); // If targetVersion is unset then use the maximum restorable version from the backup description if (targetVersion == invalidVersion) { @@ -2306,7 +2326,7 @@ ACTOR Future runRestore(Database db, BackupDescription desc = wait(bc->describeBackup()); - if (onlyAppyMutationLogs && desc.contiguousLogEnd.present()) { + if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { targetVersion = desc.contiguousLogEnd.get() - 1; } else if (desc.maxRestorableVersion.present()) { targetVersion = desc.maxRestorableVersion.get(); @@ -2330,10 +2350,11 @@ ACTOR Future runRestore(Database db, verbose, KeyRef(addPrefix), KeyRef(removePrefix), - true, - onlyAppyMutationLogs, + LockDB::TRUE, + onlyApplyMutationLogs, inconsistentSnapshotOnly, - beginVersion)); + beginVersion, + encryptionKeyFile)); if (waitForDone && verbose) { // If restore is now complete then report version restored @@ -2369,8 +2390,8 @@ ACTOR Future runFastRestoreTool(Database db, Standalone> ranges, Version dbVersion, bool performRestore, - bool verbose, - bool waitForDone) { + Verbose verbose, + WaitForComplete waitForDone) { try { state FileBackupAgent backupAgent; state Version restoreVersion = invalidVersion; @@ -2413,7 +2434,7 @@ ACTOR Future runFastRestoreTool(Database db, ranges, KeyRef(container), dbVersion, - true, + LockDB::TRUE, randomUID, LiteralStringRef(""), LiteralStringRef(""))); @@ -2512,7 +2533,8 @@ ACTOR Future expireBackupData(const char* name, Database db, bool force, Version restorableAfterVersion, - std::string restorableAfterDatetime) { + std::string restorableAfterDatetime, + Optional encryptionKeyFile) { if (!endDatetime.empty()) { Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db)); endVersion = v; @@ -2531,7 +2553,7 @@ ACTOR Future expireBackupData(const char* name, } try { - Reference c = openBackupContainer(name, destinationContainer); + Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); state IBackupContainer::ExpireProgress progress; state std::string lastProgress; @@ -2613,9 +2635,10 @@ ACTOR Future describeBackup(const char* name, std::string destinationContainer, bool deep, Optional cx, - bool json) { + bool json, + Optional encryptionKeyFile) { try { - Reference c = openBackupContainer(name, destinationContainer); + Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); state BackupDescription desc = wait(c->describeBackup(deep)); if (cx.present()) wait(desc.resolveVersionTimes(cx.get())); @@ -2645,7 +2668,7 @@ ACTOR Future queryBackup(const char* name, Version restoreVersion, std::string originalClusterFile, std::string restoreTimestamp, - bool verbose) { + Verbose verbose) { state UID operationId = deterministicRandom()->randomUniqueID(); state JsonBuilderObject result; state std::string errorMessage; @@ -2838,7 +2861,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp } state BackupConfig config(uidFlag.get().first); - EBackupState s = wait(config.stateEnum().getOrThrow(tr, false, backup_invalid_info())); + EBackupState s = wait(config.stateEnum().getOrThrow(tr, Snapshot::FALSE, backup_invalid_info())); if (!FileBackupAgent::isRunnable(s)) { fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str()); throw backup_error(); @@ -2858,7 +2881,7 @@ ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOp } if (options.activeSnapshotIntervalSeconds.present()) { - Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, false, backup_error())); + Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, Snapshot::FALSE, backup_error())); config.snapshotTargetEndVersion().set(tr, begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); @@ -3244,13 +3267,13 @@ int main(int argc, char* argv[]) { Version beginVersion = invalidVersion; Version restoreVersion = invalidVersion; std::string restoreTimestamp; - bool waitForDone = false; - bool stopWhenDone = true; - bool usePartitionedLog = false; // Set to true to use new backup system - bool incrementalBackupOnly = false; - bool onlyAppyMutationLogs = false; - bool inconsistentSnapshotOnly = false; - bool forceAction = false; + WaitForComplete waitForDone{ false }; + StopWhenDone stopWhenDone{ true }; + UsePartitionedLog usePartitionedLog{ false }; // Set to true to use new backup system + IncrementalBackupOnly incrementalBackupOnly{ false }; + OnlyApplyMutationLogs onlyApplyMutationLogs{ false }; + InconsistentSnapshotOnly inconsistentSnapshotOnly{ false }; + ForceAction forceAction{ false }; bool trace = false; bool quietDisplay = false; bool dryRun = false; @@ -3260,8 +3283,8 @@ int main(int argc, char* argv[]) { uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE; uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE; ESOError lastError; - bool partial = true; - bool dstOnly = false; + PartialBackup partial{ true }; + DstOnly dstOnly{ false }; LocalityData localities; uint64_t memLimit = 8LL << 30; Optional ti; @@ -3271,7 +3294,8 @@ int main(int argc, char* argv[]) { std::string restoreClusterFileDest; std::string restoreClusterFileOrig; bool jsonOutput = false; - bool deleteData = false; + DeleteData deleteData{ false }; + Optional encryptionKeyFile; BackupModifyOptions modifyOptions; @@ -3355,13 +3379,13 @@ int main(int argc, char* argv[]) { dryRun = true; break; case OPT_DELETE_DATA: - deleteData = true; + deleteData.set(true); break; case OPT_MIN_CLEANUP_SECONDS: knobs.emplace_back("min_cleanup_seconds", args->OptionArg()); break; case OPT_FORCE: - forceAction = true; + forceAction.set(true); break; case OPT_TRACE: trace = true; @@ -3441,10 +3465,10 @@ int main(int argc, char* argv[]) { sourceClusterFile = args->OptionArg(); break; case OPT_CLEANUP: - partial = false; + partial.set(false); break; case OPT_DSTONLY: - dstOnly = true; + dstOnly.set(true); break; case OPT_KNOB: { std::string syn = args->OptionSyntax(); @@ -3503,17 +3527,20 @@ int main(int argc, char* argv[]) { modifyOptions.verifyUID = args->OptionArg(); break; case OPT_WAITFORDONE: - waitForDone = true; + waitForDone.set(true); break; case OPT_NOSTOPWHENDONE: - stopWhenDone = false; + stopWhenDone.set(false); break; case OPT_USE_PARTITIONED_LOG: - usePartitionedLog = true; + usePartitionedLog.set(true); break; case OPT_INCREMENTALONLY: - incrementalBackupOnly = true; - onlyAppyMutationLogs = true; + incrementalBackupOnly.set(true); + onlyApplyMutationLogs.set(true); + break; + case OPT_ENCRYPTION_KEY_FILE: + encryptionKeyFile = args->OptionArg(); break; case OPT_RESTORECONTAINER: restoreContainer = args->OptionArg(); @@ -3565,7 +3592,7 @@ int main(int argc, char* argv[]) { break; } case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: { - inconsistentSnapshotOnly = true; + inconsistentSnapshotOnly.set(true); break; } #ifdef _WIN32 @@ -3704,7 +3731,7 @@ int main(int argc, char* argv[]) { } } - IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO); + IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::FALSE, IsSimulated::FALSE); auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection(); for (const auto& [knobName, knobValueString] : knobs) { try { @@ -3731,7 +3758,7 @@ int main(int argc, char* argv[]) { } // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs - g_knobs.initialize(Randomize::NO, IsSimulated::NO); + g_knobs.initialize(Randomize::FALSE, IsSimulated::FALSE); if (trace) { if (!traceLogGroup.empty()) @@ -3769,7 +3796,7 @@ int main(int argc, char* argv[]) { Reference c; try { - setupNetwork(0, true); + setupNetwork(0, UseMetrics::TRUE); } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); return FDB_EXIT_ERROR; @@ -3813,7 +3840,7 @@ int main(int argc, char* argv[]) { } try { - db = Database::createDatabase(ccf, -1, true, localities); + db = Database::createDatabase(ccf, -1, IsInternal::TRUE, localities); } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getFilename().c_str()); @@ -3833,7 +3860,7 @@ int main(int argc, char* argv[]) { } try { - sourceDb = Database::createDatabase(sourceCcf, -1, true, localities); + sourceDb = Database::createDatabase(sourceCcf, -1, IsInternal::TRUE, localities); } catch (Error& e) { fprintf(stderr, "ERROR: %s\n", e.what()); fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", sourceCcf->getFilename().c_str()); @@ -3853,7 +3880,7 @@ int main(int argc, char* argv[]) { if (!initCluster()) return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. - openBackupContainer(argv[0], destinationContainer); + openBackupContainer(argv[0], destinationContainer, encryptionKeyFile); f = stopAfter(submitBackup(db, destinationContainer, initialSnapshotIntervalSeconds, @@ -3879,7 +3906,7 @@ int main(int argc, char* argv[]) { case BackupType::STATUS: if (!initCluster()) return FDB_EXIT_ERROR; - f = stopAfter(statusBackup(db, tagName, true, jsonOutput)); + f = stopAfter(statusBackup(db, tagName, ShowErrors::TRUE, jsonOutput)); break; case BackupType::ABORT: @@ -3932,7 +3959,8 @@ int main(int argc, char* argv[]) { db, forceAction, expireRestorableAfterVersion, - expireRestorableAfterDatetime)); + expireRestorableAfterDatetime, + encryptionKeyFile)); break; case BackupType::DELETE_BACKUP: @@ -3952,7 +3980,8 @@ int main(int argc, char* argv[]) { destinationContainer, describeDeep, describeTimestamps ? Optional(db) : Optional(), - jsonOutput)); + jsonOutput, + encryptionKeyFile)); break; case BackupType::LIST: @@ -3968,7 +3997,7 @@ int main(int argc, char* argv[]) { restoreVersion, restoreClusterFileOrig, restoreTimestamp, - !quietDisplay)); + Verbose{ !quietDisplay })); break; case BackupType::DUMP: @@ -4029,15 +4058,16 @@ int main(int argc, char* argv[]) { restoreVersion, restoreTimestamp, !dryRun, - !quietDisplay, + Verbose{ !quietDisplay }, waitForDone, addPrefix, removePrefix, - onlyAppyMutationLogs, - inconsistentSnapshotOnly)); + onlyApplyMutationLogs, + inconsistentSnapshotOnly, + encryptionKeyFile)); break; case RestoreType::WAIT: - f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), true))); + f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), Verbose::TRUE))); break; case RestoreType::ABORT: f = stopAfter( @@ -4097,8 +4127,14 @@ int main(int argc, char* argv[]) { // TODO: We have not implemented the code commented out in this case switch (restoreType) { case RestoreType::START: - f = stopAfter(runFastRestoreTool( - db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, !quietDisplay, waitForDone)); + f = stopAfter(runFastRestoreTool(db, + tagName, + restoreContainer, + backupKeys, + restoreVersion, + !dryRun, + Verbose{ !quietDisplay }, + waitForDone)); break; case RestoreType::WAIT: printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n"); diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 7e7abd2e3c..d193d7ea24 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -3151,7 +3151,7 @@ struct CLIOptions { } // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs - g_knobs.initialize(Randomize::NO, IsSimulated::NO); + g_knobs.initialize(Randomize::FALSE, IsSimulated::FALSE); } int processArg(CSimpleOpt& args) { @@ -3322,7 +3322,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { TraceEvent::setNetworkThread(); try { - db = Database::createDatabase(ccf, -1, false); + db = Database::createDatabase(ccf, -1, IsInternal::FALSE); if (!opt.exec.present()) { printf("Using cluster file `%s'.\n", ccf->getFilename().c_str()); } @@ -4924,7 +4924,7 @@ int main(int argc, char** argv) { registerCrashHandler(); - IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO); + IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::CLIENT, Randomize::FALSE, IsSimulated::FALSE); #ifdef __unixish__ struct sigaction act; diff --git a/fdbclient/AsyncFileS3BlobStore.actor.h b/fdbclient/AsyncFileS3BlobStore.actor.h index bc520bda90..db436755b3 100644 --- a/fdbclient/AsyncFileS3BlobStore.actor.h +++ b/fdbclient/AsyncFileS3BlobStore.actor.h @@ -256,7 +256,7 @@ public: m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) { // Add first part - m_parts.push_back(Reference(new Part(1, m_bstore->knobs.multipart_min_part_size))); + m_parts.push_back(makeReference(1, m_bstore->knobs.multipart_min_part_size)); } }; diff --git a/fdbclient/AsyncTaskThread.actor.cpp b/fdbclient/AsyncTaskThread.actor.cpp index 2e7c6e3596..050af68c29 100644 --- a/fdbclient/AsyncTaskThread.actor.cpp +++ b/fdbclient/AsyncTaskThread.actor.cpp @@ -83,6 +83,6 @@ TEST_CASE("/asynctaskthread/add") { clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100)); } wait(waitForAll(clients)); - ASSERT(sum == 1000); + ASSERT_EQ(sum, 1000); return Void(); } diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index c8903b9fe4..69c2caa53f 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -36,6 +36,26 @@ #include "fdbclient/BackupContainer.h" #include "flow/actorcompiler.h" // has to be last include +FDB_DECLARE_BOOLEAN_PARAM(LockDB); +FDB_DECLARE_BOOLEAN_PARAM(UnlockDB); +FDB_DECLARE_BOOLEAN_PARAM(StopWhenDone); +FDB_DECLARE_BOOLEAN_PARAM(Verbose); +FDB_DECLARE_BOOLEAN_PARAM(WaitForComplete); +FDB_DECLARE_BOOLEAN_PARAM(ForceAction); +FDB_DECLARE_BOOLEAN_PARAM(Terminator); +FDB_DECLARE_BOOLEAN_PARAM(IncrementalBackupOnly); +FDB_DECLARE_BOOLEAN_PARAM(UsePartitionedLog); +FDB_DECLARE_BOOLEAN_PARAM(OnlyApplyMutationLogs); +FDB_DECLARE_BOOLEAN_PARAM(InconsistentSnapshotOnly); +FDB_DECLARE_BOOLEAN_PARAM(ShowErrors); +FDB_DECLARE_BOOLEAN_PARAM(AbortOldBackup); +FDB_DECLARE_BOOLEAN_PARAM(DstOnly); // TODO: More descriptive name? +FDB_DECLARE_BOOLEAN_PARAM(WaitForDestUID); +FDB_DECLARE_BOOLEAN_PARAM(CheckBackupUID); +FDB_DECLARE_BOOLEAN_PARAM(DeleteData); +FDB_DECLARE_BOOLEAN_PARAM(SetValidation); +FDB_DECLARE_BOOLEAN_PARAM(PartialBackup); + class BackupAgentBase : NonCopyable { public: // Time formatter for anything backup or restore related @@ -65,6 +85,7 @@ public: static const Key keyConfigStopWhenDoneKey; static const Key keyStateStatus; static const Key keyStateStop; + static const Key keyStateLogBeginVersion; static const Key keyLastUid; static const Key keyBeginKey; static const Key keyEndKey; @@ -82,151 +103,26 @@ public: static const Key keySourceStates; static const Key keySourceTagName; - static const int logHeaderSize; + static constexpr int logHeaderSize = 12; // Convert the status text to an enumerated value - static EnumState getState(std::string stateText) { - auto enState = EnumState::STATE_ERRORED; - - if (stateText.empty()) { - enState = EnumState::STATE_NEVERRAN; - } - - else if (!stateText.compare("has been submitted")) { - enState = EnumState::STATE_SUBMITTED; - } - - else if (!stateText.compare("has been started")) { - enState = EnumState::STATE_RUNNING; - } - - else if (!stateText.compare("is differential")) { - enState = EnumState::STATE_RUNNING_DIFFERENTIAL; - } - - else if (!stateText.compare("has been completed")) { - enState = EnumState::STATE_COMPLETED; - } - - else if (!stateText.compare("has been aborted")) { - enState = EnumState::STATE_ABORTED; - } - - else if (!stateText.compare("has been partially aborted")) { - enState = EnumState::STATE_PARTIALLY_ABORTED; - } - - return enState; - } + static EnumState getState(std::string const& stateText); // Convert the status enum to a text description - static const char* getStateText(EnumState enState) { - const char* stateText; - - switch (enState) { - case EnumState::STATE_ERRORED: - stateText = "has errored"; - break; - case EnumState::STATE_NEVERRAN: - stateText = "has never been started"; - break; - case EnumState::STATE_SUBMITTED: - stateText = "has been submitted"; - break; - case EnumState::STATE_RUNNING: - stateText = "has been started"; - break; - case EnumState::STATE_RUNNING_DIFFERENTIAL: - stateText = "is differential"; - break; - case EnumState::STATE_COMPLETED: - stateText = "has been completed"; - break; - case EnumState::STATE_ABORTED: - stateText = "has been aborted"; - break; - case EnumState::STATE_PARTIALLY_ABORTED: - stateText = "has been partially aborted"; - break; - default: - stateText = ""; - break; - } - - return stateText; - } + static const char* getStateText(EnumState enState); // Convert the status enum to a name - static const char* getStateName(EnumState enState) { - const char* s; - - switch (enState) { - case EnumState::STATE_ERRORED: - s = "Errored"; - break; - case EnumState::STATE_NEVERRAN: - s = "NeverRan"; - break; - case EnumState::STATE_SUBMITTED: - s = "Submitted"; - break; - case EnumState::STATE_RUNNING: - s = "Running"; - break; - case EnumState::STATE_RUNNING_DIFFERENTIAL: - s = "RunningDifferentially"; - break; - case EnumState::STATE_COMPLETED: - s = "Completed"; - break; - case EnumState::STATE_ABORTED: - s = "Aborted"; - break; - case EnumState::STATE_PARTIALLY_ABORTED: - s = "Aborting"; - break; - default: - s = ""; - break; - } - - return s; - } + static const char* getStateName(EnumState enState); // Determine if the specified state is runnable - static bool isRunnable(EnumState enState) { - bool isRunnable = false; + static bool isRunnable(EnumState enState); - switch (enState) { - case EnumState::STATE_SUBMITTED: - case EnumState::STATE_RUNNING: - case EnumState::STATE_RUNNING_DIFFERENTIAL: - case EnumState::STATE_PARTIALLY_ABORTED: - isRunnable = true; - break; - default: - break; - } + static KeyRef getDefaultTag() { return StringRef(defaultTagName); } - return isRunnable; - } - - static const KeyRef getDefaultTag() { return StringRef(defaultTagName); } - - static const std::string getDefaultTagName() { return defaultTagName; } + static std::string getDefaultTagName() { return defaultTagName; } // This is only used for automatic backup name generation - static Standalone getCurrentTime() { - double t = now(); - time_t curTime = t; - char buffer[128]; - struct tm* timeinfo; - timeinfo = localtime(&curTime); - strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo); - - std::string time(buffer); - return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime)))); - } + static Standalone getCurrentTime(); protected: static const std::string defaultTagName; @@ -249,7 +145,11 @@ public: KeyBackedProperty lastBackupTimestamp() { return config.pack(LiteralStringRef(__FUNCTION__)); } - Future run(Database cx, double* pollDelay, int maxConcurrentTasks) { + Future run(Database cx, double pollDelay, int maxConcurrentTasks) { + return taskBucket->run(cx, futureBucket, std::make_shared(pollDelay), maxConcurrentTasks); + } + + Future run(Database cx, std::shared_ptr pollDelay, int maxConcurrentTasks) { return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); } @@ -260,13 +160,13 @@ public: static Key getPauseKey(); // parallel restore - Future parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true); + Future parallelRestoreFinish(Database cx, UID randomUID, UnlockDB = UnlockDB::TRUE); Future submitParallelRestore(Database cx, Key backupTag, Standalone> backupRanges, Key bcUrl, Version targetVersion, - bool lockDB, + LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix); @@ -288,29 +188,31 @@ public: Key tagName, Key url, Standalone> ranges, - bool waitForComplete = true, - Version targetVersion = -1, - bool verbose = true, + WaitForComplete = WaitForComplete::TRUE, + Version targetVersion = ::invalidVersion, + Verbose = Verbose::TRUE, Key addPrefix = Key(), Key removePrefix = Key(), - bool lockDB = true, - bool onlyAppyMutationLogs = false, - bool inconsistentSnapshotOnly = false, - Version beginVersion = -1); + LockDB = LockDB::TRUE, + OnlyApplyMutationLogs = OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly = InconsistentSnapshotOnly::FALSE, + Version beginVersion = ::invalidVersion, + Optional const& encryptionKeyFileName = {}); Future restore(Database cx, Optional cxOrig, Key tagName, Key url, - bool waitForComplete = true, - Version targetVersion = -1, - bool verbose = true, + WaitForComplete waitForComplete = WaitForComplete::TRUE, + Version targetVersion = ::invalidVersion, + Verbose verbose = Verbose::TRUE, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key(), - bool lockDB = true, - bool onlyAppyMutationLogs = false, - bool inconsistentSnapshotOnly = false, - Version beginVersion = -1) { + LockDB lockDB = LockDB::TRUE, + OnlyApplyMutationLogs onlyApplyMutationLogs = OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly inconsistentSnapshotOnly = InconsistentSnapshotOnly::FALSE, + Version beginVersion = ::invalidVersion, + Optional const& encryptionKeyFileName = {}) { Standalone> rangeRef; rangeRef.push_back_deep(rangeRef.arena(), range); return restore(cx, @@ -324,9 +226,10 @@ public: addPrefix, removePrefix, lockDB, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, - beginVersion); + beginVersion, + encryptionKeyFileName); } Future atomicRestore(Database cx, Key tagName, @@ -347,7 +250,7 @@ public: Future abortRestore(Database cx, Key tagName); // Waits for a restore tag to reach a final (stable) state. - Future waitRestore(Database cx, Key tagName, bool verbose); + Future waitRestore(Database cx, Key tagName, Verbose); // Get a string describing the status of a tag Future restoreStatus(Reference tr, Key tagName); @@ -362,20 +265,22 @@ public: Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, - bool stopWhenDone = true, - bool partitionedLog = false, - bool incrementalBackupOnly = false); + StopWhenDone = StopWhenDone::TRUE, + UsePartitionedLog = UsePartitionedLog::FALSE, + IncrementalBackupOnly = IncrementalBackupOnly::FALSE, + Optional const& encryptionKeyFileName = {}); Future submitBackup(Database cx, Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, - bool stopWhenDone = true, - bool partitionedLog = false, - bool incrementalBackupOnly = false) { + StopWhenDone stopWhenDone = StopWhenDone::TRUE, + UsePartitionedLog partitionedLog = UsePartitionedLog::FALSE, + IncrementalBackupOnly incrementalBackupOnly = IncrementalBackupOnly::FALSE, + Optional const& encryptionKeyFileName = {}) { return runRYWTransactionFailIfLocked(cx, [=](Reference tr) { return submitBackup(tr, outContainer, @@ -385,7 +290,8 @@ public: backupRanges, stopWhenDone, partitionedLog, - incrementalBackupOnly); + incrementalBackupOnly, + encryptionKeyFileName); }); } @@ -407,19 +313,19 @@ public: return runRYWTransaction(cx, [=](Reference tr) { return abortBackup(tr, tagName); }); } - Future getStatus(Database cx, bool showErrors, std::string tagName); + Future getStatus(Database cx, ShowErrors, std::string tagName); Future getStatusJSON(Database cx, std::string tagName); Future> getLastRestorable(Reference tr, Key tagName, - bool snapshot = false); + Snapshot = Snapshot::FALSE); void setLastRestorable(Reference tr, Key tagName, Version version); // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. Future waitBackup(Database cx, std::string tagName, - bool stopWhenDone = true, + StopWhenDone = StopWhenDone::TRUE, Reference* pContainer = nullptr, UID* pUID = nullptr); @@ -478,7 +384,11 @@ public: sourceTagNames = std::move(r.sourceTagNames); } - Future run(Database cx, double* pollDelay, int maxConcurrentTasks) { + Future run(Database cx, double pollDelay, int maxConcurrentTasks) { + return taskBucket->run(cx, futureBucket, std::make_shared(pollDelay), maxConcurrentTasks); + } + + Future run(Database cx, std::shared_ptr pollDelay, int maxConcurrentTasks) { return taskBucket->run(cx, futureBucket, pollDelay, maxConcurrentTasks); } @@ -487,7 +397,7 @@ public: Standalone> backupRanges, Key addPrefix, Key removePrefix, - bool forceAction = false); + ForceAction = ForceAction::FALSE); Future unlockBackup(Reference tr, Key tagName); Future unlockBackup(Database cx, Key tagName) { @@ -506,18 +416,18 @@ public: Future submitBackup(Reference tr, Key tagName, Standalone> backupRanges, - bool stopWhenDone = true, + StopWhenDone = StopWhenDone::TRUE, Key addPrefix = StringRef(), Key removePrefix = StringRef(), - bool lockDatabase = false, + LockDB lockDatabase = LockDB::FALSE, PreBackupAction backupAction = PreBackupAction::VERIFY); Future submitBackup(Database cx, Key tagName, Standalone> backupRanges, - bool stopWhenDone = true, + StopWhenDone stopWhenDone = StopWhenDone::TRUE, Key addPrefix = StringRef(), Key removePrefix = StringRef(), - bool lockDatabase = false, + LockDB lockDatabase = LockDB::FALSE, PreBackupAction backupAction = PreBackupAction::VERIFY) { return runRYWTransaction(cx, [=](Reference tr) { return submitBackup( @@ -533,35 +443,36 @@ public: Future abortBackup(Database cx, Key tagName, - bool partial = false, - bool abortOldBackup = false, - bool dstOnly = false, - bool waitForDestUID = false); + PartialBackup = PartialBackup::FALSE, + AbortOldBackup = AbortOldBackup::FALSE, + DstOnly = DstOnly::FALSE, + WaitForDestUID = WaitForDestUID::FALSE); Future getStatus(Database cx, int errorLimit, Key tagName); - Future getStateValue(Reference tr, UID logUid, bool snapshot = false); + Future getStateValue(Reference tr, UID logUid, Snapshot = Snapshot::FALSE); Future getStateValue(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr) { return getStateValue(tr, logUid); }); } - Future getDestUid(Reference tr, UID logUid, bool snapshot = false); + Future getDestUid(Reference tr, UID logUid, Snapshot = Snapshot::FALSE); Future getDestUid(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr) { return getDestUid(tr, logUid); }); } - Future getLogUid(Reference tr, Key tagName, bool snapshot = false); + Future getLogUid(Reference tr, Key tagName, Snapshot = Snapshot::FALSE); Future getLogUid(Database cx, Key tagName) { return runRYWTransaction(cx, [=](Reference tr) { return getLogUid(tr, tagName); }); } - Future getRangeBytesWritten(Reference tr, UID logUid, bool snapshot = false); - Future getLogBytesWritten(Reference tr, UID logUid, bool snapshot = false); - + Future getRangeBytesWritten(Reference tr, + UID logUid, + Snapshot = Snapshot::FALSE); + Future getLogBytesWritten(Reference tr, UID logUid, Snapshot = Snapshot::FALSE); // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. - Future waitBackup(Database cx, Key tagName, bool stopWhenDone = true); + Future waitBackup(Database cx, Key tagName, StopWhenDone = StopWhenDone::TRUE); Future waitSubmitted(Database cx, Key tagName); Future waitUpgradeToLatestDrVersion(Database cx, Key tagName); @@ -619,7 +530,7 @@ Future eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion = Optional(), - bool checkBackupUid = false, + CheckBackupUID = CheckBackupUID::FALSE, Version backupUid = 0); Key getApplyKey(Version version, Key backupUid); Version getLogKeyVersion(Key key); @@ -631,18 +542,18 @@ ACTOR Future readCommitted(Database cx, PromiseStream results, Reference lock, KeyRangeRef range, - bool terminator = true, - bool systemAccess = false, - bool lockAware = false); + Terminator terminator = Terminator::TRUE, + AccessSystemKeys systemAccess = AccessSystemKeys::FALSE, + LockAware lockAware = LockAware::FALSE); ACTOR Future readCommitted(Database cx, PromiseStream results, Future active, Reference lock, KeyRangeRef range, std::function(Key key)> groupBy, - bool terminator = true, - bool systemAccess = false, - bool lockAware = false); + Terminator terminator = Terminator::TRUE, + AccessSystemKeys systemAccess = AccessSystemKeys::FALSE, + LockAware lockAware = LockAware::FALSE); ACTOR Future applyMutations(Database cx, Key uid, Key addPrefix, @@ -652,7 +563,7 @@ ACTOR Future applyMutations(Database cx, RequestStream commit, NotifiedVersion* committedVersion, Reference> keyVersion); -ACTOR Future cleanupBackup(Database cx, bool deleteData); +ACTOR Future cleanupBackup(Database cx, DeleteData deleteData); using EBackupState = BackupAgentBase::EnumState; template <> @@ -695,14 +606,15 @@ public: typedef KeyBackedMap TagMap; // Map of tagName to {UID, aborted_flag} located in the fileRestorePrefixRange keyspace. class TagUidMap : public KeyBackedMap { + ACTOR static Future> getAll_impl(TagUidMap* tagsMap, + Reference tr, + Snapshot snapshot); + public: TagUidMap(const StringRef& prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {} - ACTOR static Future> getAll_impl(TagUidMap* tagsMap, - Reference tr, - bool snapshot); - - Future> getAll(Reference tr, bool snapshot = false) { + Future> getAll(Reference tr, + Snapshot snapshot = Snapshot::FALSE) { return getAll_impl(this, tr, snapshot); } @@ -718,12 +630,12 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) { } static inline Future> getAllRestoreTags(Reference tr, - bool snapshot = false) { + Snapshot snapshot = Snapshot::FALSE) { return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot); } static inline Future> getAllBackupTags(Reference tr, - bool snapshot = false) { + Snapshot snapshot = Snapshot::FALSE) { return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot); } @@ -738,7 +650,9 @@ public: KeyBackedConfig(StringRef prefix, Reference task) : KeyBackedConfig(prefix, TaskParams.uid().get(task)) {} - Future toTask(Reference tr, Reference task, bool setValidation = true) { + Future toTask(Reference tr, + Reference task, + SetValidation setValidation = SetValidation::TRUE) { // Set the uid task parameter TaskParams.uid().set(task, uid); diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp index 4b00857503..fdb374d7ca 100644 --- a/fdbclient/BackupAgentBase.actor.cpp +++ b/fdbclient/BackupAgentBase.actor.cpp @@ -26,6 +26,24 @@ #include "flow/ActorCollection.h" #include "flow/actorcompiler.h" // has to be last include +FDB_DEFINE_BOOLEAN_PARAM(LockDB); +FDB_DEFINE_BOOLEAN_PARAM(UnlockDB); +FDB_DEFINE_BOOLEAN_PARAM(StopWhenDone); +FDB_DEFINE_BOOLEAN_PARAM(Verbose); +FDB_DEFINE_BOOLEAN_PARAM(WaitForComplete); +FDB_DEFINE_BOOLEAN_PARAM(ForceAction); +FDB_DEFINE_BOOLEAN_PARAM(Terminator); +FDB_DEFINE_BOOLEAN_PARAM(UsePartitionedLog); +FDB_DEFINE_BOOLEAN_PARAM(InconsistentSnapshotOnly); +FDB_DEFINE_BOOLEAN_PARAM(ShowErrors); +FDB_DEFINE_BOOLEAN_PARAM(AbortOldBackup); +FDB_DEFINE_BOOLEAN_PARAM(DstOnly); +FDB_DEFINE_BOOLEAN_PARAM(WaitForDestUID); +FDB_DEFINE_BOOLEAN_PARAM(CheckBackupUID); +FDB_DEFINE_BOOLEAN_PARAM(DeleteData); +FDB_DEFINE_BOOLEAN_PARAM(SetValidation); +FDB_DEFINE_BOOLEAN_PARAM(PartialBackup); + std::string BackupAgentBase::formatTime(int64_t epochs) { time_t curTime = (time_t)epochs; char buffer[30]; @@ -95,32 +113,33 @@ int64_t BackupAgentBase::parseTime(std::string timestamp) { return ts; } -const Key BackupAgentBase::keyFolderId = LiteralStringRef("config_folderid"); -const Key BackupAgentBase::keyBeginVersion = LiteralStringRef("beginVersion"); -const Key BackupAgentBase::keyEndVersion = LiteralStringRef("endVersion"); -const Key BackupAgentBase::keyPrevBeginVersion = LiteralStringRef("prevBeginVersion"); -const Key BackupAgentBase::keyConfigBackupTag = LiteralStringRef("config_backup_tag"); -const Key BackupAgentBase::keyConfigLogUid = LiteralStringRef("config_log_uid"); -const Key BackupAgentBase::keyConfigBackupRanges = LiteralStringRef("config_backup_ranges"); -const Key BackupAgentBase::keyConfigStopWhenDoneKey = LiteralStringRef("config_stop_when_done"); -const Key BackupAgentBase::keyStateStop = LiteralStringRef("state_stop"); -const Key BackupAgentBase::keyStateStatus = LiteralStringRef("state_status"); -const Key BackupAgentBase::keyLastUid = LiteralStringRef("last_uid"); -const Key BackupAgentBase::keyBeginKey = LiteralStringRef("beginKey"); -const Key BackupAgentBase::keyEndKey = LiteralStringRef("endKey"); -const Key BackupAgentBase::keyDrVersion = LiteralStringRef("drVersion"); -const Key BackupAgentBase::destUid = LiteralStringRef("destUid"); -const Key BackupAgentBase::backupStartVersion = LiteralStringRef("backupStartVersion"); +const Key BackupAgentBase::keyFolderId = "config_folderid"_sr; +const Key BackupAgentBase::keyBeginVersion = "beginVersion"_sr; +const Key BackupAgentBase::keyEndVersion = "endVersion"_sr; +const Key BackupAgentBase::keyPrevBeginVersion = "prevBeginVersion"_sr; +const Key BackupAgentBase::keyConfigBackupTag = "config_backup_tag"_sr; +const Key BackupAgentBase::keyConfigLogUid = "config_log_uid"_sr; +const Key BackupAgentBase::keyConfigBackupRanges = "config_backup_ranges"_sr; +const Key BackupAgentBase::keyConfigStopWhenDoneKey = "config_stop_when_done"_sr; +const Key BackupAgentBase::keyStateStop = "state_stop"_sr; +const Key BackupAgentBase::keyStateStatus = "state_status"_sr; +const Key BackupAgentBase::keyStateLogBeginVersion = "last_begin_version"_sr; +const Key BackupAgentBase::keyLastUid = "last_uid"_sr; +const Key BackupAgentBase::keyBeginKey = "beginKey"_sr; +const Key BackupAgentBase::keyEndKey = "endKey"_sr; +const Key BackupAgentBase::keyDrVersion = "drVersion"_sr; +const Key BackupAgentBase::destUid = "destUid"_sr; +const Key BackupAgentBase::backupStartVersion = "backupStartVersion"_sr; -const Key BackupAgentBase::keyTagName = LiteralStringRef("tagname"); -const Key BackupAgentBase::keyStates = LiteralStringRef("state"); -const Key BackupAgentBase::keyConfig = LiteralStringRef("config"); -const Key BackupAgentBase::keyErrors = LiteralStringRef("errors"); -const Key BackupAgentBase::keyRanges = LiteralStringRef("ranges"); -const Key BackupAgentBase::keyTasks = LiteralStringRef("tasks"); -const Key BackupAgentBase::keyFutures = LiteralStringRef("futures"); -const Key BackupAgentBase::keySourceStates = LiteralStringRef("source_states"); -const Key BackupAgentBase::keySourceTagName = LiteralStringRef("source_tagname"); +const Key BackupAgentBase::keyTagName = "tagname"_sr; +const Key BackupAgentBase::keyStates = "state"_sr; +const Key BackupAgentBase::keyConfig = "config"_sr; +const Key BackupAgentBase::keyErrors = "errors"_sr; +const Key BackupAgentBase::keyRanges = "ranges"_sr; +const Key BackupAgentBase::keyTasks = "tasks"_sr; +const Key BackupAgentBase::keyFutures = "futures"_sr; +const Key BackupAgentBase::keySourceStates = "source_states"_sr; +const Key BackupAgentBase::keySourceTagName = "source_tagname"_sr; bool copyParameter(Reference source, Reference dest, Key key) { if (source) { @@ -374,9 +393,9 @@ ACTOR Future readCommitted(Database cx, PromiseStream results, Reference lock, KeyRangeRef range, - bool terminator, - bool systemAccess, - bool lockAware) { + Terminator terminator, + AccessSystemKeys systemAccess, + LockAware lockAware) { state KeySelector begin = firstGreaterOrEqual(range.begin); state KeySelector end = firstGreaterOrEqual(range.end); state Transaction tr(cx); @@ -450,9 +469,9 @@ ACTOR Future readCommitted(Database cx, Reference lock, KeyRangeRef range, std::function(Key key)> groupBy, - bool terminator, - bool systemAccess, - bool lockAware) { + Terminator terminator, + AccessSystemKeys systemAccess, + LockAware lockAware) { state KeySelector nextKey = firstGreaterOrEqual(range.begin); state KeySelector end = firstGreaterOrEqual(range.end); @@ -559,7 +578,8 @@ Future readCommitted(Database cx, Reference lock, KeyRangeRef range, std::function(Key key)> groupBy) { - return readCommitted(cx, results, Void(), lock, range, groupBy, true, true, true); + return readCommitted( + cx, results, Void(), lock, range, groupBy, Terminator::TRUE, AccessSystemKeys::TRUE, LockAware::TRUE); } ACTOR Future dumpData(Database cx, @@ -770,7 +790,7 @@ ACTOR static Future _eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion, - bool checkBackupUid, + CheckBackupUID checkBackupUid, Version backupUid) { state Key backupLatestVersionsPath = destUidValue.withPrefix(backupLatestVersionsPrefix); state Key backupLatestVersionsKey = logUidValue.withPrefix(backupLatestVersionsPath); @@ -898,7 +918,7 @@ Future eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion, - bool checkBackupUid, + CheckBackupUID checkBackupUid, Version backupUid) { return _eraseLogData(tr, logUidValue, destUidValue, endVersion, checkBackupUid, backupUid); } @@ -995,7 +1015,7 @@ ACTOR Future cleanupLogMutations(Database cx, Value destUidValue, bool del } } -ACTOR Future cleanupBackup(Database cx, bool deleteData) { +ACTOR Future cleanupBackup(Database cx, DeleteData deleteData) { state Reference tr(new ReadYourWritesTransaction(cx)); loop { try { @@ -1014,3 +1034,124 @@ ACTOR Future cleanupBackup(Database cx, bool deleteData) { } } } + +// Convert the status text to an enumerated value +BackupAgentBase::EnumState BackupAgentBase::getState(std::string const& stateText) { + auto enState = EnumState::STATE_ERRORED; + + if (stateText.empty()) { + enState = EnumState::STATE_NEVERRAN; + } + + else if (!stateText.compare("has been submitted")) { + enState = EnumState::STATE_SUBMITTED; + } + + else if (!stateText.compare("has been started")) { + enState = EnumState::STATE_RUNNING; + } + + else if (!stateText.compare("is differential")) { + enState = EnumState::STATE_RUNNING_DIFFERENTIAL; + } + + else if (!stateText.compare("has been completed")) { + enState = EnumState::STATE_COMPLETED; + } + + else if (!stateText.compare("has been aborted")) { + enState = EnumState::STATE_ABORTED; + } + + else if (!stateText.compare("has been partially aborted")) { + enState = EnumState::STATE_PARTIALLY_ABORTED; + } + + return enState; +} + +const char* BackupAgentBase::getStateText(EnumState enState) { + const char* stateText; + + switch (enState) { + case EnumState::STATE_ERRORED: + stateText = "has errored"; + break; + case EnumState::STATE_NEVERRAN: + stateText = "has never been started"; + break; + case EnumState::STATE_SUBMITTED: + stateText = "has been submitted"; + break; + case EnumState::STATE_RUNNING: + stateText = "has been started"; + break; + case EnumState::STATE_RUNNING_DIFFERENTIAL: + stateText = "is differential"; + break; + case EnumState::STATE_COMPLETED: + stateText = "has been completed"; + break; + case EnumState::STATE_ABORTED: + stateText = "has been aborted"; + break; + case EnumState::STATE_PARTIALLY_ABORTED: + stateText = "has been partially aborted"; + break; + default: + stateText = ""; + break; + } + + return stateText; +} + +const char* BackupAgentBase::getStateName(EnumState enState) { + switch (enState) { + case EnumState::STATE_ERRORED: + return "Errored"; + case EnumState::STATE_NEVERRAN: + return "NeverRan"; + case EnumState::STATE_SUBMITTED: + return "Submitted"; + break; + case EnumState::STATE_RUNNING: + return "Running"; + case EnumState::STATE_RUNNING_DIFFERENTIAL: + return "RunningDifferentially"; + case EnumState::STATE_COMPLETED: + return "Completed"; + case EnumState::STATE_ABORTED: + return "Aborted"; + case EnumState::STATE_PARTIALLY_ABORTED: + return "Aborting"; + default: + return ""; + } +} + +bool BackupAgentBase::isRunnable(EnumState enState) { + switch (enState) { + case EnumState::STATE_SUBMITTED: + case EnumState::STATE_RUNNING: + case EnumState::STATE_RUNNING_DIFFERENTIAL: + case EnumState::STATE_PARTIALLY_ABORTED: + return true; + default: + return false; + } +} + +Standalone BackupAgentBase::getCurrentTime() { + double t = now(); + time_t curTime = t; + char buffer[128]; + struct tm* timeinfo; + timeinfo = localtime(&curTime); + strftime(buffer, 128, "%Y-%m-%d-%H-%M-%S", timeinfo); + + std::string time(buffer); + return StringRef(time + format(".%06d", (int)(1e6 * (t - curTime)))); +} + +std::string const BackupAgentBase::defaultTagName = "default"; diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index a71ab0c6ff..ce2923945e 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -58,6 +58,7 @@ ACTOR Future appendStringRefWithLen(Reference file, Standalon wait(file->append(s.begin(), s.size())); return Void(); } + } // namespace IBackupFile_impl Future IBackupFile::appendStringRefWithLen(Standalone s) { @@ -253,7 +254,8 @@ std::vector IBackupContainer::getURLFormats() { } // Get an IBackupContainer based on a container URL string -Reference IBackupContainer::openContainer(const std::string& url) { +Reference IBackupContainer::openContainer(const std::string& url, + Optional const& encryptionKeyFileName) { static std::map> m_cache; Reference& r = m_cache[url]; @@ -262,9 +264,9 @@ Reference IBackupContainer::openContainer(const std::string& u try { StringRef u(url); - if (u.startsWith(LiteralStringRef("file://"))) { - r = Reference(new BackupContainerLocalDirectory(url)); - } else if (u.startsWith(LiteralStringRef("blobstore://"))) { + if (u.startsWith("file://"_sr)) { + r = makeReference(url, encryptionKeyFileName); + } else if (u.startsWith("blobstore://"_sr)) { std::string resource; // The URL parameters contain blobstore endpoint tunables as well as possible backup-specific options. @@ -277,15 +279,16 @@ Reference IBackupContainer::openContainer(const std::string& u for (auto c : resource) if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/') throw backup_invalid_url(); - r = Reference(new BackupContainerS3BlobStore(bstore, resource, backupParams)); + r = makeReference(bstore, resource, backupParams, encryptionKeyFileName); } #ifdef BUILD_AZURE_BACKUP - else if (u.startsWith(LiteralStringRef("azure://"))) { - u.eat(LiteralStringRef("azure://")); - auto address = NetworkAddress::parse(u.eat(LiteralStringRef("/")).toString()); - auto containerName = u.eat(LiteralStringRef("/")).toString(); - auto accountName = u.eat(LiteralStringRef("/")).toString(); - r = Reference(new BackupContainerAzureBlobStore(address, containerName, accountName)); + else if (u.startsWith("azure://"_sr)) { + u.eat("azure://"_sr); + auto address = NetworkAddress::parse(u.eat("/"_sr).toString()); + auto containerName = u.eat("/"_sr).toString(); + auto accountName = u.eat("/"_sr).toString(); + r = makeReference( + address, containerName, accountName, encryptionKeyFileName); } #endif else { @@ -315,10 +318,10 @@ Reference IBackupContainer::openContainer(const std::string& u ACTOR Future> listContainers_impl(std::string baseURL) { try { StringRef u(baseURL); - if (u.startsWith(LiteralStringRef("file://"))) { + if (u.startsWith("file://"_sr)) { std::vector results = wait(BackupContainerLocalDirectory::listURLs(baseURL)); return results; - } else if (u.startsWith(LiteralStringRef("blobstore://"))) { + } else if (u.startsWith("blobstore://"_sr)) { std::string resource; S3BlobStoreEndpoint::ParametersT backupParams; @@ -333,14 +336,14 @@ ACTOR Future> listContainers_impl(std::string baseURL) } // Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name - BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams); + BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {}); std::vector results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket())); return results; } // TODO: Enable this when Azure backups are ready /* - else if (u.startsWith(LiteralStringRef("azure://"))) { + else if (u.startsWith("azure://"_sr)) { std::vector results = wait(BackupContainerAzureBlobStore::listURLs(baseURL)); return results; } @@ -386,7 +389,7 @@ ACTOR Future timeKeeperVersionFromDatetime(std::string datetime, Databa tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state std::vector> results = - wait(versionMap.getRange(tr, 0, time, 1, false, true)); + wait(versionMap.getRange(tr, 0, time, 1, Snapshot::FALSE, Reverse::TRUE)); if (results.size() != 1) { // No key less than time was found in the database // Look for a key >= time. @@ -425,7 +428,7 @@ ACTOR Future> timeKeeperEpochsFromVersion(Version v, Reference // Find the highest time < mid state std::vector> results = - wait(versionMap.getRange(tr, min, mid, 1, false, true)); + wait(versionMap.getRange(tr, min, mid, 1, Snapshot::FALSE, Reverse::TRUE)); if (results.size() != 1) { if (mid == min) { diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index 2da1e50985..5a9af3d1d9 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -293,7 +293,8 @@ public: Version beginVersion = -1) = 0; // Get an IBackupContainer based on a container spec string - static Reference openContainer(const std::string& url); + static Reference openContainer(const std::string& url, + const Optional& encryptionKeyFileName = {}); static std::vector getURLFormats(); static Future> listContainers(const std::string& baseURL); diff --git a/fdbclient/BackupContainerAzureBlobStore.actor.cpp b/fdbclient/BackupContainerAzureBlobStore.actor.cpp index dea07c382e..4ee3a7ebf5 100644 --- a/fdbclient/BackupContainerAzureBlobStore.actor.cpp +++ b/fdbclient/BackupContainerAzureBlobStore.actor.cpp @@ -19,6 +19,7 @@ */ #include "fdbclient/BackupContainerAzureBlobStore.h" +#include "fdbrpc/AsyncFileEncrypted.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -167,8 +168,12 @@ public: if (!exists) { throw file_not_found(); } - return Reference( - new ReadFile(self->asyncTaskThread, self->containerName, fileName, self->client.get())); + Reference f = + makeReference(self->asyncTaskThread, self->containerName, fileName, self->client.get()); + if (self->usesEncryption()) { + f = makeReference(f, false); + } + return f; } ACTOR static Future> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) { @@ -177,10 +182,11 @@ public: auto outcome = client->create_append_blob(containerName, fileName).get(); return Void(); })); - return Reference( - new BackupFile(fileName, - Reference(new WriteFile( - self->asyncTaskThread, self->containerName, fileName, self->client.get())))); + auto f = makeReference(self->asyncTaskThread, self->containerName, fileName, self->client.get()); + if (self->usesEncryption()) { + f = makeReference(f, true); + } + return makeReference(fileName, f); } static void listFiles(AzureClient* client, @@ -213,6 +219,16 @@ public: } return Void(); } + + ACTOR static Future create(BackupContainerAzureBlobStore* self) { + state Future f1 = + self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] { + client->create_container(containerName).wait(); + return Void(); + }); + state Future f2 = self->usesEncryption() ? self->encryptionSetupComplete() : Void(); + return f1 && f2; + } }; Future BackupContainerAzureBlobStore::blobExists(const std::string& fileName) { @@ -225,10 +241,11 @@ Future BackupContainerAzureBlobStore::blobExists(const std::string& fileNa BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName, - const std::string& containerName) + const std::string& containerName, + const Optional& encryptionKeyFileName) : containerName(containerName) { + setEncryptionKey(encryptionKeyFileName); std::string accountKey = std::getenv("AZURE_KEY"); - auto credential = std::make_shared(accountName, accountKey); auto storageAccount = std::make_shared( accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str())); @@ -244,10 +261,7 @@ void BackupContainerAzureBlobStore::delref() { } Future BackupContainerAzureBlobStore::create() { - return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] { - client->create_container(containerName).wait(); - return Void(); - }); + return BackupContainerAzureBlobStoreImpl::create(this); } Future BackupContainerAzureBlobStore::exists() { return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] { diff --git a/fdbclient/BackupContainerAzureBlobStore.h b/fdbclient/BackupContainerAzureBlobStore.h index 193fe4a301..aae378fcf4 100644 --- a/fdbclient/BackupContainerAzureBlobStore.h +++ b/fdbclient/BackupContainerAzureBlobStore.h @@ -44,7 +44,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem, public: BackupContainerAzureBlobStore(const NetworkAddress& address, const std::string& accountName, - const std::string& containerName); + const std::string& containerName, + const Optional& encryptionKeyFileName); void addref() override; void delref() override; diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 87e4ffcbf0..a4eb1f6e34 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -23,6 +23,7 @@ #include "fdbclient/BackupContainerFileSystem.h" #include "fdbclient/BackupContainerLocalDirectory.h" #include "fdbclient/JsonBuilder.h" +#include "flow/StreamCipher.h" #include "flow/UnitTest.h" #include @@ -290,13 +291,13 @@ public: std::map> tagIndices; // tagId -> indices in files for (int i = 0; i < logs.size(); i++) { - ASSERT(logs[i].tagId >= 0); - ASSERT(logs[i].tagId < logs[i].totalTags); + ASSERT_GE(logs[i].tagId, 0); + ASSERT_LT(logs[i].tagId, logs[i].totalTags); auto& indices = tagIndices[logs[i].tagId]; // filter out if indices.back() is subset of files[i] or vice versa if (!indices.empty()) { if (logs[indices.back()].isSubset(logs[i])) { - ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[indices.back()].fileSize, logs[i].fileSize); indices.back() = i; } else if (!logs[i].isSubset(logs[indices.back()])) { indices.push_back(i); @@ -864,7 +865,7 @@ public: int i = 0; for (int j = 1; j < logs.size(); j++) { if (logs[j].isSubset(logs[i])) { - ASSERT(logs[j].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[j].fileSize, logs[i].fileSize); continue; } @@ -1032,10 +1033,10 @@ public: } static std::string versionFolderString(Version v, int smallestBucket) { - ASSERT(smallestBucket < 14); + ASSERT_LT(smallestBucket, 14); // Get a 0-padded fixed size representation of v std::string vFixedPrecision = format("%019lld", v); - ASSERT(vFixedPrecision.size() == 19); + ASSERT_EQ(vFixedPrecision.size(), 19); // Truncate smallestBucket from the fixed length representation vFixedPrecision.resize(vFixedPrecision.size() - smallestBucket); @@ -1126,6 +1127,42 @@ public: return false; } + ACTOR static Future createTestEncryptionKeyFile(std::string filename) { + state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( + filename, + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, + 0600)); + StreamCipher::Key::RawKeyType testKey; + generateRandomData(testKey.data(), testKey.size()); + keyFile->write(testKey.data(), testKey.size(), 0); + wait(keyFile->sync()); + return Void(); + } + + ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { + state Reference keyFile; + state StreamCipher::Key::RawKeyType key; + try { + Reference _keyFile = + wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + keyFile = _keyFile; + } catch (Error& e) { + TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile") + .detail("FileName", encryptionKeyFileName) + .error(e); + throw e; + } + int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); + if (bytesRead != key.size()) { + TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize") + .detail("ExpectedSize", key.size()) + .detail("ActualSize", bytesRead); + throw invalid_encryption_key_file(); + } + ASSERT_EQ(bytesRead, key.size()); + StreamCipher::Key::initializeKey(std::move(key)); + return Void(); + } }; // class BackupContainerFileSystemImpl Future> BackupContainerFileSystem::writeLogFile(Version beginVersion, @@ -1432,6 +1469,20 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() { return { Reference::addRef(this), "mutation_log_type" }; } +bool BackupContainerFileSystem::usesEncryption() const { + return encryptionSetupFuture.isValid(); +} +Future BackupContainerFileSystem::encryptionSetupComplete() const { + return encryptionSetupFuture; +} +void BackupContainerFileSystem::setEncryptionKey(Optional const& encryptionKeyFileName) { + if (encryptionKeyFileName.present()) { + encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get()); + } +} +Future BackupContainerFileSystem::createTestEncryptionKeyFile(std::string const &filename) { + return BackupContainerFileSystemImpl::createTestEncryptionKeyFile(filename); +} namespace backup_test { @@ -1466,12 +1517,12 @@ ACTOR Future writeAndVerifyFile(Reference c, Reference inputFile = wait(c->readFile(f->getFileName())); int64_t fileSize = wait(inputFile->size()); - ASSERT(size == fileSize); + ASSERT_EQ(size, fileSize); if (size > 0) { state Standalone> buf; buf.resize(buf.arena(), fileSize); int b = wait(inputFile->read(buf.begin(), buf.size(), 0)); - ASSERT(b == buf.size()); + ASSERT_EQ(b, buf.size()); ASSERT(buf == content); } return Void(); @@ -1485,7 +1536,7 @@ Version nextVersion(Version v) { // Write a snapshot file with only begin & end key ACTOR static Future testWriteSnapshotFile(Reference file, Key begin, Key end, uint32_t blockSize) { - ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size()); + ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size()); uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION; // write Header @@ -1506,12 +1557,16 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key return Void(); } -ACTOR static Future testBackupContainer(std::string url) { +ACTOR Future testBackupContainer(std::string url, Optional encryptionKeyFileName) { state FlowLock lock(100e6); + if (encryptionKeyFileName.present()) { + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get())); + } + printf("BackupContainerTest URL %s\n", url.c_str()); - state Reference c = IBackupContainer::openContainer(url); + state Reference c = IBackupContainer::openContainer(url, encryptionKeyFileName); // Make sure container doesn't exist, then create it. try { @@ -1597,9 +1652,9 @@ ACTOR static Future testBackupContainer(std::string url) { wait(waitForAll(writes)); state BackupFileList listing = wait(c->dumpFileList()); - ASSERT(listing.ranges.size() == nRangeFiles); - ASSERT(listing.logs.size() == logs.size()); - ASSERT(listing.snapshots.size() == snapshots.size()); + ASSERT_EQ(listing.ranges.size(), nRangeFiles); + ASSERT_EQ(listing.logs.size(), logs.size()); + ASSERT_EQ(listing.snapshots.size(), snapshots.size()); state BackupDescription desc = wait(c->describeBackup()); printf("\n%s\n", desc.toString().c_str()); @@ -1629,8 +1684,8 @@ ACTOR static Future testBackupContainer(std::string url) { // If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot if (f.isError()) { - ASSERT(f.getError().code() == error_code_backup_cannot_expire); - ASSERT(i == listing.snapshots.size() - 1); + ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire); + ASSERT_EQ(i, listing.snapshots.size() - 1); wait(c->expireData(expireVersion, true)); } @@ -1646,31 +1701,34 @@ ACTOR static Future testBackupContainer(std::string url) { ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist); BackupFileList empty = wait(c->dumpFileList()); - ASSERT(empty.ranges.size() == 0); - ASSERT(empty.logs.size() == 0); - ASSERT(empty.snapshots.size() == 0); + ASSERT_EQ(empty.ranges.size(), 0); + ASSERT_EQ(empty.logs.size(), 0); + ASSERT_EQ(empty.snapshots.size(), 0); printf("BackupContainerTest URL=%s PASSED.\n", url.c_str()); return Void(); } -TEST_CASE("/backup/containers/localdir") { - if (g_network->isSimulated()) - wait(testBackupContainer(format("file://simfdb/backups/%llx", timer_int()))); - else - wait(testBackupContainer(format("file:///private/tmp/fdb_backups/%llx", timer_int()))); +TEST_CASE("/backup/containers/localdir/unencrypted") { + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), {})); return Void(); -}; +} + +TEST_CASE("/backup/containers/localdir/encrypted") { + wait(testBackupContainer(format("file://%s/fdb_backups/%llx", params.getDataDir().c_str(), timer_int()), + format("%s/test_encryption_key", params.getDataDir().c_str()))); + return Void(); +} TEST_CASE("/backup/containers/url") { if (!g_network->isSimulated()) { const char* url = getenv("FDB_TEST_BACKUP_URL"); ASSERT(url != nullptr); - wait(testBackupContainer(url)); + wait(testBackupContainer(url, {})); } return Void(); -}; +} TEST_CASE("/backup/containers_list") { if (!g_network->isSimulated()) { @@ -1683,7 +1741,7 @@ TEST_CASE("/backup/containers_list") { } } return Void(); -}; +} TEST_CASE("/backup/time") { // test formatTime() diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h index cd0ddf4435..6acf2d87a7 100644 --- a/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/BackupContainerFileSystem.h @@ -153,6 +153,13 @@ public: bool logsOnly, Version beginVersion) final; + static Future createTestEncryptionKeyFile(std::string const& filename); + +protected: + bool usesEncryption() const; + void setEncryptionKey(Optional const& encryptionKeyFileName); + Future encryptionSetupComplete() const; + private: struct VersionProperty { VersionProperty(Reference bc, const std::string& name) @@ -186,6 +193,8 @@ private: Future> old_listRangeFiles(Version beginVersion, Version endVersion); friend class BackupContainerFileSystemImpl; + + Future encryptionSetupFuture; }; #endif diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp index e0c78a31bf..b89d085a64 100644 --- a/fdbclient/BackupContainerLocalDirectory.actor.cpp +++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp @@ -131,7 +131,10 @@ std::string BackupContainerLocalDirectory::getURLFormat() { return "file://"; } -BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url) { +BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url, + const Optional& encryptionKeyFileName) { + setEncryptionKey(encryptionKeyFileName); + std::string path; if (url.find("file://") != 0) { TraceEvent(SevWarn, "BackupContainerLocalDirectory") @@ -193,7 +196,10 @@ Future> BackupContainerLocalDirectory::listURLs(const s } Future BackupContainerLocalDirectory::create() { - // Nothing should be done here because create() can be called by any process working with the container URL, + if (usesEncryption()) { + return encryptionSetupComplete(); + } + // No directory should be created here because create() can be called by any process working with the container URL, // such as fdbbackup. Since "local directory" containers are by definition local to the machine they are // accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to // every file creation, which is done in openFile(). Creating the directory here will result in unnecessary @@ -207,6 +213,9 @@ Future BackupContainerLocalDirectory::exists() { Future> BackupContainerLocalDirectory::readFile(const std::string& path) { int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED; + if (usesEncryption()) { + flags |= IAsyncFile::OPEN_ENCRYPTED; + } // Simulation does not properly handle opening the same file from multiple machines using a shared filesystem, // so create a symbolic link to make each file opening appear to be unique. This could also work in production // but only if the source directory is writeable which shouldn't be required for a restore. @@ -258,8 +267,11 @@ Future> BackupContainerLocalDirectory::readFile(const std: } Future> BackupContainerLocalDirectory::writeFile(const std::string& path) { - int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | - IAsyncFile::OPEN_READWRITE; + int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; + if (usesEncryption()) { + flags |= IAsyncFile::OPEN_ENCRYPTED; + } std::string fullPath = joinPath(m_path, path); platform::createDirectory(parentDirectory(fullPath)); std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp"; diff --git a/fdbclient/BackupContainerLocalDirectory.h b/fdbclient/BackupContainerLocalDirectory.h index 9db8e07aef..f7c77e4636 100644 --- a/fdbclient/BackupContainerLocalDirectory.h +++ b/fdbclient/BackupContainerLocalDirectory.h @@ -33,7 +33,7 @@ public: static std::string getURLFormat(); - BackupContainerLocalDirectory(const std::string& url); + BackupContainerLocalDirectory(const std::string& url, Optional const& encryptionKeyFileName); static Future> listURLs(const std::string& url); diff --git a/fdbclient/BackupContainerS3BlobStore.actor.cpp b/fdbclient/BackupContainerS3BlobStore.actor.cpp index 4e89402ae0..02112c2f58 100644 --- a/fdbclient/BackupContainerS3BlobStore.actor.cpp +++ b/fdbclient/BackupContainerS3BlobStore.actor.cpp @@ -20,6 +20,7 @@ #include "fdbclient/AsyncFileS3BlobStore.actor.h" #include "fdbclient/BackupContainerS3BlobStore.h" +#include "fdbrpc/AsyncFileEncrypted.h" #include "fdbrpc/AsyncFileReadAhead.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -103,6 +104,10 @@ public: wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), "")); } + if (bc->usesEncryption()) { + wait(bc->encryptionSetupComplete()); + } + return Void(); } @@ -137,9 +142,10 @@ std::string BackupContainerS3BlobStore::indexEntry() { BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference bstore, const std::string& name, - const S3BlobStoreEndpoint::ParametersT& params) + const S3BlobStoreEndpoint::ParametersT& params, + const Optional& encryptionKeyFileName) : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") { - + setEncryptionKey(encryptionKeyFileName); // Currently only one parameter is supported, "bucket" for (const auto& [name, value] : params) { if (name == "bucket") { @@ -164,12 +170,16 @@ std::string BackupContainerS3BlobStore::getURLFormat() { } Future> BackupContainerS3BlobStore::readFile(const std::string& path) { - return Reference(new AsyncFileReadAheadCache( - Reference(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))), - m_bstore->knobs.read_block_size, - m_bstore->knobs.read_ahead_blocks, - m_bstore->knobs.concurrent_reads_per_file, - m_bstore->knobs.read_cache_blocks_per_file)); + Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); + if (usesEncryption()) { + f = makeReference(f, AsyncFileEncrypted::Mode::READ_ONLY); + } + f = makeReference(f, + m_bstore->knobs.read_block_size, + m_bstore->knobs.read_ahead_blocks, + m_bstore->knobs.concurrent_reads_per_file, + m_bstore->knobs.read_cache_blocks_per_file); + return f; } Future> BackupContainerS3BlobStore::listURLs(Reference bstore, @@ -178,8 +188,11 @@ Future> BackupContainerS3BlobStore::listURLs(Reference< } Future> BackupContainerS3BlobStore::writeFile(const std::string& path) { - return Reference(new BackupContainerS3BlobStoreImpl::BackupFile( - path, Reference(new AsyncFileS3BlobStoreWrite(m_bstore, m_bucket, dataPath(path))))); + Reference f = makeReference(m_bstore, m_bucket, dataPath(path)); + if (usesEncryption()) { + f = makeReference(f, AsyncFileEncrypted::Mode::APPEND_ONLY); + } + return Future>(makeReference(path, f)); } Future BackupContainerS3BlobStore::deleteFile(const std::string& path) { diff --git a/fdbclient/BackupContainerS3BlobStore.h b/fdbclient/BackupContainerS3BlobStore.h index 57199fcb85..9e47483adf 100644 --- a/fdbclient/BackupContainerS3BlobStore.h +++ b/fdbclient/BackupContainerS3BlobStore.h @@ -43,7 +43,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem, public: BackupContainerS3BlobStore(Reference bstore, const std::string& name, - const S3BlobStoreEndpoint::ParametersT& params); + const S3BlobStoreEndpoint::ParametersT& params, + const Optional& encryptionKeyFileName); void addref() override; void delref() override; diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index 998a39f38a..7b602bc1ea 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -15,6 +15,8 @@ set(FDBCLIENT_SRCS BackupContainerLocalDirectory.h BackupContainerS3BlobStore.actor.cpp BackupContainerS3BlobStore.h + ClientBooleanParams.cpp + ClientBooleanParams.h ClientKnobCollection.cpp ClientKnobCollection.h ClientKnobs.cpp diff --git a/fdbclient/ClientBooleanParams.cpp b/fdbclient/ClientBooleanParams.cpp new file mode 100644 index 0000000000..1027fdece6 --- /dev/null +++ b/fdbclient/ClientBooleanParams.cpp @@ -0,0 +1,30 @@ +/* + * ClientBooleanParams.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/ClientBooleanParams.h" + +FDB_DEFINE_BOOLEAN_PARAM(EnableLocalityLoadBalance); +FDB_DEFINE_BOOLEAN_PARAM(LockAware); +FDB_DEFINE_BOOLEAN_PARAM(Reverse); +FDB_DEFINE_BOOLEAN_PARAM(Snapshot); +FDB_DEFINE_BOOLEAN_PARAM(IsInternal); +FDB_DEFINE_BOOLEAN_PARAM(AddConflictRange); +FDB_DEFINE_BOOLEAN_PARAM(UseMetrics); +FDB_DEFINE_BOOLEAN_PARAM(IsSwitchable); diff --git a/fdbclient/ClientBooleanParams.h b/fdbclient/ClientBooleanParams.h new file mode 100644 index 0000000000..c078c6575e --- /dev/null +++ b/fdbclient/ClientBooleanParams.h @@ -0,0 +1,32 @@ +/* + * ClientBooleanParams.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "flow/BooleanParam.h" + +FDB_DECLARE_BOOLEAN_PARAM(EnableLocalityLoadBalance); +FDB_DECLARE_BOOLEAN_PARAM(LockAware); +FDB_DECLARE_BOOLEAN_PARAM(Reverse); +FDB_DECLARE_BOOLEAN_PARAM(Snapshot); +FDB_DECLARE_BOOLEAN_PARAM(IsInternal); +FDB_DECLARE_BOOLEAN_PARAM(AddConflictRange); +FDB_DECLARE_BOOLEAN_PARAM(UseMetrics); +FDB_DECLARE_BOOLEAN_PARAM(IsSwitchable); diff --git a/fdbclient/ClientKnobs.cpp b/fdbclient/ClientKnobs.cpp index c7090a374d..4c735ef9c4 100644 --- a/fdbclient/ClientKnobs.cpp +++ b/fdbclient/ClientKnobs.cpp @@ -29,8 +29,7 @@ ClientKnobs::ClientKnobs(Randomize randomize) { initialize(randomize); } -void ClientKnobs::initialize(Randomize _randomize) { - bool const randomize = (_randomize == Randomize::YES); +void ClientKnobs::initialize(Randomize randomize) { // clang-format off init( TOO_MANY, 1000000 ); @@ -253,13 +252,13 @@ void ClientKnobs::initialize(Randomize _randomize) { TEST_CASE("/fdbclient/knobs/initialize") { // This test depends on TASKBUCKET_TIMEOUT_VERSIONS being defined as a constant multiple of CORE_VERSIONSPERSECOND - ClientKnobs clientKnobs(Randomize::NO); + ClientKnobs clientKnobs(Randomize::FALSE); int64_t initialCoreVersionsPerSecond = clientKnobs.CORE_VERSIONSPERSECOND; int initialTaskBucketTimeoutVersions = clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS; clientKnobs.setKnob("core_versionspersecond", initialCoreVersionsPerSecond * 2); ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2); ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions); - clientKnobs.initialize(Randomize::NO); + clientKnobs.initialize(Randomize::FALSE); ASSERT_EQ(clientKnobs.CORE_VERSIONSPERSECOND, initialCoreVersionsPerSecond * 2); ASSERT_EQ(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS, initialTaskBucketTimeoutVersions * 2); return Void(); diff --git a/fdbclient/ClientKnobs.h b/fdbclient/ClientKnobs.h index d8a26deb4a..08c00fc9fa 100644 --- a/fdbclient/ClientKnobs.h +++ b/fdbclient/ClientKnobs.h @@ -22,9 +22,13 @@ #define FDBCLIENT_KNOBS_H #pragma once +#include "flow/BooleanParam.h" #include "flow/Knobs.h" #include "flow/flow.h" +FDB_DECLARE_BOOLEAN_PARAM(Randomize); +FDB_DECLARE_BOOLEAN_PARAM(IsSimulated); + class ClientKnobs : public KnobsImpl { public: int TOO_MANY; // FIXME: this should really be split up so we can control these more specifically diff --git a/fdbclient/CommitProxyInterface.h b/fdbclient/CommitProxyInterface.h index 78d4c24463..9ec8b908b2 100644 --- a/fdbclient/CommitProxyInterface.h +++ b/fdbclient/CommitProxyInterface.h @@ -64,6 +64,7 @@ struct CommitProxyInterface { bool operator==(CommitProxyInterface const& r) const { return id() == r.id(); } bool operator!=(CommitProxyInterface const& r) const { return id() != r.id(); } NetworkAddress address() const { return commit.getEndpoint().getPrimaryAddress(); } + NetworkAddressList addresses() const { return commit.getEndpoint().addresses; } template void serialize(Archive& ar) { diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp index 20f9c6bcf2..356b9538e2 100644 --- a/fdbclient/DatabaseBackupAgent.actor.cpp +++ b/fdbclient/DatabaseBackupAgent.actor.cpp @@ -47,8 +47,11 @@ DatabaseBackupAgent::DatabaseBackupAgent() : subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)), states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)), errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)), - taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)), - futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)), + taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), + AccessSystemKeys::TRUE, + PriorityBatch::FALSE, + LockAware::TRUE)), + futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::TRUE, LockAware::TRUE)), sourceStates(subspace.get(BackupAgentBase::keySourceStates)), sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) {} @@ -56,8 +59,11 @@ DatabaseBackupAgent::DatabaseBackupAgent(Database src) : subspace(Subspace(databaseBackupPrefixRange.begin)), tagNames(subspace.get(BackupAgentBase::keyTagName)), states(subspace.get(BackupAgentBase::keyStates)), config(subspace.get(BackupAgentBase::keyConfig)), errors(subspace.get(BackupAgentBase::keyErrors)), ranges(subspace.get(BackupAgentBase::keyRanges)), - taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)), - futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)), + taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), + AccessSystemKeys::TRUE, + PriorityBatch::FALSE, + LockAware::TRUE)), + futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::TRUE, LockAware::TRUE)), sourceStates(subspace.get(BackupAgentBase::keySourceStates)), sourceTagNames(subspace.get(BackupAgentBase::keyTagName)) { taskBucket->src = src; @@ -234,7 +240,8 @@ struct BackupRangeTaskFunc : TaskFuncBase { // retrieve kvData state PromiseStream results; - state Future rc = readCommitted(taskBucket->src, results, lock, range, true, true, true); + state Future rc = readCommitted( + taskBucket->src, results, lock, range, Terminator::TRUE, AccessSystemKeys::TRUE, LockAware::TRUE); state Key rangeBegin = range.begin; state Key rangeEnd; state bool endOfStream = false; @@ -316,16 +323,20 @@ struct BackupRangeTaskFunc : TaskFuncBase { applyMutationsKeyVersionCountRange.begin); state Future backupVersions = krmGetRanges(tr, prefix, KeyRangeRef(rangeBegin, rangeEnd), BUGGIFY ? 2 : 2000, 1e5); - state Future> logVersionValue = tr->get( - task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin), true); - state Future> rangeCountValue = tr->get(rangeCountKey, true); - state Future prevRange = tr->getRange( - firstGreaterOrEqual(prefix), lastLessOrEqual(rangeBegin.withPrefix(prefix)), 1, true, true); + state Future> logVersionValue = + tr->get(task->params[BackupAgentBase::keyConfigLogUid].withPrefix(applyMutationsEndRange.begin), + Snapshot::TRUE); + state Future> rangeCountValue = tr->get(rangeCountKey, Snapshot::TRUE); + state Future prevRange = tr->getRange(firstGreaterOrEqual(prefix), + lastLessOrEqual(rangeBegin.withPrefix(prefix)), + 1, + Snapshot::TRUE, + Reverse::TRUE); state Future nextRange = tr->getRange(firstGreaterOrEqual(rangeEnd.withPrefix(prefix)), firstGreaterOrEqual(strinc(prefix)), 1, - true, - false); + Snapshot::TRUE, + Reverse::FALSE); state Future verified = taskBucket->keepRunning(tr, task); wait(checkDatabaseLock(tr, @@ -363,7 +374,7 @@ struct BackupRangeTaskFunc : TaskFuncBase { Version logVersion = logVersionValue.get().present() ? BinaryReader::fromStringRef(logVersionValue.get().get(), Unversioned()) - : -1; + : ::invalidVersion; if (logVersion >= values.second) { task->params[BackupRangeTaskFunc::keyBackupRangeBeginKey] = rangeBegin; return Void(); @@ -633,7 +644,7 @@ struct EraseLogRangeTaskFunc : TaskFuncBase { task->params[BackupAgentBase::keyConfigLogUid], task->params[BackupAgentBase::destUid], Optional(endVersion), - true, + CheckBackupUID::TRUE, BinaryReader::fromStringRef(task->params[BackupAgentBase::keyFolderId], Unversioned()))); wait(tr->commit()); return Void(); @@ -886,9 +897,9 @@ struct CopyLogRangeTaskFunc : TaskFuncBase { locks[j], ranges[j], decodeBKMutationLogKey, - true, - true, - true)); + Terminator::TRUE, + AccessSystemKeys::TRUE, + LockAware::TRUE)); } // copy the range @@ -1191,7 +1202,7 @@ struct FinishedFullBackupTaskFunc : TaskFuncBase { task->params[DatabaseBackupAgent::keyFolderId], Unversioned())) return Void(); - wait(eraseLogData(tr, logUidValue, destUidValue, Optional(), true, backupUid)); + wait(eraseLogData(tr, logUidValue, destUidValue, Optional(), CheckBackupUID::TRUE, backupUid)); wait(tr->commit()); return Void(); } catch (Error& e) { @@ -1321,6 +1332,10 @@ struct CopyDiffLogsTaskFunc : TaskFuncBase { .detail("LogUID", task->params[BackupAgentBase::keyConfigLogUid]); } + // set the log version to the state + tr->set(StringRef(states.pack(DatabaseBackupAgent::keyStateLogBeginVersion)), + BinaryWriter::toValue(beginVersion, Unversioned())); + if (!stopWhenDone.present()) { state Reference allPartsDone = futureBucket->future(tr); std::vector> addTaskVector; @@ -1592,9 +1607,9 @@ struct OldCopyLogRangeTaskFunc : TaskFuncBase { lock, ranges[i], decodeBKMutationLogKey, - true, - true, - true)); + Terminator::TRUE, + AccessSystemKeys::TRUE, + LockAware::TRUE)); dump.push_back(dumpData(cx, task, results[i], lock.getPtr(), taskBucket)); } @@ -1701,7 +1716,7 @@ struct AbortOldBackupTaskFunc : TaskFuncBase { } TraceEvent("DBA_AbortOldBackup").detail("TagName", tagNameKey.printable()); - wait(srcDrAgent.abortBackup(cx, tagNameKey, false, true)); + wait(srcDrAgent.abortBackup(cx, tagNameKey, PartialBackup::FALSE, AbortOldBackup::TRUE)); return Void(); } @@ -2445,7 +2460,7 @@ public: ACTOR static Future waitBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, - bool stopWhenDone) { + StopWhenDone stopWhenDone) { state std::string backTrace; state UID logUid = wait(backupAgent->getLogUid(cx, tagName)); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) @@ -2510,10 +2525,10 @@ public: Reference tr, Key tagName, Standalone> backupRanges, - bool stopWhenDone, + StopWhenDone stopWhenDone, Key addPrefix, Key removePrefix, - bool lockDB, + LockDB lockDB, DatabaseBackupAgent::PreBackupAction backupAction) { state UID logUid = deterministicRandom()->randomUniqueID(); state Key logUidValue = BinaryWriter::toValue(logUid, Unversioned()); @@ -2667,7 +2682,7 @@ public: Standalone> backupRanges, Key addPrefix, Key removePrefix, - bool forceAction) { + ForceAction forceAction) { state DatabaseBackupAgent drAgent(dest); state UID destlogUid = wait(backupAgent->getLogUid(dest, tagName)); state EBackupState status = wait(backupAgent->getStateValue(dest, destlogUid)); @@ -2751,7 +2766,7 @@ public: throw; } - wait(success(backupAgent->waitBackup(dest, tagName, true))); + wait(success(backupAgent->waitBackup(dest, tagName, StopWhenDone::TRUE))); TraceEvent("DBA_SwitchoverStopped"); @@ -2780,10 +2795,10 @@ public: wait(drAgent.submitBackup(backupAgent->taskBucket->src, tagName, backupRanges, - false, + StopWhenDone::FALSE, addPrefix, removePrefix, - true, + LockDB::TRUE, DatabaseBackupAgent::PreBackupAction::NONE)); } catch (Error& e) { if (e.code() != error_code_backup_duplicate) @@ -2835,10 +2850,10 @@ public: ACTOR static Future abortBackup(DatabaseBackupAgent* backupAgent, Database cx, Key tagName, - bool partial, - bool abortOldBackup, - bool dstOnly, - bool waitForDestUID) { + PartialBackup partial, + AbortOldBackup abortOldBackup, + DstOnly dstOnly, + WaitForDestUID waitForDestUID) { state Reference tr(new ReadYourWritesTransaction(cx)); state Key logUidValue, destUidValue; state UID logUid, destUid; @@ -3063,8 +3078,8 @@ public: errorLimit > 0 ? tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(), errorLimit, - false, - true) + Snapshot::FALSE, + Reverse::TRUE) : Future(); state Future> fBackupUid = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) @@ -3080,6 +3095,9 @@ public: state Future> fBackupKeysPacked = tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())) .pack(BackupAgentBase::keyConfigBackupRanges)); + state Future> flogVersionKey = + tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) + .pack(BackupAgentBase::keyStateLogBeginVersion)); state EBackupState backupState = wait(backupAgent->getStateValue(tr, logUid)); @@ -3095,7 +3113,14 @@ public: } state Optional stopVersionKey = wait(fStopVersionKey); - + Optional logVersionKey = wait(flogVersionKey); + state std::string logVersionText + = ". Last log version is " + + ( + logVersionKey.present() + ? format("%lld", BinaryReader::fromStringRef(logVersionKey.get(), Unversioned())) + : "unset" + ); Optional backupKeysPacked = wait(fBackupKeysPacked); state Standalone> backupRanges; @@ -3115,7 +3140,7 @@ public: break; case EBackupState::STATE_RUNNING_DIFFERENTIAL: statusText += - "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database.\n"; + "The DR on tag `" + tagNameDisplay + "' is a complete copy of the primary database" + logVersionText + ".\n"; break; case EBackupState::STATE_COMPLETED: { Version stopVersion = @@ -3127,13 +3152,13 @@ public: } break; case EBackupState::STATE_PARTIALLY_ABORTED: { statusText += "The previous DR on tag `" + tagNameDisplay + "' " + - BackupAgentBase::getStateText(backupState) + ".\n"; + BackupAgentBase::getStateText(backupState) + logVersionText + ".\n"; statusText += "Abort the DR with --cleanup before starting a new DR.\n"; break; } default: statusText += "The previous DR on tag `" + tagNameDisplay + "' " + - BackupAgentBase::getStateText(backupState) + ".\n"; + BackupAgentBase::getStateText(backupState) + logVersionText + ".\n"; break; } } @@ -3191,7 +3216,7 @@ public: ACTOR static Future getStateValue(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())) @@ -3204,7 +3229,7 @@ public: ACTOR static Future getDestUid(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key destUidKey = @@ -3217,7 +3242,7 @@ public: ACTOR static Future getLogUid(DatabaseBackupAgent* backupAgent, Reference tr, Key tagName, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Optional logUid = wait(tr->get(backupAgent->tagNames.pack(tagName), snapshot)); @@ -3235,7 +3260,7 @@ Future DatabaseBackupAgent::atomicSwitchover(Database dest, Standalone> backupRanges, Key addPrefix, Key removePrefix, - bool forceAction) { + ForceAction forceAction) { return DatabaseBackupAgentImpl::atomicSwitchover( this, dest, tagName, backupRanges, addPrefix, removePrefix, forceAction); } @@ -3243,10 +3268,10 @@ Future DatabaseBackupAgent::atomicSwitchover(Database dest, Future DatabaseBackupAgent::submitBackup(Reference tr, Key tagName, Standalone> backupRanges, - bool stopWhenDone, + StopWhenDone stopWhenDone, Key addPrefix, Key removePrefix, - bool lockDatabase, + LockDB lockDatabase, PreBackupAction backupAction) { return DatabaseBackupAgentImpl::submitBackup( this, tr, tagName, backupRanges, stopWhenDone, addPrefix, removePrefix, lockDatabase, backupAction); @@ -3258,10 +3283,10 @@ Future DatabaseBackupAgent::discontinueBackup(Reference DatabaseBackupAgent::abortBackup(Database cx, Key tagName, - bool partial, - bool abortOldBackup, - bool dstOnly, - bool waitForDestUID) { + PartialBackup partial, + AbortOldBackup abortOldBackup, + DstOnly dstOnly, + WaitForDestUID waitForDestUID) { return DatabaseBackupAgentImpl::abortBackup(this, cx, tagName, partial, abortOldBackup, dstOnly, waitForDestUID); } @@ -3271,15 +3296,15 @@ Future DatabaseBackupAgent::getStatus(Database cx, int errorLimit, Future DatabaseBackupAgent::getStateValue(Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid, snapshot); } -Future DatabaseBackupAgent::getDestUid(Reference tr, UID logUid, bool snapshot) { +Future DatabaseBackupAgent::getDestUid(Reference tr, UID logUid, Snapshot snapshot) { return DatabaseBackupAgentImpl::getDestUid(this, tr, logUid, snapshot); } -Future DatabaseBackupAgent::getLogUid(Reference tr, Key tagName, bool snapshot) { +Future DatabaseBackupAgent::getLogUid(Reference tr, Key tagName, Snapshot snapshot) { return DatabaseBackupAgentImpl::getLogUid(this, tr, tagName, snapshot); } @@ -3287,7 +3312,7 @@ Future DatabaseBackupAgent::waitUpgradeToLatestDrVersion(Database cx, Key return DatabaseBackupAgentImpl::waitUpgradeToLatestDrVersion(this, cx, tagName); } -Future DatabaseBackupAgent::waitBackup(Database cx, Key tagName, bool stopWhenDone) { +Future DatabaseBackupAgent::waitBackup(Database cx, Key tagName, StopWhenDone stopWhenDone) { return DatabaseBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone); } @@ -3297,12 +3322,12 @@ Future DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName Future DatabaseBackupAgent::getRangeBytesWritten(Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { return DRConfig(logUid).rangeBytesWritten().getD(tr, snapshot); } Future DatabaseBackupAgent::getLogBytesWritten(Reference tr, UID logUid, - bool snapshot) { + Snapshot snapshot) { return DRConfig(logUid).logBytesWritten().getD(tr, snapshot); } diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 7679b51c7e..d95ca71c32 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -157,11 +157,11 @@ public: static Database create(Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, - bool enableLocalityLoadBalance, + EnableLocalityLoadBalance, TaskPriority taskID = TaskPriority::DefaultEndpoint, - bool lockAware = false, + LockAware = LockAware::FALSE, int apiVersion = Database::API_VERSION_LATEST, - bool switchable = false); + IsSwitchable = IsSwitchable::FALSE); ~DatabaseContext(); @@ -180,13 +180,13 @@ public: switchable)); } - std::pair> getCachedLocation(const KeyRef&, bool isBackward = false); + std::pair> getCachedLocation(const KeyRef&, Reverse isBackward = Reverse::FALSE); bool getCachedLocations(const KeyRangeRef&, vector>>&, int limit, - bool reverse); + Reverse reverse); Reference setCachedLocation(const KeyRangeRef&, const vector&); - void invalidateCache(const KeyRef&, bool isBackward = false); + void invalidateCache(const KeyRef&, Reverse isBackward = Reverse::FALSE); void invalidateCache(const KeyRangeRef&); bool sampleReadTags() const; @@ -217,7 +217,7 @@ public: void setOption(FDBDatabaseOptions::Option option, Optional value); Error deferredError; - bool lockAware; + LockAware lockAware{ LockAware::FALSE }; bool isError() const { return deferredError.code() != invalid_error_code; } @@ -242,7 +242,7 @@ public: // new cluster. Future switchConnectionFile(Reference standby); Future connectionFileChanged(); - bool switchable = false; + IsSwitchable switchable{ false }; // Management API, Attempt to kill or suspend a process, return 1 for request sent out, 0 for failure Future rebootWorker(StringRef address, bool check = false, int duration = 0); @@ -259,11 +259,11 @@ public: Future clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, - bool enableLocalityLoadBalance, - bool lockAware, - bool internal = true, + EnableLocalityLoadBalance, + LockAware, + IsInternal = IsInternal::TRUE, int apiVersion = Database::API_VERSION_LATEST, - bool switchable = false); + IsSwitchable = IsSwitchable::FALSE); explicit DatabaseContext(const Error& err); @@ -282,7 +282,7 @@ public: UID proxiesLastChange; LocalityData clientLocality; QueueModel queueModel; - bool enableLocalityLoadBalance; + EnableLocalityLoadBalance enableLocalityLoadBalance{ EnableLocalityLoadBalance::FALSE }; struct VersionRequest { SpanID spanContext; @@ -329,7 +329,7 @@ public: std::unordered_map> tssMetrics; UID dbId; - bool internal; // Only contexts created through the C client and fdbcli are non-internal + IsInternal internal; // Only contexts created through the C client and fdbcli are non-internal PrioritizedTransactionTagMap throttledTags; diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 35d6743821..63d4f1aee3 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -42,6 +42,9 @@ #include "flow/actorcompiler.h" // This must be the last #include. +FDB_DEFINE_BOOLEAN_PARAM(IncrementalBackupOnly); +FDB_DEFINE_BOOLEAN_PARAM(OnlyApplyMutationLogs); + #define SevFRTestInfo SevVerbose //#define SevFRTestInfo SevInfo @@ -117,7 +120,7 @@ Key FileBackupAgent::getPauseKey() { ACTOR Future> TagUidMap::getAll_impl(TagUidMap* tagsMap, Reference tr, - bool snapshot) { + Snapshot snapshot) { state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot)); std::vector results; @@ -142,7 +145,7 @@ public: } KeyBackedProperty addPrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } KeyBackedProperty removePrefix() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } - KeyBackedProperty onlyAppyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } + KeyBackedProperty onlyApplyMutationLogs() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } KeyBackedProperty inconsistentSnapshotOnly() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } // XXX: Remove restoreRange() once it is safe to remove. It has been changed to restoreRanges KeyBackedProperty restoreRange() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); } @@ -248,9 +251,9 @@ public: Key applyMutationsMapPrefix() { return uidPrefixKey(applyMutationsKeyVersionMapRange.begin, uid); } ACTOR static Future getApplyVersionLag_impl(Reference tr, UID uid) { - // Both of these are snapshot reads - state Future> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), true); - state Future> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), true); + state Future> beginVal = + tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::TRUE); + state Future> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::TRUE); wait(success(beginVal) && success(endVal)); if (!beginVal.get().present() || !endVal.get().present()) @@ -440,8 +443,12 @@ FileBackupAgent::FileBackupAgent() // The other subspaces have logUID -> value , config(subspace.get(BackupAgentBase::keyConfig)), lastRestorable(subspace.get(FileBackupAgent::keyLastRestorable)), - taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), true, false, true)), - futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), true, true)) {} + taskBucket(new TaskBucket(subspace.get(BackupAgentBase::keyTasks), + AccessSystemKeys::TRUE, + PriorityBatch::FALSE, + LockAware::TRUE)), + futureBucket(new FutureBucket(subspace.get(BackupAgentBase::keyFutures), AccessSystemKeys::TRUE, LockAware::TRUE)) { +} namespace fileBackup { @@ -863,10 +870,10 @@ ACTOR static Future abortFiveOneBackup(FileBackupAgent* backupAgent, tr->setOption(FDBTransactionOptions::LOCK_AWARE); state KeyBackedTag tag = makeBackupTag(tagName); - state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded())); + state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::FALSE, backup_unneeded())); state BackupConfig config(current.first); - EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); if (!backupAgent->isRunnable(status)) { throw backup_unneeded(); @@ -952,7 +959,7 @@ ACTOR static Future addBackupTask(StringRef name, Reference waitFor = Reference(), std::function)> setupTaskFn = NOP_SETUP_TASK_FN, int priority = 0, - bool setValidation = true) { + SetValidation setValidation = SetValidation::TRUE) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1107,7 +1114,7 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase { Params.beginKey().set(task, range.end); // Save and extend the task with the new begin parameter - state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, true)); + state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::TRUE)); // Update the range bytes written in the backup config backup.rangeBytesWritten().atomicOp(tr, file->size(), MutationRef::AddValue); @@ -1201,7 +1208,13 @@ struct BackupRangeTaskFunc : BackupTaskFuncBase { // retrieve kvData state PromiseStream results; - state Future rc = readCommitted(cx, results, lock, KeyRangeRef(beginKey, endKey), true, true, true); + state Future rc = readCommitted(cx, + results, + lock, + KeyRangeRef(beginKey, endKey), + Terminator::TRUE, + AccessSystemKeys::TRUE, + LockAware::TRUE); state RangeFileWriter rangeFile; state BackupConfig backup(task); @@ -2044,7 +2057,8 @@ struct BackupLogRangeTaskFunc : BackupTaskFuncBase { state std::vector> rc; for (auto& range : ranges) { - rc.push_back(readCommitted(cx, results, lock, range, false, true, true)); + rc.push_back( + readCommitted(cx, results, lock, range, Terminator::FALSE, AccessSystemKeys::TRUE, LockAware::TRUE)); } state Future sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr const& result) { @@ -2222,7 +2236,7 @@ struct EraseLogRangeTaskFunc : BackupTaskFuncBase { Params.destUidValue().set(task, destUidValue); }, 0, - false)); + SetValidation::FALSE)); return key; } @@ -3580,9 +3594,9 @@ struct RestoreDispatchTaskFunc : RestoreTaskFuncBase { state int64_t remainingInBatch = Params.remainingInBatch().get(task); state bool addingToExistingBatch = remainingInBatch > 0; state Version restoreVersion; - state Future> onlyAppyMutationLogs = restore.onlyAppyMutationLogs().get(tr); + state Future> onlyApplyMutationLogs = restore.onlyApplyMutationLogs().get(tr); - wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) && success(onlyAppyMutationLogs) && + wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr)) && success(onlyApplyMutationLogs) && checkTaskVersion(tr->getDatabase(), task, name, version)); // If not adding to an existing batch then update the apply mutations end version so the mutations from the @@ -4058,12 +4072,13 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase { tr->setOption(FDBTransactionOptions::LOCK_AWARE); wait(checkTaskVersion(tr->getDatabase(), task, name, version)); - wait(store(beginVersion, restore.beginVersion().getD(tr, false, invalidVersion))); + wait(store(beginVersion, restore.beginVersion().getD(tr, Snapshot::FALSE, ::invalidVersion))); wait(store(restoreVersion, restore.restoreVersion().getOrThrow(tr))); wait(store(ranges, restore.getRestoreRangesOrDefault(tr))); - wait(store(logsOnly, restore.onlyAppyMutationLogs().getD(tr, false, false))); - wait(store(inconsistentSnapshotOnly, restore.inconsistentSnapshotOnly().getD(tr, false, false))); + wait(store(logsOnly, restore.onlyApplyMutationLogs().getD(tr, Snapshot::FALSE, false))); + wait(store(inconsistentSnapshotOnly, + restore.inconsistentSnapshotOnly().getD(tr, Snapshot::FALSE, false))); wait(taskBucket->keepRunning(tr, task)); @@ -4245,7 +4260,7 @@ struct StartFullRestoreTaskFunc : RestoreTaskFuncBase { tr, taskBucket, task, 0, "", 0, CLIENT_KNOBS->RESTORE_DISPATCH_BATCH_SIZE))); wait(taskBucket->finish(tr, task)); - state Future> logsOnly = restore.onlyAppyMutationLogs().get(tr); + state Future> logsOnly = restore.onlyApplyMutationLogs().get(tr); wait(success(logsOnly)); if (logsOnly.get().present() && logsOnly.get().get()) { // If this is an incremental restore, we need to set the applyMutationsMapPrefix @@ -4314,7 +4329,7 @@ public: static constexpr int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; // Parallel restore - ACTOR static Future parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB = true) { + ACTOR static Future parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB = UnlockDB::TRUE) { state ReadYourWritesTransaction tr(cx); state Optional restoreRequestDoneKeyValue; TraceEvent("FastRestoreToolWaitForRestoreToFinish").detail("DBLock", randomUID); @@ -4365,7 +4380,7 @@ public: Standalone> backupRanges, Key bcUrl, Version targetVersion, - bool lockDB, + LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix) { @@ -4458,7 +4473,7 @@ public: ACTOR static Future waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, - bool stopWhenDone, + StopWhenDone stopWhenDone, Reference* pContainer = nullptr, UID* pUID = nullptr) { state std::string backTrace; @@ -4476,7 +4491,8 @@ public: } state BackupConfig config(oldUidAndAborted.get().first); - state EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + state EBackupState status = + wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); // Break, if one of the following is true // - no longer runnable @@ -4486,7 +4502,7 @@ public: if (pContainer != nullptr) { Reference c = - wait(config.backupContainer().getOrThrow(tr, false, backup_invalid_info())); + wait(config.backupContainer().getOrThrow(tr, Snapshot::FALSE, backup_invalid_info())); *pContainer = c; } @@ -4506,6 +4522,7 @@ public: } } + // TODO: Get rid of all of these confusing boolean flags ACTOR static Future submitBackup(FileBackupAgent* backupAgent, Reference tr, Key outContainer, @@ -4513,9 +4530,10 @@ public: int snapshotIntervalSeconds, std::string tagName, Standalone> backupRanges, - bool stopWhenDone, - bool partitionedLog, - bool incrementalBackupOnly) { + StopWhenDone stopWhenDone, + UsePartitionedLog partitionedLog, + IncrementalBackupOnly incrementalBackupOnly, + Optional encryptionKeyFileName) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); @@ -4531,7 +4549,7 @@ public: if (uidAndAbortedFlag.present()) { state BackupConfig prevConfig(uidAndAbortedFlag.get().first); state EBackupState prevBackupStatus = - wait(prevConfig.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + wait(prevConfig.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); if (FileBackupAgent::isRunnable(prevBackupStatus)) { throw backup_duplicate(); } @@ -4553,7 +4571,7 @@ public: backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString()); } - state Reference bc = IBackupContainer::openContainer(backupContainer); + state Reference bc = IBackupContainer::openContainer(backupContainer, encryptionKeyFileName); try { wait(timeoutError(bc->create(), 30)); } catch (Error& e) { @@ -4644,9 +4662,9 @@ public: Version restoreVersion, Key addPrefix, Key removePrefix, - bool lockDB, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly, + LockDB lockDB, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, Version beginVersion, UID uid) { KeyRangeMap restoreRangeSet; @@ -4698,7 +4716,7 @@ public: .removePrefix(removePrefix) .withPrefix(addPrefix); RangeResult existingRows = wait(tr->getRange(restoreIntoRange, 1)); - if (existingRows.size() > 0 && !onlyAppyMutationLogs) { + if (existingRows.size() > 0 && !onlyApplyMutationLogs) { throw restore_destination_not_empty(); } } @@ -4715,7 +4733,7 @@ public: restore.sourceContainer().set(tr, bc); restore.stateEnum().set(tr, ERestoreState::QUEUED); restore.restoreVersion().set(tr, restoreVersion); - restore.onlyAppyMutationLogs().set(tr, onlyAppyMutationLogs); + restore.onlyApplyMutationLogs().set(tr, onlyApplyMutationLogs); restore.inconsistentSnapshotOnly().set(tr, inconsistentSnapshotOnly); restore.beginVersion().set(tr, beginVersion); if (BUGGIFY && restoreRanges.size() == 1) { @@ -4738,7 +4756,7 @@ public: } // This method will return the final status of the backup - ACTOR static Future waitRestore(Database cx, Key tagName, bool verbose) { + ACTOR static Future waitRestore(Database cx, Key tagName, Verbose verbose) { state ERestoreState status; loop { state Reference tr(new ReadYourWritesTransaction(cx)); @@ -4794,9 +4812,9 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); state KeyBackedTag tag = makeBackupTag(tagName.toString()); - state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded())); + state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::FALSE, backup_unneeded())); state BackupConfig config(current.first); - state EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + state EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); if (!FileBackupAgent::isRunnable(status)) { throw backup_unneeded(); @@ -4845,11 +4863,11 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); state KeyBackedTag tag = makeBackupTag(tagName); - state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, false, backup_unneeded())); + state UidAndAbortedFlagT current = wait(tag.getOrThrow(tr, Snapshot::FALSE, backup_unneeded())); state BackupConfig config(current.first); state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); - EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + EBackupState status = wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); if (!backupAgent->isRunnable(status)) { throw backup_unneeded(); @@ -4951,7 +4969,7 @@ public: state BackupConfig config(uidAndAbortedFlag.get().first); state EBackupState backupState = - wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); JsonBuilderObject statusDoc; statusDoc.setKey("Name", BackupAgentBase::getStateName(backupState)); statusDoc.setKey("Description", BackupAgentBase::getStateText(backupState)); @@ -5075,7 +5093,7 @@ public: ACTOR static Future getStatus(FileBackupAgent* backupAgent, Database cx, - bool showErrors, + ShowErrors showErrors, std::string tagName) { state Reference tr(new ReadYourWritesTransaction(cx)); state std::string statusText; @@ -5095,7 +5113,8 @@ public: state Future> fPaused = tr->get(backupAgent->taskBucket->getPauseKey()); if (uidAndAbortedFlag.present()) { config = BackupConfig(uidAndAbortedFlag.get().first); - EBackupState status = wait(config.stateEnum().getD(tr, false, EBackupState::STATE_NEVERRAN)); + EBackupState status = + wait(config.stateEnum().getD(tr, Snapshot::FALSE, EBackupState::STATE_NEVERRAN)); backupState = status; } @@ -5257,7 +5276,7 @@ public: ACTOR static Future> getLastRestorable(FileBackupAgent* backupAgent, Reference tr, Key tagName, - bool snapshot) { + Snapshot snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Optional version = wait(tr->get(backupAgent->lastRestorable.pack(tagName), snapshot)); @@ -5290,7 +5309,7 @@ public: // removePrefix: for each key to be restored, remove this prefix first. // lockDB: if set lock the database with randomUid before performing restore; // otherwise, check database is locked with the randomUid - // onlyAppyMutationLogs: only perform incremental restore, by only applying mutation logs + // onlyApplyMutationLogs: only perform incremental restore, by only applying mutation logs // inconsistentSnapshotOnly: Ignore mutation log files during the restore to speedup the process. // When set to true, gives an inconsistent snapshot, thus not recommended // beginVersion: restore's begin version @@ -5301,15 +5320,16 @@ public: Key tagName, Key url, Standalone> ranges, - bool waitForComplete, + WaitForComplete waitForComplete, Version targetVersion, - bool verbose, + Verbose verbose, Key addPrefix, Key removePrefix, - bool lockDB, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly, + LockDB lockDB, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, Version beginVersion, + Optional encryptionKeyFileName, UID randomUid) { // The restore command line tool won't allow ranges to be empty, but correctness workloads somehow might. if (ranges.empty()) { @@ -5327,12 +5347,12 @@ public: if (targetVersion == invalidVersion && desc.maxRestorableVersion.present()) targetVersion = desc.maxRestorableVersion.get(); - if (targetVersion == invalidVersion && onlyAppyMutationLogs && desc.contiguousLogEnd.present()) { + if (targetVersion == invalidVersion && onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { targetVersion = desc.contiguousLogEnd.get() - 1; } Optional restoreSet = - wait(bc->getRestoreSet(targetVersion, ranges, onlyAppyMutationLogs, beginVersion)); + wait(bc->getRestoreSet(targetVersion, ranges, onlyApplyMutationLogs, beginVersion)); if (!restoreSet.present()) { TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible") @@ -5364,7 +5384,7 @@ public: addPrefix, removePrefix, lockDB, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, beginVersion, randomUid)); @@ -5395,7 +5415,7 @@ public: Standalone> ranges, Key addPrefix, Key removePrefix, - bool fastRestore) { + UsePartitionedLog fastRestore) { state Reference ryw_tr = Reference(new ReadYourWritesTransaction(cx)); state BackupConfig backupConfig; @@ -5468,7 +5488,7 @@ public: } } - wait(success(waitBackup(backupAgent, cx, tagName.toString(), true))); + wait(success(waitBackup(backupAgent, cx, tagName.toString(), StopWhenDone::TRUE))); TraceEvent("AS_BackupStopped"); ryw_tr->reset(); @@ -5493,13 +5513,19 @@ public: if (fastRestore) { TraceEvent("AtomicParallelRestoreStartRestore"); - Version targetVersion = -1; - bool lockDB = true; - wait(submitParallelRestore( - cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, lockDB, randomUid, addPrefix, removePrefix)); + Version targetVersion = ::invalidVersion; + wait(submitParallelRestore(cx, + tagName, + ranges, + KeyRef(bc->getURL()), + targetVersion, + LockDB::TRUE, + randomUid, + addPrefix, + removePrefix)); state bool hasPrefix = (addPrefix.size() > 0 || removePrefix.size() > 0); TraceEvent("AtomicParallelRestoreWaitForRestoreFinish").detail("HasPrefix", hasPrefix); - wait(parallelRestoreFinish(cx, randomUid, !hasPrefix)); + wait(parallelRestoreFinish(cx, randomUid, UnlockDB{ !hasPrefix })); // If addPrefix or removePrefix set, we want to transform the effect by copying data if (hasPrefix) { wait(transformRestoredDatabase(cx, ranges, addPrefix, removePrefix)); @@ -5514,15 +5540,16 @@ public: tagName, KeyRef(bc->getURL()), ranges, - true, - -1, - true, + WaitForComplete::TRUE, + ::invalidVersion, + Verbose::TRUE, addPrefix, removePrefix, - true, - false, - false, - invalidVersion, + LockDB::TRUE, + OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly::FALSE, + ::invalidVersion, + {}, randomUid)); return ver; } @@ -5537,16 +5564,15 @@ public: Standalone> ranges, Key addPrefix, Key removePrefix) { - return success(atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, true)); + return success( + atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::TRUE)); } }; -const std::string BackupAgentBase::defaultTagName = "default"; -const int BackupAgentBase::logHeaderSize = 12; const int FileBackupAgent::dataFooterSize = 20; // Return if parallel restore has finished -Future FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, bool unlockDB) { +Future FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID, UnlockDB unlockDB) { return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID, unlockDB); } @@ -5555,7 +5581,7 @@ Future FileBackupAgent::submitParallelRestore(Database cx, Standalone> backupRanges, Key bcUrl, Version targetVersion, - bool lockDB, + LockDB lockDB, UID randomUID, Key addPrefix, Key removePrefix) { @@ -5576,15 +5602,16 @@ Future FileBackupAgent::restore(Database cx, Key tagName, Key url, Standalone> ranges, - bool waitForComplete, + WaitForComplete waitForComplete, Version targetVersion, - bool verbose, + Verbose verbose, Key addPrefix, Key removePrefix, - bool lockDB, - bool onlyAppyMutationLogs, - bool inconsistentSnapshotOnly, - Version beginVersion) { + LockDB lockDB, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, + Version beginVersion, + Optional const& encryptionKeyFileName) { return FileBackupAgentImpl::restore(this, cx, cxOrig, @@ -5597,9 +5624,10 @@ Future FileBackupAgent::restore(Database cx, addPrefix, removePrefix, lockDB, - onlyAppyMutationLogs, + onlyApplyMutationLogs, inconsistentSnapshotOnly, beginVersion, + encryptionKeyFileName, deterministicRandom()->randomUniqueID()); } @@ -5608,7 +5636,8 @@ Future FileBackupAgent::atomicRestore(Database cx, Standalone> ranges, Key addPrefix, Key removePrefix) { - return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix, false); + return FileBackupAgentImpl::atomicRestore( + this, cx, tagName, ranges, addPrefix, removePrefix, UsePartitionedLog::FALSE); } Future FileBackupAgent::abortRestore(Reference tr, Key tagName) { @@ -5623,7 +5652,7 @@ Future FileBackupAgent::restoreStatus(Reference FileBackupAgent::waitRestore(Database cx, Key tagName, bool verbose) { +Future FileBackupAgent::waitRestore(Database cx, Key tagName, Verbose verbose) { return FileBackupAgentImpl::waitRestore(cx, tagName, verbose); }; @@ -5631,11 +5660,12 @@ Future FileBackupAgent::submitBackup(Reference Key outContainer, int initialSnapshotIntervalSeconds, int snapshotIntervalSeconds, - std::string tagName, + std::string const& tagName, Standalone> backupRanges, - bool stopWhenDone, - bool partitionedLog, - bool incrementalBackupOnly) { + StopWhenDone stopWhenDone, + UsePartitionedLog partitionedLog, + IncrementalBackupOnly incrementalBackupOnly, + Optional const& encryptionKeyFileName) { return FileBackupAgentImpl::submitBackup(this, tr, outContainer, @@ -5645,7 +5675,8 @@ Future FileBackupAgent::submitBackup(Reference backupRanges, stopWhenDone, partitionedLog, - incrementalBackupOnly); + incrementalBackupOnly, + encryptionKeyFileName); } Future FileBackupAgent::discontinueBackup(Reference tr, Key tagName) { @@ -5656,7 +5687,7 @@ Future FileBackupAgent::abortBackup(Reference t return FileBackupAgentImpl::abortBackup(this, tr, tagName); } -Future FileBackupAgent::getStatus(Database cx, bool showErrors, std::string tagName) { +Future FileBackupAgent::getStatus(Database cx, ShowErrors showErrors, std::string tagName) { return FileBackupAgentImpl::getStatus(this, cx, showErrors, tagName); } @@ -5666,7 +5697,7 @@ Future FileBackupAgent::getStatusJSON(Database cx, std::string tagN Future> FileBackupAgent::getLastRestorable(Reference tr, Key tagName, - bool snapshot) { + Snapshot snapshot) { return FileBackupAgentImpl::getLastRestorable(this, tr, tagName, snapshot); } @@ -5678,7 +5709,7 @@ void FileBackupAgent::setLastRestorable(Reference tr, Future FileBackupAgent::waitBackup(Database cx, std::string tagName, - bool stopWhenDone, + StopWhenDone stopWhenDone, Reference* pContainer, UID* pUID) { return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID); @@ -5739,8 +5770,8 @@ ACTOR static Future writeKVs(Database cx, Standalone void serialize(Archive& ar) { diff --git a/fdbclient/IConfigTransaction.h b/fdbclient/IConfigTransaction.h index 007dd9e2d3..9dfe139c8a 100644 --- a/fdbclient/IConfigTransaction.h +++ b/fdbclient/IConfigTransaction.h @@ -45,7 +45,9 @@ public: // Not implemented: void setVersion(Version) override { throw client_invalid_operation(); } - Future getKey(KeySelector const& key, bool snapshot = false) override { throw client_invalid_operation(); } + Future getKey(KeySelector const& key, Snapshot snapshot = Snapshot::FALSE) override { + throw client_invalid_operation(); + } Future>> getAddressesForKey(Key const& key) override { throw client_invalid_operation(); } diff --git a/fdbclient/IKnobCollection.cpp b/fdbclient/IKnobCollection.cpp index 7f3a595763..aee94dbc97 100644 --- a/fdbclient/IKnobCollection.cpp +++ b/fdbclient/IKnobCollection.cpp @@ -56,17 +56,17 @@ KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::stri static std::unique_ptr clientKnobCollection, serverKnobCollection, testKnobCollection; if (type == Type::CLIENT) { if (!clientKnobCollection) { - clientKnobCollection = create(type, Randomize::NO, IsSimulated::NO); + clientKnobCollection = create(type, Randomize::FALSE, IsSimulated::FALSE); } return clientKnobCollection->parseKnobValue(knobName, knobValue); } else if (type == Type::SERVER) { if (!serverKnobCollection) { - serverKnobCollection = create(type, Randomize::NO, IsSimulated::NO); + serverKnobCollection = create(type, Randomize::FALSE, IsSimulated::FALSE); } return serverKnobCollection->parseKnobValue(knobName, knobValue); } else if (type == Type::TEST) { if (!testKnobCollection) { - testKnobCollection = create(type, Randomize::NO, IsSimulated::NO); + testKnobCollection = create(type, Randomize::FALSE, IsSimulated::FALSE); } return testKnobCollection->parseKnobValue(knobName, knobValue); } @@ -74,7 +74,7 @@ KnobValue IKnobCollection::parseKnobValue(std::string const& knobName, std::stri } std::unique_ptr IKnobCollection::globalKnobCollection = - IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::NO, IsSimulated::NO); + IKnobCollection::create(IKnobCollection::Type::CLIENT, Randomize::FALSE, IsSimulated::FALSE); void IKnobCollection::setGlobalKnobCollection(Type type, Randomize randomize, IsSimulated isSimulated) { globalKnobCollection = create(type, randomize, isSimulated); diff --git a/fdbclient/ISingleThreadTransaction.h b/fdbclient/ISingleThreadTransaction.h index 80dd184e74..407bf97ca1 100644 --- a/fdbclient/ISingleThreadTransaction.h +++ b/fdbclient/ISingleThreadTransaction.h @@ -23,6 +23,7 @@ #include "fdbclient/FDBOptions.g.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/KeyRangeMap.h" +#include "fdbclient/NativeAPI.actor.h" #include "flow/Error.h" #include "flow/FastRef.h" @@ -49,18 +50,18 @@ public: virtual void setVersion(Version v) = 0; virtual Future getReadVersion() = 0; virtual Optional getCachedReadVersion() const = 0; - virtual Future> get(const Key& key, bool snapshot = false) = 0; - virtual Future getKey(const KeySelector& key, bool snapshot = false) = 0; + virtual Future> get(const Key& key, Snapshot = Snapshot::FALSE) = 0; + virtual Future getKey(const KeySelector& key, Snapshot = Snapshot::FALSE) = 0; virtual Future> getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot = false, - bool reverse = false) = 0; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) = 0; virtual Future> getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) = 0; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) = 0; virtual Future>> getAddressesForKey(Key const& key) = 0; virtual Future>> getRangeSplitPoints(KeyRange const& range, int64_t chunkSize) = 0; virtual Future getEstimatedRangeSizeBytes(KeyRange const& keys) = 0; diff --git a/fdbclient/KeyBackedTypes.h b/fdbclient/KeyBackedTypes.h index f92324e4ab..7fa26cbea9 100644 --- a/fdbclient/KeyBackedTypes.h +++ b/fdbclient/KeyBackedTypes.h @@ -150,7 +150,7 @@ template class KeyBackedProperty { public: KeyBackedProperty(KeyRef key) : key(key) {} - Future> get(Reference tr, bool snapshot = false) const { + Future> get(Reference tr, Snapshot snapshot = Snapshot::FALSE) const { return map(tr->get(key, snapshot), [](Optional const& val) -> Optional { if (val.present()) return Codec::unpack(Tuple::unpack(val.get())); @@ -158,12 +158,14 @@ public: }); } // Get property's value or defaultValue if it doesn't exist - Future getD(Reference tr, bool snapshot = false, T defaultValue = T()) const { + Future getD(Reference tr, + Snapshot snapshot = Snapshot::FALSE, + T defaultValue = T()) const { return map(get(tr, snapshot), [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); } // Get property's value or throw error if it doesn't exist Future getOrThrow(Reference tr, - bool snapshot = false, + Snapshot snapshot = Snapshot::FALSE, Error err = key_not_found()) const { auto keyCopy = key; auto backtrace = platform::get_backtrace(); @@ -180,7 +182,7 @@ public: }); } - Future> get(Database cx, bool snapshot = false) const { + Future> get(Database cx, Snapshot snapshot = Snapshot::FALSE) const { auto& copy = *this; return runRYWTransaction(cx, [=](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -190,7 +192,7 @@ public: }); } - Future getD(Database cx, bool snapshot = false, T defaultValue = T()) const { + Future getD(Database cx, Snapshot snapshot = Snapshot::FALSE, T defaultValue = T()) const { auto& copy = *this; return runRYWTransaction(cx, [=](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -200,7 +202,7 @@ public: }); } - Future getOrThrow(Database cx, bool snapshot = false, Error err = key_not_found()) const { + Future getOrThrow(Database cx, Snapshot snapshot = Snapshot::FALSE, Error err = key_not_found()) const { auto& copy = *this; return runRYWTransaction(cx, [=](Reference tr) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -235,7 +237,7 @@ template class KeyBackedBinaryValue { public: KeyBackedBinaryValue(KeyRef key) : key(key) {} - Future> get(Reference tr, bool snapshot = false) const { + Future> get(Reference tr, Snapshot snapshot = Snapshot::FALSE) const { return map(tr->get(key, snapshot), [](Optional const& val) -> Optional { if (val.present()) return BinaryReader::fromStringRef(val.get(), Unversioned()); @@ -243,8 +245,11 @@ public: }); } // Get property's value or defaultValue if it doesn't exist - Future getD(Reference tr, bool snapshot = false, T defaultValue = T()) const { - return map(get(tr, false), [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); + Future getD(Reference tr, + Snapshot snapshot = Snapshot::FALSE, + T defaultValue = T()) const { + return map(get(tr, Snapshot::FALSE), + [=](Optional val) -> T { return val.present() ? val.get() : defaultValue; }); } void set(Reference tr, T const& val) { return tr->set(key, BinaryWriter::toValue(val, Unversioned())); @@ -273,8 +278,8 @@ public: KeyType const& begin, Optional const& end, int limit, - bool snapshot = false, - bool reverse = false) const { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) const { Subspace s = space; // 'this' could be invalid inside lambda Key endKey = end.present() ? s.pack(Codec::pack(end.get())) : space.range().end; return map( @@ -293,7 +298,7 @@ public: Future> get(Reference tr, KeyType const& key, - bool snapshot = false) const { + Snapshot snapshot = Snapshot::FALSE) const { return map(tr->get(space.pack(Codec::pack(key)), snapshot), [](Optional const& val) -> Optional { if (val.present()) @@ -339,7 +344,7 @@ public: ValueType const& begin, Optional const& end, int limit, - bool snapshot = false) const { + Snapshot snapshot = Snapshot::FALSE) const { Subspace s = space; // 'this' could be invalid inside lambda Key endKey = end.present() ? s.pack(Codec::pack(end.get())) : space.range().end; return map( @@ -353,7 +358,9 @@ public: }); } - Future exists(Reference tr, ValueType const& val, bool snapshot = false) const { + Future exists(Reference tr, + ValueType const& val, + Snapshot snapshot = Snapshot::FALSE) const { return map(tr->get(space.pack(Codec::pack(val)), snapshot), [](Optional const& val) -> bool { return val.present(); }); } diff --git a/fdbclient/KeyRangeMap.actor.cpp b/fdbclient/KeyRangeMap.actor.cpp index 7b7dcdf1e3..3ca129872e 100644 --- a/fdbclient/KeyRangeMap.actor.cpp +++ b/fdbclient/KeyRangeMap.actor.cpp @@ -119,7 +119,8 @@ void krmSetPreviouslyEmptyRange(CommitTransactionRef& tr, ACTOR Future krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, Value value) { state KeyRange withPrefix = KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString()); - RangeResult old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); + RangeResult old = + wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::TRUE)); Value oldValue; bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); @@ -140,7 +141,8 @@ ACTOR Future krmSetRange(Transaction* tr, Key mapPrefix, KeyRange range, V ACTOR Future krmSetRange(Reference tr, Key mapPrefix, KeyRange range, Value value) { state KeyRange withPrefix = KeyRangeRef(mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString()); - RangeResult old = wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, true)); + RangeResult old = + wait(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end), 1, Snapshot::TRUE)); Value oldValue; bool hasResult = old.size() > 0 && old[0].key.startsWith(mapPrefix); @@ -175,8 +177,10 @@ static Future krmSetRangeCoalescing_(Transaction* tr, KeyRangeRef(mapPrefix.toString() + maxRange.begin.toString(), mapPrefix.toString() + maxRange.end.toString()); state vector> keys; - keys.push_back(tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, true)); - keys.push_back(tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, true)); + keys.push_back( + tr->getRange(lastLessThan(withPrefix.begin), firstGreaterOrEqual(withPrefix.begin), 1, Snapshot::TRUE)); + keys.push_back( + tr->getRange(lastLessOrEqual(withPrefix.end), firstGreaterThan(withPrefix.end) + 1, 2, Snapshot::TRUE)); wait(waitForAll(keys)); // Determine how far to extend this range at the beginning diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index f5d3bd20ce..38f79e8ac1 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -143,7 +143,7 @@ std::map configForToken(std::string const& mode) { } if (key == "perpetual_storage_wiggle" && isInteger(value)) { - int ppWiggle = atoi(value.c_str()); + int ppWiggle = std::stoi(value); if (ppWiggle >= 2 || ppWiggle < 0) { printf("Error: Only 0 and 1 are valid values of perpetual_storage_wiggle at present.\n"); return out; @@ -2473,7 +2473,8 @@ ACTOR Future changeCachedRange(Database cx, KeyRangeRef range, bool add) { tr.clear(sysRangeClear); tr.clear(privateRange); tr.addReadConflictRange(privateRange); - RangeResult previous = wait(tr.getRange(KeyRangeRef(storageCachePrefix, sysRange.begin), 1, true)); + RangeResult previous = + wait(tr.getRange(KeyRangeRef(storageCachePrefix, sysRange.begin), 1, Snapshot::TRUE)); bool prevIsCached = false; if (!previous.empty()) { std::vector prevVal; @@ -2489,7 +2490,7 @@ ACTOR Future changeCachedRange(Database cx, KeyRangeRef range, bool add) { tr.set(sysRange.begin, trueValue); tr.set(privateRange.begin, serverKeysTrue); } - RangeResult after = wait(tr.getRange(KeyRangeRef(sysRange.end, storageCacheKeys.end), 1, false)); + RangeResult after = wait(tr.getRange(KeyRangeRef(sysRange.end, storageCacheKeys.end), 1, Snapshot::FALSE)); bool afterIsCached = false; if (!after.empty()) { std::vector afterVal; diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 1c5124c12a..ffaf51f73e 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -1884,8 +1884,6 @@ bool ClientInfo::canReplace(Reference other) const { } // UNIT TESTS -extern bool noUnseed; - TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing") { auto vals = parseOptionValues("a"); ASSERT(vals.size() == 1 && vals[0] == "a"); diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 8bfba1ee9f..f588f31418 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -94,7 +94,8 @@ Future loadBalance( RequestStream Interface::*channel, const Request& request = Request(), TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint, - bool atMostOnce = false, // if true, throws request_maybe_delivered() instead of retrying automatically + AtMostOnce atMostOnce = + AtMostOnce::FALSE, // if true, throws request_maybe_delivered() instead of retrying automatically QueueModel* model = nullptr) { if (alternatives->hasCaches) { return loadBalance(alternatives->locations(), channel, request, taskID, atMostOnce, model); @@ -154,6 +155,8 @@ void DatabaseContext::addTssMapping(StorageServerInterface const& ssi, StorageSe TSSEndpointData(tssi.id(), tssi.getKeyValues.getEndpoint(), metrics)); queueModel.updateTssEndpoint(ssi.watchValue.getEndpoint().token.first(), TSSEndpointData(tssi.id(), tssi.watchValue.getEndpoint(), metrics)); + queueModel.updateTssEndpoint(ssi.getKeyValuesStream.getEndpoint().token.first(), + TSSEndpointData(tssi.id(), tssi.getKeyValuesStream.getEndpoint(), metrics)); } } @@ -166,6 +169,7 @@ void DatabaseContext::removeTssMapping(StorageServerInterface const& ssi) { queueModel.removeTssEndpoint(ssi.getKey.getEndpoint().token.first()); queueModel.removeTssEndpoint(ssi.getKeyValues.getEndpoint().token.first()); queueModel.removeTssEndpoint(ssi.watchValue.getEndpoint().token.first()); + queueModel.removeTssEndpoint(ssi.getKeyValuesStream.getEndpoint().token.first()); } } @@ -296,11 +300,16 @@ void DatabaseContext::validateVersion(Version version) { ASSERT(version > 0 || version == latestVersion); } -void validateOptionValue(Optional value, bool shouldBePresent) { - if (shouldBePresent && !value.present()) +void validateOptionValuePresent(Optional value) { + if (!value.present()) { throw invalid_option_value(); - if (!shouldBePresent && value.present() && value.get().size() > 0) + } +} + +void validateOptionValueNotPresent(Optional value) { + if (value.present() && value.get().size() > 0) { throw invalid_option_value(); + } } void dumpMutations(const MutationListRef& mutations) { @@ -485,7 +494,7 @@ ACTOR static Future delExcessClntTxnEntriesActor(Transaction* tr, int64_t tr->reset(); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - Optional ctrValue = wait(tr->get(KeyRef(clientLatencyAtomicCtr), true)); + Optional ctrValue = wait(tr->get(KeyRef(clientLatencyAtomicCtr), Snapshot::TRUE)); if (!ctrValue.present()) { TraceEvent(SevInfo, "NumClntTxnEntriesNotFound"); return Void(); @@ -1080,11 +1089,11 @@ DatabaseContext::DatabaseContext(Reference clientInfoMonitor, TaskPriority taskID, LocalityData const& clientLocality, - bool enableLocalityLoadBalance, - bool lockAware, - bool internal, + EnableLocalityLoadBalance enableLocalityLoadBalance, + LockAware lockAware, + IsInternal internal, int apiVersion, - bool switchable) + IsSwitchable switchable) : connectionFile(connectionFile), clientInfo(clientInfo), coordinator(coordinator), clientInfoMonitor(clientInfoMonitor), taskID(taskID), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware), apiVersion(apiVersion), @@ -1360,7 +1369,7 @@ DatabaseContext::DatabaseContext(const Error& err) transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT), - transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), internal(false), + transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc), internal(IsInternal::FALSE), transactionTracingEnabled(true) {} // Static constructor used by server processes to create a DatabaseContext @@ -1368,11 +1377,11 @@ DatabaseContext::DatabaseContext(const Error& err) Database DatabaseContext::create(Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, - bool enableLocalityLoadBalance, + EnableLocalityLoadBalance enableLocalityLoadBalance, TaskPriority taskID, - bool lockAware, + LockAware lockAware, int apiVersion, - bool switchable) { + IsSwitchable switchable) { return Database(new DatabaseContext(Reference>>(), clientInfo, makeReference>>(), @@ -1381,7 +1390,7 @@ Database DatabaseContext::create(Reference> clientInfo, clientLocality, enableLocalityLoadBalance, lockAware, - true, + IsInternal::TRUE, apiVersion, switchable)); } @@ -1397,7 +1406,7 @@ DatabaseContext::~DatabaseContext() { locationCache.insert(allKeys, Reference()); } -pair> DatabaseContext::getCachedLocation(const KeyRef& key, bool isBackward) { +pair> DatabaseContext::getCachedLocation(const KeyRef& key, Reverse isBackward) { if (isBackward) { auto range = locationCache.rangeContainingKeyBefore(key); return std::make_pair(range->range(), range->value()); @@ -1410,7 +1419,7 @@ pair> DatabaseContext::getCachedLocation(const bool DatabaseContext::getCachedLocations(const KeyRangeRef& range, vector>>& result, int limit, - bool reverse) { + Reverse reverse) { result.clear(); auto begin = locationCache.rangeContaining(range.begin); @@ -1458,7 +1467,7 @@ Reference DatabaseContext::setCachedLocation(const KeyRangeRef& ke return loc; } -void DatabaseContext::invalidateCache(const KeyRef& key, bool isBackward) { +void DatabaseContext::invalidateCache(const KeyRef& key, Reverse isBackward) { if (isBackward) { locationCache.rangeContainingKeyBefore(key)->value() = Reference(); } else { @@ -1491,7 +1500,7 @@ bool DatabaseContext::sampleOnCost(uint64_t cost) const { } int64_t extractIntOption(Optional value, int64_t minValue, int64_t maxValue) { - validateOptionValue(value, true); + validateOptionValuePresent(value); if (value.get().size() != 8) { throw invalid_option_value(); } @@ -1553,23 +1562,23 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional()); break; case FDBDatabaseOptions::SNAPSHOT_RYW_ENABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); snapshotRywEnabled++; break; case FDBDatabaseOptions::SNAPSHOT_RYW_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); snapshotRywEnabled--; break; case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_ENABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); transactionTracingEnabled++; break; case FDBDatabaseOptions::DISTRIBUTED_TRANSACTION_TRACE_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); transactionTracingEnabled--; break; case FDBDatabaseOptions::USE_CONFIG_DATABASE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); useConfigDatabase = true; break; default: @@ -1669,7 +1678,7 @@ extern IPAddress determinePublicIPAutomatically(ClusterConnectionString const& c // on another thread Database Database::createDatabase(Reference connFile, int apiVersion, - bool internal, + IsInternal internal, LocalityData const& clientLocality, DatabaseContext* preallocatedDb) { if (!g_network) @@ -1730,11 +1739,11 @@ Database Database::createDatabase(Reference connFile, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, - true, - false, + EnableLocalityLoadBalance::TRUE, + LockAware::FALSE, internal, apiVersion, - /*switchable*/ true); + IsSwitchable::TRUE); } else { db = new DatabaseContext(connectionFile, clientInfo, @@ -1742,11 +1751,11 @@ Database Database::createDatabase(Reference connFile, clientInfoMonitor, TaskPriority::DefaultEndpoint, clientLocality, - true, - false, + EnableLocalityLoadBalance::TRUE, + LockAware::FALSE, internal, apiVersion, - /*switchable*/ true); + IsSwitchable::TRUE); } auto database = Database(db); @@ -1756,7 +1765,7 @@ Database Database::createDatabase(Reference connFile, Database Database::createDatabase(std::string connFileName, int apiVersion, - bool internal, + IsInternal internal, LocalityData const& clientLocality) { Reference rccf = Reference( new ClusterConnectionFile(ClusterConnectionFile::lookupClusterFileName(connFileName).first)); @@ -1803,15 +1812,15 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu networkOptions.traceDirectory = value.present() ? value.get().toString() : ""; break; case FDBNetworkOptions::TRACE_ROLL_SIZE: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceRollSize = extractIntOption(value, 0, std::numeric_limits::max()); break; case FDBNetworkOptions::TRACE_MAX_LOGS_SIZE: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceMaxLogsSize = extractIntOption(value, 0, std::numeric_limits::max()); break; case FDBNetworkOptions::TRACE_FORMAT: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceFormat = value.get().toString(); if (!validateTraceFormat(networkOptions.traceFormat)) { fprintf(stderr, "Unrecognized trace format: `%s'\n", networkOptions.traceFormat.c_str()); @@ -1819,7 +1828,7 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } break; case FDBNetworkOptions::TRACE_FILE_IDENTIFIER: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceFileIdentifier = value.get().toString(); if (networkOptions.traceFileIdentifier.length() > CLIENT_KNOBS->TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH) { fprintf(stderr, "Trace file identifier provided is too long.\n"); @@ -1840,7 +1849,7 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } break; case FDBNetworkOptions::TRACE_CLOCK_SOURCE: - validateOptionValue(value, true); + validateOptionValuePresent(value); networkOptions.traceClockSource = value.get().toString(); if (!validateTraceClockSource(networkOptions.traceClockSource)) { fprintf(stderr, "Unrecognized trace clock source: `%s'\n", networkOptions.traceClockSource.c_str()); @@ -1848,7 +1857,7 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu } break; case FDBNetworkOptions::KNOB: { - validateOptionValue(value, true); + validateOptionValuePresent(value); std::string optionValue = value.get().toString(); TraceEvent("SetKnob").detail("KnobString", optionValue); @@ -1872,42 +1881,42 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu break; } case FDBNetworkOptions::TLS_PLUGIN: - validateOptionValue(value, true); + validateOptionValuePresent(value); break; case FDBNetworkOptions::TLS_CERT_PATH: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCertificatePath(value.get().toString()); break; case FDBNetworkOptions::TLS_CERT_BYTES: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCertificateBytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_CA_PATH: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCAPath(value.get().toString()); break; } case FDBNetworkOptions::TLS_CA_BYTES: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setCABytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_PASSWORD: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setPassword(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_PATH: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setKeyPath(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_BYTES: { - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.setKeyBytes(value.get().toString()); break; } case FDBNetworkOptions::TLS_VERIFY_PEERS: - validateOptionValue(value, true); + validateOptionValuePresent(value); tlsConfig.clearVerifyPeers(); tlsConfig.addVerifyPeers(value.get().toString()); break; @@ -1918,16 +1927,16 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu enableBuggify(false, BuggifyType::Client); break; case FDBNetworkOptions::CLIENT_BUGGIFY_SECTION_ACTIVATED_PROBABILITY: - validateOptionValue(value, true); + validateOptionValuePresent(value); clearBuggifySections(BuggifyType::Client); P_BUGGIFIED_SECTION_ACTIVATED[int(BuggifyType::Client)] = double(extractIntOption(value, 0, 100)) / 100.0; break; case FDBNetworkOptions::CLIENT_BUGGIFY_SECTION_FIRED_PROBABILITY: - validateOptionValue(value, true); + validateOptionValuePresent(value); P_BUGGIFIED_SECTION_FIRES[int(BuggifyType::Client)] = double(extractIntOption(value, 0, 100)) / 100.0; break; case FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); networkOptions.logClientInfo = false; break; case FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS: { @@ -1947,11 +1956,11 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu break; } case FDBNetworkOptions::ENABLE_RUN_LOOP_PROFILING: // Same as ENABLE_SLOW_TASK_PROFILING - validateOptionValue(value, false); + validateOptionValueNotPresent(value); networkOptions.runLoopProfilingEnabled = true; break; case FDBNetworkOptions::DISTRIBUTED_CLIENT_TRACER: { - validateOptionValue(value, true); + validateOptionValuePresent(value); std::string tracer = value.get().toString(); if (tracer == "none" || tracer == "disabled") { openTracer(TracerType::DISABLED); @@ -1994,7 +2003,7 @@ ACTOR Future monitorNetworkBusyness() { } // Setup g_network and start monitoring for network busyness -void setupNetwork(uint64_t transportId, bool useMetrics) { +void setupNetwork(uint64_t transportId, UseMetrics useMetrics) { if (g_network) throw network_already_setup(); @@ -2163,7 +2172,7 @@ Future getRange(Database const& cx, KeySelector const& begin, KeySelector const& end, GetRangeLimits const& limits, - bool const& reverse, + Reverse const& reverse, TransactionInfo const& info, TagSet const& tags); @@ -2239,7 +2248,7 @@ void updateTssMappings(Database cx, const GetKeyServerLocationsReply& reply) { ACTOR Future>> getKeyLocation_internal(Database cx, Key key, TransactionInfo info, - bool isBackward = false) { + Reverse isBackward = Reverse::FALSE) { state Span span("NAPI:getKeyLocation"_loc, info.spanID); if (isBackward) { ASSERT(key != allKeys.begin && key <= allKeys.end); @@ -2278,7 +2287,7 @@ Future>> getKeyLocation(Database const& c Key const& key, F StorageServerInterface::*member, TransactionInfo const& info, - bool isBackward = false) { + Reverse isBackward = Reverse::FALSE) { // we first check whether this range is cached auto ssi = cx->getCachedLocation(key, isBackward); if (!ssi.second) { @@ -2299,7 +2308,7 @@ Future>> getKeyLocation(Database const& c ACTOR Future>>> getKeyRangeLocations_internal(Database cx, KeyRange keys, int limit, - bool reverse, + Reverse reverse, TransactionInfo info) { state Span span("NAPI:getKeyRangeLocations"_loc, info.spanID); if (info.debugID.present()) @@ -2348,7 +2357,7 @@ template Future>>> getKeyRangeLocations(Database const& cx, KeyRange const& keys, int limit, - bool reverse, + Reverse reverse, F StorageServerInterface::*member, TransactionInfo const& info) { ASSERT(!keys.empty()); @@ -2385,8 +2394,8 @@ ACTOR Future warmRange_impl(Transaction* self, Database cx, KeyRange keys) state int totalRanges = 0; state int totalRequests = 0; loop { - vector>> locations = - wait(getKeyRangeLocations_internal(cx, keys, CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT, false, self->info)); + vector>> locations = wait( + getKeyRangeLocations_internal(cx, keys, CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT, Reverse::FALSE, self->info)); totalRanges += CLIENT_KNOBS->WARM_RANGE_SHARD_LIMIT; totalRequests++; if (locations.size() == 0 || totalRanges >= cx->locationCacheSize || @@ -2469,7 +2478,7 @@ ACTOR Future> getValue(Future version, GetValueRequest( span.context, key, ver, cx->sampleReadTags() ? tags : Optional(), getValueID), TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::FALSE, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { reply = _reply; } @@ -2556,7 +2565,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr Key locationKey(k.getKey(), k.arena()); state pair> ssi = - wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, k.isBackward())); + wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKey, info, Reverse{ k.isBackward() })); try { if (info.debugID.present()) @@ -2581,7 +2590,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr &StorageServerInterface::getKey, req, TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::FALSE, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { reply = _reply; } @@ -2604,7 +2613,7 @@ ACTOR Future getKey(Database cx, KeySelector k, Future version, Tr if (info.debugID.present()) g_traceBatch.addEvent("GetKeyDebug", getKeyID.get().first(), "NativeAPI.getKey.Error"); if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed) { - cx->invalidateCache(k.getKey(), k.isBackward()); + cx->invalidateCache(k.getKey(), Reverse{ k.isBackward() }); wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, info.taskID)); } else { @@ -2895,7 +2904,7 @@ ACTOR Future watchValueMap(Future version, return Void(); } -void transformRangeLimits(GetRangeLimits limits, bool reverse, GetKeyValuesRequest& req) { +void transformRangeLimits(GetRangeLimits limits, Reverse reverse, GetKeyValuesRequest& req) { if (limits.bytes != 0) { if (!limits.hasRowLimit()) req.limit = CLIENT_KNOBS->REPLY_BYTE_LIMIT; // Can't get more than this many rows anyway @@ -2919,7 +2928,7 @@ ACTOR Future getExactRange(Database cx, Version version, KeyRange keys, GetRangeLimits limits, - bool reverse, + Reverse reverse, TransactionInfo info, TagSet tags) { state RangeResult output; @@ -2974,7 +2983,7 @@ ACTOR Future getExactRange(Database cx, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::FALSE, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { rep = _rep; } @@ -3102,7 +3111,7 @@ ACTOR Future getRangeFallback(Database cx, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse, + Reverse reverse, TransactionInfo info, TagSet tags) { if (version == latestVersion) { @@ -3157,9 +3166,9 @@ void getRangeFinished(Database cx, double startTime, KeySelector begin, KeySelector end, - bool snapshot, + Snapshot snapshot, Promise> conflictRange, - bool reverse, + Reverse reverse, RangeResult result) { int64_t bytes = 0; for (const KeyValueRef& kv : result) { @@ -3213,8 +3222,8 @@ ACTOR Future getRange(Database cx, KeySelector end, GetRangeLimits limits, Promise> conflictRange, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, TransactionInfo info, TagSet tags) { state GetRangeLimits originalLimits(limits); @@ -3249,7 +3258,7 @@ ACTOR Future getRange(Database cx, } Key locationKey = reverse ? Key(end.getKey(), end.arena()) : Key(begin.getKey(), begin.arena()); - bool locationBackward = reverse ? (end - 1).isBackward() : begin.isBackward(); + Reverse locationBackward{ reverse ? (end - 1).isBackward() : begin.isBackward() }; state pair> beginServer = wait(getKeyLocation(cx, locationKey, &StorageServerInterface::getKeyValues, info, locationBackward)); state KeyRange shard = beginServer.first; @@ -3325,7 +3334,7 @@ ACTOR Future getRange(Database cx, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, - false, + AtMostOnce::FALSE, cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr)); rep = _rep; ++cx->transactionPhysicalReadsCompleted; @@ -3459,7 +3468,7 @@ ACTOR Future getRange(Database cx, if (e.code() == error_code_wrong_shard_server || e.code() == error_code_all_alternatives_failed || (e.code() == error_code_transaction_too_old && readVersion == latestVersion)) { cx->invalidateCache(reverse ? end.getKey() : begin.getKey(), - reverse ? (end - 1).isBackward() : begin.isBackward()); + Reverse{ reverse ? (end - 1).isBackward() : begin.isBackward() }); if (e.code() == error_code_wrong_shard_server) { RangeResult result = wait(getRangeFallback( @@ -3498,6 +3507,174 @@ ACTOR Future getRange(Database cx, } } +template +struct TSSDuplicateStreamData { + PromiseStream stream; + Promise tssComparisonDone; + + // empty constructor for optional? + TSSDuplicateStreamData() {} + + TSSDuplicateStreamData(PromiseStream stream) : stream(stream) {} + + bool done() { return tssComparisonDone.getFuture().isReady(); } + + void setDone() { + if (tssComparisonDone.canBeSet()) { + tssComparisonDone.send(Void()); + } + } + + ~TSSDuplicateStreamData() {} +}; + +// Error tracking here is weird, and latency doesn't really mean the same thing here as it does with normal tss +// comparisons, so this is pretty much just counting mismatches +ACTOR template +static Future tssStreamComparison(Request request, + TSSDuplicateStreamData streamData, + ReplyPromiseStream tssReplyStream, + TSSEndpointData tssData) { + state bool ssEndOfStream = false; + state bool tssEndOfStream = false; + state Optional ssReply = Optional(); + state Optional tssReply = Optional(); + + loop { + // reset replies + ssReply = Optional(); + tssReply = Optional(); + + state double startTime = now(); + // wait for ss response + try { + REPLYSTREAM_TYPE(Request) _ssReply = waitNext(streamData.stream.getFuture()); + ssReply = _ssReply; + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + streamData.setDone(); + throw; + } + if (e.code() == error_code_end_of_stream) { + // ss response will be set to empty, to compare to the SS response if it wasn't empty and cause a + // mismatch + ssEndOfStream = true; + } else { + tssData.metrics->ssError(e.code()); + } + TEST(e.code() != error_code_end_of_stream); // SS got error in TSS stream comparison + } + + state double sleepTime = std::max(startTime + FLOW_KNOBS->LOAD_BALANCE_TSS_TIMEOUT - now(), 0.0); + // wait for tss response + try { + choose { + when(REPLYSTREAM_TYPE(Request) _tssReply = waitNext(tssReplyStream.getFuture())) { + tssReply = _tssReply; + } + when(wait(delay(sleepTime))) { + ++tssData.metrics->tssTimeouts; + TEST(true); // Got TSS timeout in stream comparison + } + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) { + streamData.setDone(); + throw; + } + if (e.code() == error_code_end_of_stream) { + // tss response will be set to empty, to compare to the SS response if it wasn't empty and cause a + // mismatch + tssEndOfStream = true; + } else { + tssData.metrics->tssError(e.code()); + } + TEST(e.code() != error_code_end_of_stream); // TSS got error in TSS stream comparison + } + + if (!ssEndOfStream || !tssEndOfStream) { + ++tssData.metrics->streamComparisons; + } + + // if both are successful, compare + if (ssReply.present() && tssReply.present()) { + // compare results + // FIXME: this code is pretty much identical to LoadBalance.h + // TODO could add team check logic in if we added synchronous way to turn this into a fixed getRange request + // and send it to the whole team and compare? I think it's fine to skip that for streaming though + TEST(ssEndOfStream != tssEndOfStream); // SS or TSS stream finished early! + + // skip tss comparison if both are end of stream + if ((!ssEndOfStream || !tssEndOfStream) && !TSS_doCompare(ssReply.get(), tssReply.get())) { + TEST(true); // TSS mismatch in stream comparison + TraceEvent mismatchEvent( + (g_network->isSimulated() && g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) + ? SevWarnAlways + : SevError, + TSS_mismatchTraceName(request)); + mismatchEvent.setMaxEventLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE); + mismatchEvent.detail("TSSID", tssData.tssId); + + if (tssData.metrics->shouldRecordDetailedMismatch()) { + TSS_traceMismatch(mismatchEvent, request, ssReply.get(), tssReply.get()); + + TEST(FLOW_KNOBS + ->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Full TSS Mismatch in stream comparison + TEST(!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL); // Tracing Partial TSS Mismatch in stream + // comparison and storing the rest in FDB + + if (!FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL) { + mismatchEvent.disable(); + UID mismatchUID = deterministicRandom()->randomUniqueID(); + tssData.metrics->recordDetailedMismatchData(mismatchUID, mismatchEvent.getFields().toString()); + + // record a summarized trace event instead + TraceEvent summaryEvent((g_network->isSimulated() && + g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations) + ? SevWarnAlways + : SevError, + TSS_mismatchTraceName(request)); + summaryEvent.detail("TSSID", tssData.tssId).detail("MismatchId", mismatchUID); + } + } else { + // don't record trace event + mismatchEvent.disable(); + } + streamData.setDone(); + return Void(); + } + } + if (!ssReply.present() || !tssReply.present() || ssEndOfStream || tssEndOfStream) { + // if both streams don't still have more data, stop comparison + streamData.setDone(); + return Void(); + } + } +} + +// Currently only used for GetKeyValuesStream but could easily be plugged for other stream types +// User of the stream has to forward the SS's responses to the returned promise stream, if it is set +template +Optional> +maybeDuplicateTSSStreamFragment(Request& req, QueueModel* model, RequestStream const* ssStream) { + if (model) { + Optional tssData = model->getTssData(ssStream->getEndpoint().token.first()); + + if (tssData.present()) { + TEST(true); // duplicating stream to TSS + resetReply(req); + // FIXME: optimize to avoid creating new netNotifiedQueueWithAcknowledgements for each stream duplication + RequestStream tssRequestStream(tssData.get().endpoint); + ReplyPromiseStream tssReplyStream = tssRequestStream.getReplyStream(req); + PromiseStream ssDuplicateReplyStream; + TSSDuplicateStreamData streamData(ssDuplicateReplyStream); + model->addActor.send(tssStreamComparison(req, streamData, tssReplyStream, tssData.get())); + return Optional>(streamData); + } + } + return Optional>(); +} + // Streams all of the KV pairs in a target key range into a ParallelStream fragment ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* results, Database cx, @@ -3505,8 +3682,8 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* Version version, KeyRange keys, GetRangeLimits limits, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, TransactionInfo info, TagSet tags, SpanID spanContext) { @@ -3518,6 +3695,7 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* loop { const KeyRange& range = locations[shard].first; + state Optional> tssDuplicateStream; state GetKeyValuesStreamRequest req; req.version = version; req.begin = firstGreaterOrEqual(range.begin); @@ -3526,6 +3704,9 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* req.limit = reverse ? -CLIENT_KNOBS->REPLY_BYTE_LIMIT : CLIENT_KNOBS->REPLY_BYTE_LIMIT; req.limitBytes = std::numeric_limits::max(); + // keep shard's arena around in case of async tss comparison + req.arena.dependsOn(range.arena()); + ASSERT(req.limitBytes > 0 && req.limit != 0 && req.limit < 0 == reverse); // FIXME: buggify byte limits on internal functions that use them, instead of globally @@ -3589,6 +3770,12 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* locations[shard] .second->get(useIdx, &StorageServerInterface::getKeyValuesStream) .getReplyStream(req); + + tssDuplicateStream = maybeDuplicateTSSStreamFragment( + req, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr, + &locations[shard].second->get(useIdx, &StorageServerInterface::getKeyValuesStream)); + state bool breakAgain = false; loop { wait(results->onEmpty()); @@ -3596,6 +3783,9 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* choose { when(wait(cx->connectionFileChanged())) { results->sendError(transaction_too_old()); + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(transaction_too_old()); + } return Void(); } @@ -3605,9 +3795,15 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* } catch (Error& e) { ++cx->transactionPhysicalReadsCompleted; if (e.code() == error_code_broken_promise) { + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(connection_failed()); + } throw connection_failed(); } if (e.code() != error_code_end_of_stream) { + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(e); + } throw; } rep = GetKeyValuesStreamReply(); @@ -3617,6 +3813,17 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* "TransactionDebug", info.debugID.get().first(), "NativeAPI.getExactRange.After"); RangeResult output(RangeResultRef(rep.data, rep.more), rep.arena); + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + // shallow copy the reply with an arena depends, and send it to the duplicate stream for TSS + GetKeyValuesStreamReply replyCopy; + replyCopy.version = rep.version; + replyCopy.more = rep.more; + replyCopy.cached = rep.cached; + replyCopy.arena.dependsOn(rep.arena); + replyCopy.data.append(replyCopy.arena, rep.data.begin(), rep.data.size()); + tssDuplicateStream.get().stream.send(replyCopy); + } + int64_t bytes = 0; for (const KeyValueRef& kv : output) { bytes += kv.key.size() + kv.value.size(); @@ -3674,6 +3881,9 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* output.readThrough = reverse ? keys.begin : keys.end; results->send(std::move(output)); results->finish(); + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(end_of_stream()); + } return Void(); } keys = KeyRangeRef(begin, end); @@ -3700,6 +3910,10 @@ ACTOR Future getRangeStreamFragment(ParallelStream::Fragment* break; } } catch (Error& e) { + // send errors to tss duplicate stream, including actor_cancelled + if (tssDuplicateStream.present() && !tssDuplicateStream.get().done()) { + tssDuplicateStream.get().stream.sendError(e); + } if (e.code() == error_code_actor_cancelled) { throw; } @@ -3740,8 +3954,8 @@ ACTOR Future getRangeStream(PromiseStream _results, KeySelector end, GetRangeLimits limits, Promise> conflictRange, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, TransactionInfo info, TagSet tags) { @@ -3821,7 +4035,7 @@ Future getRange(Database const& cx, KeySelector const& begin, KeySelector const& end, GetRangeLimits const& limits, - bool const& reverse, + Reverse const& reverse, TransactionInfo const& info, TagSet const& tags) { return getRange(cx, @@ -3831,7 +4045,7 @@ Future getRange(Database const& cx, end, limits, Promise>(), - true, + Snapshot::TRUE, reverse, info, tags); @@ -3927,7 +4141,7 @@ void Transaction::setVersion(Version v) { readVersion = v; } -Future> Transaction::get(const Key& key, bool snapshot) { +Future> Transaction::get(const Key& key, Snapshot snapshot) { ++cx->transactionLogicalReads; ++cx->transactionGetValueRequests; // ASSERT (key < allKeys.end); @@ -4050,12 +4264,18 @@ ACTOR Future>> getAddressesForKeyActor(Key key lastLessOrEqual(serverTagKeys.begin), firstGreaterThan(serverTagKeys.end), GetRangeLimits(CLIENT_KNOBS->TOO_MANY), - false, + Reverse::FALSE, info, options.readTags)); ASSERT(!serverTagResult.more && serverTagResult.size() < CLIENT_KNOBS->TOO_MANY); - Future futureServerUids = getRange( - cx, ver, lastLessOrEqual(ksKey), firstGreaterThan(ksKey), GetRangeLimits(1), false, info, options.readTags); + Future futureServerUids = getRange(cx, + ver, + lastLessOrEqual(ksKey), + firstGreaterThan(ksKey), + GetRangeLimits(1), + Reverse::FALSE, + info, + options.readTags); RangeResult serverUids = wait(futureServerUids); ASSERT(serverUids.size()); // every shard needs to have a team @@ -4110,7 +4330,7 @@ ACTOR Future getKeyAndConflictRange(Database cx, } } -Future Transaction::getKey(const KeySelector& key, bool snapshot) { +Future Transaction::getKey(const KeySelector& key, Snapshot snapshot) { ++cx->transactionLogicalReads; ++cx->transactionGetKeyRequests; if (snapshot) @@ -4124,8 +4344,8 @@ Future Transaction::getKey(const KeySelector& key, bool snapshot) { Future Transaction::getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { ++cx->transactionLogicalReads; ++cx->transactionGetRangeRequests; @@ -4166,8 +4386,8 @@ Future Transaction::getRange(const KeySelector& begin, Future Transaction::getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse); } @@ -4177,8 +4397,8 @@ Future Transaction::getRangeStream(const PromiseStream& resul const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { ++cx->transactionLogicalReads; ++cx->transactionGetRangeStreamRequests; @@ -4227,8 +4447,8 @@ Future Transaction::getRangeStream(const PromiseStream& resul const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return getRangeStream(results, begin, end, GetRangeLimits(limit), snapshot, reverse); } @@ -4271,7 +4491,7 @@ void Transaction::makeSelfConflicting() { tr.transaction.write_conflict_ranges.push_back(tr.arena, r); } -void Transaction::set(const KeyRef& key, const ValueRef& value, bool addConflictRange) { +void Transaction::set(const KeyRef& key, const ValueRef& value, AddConflictRange addConflictRange) { ++cx->transactionSetMutations; if (key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) @@ -4293,7 +4513,7 @@ void Transaction::set(const KeyRef& key, const ValueRef& value, bool addConflict void Transaction::atomicOp(const KeyRef& key, const ValueRef& operand, MutationRef::Type operationType, - bool addConflictRange) { + AddConflictRange addConflictRange) { ++cx->transactionAtomicMutations; if (key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) @@ -4321,7 +4541,7 @@ void Transaction::atomicOp(const KeyRef& key, TEST(true); // NativeAPI atomic operation } -void Transaction::clear(const KeyRangeRef& range, bool addConflictRange) { +void Transaction::clear(const KeyRangeRef& range, AddConflictRange addConflictRange) { ++cx->transactionClearMutations; auto& req = tr; auto& t = req.transaction; @@ -4353,7 +4573,7 @@ void Transaction::clear(const KeyRangeRef& range, bool addConflictRange) { if (addConflictRange) t.write_conflict_ranges.push_back(req.arena, r); } -void Transaction::clear(const KeyRef& key, bool addConflictRange) { +void Transaction::clear(const KeyRef& key, AddConflictRange addConflictRange) { ++cx->transactionClearMutations; // There aren't any keys in the database with size larger than KEY_SIZE_LIMIT if (key.size() > @@ -4697,11 +4917,12 @@ ACTOR Future> estimateCommitCosts(Transac wait(getKeyRangeLocations(self->getDatabase(), keyRange, CLIENT_KNOBS->TOO_MANY, - false, + Reverse::FALSE, &StorageServerInterface::getShardState, self->info)); - if (locations.empty()) + if (locations.empty()) { continue; + } uint64_t bytes = 0; if (locations.size() == 1) { @@ -4797,7 +5018,7 @@ ACTOR static Future tryCommit(Database cx, &CommitProxyInterface::commit, req, TaskPriority::DefaultPromiseEndpoint, - true); + AtMostOnce::TRUE); } choose { @@ -5047,7 +5268,7 @@ Future Transaction::commit() { void Transaction::setOption(FDBTransactionOptions::Option option, Optional value) { switch (option) { case FDBTransactionOptions::INITIALIZE_NEW_DATABASE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); if (readVersion.isValid()) throw read_version_already_set(); readVersion = Version(0); @@ -5055,37 +5276,37 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional 100 || value.get().size() == 0) { throw invalid_option_value(); @@ -5122,7 +5343,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optionalidentifier.empty()) { trLogInfo->logTo(TransactionLogInfo::TRACE_LOG); } else { @@ -5133,7 +5354,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional::max()); if (maxFieldLength == 0) { @@ -5147,7 +5368,7 @@ void Transaction::setOption(FDBTransactionOptions::Option option, OptionalrandomUniqueID()); if (trLogInfo && !trLogInfo->identifier.empty()) { TraceEvent(SevInfo, "TransactionBeingTraced") @@ -5157,23 +5378,23 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional::max()) / 1000.0; break; case FDBTransactionOptions::SIZE_LIMIT: - validateOptionValue(value, true); + validateOptionValuePresent(value); options.sizeLimit = extractIntOption(value, 32, CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT); break; case FDBTransactionOptions::LOCK_AWARE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.lockAware = true; options.readOnly = false; break; case FDBTransactionOptions::READ_LOCK_AWARE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); if (!options.lockAware) { options.lockAware = true; options.readOnly = true; @@ -5181,34 +5402,34 @@ void Transaction::setOption(FDBTransactionOptions::Option option, Optional extractReadVersion(Location location, TransactionPriority priority, Reference trLogInfo, Future f, - bool lockAware, + LockAware lockAware, double startTime, Promise> metadataVersion, TagSet tags) { @@ -5500,7 +5721,7 @@ Future Transaction::getReadVersion(uint32_t flags) { options.priority, trLogInfo, req.reply.getFuture(), - options.lockAware, + LockAware{ options.lockAware }, startTime, metadataVersion, options.tags); @@ -5690,7 +5911,7 @@ ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRang wait(getKeyRangeLocations(cx, keys, std::numeric_limits::max(), - false, + Reverse::FALSE, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); state int nLocs = locations.size(); @@ -5789,7 +6010,7 @@ ACTOR Future>> getReadHotRanges(Da wait(getKeyRangeLocations(cx, keys, shardLimit, - false, + Reverse::FALSE, &StorageServerInterface::getReadHotRanges, TransactionInfo(TaskPriority::DataDistribution, span.context))); try { @@ -5857,7 +6078,7 @@ ACTOR Future, int>> waitStorageMetrics(Databa wait(getKeyRangeLocations(cx, keys, shardLimit, - false, + Reverse::FALSE, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); if (expectedShardCount >= 0 && locations.size() != expectedShardCount) { @@ -5949,7 +6170,7 @@ ACTOR Future>> getRangeSplitPoints(Database cx, Key wait(getKeyRangeLocations(cx, keys, CLIENT_KNOBS->TOO_MANY, - false, + Reverse::FALSE, &StorageServerInterface::getRangeSplitPoints, TransactionInfo(TaskPriority::DataDistribution, span.context))); try { @@ -6010,7 +6231,7 @@ ACTOR Future>> splitStorageMetrics(Database cx, wait(getKeyRangeLocations(cx, keys, CLIENT_KNOBS->STORAGE_METRICS_SHARD_LIMIT, - false, + Reverse::FALSE, &StorageServerInterface::splitMetrics, TransactionInfo(TaskPriority::DataDistribution, span.context))); state StorageMetrics used; @@ -6117,7 +6338,7 @@ ACTOR Future snapCreate(Database cx, Standalone snapCmd, UID sn &CommitProxyInterface::proxySnapReq, ProxySnapRequest(snapCmd, snapUID, snapUID), cx->taskID, - true /*atmostOnce*/))) { + AtMostOnce::TRUE))) { TraceEvent("SnapCreateExit").detail("SnapCmd", snapCmd.toString()).detail("UID", snapUID); return Void(); } @@ -6280,16 +6501,16 @@ Future DatabaseContext::createSnapshot(StringRef uid, StringRef snapshot_c return createSnapshotActor(this, UID::fromString(uid_str), snapshot_command); } -ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, bool lock_aware) { +ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware) { state ReadYourWritesTransaction tr(cx); loop { try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (lock_aware) { + if (lockAware) { tr.setOption(FDBTransactionOptions::LOCK_AWARE); } - tr.set(perpetualStorageWiggleKey, enable ? LiteralStringRef("1") : LiteralStringRef("0")); + tr.set(perpetualStorageWiggleKey, enable ? "1"_sr : "0"_sr); wait(tr.commit()); break; } catch (Error& e) { @@ -6297,4 +6518,4 @@ ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, bool lock } } return Void(); -} \ No newline at end of file +} diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index 043bcaf4f2..9c64af215f 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -27,10 +27,12 @@ #elif !defined(FDBCLIENT_NATIVEAPI_ACTOR_H) #define FDBCLIENT_NATIVEAPI_ACTOR_H +#include "flow/BooleanParam.h" #include "flow/flow.h" #include "flow/TDMetric.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/CommitProxyInterface.h" +#include "fdbclient/ClientBooleanParams.h" #include "fdbclient/FDBOptions.g.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" @@ -51,7 +53,8 @@ void addref(DatabaseContext* ptr); template <> void delref(DatabaseContext* ptr); -void validateOptionValue(Optional value, bool shouldBePresent); +void validateOptionValuePresent(Optional value); +void validateOptionValueNotPresent(Optional value); void enableClientInfoLogging(); @@ -81,13 +84,13 @@ public: // on another thread static Database createDatabase(Reference connFile, int apiVersion, - bool internal = true, + IsInternal internal = IsInternal::TRUE, LocalityData const& clientLocality = LocalityData(), DatabaseContext* preallocatedDb = nullptr); static Database createDatabase(std::string connFileName, int apiVersion, - bool internal = true, + IsInternal internal = IsInternal::TRUE, LocalityData const& clientLocality = LocalityData()); Database() {} // an uninitialized database can be destructed or reassigned safely; that's it @@ -112,7 +115,7 @@ private: void setNetworkOption(FDBNetworkOptions::Option option, Optional value = Optional()); // Configures the global networking machinery -void setupNetwork(uint64_t transportId = 0, bool useMetrics = false); +void setupNetwork(uint64_t transportId = 0, UseMetrics = UseMetrics::FALSE); // This call blocks while the network is running. To use the API in a single-threaded // environment, the calling program must have ACTORs already launched that are waiting @@ -248,24 +251,24 @@ public: Future getRawReadVersion(); Optional getCachedReadVersion() const; - [[nodiscard]] Future> get(const Key& key, bool snapshot = false); + [[nodiscard]] Future> get(const Key& key, Snapshot = Snapshot::FALSE); [[nodiscard]] Future watch(Reference watch); - [[nodiscard]] Future getKey(const KeySelector& key, bool snapshot = false); + [[nodiscard]] Future getKey(const KeySelector& key, Snapshot = Snapshot::FALSE); // Future< Optional > get( const KeySelectorRef& key ); [[nodiscard]] Future getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE); [[nodiscard]] Future getRange(const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE); [[nodiscard]] Future getRange(const KeyRange& keys, int limit, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, @@ -274,8 +277,8 @@ public: } [[nodiscard]] Future getRange(const KeyRange& keys, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, @@ -289,19 +292,19 @@ public: const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE); [[nodiscard]] Future getRangeStream(const PromiseStream>& results, const KeySelector& begin, const KeySelector& end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false); + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE); [[nodiscard]] Future getRangeStream(const PromiseStream>& results, const KeyRange& keys, int limit, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) { return getRangeStream(results, KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), @@ -312,8 +315,8 @@ public: [[nodiscard]] Future getRangeStream(const PromiseStream>& results, const KeyRange& keys, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) { return getRangeStream(results, KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), @@ -348,13 +351,13 @@ public: // The returned list would still be in form of [keys.begin, splitPoint1, splitPoint2, ... , keys.end] Future>> getRangeSplitPoints(KeyRange const& keys, int64_t chunkSize); // If checkWriteConflictRanges is true, existing write conflict ranges will be searched for this key - void set(const KeyRef& key, const ValueRef& value, bool addConflictRange = true); + void set(const KeyRef& key, const ValueRef& value, AddConflictRange = AddConflictRange::TRUE); void atomicOp(const KeyRef& key, const ValueRef& value, MutationRef::Type operationType, - bool addConflictRange = true); - void clear(const KeyRangeRef& range, bool addConflictRange = true); - void clear(const KeyRef& key, bool addConflictRange = true); + AddConflictRange = AddConflictRange::TRUE); + void clear(const KeyRangeRef& range, AddConflictRange = AddConflictRange::TRUE); + void clear(const KeyRef& key, AddConflictRange = AddConflictRange::TRUE); [[nodiscard]] Future commit(); // Throws not_committed or commit_unknown_result errors in normal operation void setOption(FDBTransactionOptions::Option option, Optional value = Optional()); @@ -451,7 +454,7 @@ inline uint64_t getWriteOperationCost(uint64_t bytes) { // Create a transaction to set the value of system key \xff/conf/perpetual_storage_wiggle. If enable == true, the value // will be 1. Otherwise, the value will be 0. -ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, bool lock_aware = false); +ACTOR Future setPerpetualStorageWiggle(Database cx, bool enable, LockAware lockAware = LockAware::FALSE); #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/PaxosConfigTransaction.actor.cpp b/fdbclient/PaxosConfigTransaction.actor.cpp index 8b7ef9f06d..907c1cc449 100644 --- a/fdbclient/PaxosConfigTransaction.actor.cpp +++ b/fdbclient/PaxosConfigTransaction.actor.cpp @@ -33,7 +33,7 @@ Optional PaxosConfigTransaction::getCachedReadVersion() const { return ::invalidVersion; } -Future> PaxosConfigTransaction::get(Key const& key, bool snapshot) { +Future> PaxosConfigTransaction::get(Key const& key, Snapshot snapshot) { // TODO: Implement return Optional{}; } @@ -41,8 +41,8 @@ Future> PaxosConfigTransaction::get(Key const& key, bool snapsho Future> PaxosConfigTransaction::getRange(KeySelector const& begin, KeySelector const& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { // TODO: Implement ASSERT(false); return Standalone{}; @@ -51,9 +51,9 @@ Future> PaxosConfigTransaction::getRange(KeySelector Future> PaxosConfigTransaction::getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot, - bool reverse) { - // TODO: Implememnt + Snapshot snapshot, + Reverse reverse) { + // TODO: Implement ASSERT(false); return Standalone{}; } diff --git a/fdbclient/PaxosConfigTransaction.h b/fdbclient/PaxosConfigTransaction.h index 884afdb2d1..f3af19bb98 100644 --- a/fdbclient/PaxosConfigTransaction.h +++ b/fdbclient/PaxosConfigTransaction.h @@ -38,17 +38,17 @@ public: Future getReadVersion() override; Optional getCachedReadVersion() const override; - Future> get(Key const& key, bool snapshot = false) override; + Future> get(Key const& key, Snapshot = Snapshot::FALSE) override; Future> getRange(KeySelector const& begin, KeySelector const& end, int limit, - bool snapshot = false, - bool reverse = false) override; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) override; Future> getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) override; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) override; void set(KeyRef const& key, ValueRef const& value) override; void clear(KeyRangeRef const&) override { throw client_invalid_operation(); } void clear(KeyRef const&) override; diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 4db07f527b..dcb5d322a1 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -65,7 +65,7 @@ public: typedef Key Result; }; - template + template struct GetRangeReq { GetRangeReq(KeySelector begin, KeySelector end, GetRangeLimits limits) : begin(begin), end(end), limits(limits) {} @@ -99,7 +99,7 @@ public: } else if (it->is_empty_range()) { return Optional(); } else { - Optional res = wait(ryw->tr.get(read.key, true)); + Optional res = wait(ryw->tr.get(read.key, Snapshot::TRUE)); KeyRef k(ryw->arena, read.key); if (res.present()) { @@ -162,20 +162,22 @@ public: // transaction. Responsible for clipping results to the non-system keyspace when appropriate, since NativeAPI // doesn't do that. - static Future> readThrough(ReadYourWritesTransaction* ryw, GetValueReq read, bool snapshot) { + static Future> readThrough(ReadYourWritesTransaction* ryw, GetValueReq read, Snapshot snapshot) { return ryw->tr.get(read.key, snapshot); } - ACTOR static Future readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, bool snapshot) { + ACTOR static Future readThrough(ReadYourWritesTransaction* ryw, GetKeyReq read, Snapshot snapshot) { Key key = wait(ryw->tr.getKey(read.key, snapshot)); if (ryw->getMaxReadKey() < key) return ryw->getMaxReadKey(); // Filter out results in the system keys if they are not accessible return key; } - ACTOR template - static Future readThrough(ReadYourWritesTransaction* ryw, GetRangeReq read, bool snapshot) { - if (Reverse && read.end.offset > 1) { + ACTOR template + static Future readThrough(ReadYourWritesTransaction* ryw, + GetRangeReq read, + Snapshot snapshot) { + if (backwards && read.end.offset > 1) { // FIXME: Optimistically assume that this will not run into the system keys, and only reissue if the result // actually does. Key key = wait(ryw->tr.getKey(read.end, snapshot)); @@ -185,10 +187,11 @@ public: read.end = KeySelector(firstGreaterOrEqual(key), key.arena()); } - RangeResult v = wait(ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, Reverse)); + RangeResult v = wait( + ryw->tr.getRange(read.begin, read.end, read.limits, snapshot, backwards ? Reverse::TRUE : Reverse::FALSE)); KeyRef maxKey = ryw->getMaxReadKey(); if (v.size() > 0) { - if (!Reverse && v[v.size() - 1].key >= maxKey) { + if (!backwards && v[v.size() - 1].key >= maxKey) { state RangeResult _v = v; int i = _v.size() - 2; for (; i >= 0 && _v[i].key >= maxKey; --i) { @@ -299,7 +302,7 @@ public: ACTOR template static Future readWithConflictRangeThrough(ReadYourWritesTransaction* ryw, Req req, - bool snapshot) { + Snapshot snapshot) { choose { when(typename Req::Result result = wait(readThrough(ryw, req, snapshot))) { return result; } when(wait(ryw->resetPromise.getFuture())) { throw internal_error(); } @@ -316,7 +319,7 @@ public: ACTOR template static Future readWithConflictRangeRYW(ReadYourWritesTransaction* ryw, Req req, - bool snapshot) { + Snapshot snapshot) { state RYWIterator it(&ryw->cache, &ryw->writes); choose { when(typename Req::Result result = wait(read(ryw, req, &it))) { @@ -332,7 +335,7 @@ public: template static inline Future readWithConflictRange(ReadYourWritesTransaction* ryw, Req const& req, - bool snapshot) { + Snapshot snapshot) { if (ryw->options.readYourWritesDisabled) { return readWithConflictRangeThrough(ryw, req, snapshot); } else if (snapshot && ryw->options.snapshotRywEnabled <= 0) { @@ -690,7 +693,8 @@ public: //TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", ucEnd.beginKey()).detail("MinRows", requestLimit.minRows); additionalRows = 0; - RangeResult snapshot_read = wait(ryw->tr.getRange(read_begin, read_end, requestLimit, true, false)); + RangeResult snapshot_read = + wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::TRUE, Reverse::FALSE)); KeyRangeRef range = getKnownKeyRange(snapshot_read, read_begin, read_end, ryw->arena); //TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough); @@ -993,7 +997,8 @@ public: //TraceEvent("RYWIssuing", randomID).detail("Begin", read_begin.toString()).detail("End", read_end.toString()).detail("Bytes", requestLimit.bytes).detail("Rows", requestLimit.rows).detail("Limits", limits.bytes).detail("Reached", limits.isReached()).detail("RequestCount", requestCount).detail("SingleClears", singleClears).detail("UcEnd", ucEnd.beginKey()).detail("MinRows", requestLimit.minRows); additionalRows = 0; - RangeResult snapshot_read = wait(ryw->tr.getRange(read_begin, read_end, requestLimit, true, true)); + RangeResult snapshot_read = + wait(ryw->tr.getRange(read_begin, read_end, requestLimit, Snapshot::TRUE, Reverse::TRUE)); KeyRangeRef range = getKnownKeyRangeBack(snapshot_read, read_begin, read_end, ryw->arena); //TraceEvent("RYWCacheInsert", randomID).detail("Range", range).detail("ExpectedSize", snapshot_read.expectedSize()).detail("Rows", snapshot_read.size()).detail("Results", snapshot_read).detail("More", snapshot_read.more).detail("ReadToBegin", snapshot_read.readToBegin).detail("ReadThroughEnd", snapshot_read.readThroughEnd).detail("ReadThrough", snapshot_read.readThrough); @@ -1110,7 +1115,7 @@ public: if (!ryw->options.readYourWritesDisabled) { ryw->watchMap[key].push_back(watch); - val = readWithConflictRange(ryw, GetValueReq(key), false); + val = readWithConflictRange(ryw, GetValueReq(key), Snapshot::FALSE); } else { ryw->approximateSize += 2 * key.expectedSize() + 1; val = ryw->tr.get(key); @@ -1352,7 +1357,7 @@ ACTOR Future getWorkerInterfaces(Reference c } } -Future> ReadYourWritesTransaction::get(const Key& key, bool snapshot) { +Future> ReadYourWritesTransaction::get(const Key& key, Snapshot snapshot) { TEST(true); // ReadYourWritesTransaction::get if (getDatabase()->apiVersionAtLeast(630)) { @@ -1416,7 +1421,7 @@ Future> ReadYourWritesTransaction::get(const Key& key, bool snap return result; } -Future ReadYourWritesTransaction::getKey(const KeySelector& key, bool snapshot) { +Future ReadYourWritesTransaction::getKey(const KeySelector& key, Snapshot snapshot) { if (checkUsedDuringCommit()) { return used_during_commit(); } @@ -1435,8 +1440,8 @@ Future ReadYourWritesTransaction::getKey(const KeySelector& key, bool snaps Future ReadYourWritesTransaction::getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { if (getDatabase()->apiVersionAtLeast(630)) { if (specialKeys.contains(begin.getKey()) && specialKeys.begin <= end.getKey() && end.getKey() <= specialKeys.end) { @@ -1495,8 +1500,8 @@ Future ReadYourWritesTransaction::getRange(KeySelector begin, Future ReadYourWritesTransaction::getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return getRange(begin, end, GetRangeLimits(limit), snapshot, reverse); } @@ -1627,13 +1632,14 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& clearBegin = std::max(ExtStringRef(keys.begin), it.beginKey()); inClearRange = true; } else if (!it.is_cleared_range() && inClearRange) { - tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)), false); + tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), it.beginKey().toArenaOrRef(arena)), + AddConflictRange::FALSE); inClearRange = false; } } if (inClearRange) { - tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), false); + tr.clear(KeyRangeRef(clearBegin.toArenaOrRef(arena), keys.end), AddConflictRange::FALSE); } it.skip(keys.begin); @@ -1657,9 +1663,9 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& switch (op[i].type) { case MutationRef::SetValue: if (op[i].value.present()) { - tr.set(it.beginKey().assertRef(), op[i].value.get(), false); + tr.set(it.beginKey().assertRef(), op[i].value.get(), AddConflictRange::FALSE); } else { - tr.clear(it.beginKey().assertRef(), false); + tr.clear(it.beginKey().assertRef(), AddConflictRange::FALSE); } break; case MutationRef::AddValue: @@ -1676,7 +1682,7 @@ void ReadYourWritesTransaction::writeRangeToNativeTransaction(KeyRangeRef const& case MutationRef::MinV2: case MutationRef::AndV2: case MutationRef::CompareAndClear: - tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, false); + tr.atomicOp(it.beginKey().assertRef(), op[i].value.get(), op[i].type, AddConflictRange::FALSE); break; default: break; @@ -1845,7 +1851,7 @@ RangeResult ReadYourWritesTransaction::getWriteConflictRangeIntersecting(KeyRang } void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& operand, uint32_t operationType) { - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -1893,7 +1899,7 @@ void ReadYourWritesTransaction::atomicOp(const KeyRef& key, const ValueRef& oper // this does validation of the key and needs to be performed before the readYourWritesDisabled path KeyRangeRef range = getVersionstampKeyRange(arena, k, tr.getCachedReadVersion().orDefault(0), getMaxReadKey()); versionStampKeys.push_back(arena, k); - addWriteConflict = false; + addWriteConflict = AddConflictRange::FALSE; if (!options.readYourWritesDisabled) { writeRangeToNativeTransaction(range); writes.addUnmodifiedAndUnreadableRange(range); @@ -1953,7 +1959,7 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) { } } - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -1983,7 +1989,7 @@ void ReadYourWritesTransaction::set(const KeyRef& key, const ValueRef& value) { } void ReadYourWritesTransaction::clear(const KeyRangeRef& range) { - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -2036,7 +2042,7 @@ void ReadYourWritesTransaction::clear(const KeyRangeRef& range) { } void ReadYourWritesTransaction::clear(const KeyRef& key) { - bool addWriteConflict = !options.getAndResetWriteConflictDisabled(); + AddConflictRange addWriteConflict{ !options.getAndResetWriteConflictDisabled() }; if (checkUsedDuringCommit()) { throw used_during_commit(); @@ -2165,7 +2171,7 @@ void ReadYourWritesTransaction::setOption(FDBTransactionOptions::Option option, void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option option, Optional value) { switch (option) { case FDBTransactionOptions::READ_YOUR_WRITES_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); if (!reading.isReady() || !cache.empty() || !writes.empty()) throw client_invalid_operation(); @@ -2174,26 +2180,26 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti break; case FDBTransactionOptions::READ_AHEAD_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.readAheadDisabled = true; break; case FDBTransactionOptions::NEXT_WRITE_NO_WRITE_CONFLICT_RANGE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.nextWriteDisableConflictRange = true; break; case FDBTransactionOptions::ACCESS_SYSTEM_KEYS: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.readSystemKeys = true; options.writeSystemKeys = true; break; case FDBTransactionOptions::READ_SYSTEM_KEYS: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.readSystemKeys = true; break; @@ -2217,30 +2223,30 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti transactionDebugInfo->transactionName = value.present() ? value.get().toString() : ""; break; case FDBTransactionOptions::SNAPSHOT_RYW_ENABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.snapshotRywEnabled++; break; case FDBTransactionOptions::SNAPSHOT_RYW_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.snapshotRywEnabled--; break; case FDBTransactionOptions::USED_DURING_COMMIT_PROTECTION_DISABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.disableUsedDuringCommitProtection = true; break; case FDBTransactionOptions::SPECIAL_KEY_SPACE_RELAXED: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.specialKeySpaceRelaxed = true; break; case FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.specialKeySpaceChangeConfiguration = true; break; case FDBTransactionOptions::BYPASS_UNREADABLE: - validateOptionValue(value, false); + validateOptionValueNotPresent(value); options.bypassUnreadable = true; break; default: diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index 65bb972da9..7a3afecbe0 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -71,22 +71,22 @@ public: void setVersion(Version v) override { tr.setVersion(v); } Future getReadVersion() override; Optional getCachedReadVersion() const override { return tr.getCachedReadVersion(); } - Future> get(const Key& key, bool snapshot = false) override; - Future getKey(const KeySelector& key, bool snapshot = false) override; + Future> get(const Key& key, Snapshot = Snapshot::FALSE) override; + Future getKey(const KeySelector& key, Snapshot = Snapshot::FALSE) override; Future> getRange(const KeySelector& begin, const KeySelector& end, int limit, - bool snapshot = false, - bool reverse = false) override; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) override; Future> getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) override; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) override; Future> getRange(const KeyRange& keys, int limit, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limit, @@ -95,8 +95,8 @@ public: } Future getRange(const KeyRange& keys, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) { + Snapshot snapshot = Snapshot::FALSE, + Reverse reverse = Reverse::FALSE) { return getRange(KeySelector(firstGreaterOrEqual(keys.begin), keys.arena()), KeySelector(firstGreaterOrEqual(keys.end), keys.arena()), limits, diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 2b5a46cba5..7d84f1f851 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -26,9 +26,7 @@ ServerKnobs::ServerKnobs(Randomize randomize, ClientKnobs* clientKnobs, IsSimula initialize(randomize, clientKnobs, isSimulated); } -void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsSimulated _isSimulated) { - bool const randomize = _randomize == Randomize::YES; - bool const isSimulated = _isSimulated == IsSimulated::YES; +void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSimulated isSimulated) { // clang-format off // Versions init( VERSIONS_PER_SECOND, 1e6 ); @@ -103,6 +101,8 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( PUSH_STATS_SLOW_AMOUNT, 2 ); init( PUSH_STATS_SLOW_RATIO, 0.5 ); init( TLOG_POP_BATCH_SIZE, 1000 ); if ( randomize && BUGGIFY ) TLOG_POP_BATCH_SIZE = 10; + init( TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE, 250e6 ); + init( ENABLE_DETAILED_TLOG_POP_TRACE, true ); // disk snapshot max timeout, to be put in TLog, storage and coordinator nodes init( MAX_FORKED_PROCESS_OUTPUT, 1024 ); @@ -256,6 +256,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( DD_TEAMS_INFO_PRINT_YIELD_COUNT, 100 ); if( randomize && BUGGIFY ) DD_TEAMS_INFO_PRINT_YIELD_COUNT = deterministicRandom()->random01() * 1000 + 1; init( DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY, 120 ); if( randomize && BUGGIFY ) DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY = 5; init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD, 1 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 10; + init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 50 ); // TeamRemover init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER, false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true @@ -463,7 +464,15 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( REPLACE_INTERFACE_CHECK_DELAY, 5.0 ); init( COORDINATOR_REGISTER_INTERVAL, 5.0 ); init( CLIENT_REGISTER_INTERVAL, 600.0 ); - init( CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR, false ); + init( CC_ENABLE_WORKER_HEALTH_MONITOR, false ); + init( CC_WORKER_HEALTH_CHECKING_INTERVAL, 60.0 ); + init( CC_DEGRADED_LINK_EXPIRATION_INTERVAL, 300.0 ); + init( CC_MIN_DEGRADATION_INTERVAL, 120.0 ); + init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, 3 ); + init( CC_MAX_EXCLUSION_DUE_TO_HEALTH, 2 ); + init( CC_HEALTH_TRIGGER_RECOVERY, false ); + init( CC_TRACKING_HEALTH_RECOVERY_INTERVAL, 3600.0 ); + init( CC_MAX_HEALTH_RECOVERY_COUNT, 2 ); init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0; init( EXPECTED_MASTER_FITNESS, ProcessClass::UnsetFit ); @@ -721,6 +730,7 @@ void ServerKnobs::initialize(Randomize _randomize, ClientKnobs* clientKnobs, IsS init( REDWOOD_DEFAULT_EXTENT_READ_SIZE, 1024 * 1024 ); init( REDWOOD_EXTENT_CONCURRENT_READS, 4 ); init( REDWOOD_KVSTORE_CONCURRENT_READS, 64 ); + init( REDWOOD_KVSTORE_RANGE_PREFETCH, true ); init( REDWOOD_PAGE_REBUILD_MAX_SLACK, 0.33 ); init( REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES, 10 ); init( REDWOOD_LAZY_CLEAR_MIN_PAGES, 0 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index fc4ef2c8a8..14870283a9 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -20,6 +20,7 @@ #pragma once +#include "flow/BooleanParam.h" #include "flow/Knobs.h" #include "fdbrpc/fdbrpc.h" #include "fdbrpc/Locality.h" @@ -65,6 +66,8 @@ public: // message (measured in 1/1024ths, e.g. a value of 2048 yields a // factor of 2). int64_t VERSION_MESSAGES_ENTRY_BYTES_WITH_OVERHEAD; + int64_t TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE; + bool ENABLE_DETAILED_TLOG_POP_TRACE; double TLOG_MESSAGE_BLOCK_OVERHEAD_FACTOR; int64_t TLOG_MESSAGE_BLOCK_BYTES; int64_t MAX_MESSAGE_SIZE; @@ -206,6 +209,7 @@ public: int DD_TEAMS_INFO_PRINT_YIELD_COUNT; int DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY; int DD_STORAGE_WIGGLE_PAUSE_THRESHOLD; // How many unhealthy relocations are ongoing will pause storage wiggle + int DD_STORAGE_WIGGLE_STUCK_THRESHOLD; // How many times bestTeamStuck accumulate will pause storage wiggle // TeamRemover to remove redundant teams bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor @@ -223,10 +227,6 @@ public: double DD_FAILURE_TIME; double DD_ZERO_HEALTHY_TEAM_DELAY; - // Redwood Storage Engine - int PREFIX_TREE_IMMEDIATE_KEY_SIZE_LIMIT; - int PREFIX_TREE_IMMEDIATE_KEY_SIZE_MIN; - // KeyValueStore SQLITE int CLEAR_BUFFER_SIZE; double READ_VALUE_TIME_ESTIMATE; @@ -390,7 +390,23 @@ public: double REPLACE_INTERFACE_CHECK_DELAY; double COORDINATOR_REGISTER_INTERVAL; double CLIENT_REGISTER_INTERVAL; - bool CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR; + bool CC_ENABLE_WORKER_HEALTH_MONITOR; + double CC_WORKER_HEALTH_CHECKING_INTERVAL; // The interval of refreshing the degraded server list. + double CC_DEGRADED_LINK_EXPIRATION_INTERVAL; // The time period from the last degradation report after which a + // degraded server is considered healthy. + double CC_MIN_DEGRADATION_INTERVAL; // The minimum interval that a server is reported as degraded to be considered + // as degraded by Cluster Controller. + int CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE; // The maximum number of degraded peers when excluding a server. When the + // number of degraded peers is more than this value, we will not exclude + // this server since it may because of server overload. + int CC_MAX_EXCLUSION_DUE_TO_HEALTH; // The max number of degraded servers to exclude by Cluster Controller due to + // degraded health. + bool CC_HEALTH_TRIGGER_RECOVERY; // If true, cluster controller will kill the master to trigger recovery when + // detecting degraded servers. If false, cluster controller only prints a warning. + double CC_TRACKING_HEALTH_RECOVERY_INTERVAL; // The number of recovery count should not exceed + // CC_MAX_HEALTH_RECOVERY_COUNT within + // CC_TRACKING_HEALTH_RECOVERY_INTERVAL. + int CC_MAX_HEALTH_RECOVERY_COUNT; // Knobs used to select the best policy (via monte carlo) int POLICY_RATING_TESTS; // number of tests per policy (in order to compare) @@ -660,7 +676,7 @@ public: int REDWOOD_DEFAULT_EXTENT_READ_SIZE; // Extent read size for Redwood files int REDWOOD_EXTENT_CONCURRENT_READS; // Max number of simultaneous extent disk reads in progress. int REDWOOD_KVSTORE_CONCURRENT_READS; // Max number of simultaneous point or range reads in progress. - int REDWOOD_COMMIT_CONCURRENT_READS; // Max number of concurrent reads done to support commit operations + bool REDWOOD_KVSTORE_RANGE_PREFETCH; // Whether to use range read prefetching double REDWOOD_PAGE_REBUILD_MAX_SLACK; // When rebuilding pages, max slack to allow in page int REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES; // Number of pages to try to pop from the lazy delete queue and process at // once diff --git a/fdbclient/SimpleConfigTransaction.actor.cpp b/fdbclient/SimpleConfigTransaction.actor.cpp index 8b03fb0d91..f81cabbcdc 100644 --- a/fdbclient/SimpleConfigTransaction.actor.cpp +++ b/fdbclient/SimpleConfigTransaction.actor.cpp @@ -221,23 +221,23 @@ Optional SimpleConfigTransaction::getCachedReadVersion() const { return impl().getCachedReadVersion(); } -Future> SimpleConfigTransaction::get(Key const& key, bool snapshot) { +Future> SimpleConfigTransaction::get(Key const& key, Snapshot snapshot) { return impl().get(key); } Future> SimpleConfigTransaction::getRange(KeySelector const& begin, KeySelector const& end, int limit, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey())); } Future> SimpleConfigTransaction::getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot, - bool reverse) { + Snapshot snapshot, + Reverse reverse) { return impl().getRange(KeyRangeRef(begin.getKey(), end.getKey())); } diff --git a/fdbclient/SimpleConfigTransaction.h b/fdbclient/SimpleConfigTransaction.h index dd779922bd..ced40721af 100644 --- a/fdbclient/SimpleConfigTransaction.h +++ b/fdbclient/SimpleConfigTransaction.h @@ -47,17 +47,17 @@ public: Future getReadVersion() override; Optional getCachedReadVersion() const override; - Future> get(Key const& key, bool snapshot = false) override; + Future> get(Key const& key, Snapshot = Snapshot::FALSE) override; Future> getRange(KeySelector const& begin, KeySelector const& end, int limit, - bool snapshot = false, - bool reverse = false) override; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) override; Future> getRange(KeySelector begin, KeySelector end, GetRangeLimits limits, - bool snapshot = false, - bool reverse = false) override; + Snapshot = Snapshot::FALSE, + Reverse = Reverse::FALSE) override; Future commit() override; Version getCommittedVersion() const override; void setOption(FDBTransactionOptions::Option option, Optional value = Optional()) override; diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index fe1a2d5409..11997ca5c5 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -277,7 +277,7 @@ ACTOR Future SpecialKeySpace::checkRYWValid(SpecialKeySpace* sks, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse) { + Reverse reverse) { ASSERT(ryw); choose { when(RangeResult result = @@ -293,7 +293,7 @@ ACTOR Future SpecialKeySpace::getRangeAggregationActor(SpecialKeySp KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse) { + Reverse reverse) { // This function handles ranges which cover more than one keyrange and aggregates all results // KeySelector, GetRangeLimits and reverse are all handled here state RangeResult result; @@ -413,7 +413,7 @@ Future SpecialKeySpace::getRange(ReadYourWritesTransaction* ryw, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse) { + Reverse reverse) { // validate limits here if (!limits.isValid()) return range_limits_invalid(); @@ -441,7 +441,7 @@ ACTOR Future> SpecialKeySpace::getActor(SpecialKeySpace* sks, KeySelector(firstGreaterOrEqual(key)), KeySelector(firstGreaterOrEqual(keyAfter(key))), GetRangeLimits(CLIENT_KNOBS->TOO_MANY), - false)); + Reverse::FALSE)); ASSERT(result.size() <= 1); if (result.size()) { return Optional(result[0].value); diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h index 8076c320b9..88dc855769 100644 --- a/fdbclient/SpecialKeySpace.actor.h +++ b/fdbclient/SpecialKeySpace.actor.h @@ -168,7 +168,7 @@ public: KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse = false); + Reverse = Reverse::FALSE); void set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value); @@ -209,13 +209,13 @@ private: KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse); + Reverse reverse); ACTOR static Future getRangeAggregationActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeySelector begin, KeySelector end, GetRangeLimits limits, - bool reverse); + Reverse reverse); KeyRangeMap readImpls; KeyRangeMap modules; diff --git a/fdbclient/StorageServerInterface.cpp b/fdbclient/StorageServerInterface.cpp index 404322e7aa..d379a0fa69 100644 --- a/fdbclient/StorageServerInterface.cpp +++ b/fdbclient/StorageServerInterface.cpp @@ -145,6 +145,47 @@ void TSS_traceMismatch(TraceEvent& event, .detail("Version", req.version) .detail("Limit", req.limit) .detail("LimitBytes", req.limitBytes) + .setMaxFieldLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE * 4 / 10) + .detail("SSReply", ssResultsString) + .detail("TSSReply", tssResultsString); +} + +// streaming range reads +template <> +bool TSS_doCompare(const GetKeyValuesStreamReply& src, const GetKeyValuesStreamReply& tss) { + return src.more == tss.more && src.data == tss.data; +} + +template <> +const char* TSS_mismatchTraceName(const GetKeyValuesStreamRequest& req) { + return "TSSMismatchGetKeyValuesStream"; +} + +// TODO this is all duplicated from above, simplify? +template <> +void TSS_traceMismatch(TraceEvent& event, + const GetKeyValuesStreamRequest& req, + const GetKeyValuesStreamReply& src, + const GetKeyValuesStreamReply& tss) { + std::string ssResultsString = format("(%d)%s:\n", src.data.size(), src.more ? "+" : ""); + for (auto& it : src.data) { + ssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); + } + + std::string tssResultsString = format("(%d)%s:\n", tss.data.size(), tss.more ? "+" : ""); + for (auto& it : tss.data) { + tssResultsString += "\n" + it.key.printable() + "=" + traceChecksumValue(it.value); + } + event + .detail( + "Begin", + format("%s%s:%d", req.begin.orEqual ? "=" : "", req.begin.getKey().printable().c_str(), req.begin.offset)) + .detail("End", + format("%s%s:%d", req.end.orEqual ? "=" : "", req.end.getKey().printable().c_str(), req.end.offset)) + .detail("Version", req.version) + .detail("Limit", req.limit) + .detail("LimitBytes", req.limitBytes) + .setMaxFieldLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE * 4 / 10) .detail("SSReply", ssResultsString) .detail("TSSReply", tssResultsString); } @@ -290,6 +331,9 @@ void TSSMetrics::recordLatency(const ReadHotSubRangeRequest& req, double ssLaten template <> void TSSMetrics::recordLatency(const SplitRangeRequest& req, double ssLatency, double tssLatency) {} +template <> +void TSSMetrics::recordLatency(const GetKeyValuesStreamRequest& req, double ssLatency, double tssLatency) {} + // ------------------- TEST_CASE("/StorageServerInterface/TSSCompare/TestComparison") { diff --git a/fdbclient/TaskBucket.actor.cpp b/fdbclient/TaskBucket.actor.cpp index 4e17a1c9f7..759bf82842 100644 --- a/fdbclient/TaskBucket.actor.cpp +++ b/fdbclient/TaskBucket.actor.cpp @@ -22,6 +22,11 @@ #include "fdbclient/ReadYourWrites.h" #include "flow/actorcompiler.h" // has to be last include +FDB_DEFINE_BOOLEAN_PARAM(AccessSystemKeys); +FDB_DEFINE_BOOLEAN_PARAM(PriorityBatch); +FDB_DEFINE_BOOLEAN_PARAM(VerifyTask); +FDB_DEFINE_BOOLEAN_PARAM(UpdateParams); + Reference Task::getDoneFuture(Reference fb) { return fb->unpack(params[reservedTaskParamKeyDone]); } @@ -168,14 +173,14 @@ public: { // Get a task key that is <= a random UID task key, if successful then return it - Key k = wait(tr->getKey(lastLessOrEqual(space.pack(uid)), true)); + Key k = wait(tr->getKey(lastLessOrEqual(space.pack(uid)), Snapshot::TRUE)); if (space.contains(k)) return Optional(k); } { // Get a task key that is <= the maximum possible UID, if successful return it. - Key k = wait(tr->getKey(lastLessOrEqual(space.pack(maxUIDKey)), true)); + Key k = wait(tr->getKey(lastLessOrEqual(space.pack(maxUIDKey)), Snapshot::TRUE)); if (space.contains(k)) return Optional(k); } @@ -328,7 +333,7 @@ public: Reference futureBucket, Reference task, Reference taskFunc, - bool verifyTask) { + VerifyTask verifyTask) { bool isFinished = wait(taskBucket->isFinished(tr, task)); if (isFinished) { return Void(); @@ -390,7 +395,7 @@ public: taskBucket->setOptions(tr); // Attempt to extend the task's timeout - state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, false)); + state Version newTimeout = wait(taskBucket->extendTimeout(tr, task, UpdateParams::FALSE)); wait(tr->commit()); task->timeoutVersion = newTimeout; versionNow = tr->getCommittedVersion(); @@ -406,15 +411,16 @@ public: Reference taskBucket, Reference futureBucket, Reference task) { + state Reference taskFunc; + state VerifyTask verifyTask = false; + if (!task || !TaskFuncBase::isValidTask(task)) return false; - state Reference taskFunc; - try { taskFunc = TaskFuncBase::create(task->params[Task::reservedTaskParamKeyType]); if (taskFunc) { - state bool verifyTask = (task->params.find(Task::reservedTaskParamValidKey) != task->params.end()); + verifyTask.set(task->params.find(Task::reservedTaskParamValidKey) != task->params.end()); if (verifyTask) { loop { @@ -472,7 +478,7 @@ public: ACTOR static Future dispatch(Database cx, Reference taskBucket, Reference futureBucket, - double* pollDelay, + std::shared_ptr pollDelay, int maxConcurrentTasks) { state std::vector> tasks(maxConcurrentTasks); for (auto& f : tasks) @@ -569,7 +575,7 @@ public: ACTOR static Future run(Database cx, Reference taskBucket, Reference futureBucket, - double* pollDelay, + std::shared_ptr pollDelay, int maxConcurrentTasks) { state Reference> paused = makeReference>(true); state Future watchPausedFuture = watchPaused(cx, taskBucket, paused); @@ -812,7 +818,7 @@ public: ACTOR static Future extendTimeout(Reference tr, Reference taskBucket, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion) { taskBucket->setOptions(tr); @@ -863,11 +869,14 @@ public: } }; -TaskBucket::TaskBucket(const Subspace& subspace, bool sysAccess, bool priorityBatch, bool lockAware) +TaskBucket::TaskBucket(const Subspace& subspace, + AccessSystemKeys sysAccess, + PriorityBatch priorityBatch, + LockAware lockAware) : prefix(subspace), active(prefix.get(LiteralStringRef("ac"))), available(prefix.get(LiteralStringRef("av"))), available_prioritized(prefix.get(LiteralStringRef("avp"))), timeouts(prefix.get(LiteralStringRef("to"))), pauseKey(prefix.pack(LiteralStringRef("pause"))), timeout(CLIENT_KNOBS->TASKBUCKET_TIMEOUT_VERSIONS), - system_access(sysAccess), priority_batch(priorityBatch), lock_aware(lockAware), cc("TaskBucket"), + system_access(sysAccess), priority_batch(priorityBatch), lockAware(lockAware), cc("TaskBucket"), dbgid(deterministicRandom()->randomUniqueID()), dispatchSlotChecksStarted("DispatchSlotChecksStarted", cc), dispatchErrors("DispatchErrors", cc), dispatchDoTasks("DispatchDoTasks", cc), dispatchEmptyTasks("DispatchEmptyTasks", cc), dispatchSlotChecksComplete("DispatchSlotChecksComplete", cc) {} @@ -971,7 +980,7 @@ Future TaskBucket::doTask(Database cx, Reference futureBucke Future TaskBucket::run(Database cx, Reference futureBucket, - double* pollDelay, + std::shared_ptr pollDelay, int maxConcurrentTasks) { return TaskBucketImpl::run(cx, Reference::addRef(this), futureBucket, pollDelay, maxConcurrentTasks); } @@ -1001,7 +1010,7 @@ Future TaskBucket::finish(Reference tr, Referen Future TaskBucket::extendTimeout(Reference tr, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion) { return TaskBucketImpl::extendTimeout( tr, Reference::addRef(this), task, updateParams, newTimeoutVersion); @@ -1041,8 +1050,8 @@ public: } }; -FutureBucket::FutureBucket(const Subspace& subspace, bool sysAccess, bool lockAware) - : prefix(subspace), system_access(sysAccess), lock_aware(lockAware) {} +FutureBucket::FutureBucket(const Subspace& subspace, AccessSystemKeys sysAccess, LockAware lockAware) + : prefix(subspace), system_access(sysAccess), lockAware(lockAware) {} FutureBucket::~FutureBucket() {} diff --git a/fdbclient/TaskBucket.h b/fdbclient/TaskBucket.h index dcdf0dad0c..3158d0a1ae 100644 --- a/fdbclient/TaskBucket.h +++ b/fdbclient/TaskBucket.h @@ -35,6 +35,11 @@ class FutureBucket; class TaskFuture; +FDB_DECLARE_BOOLEAN_PARAM(AccessSystemKeys); +FDB_DECLARE_BOOLEAN_PARAM(PriorityBatch); +FDB_DECLARE_BOOLEAN_PARAM(VerifyTask); +FDB_DECLARE_BOOLEAN_PARAM(UpdateParams); + // A Task is a set of key=value parameters that constitute a unit of work for a TaskFunc to perform. // The parameter keys are specific to the TaskFunc that the Task is for, except for a set of reserved // parameter keys which are used by TaskBucket to determine which TaskFunc to run and provide @@ -134,13 +139,16 @@ class FutureBucket; // instance may declare the Task a failure and move it back to the available subspace. class TaskBucket : public ReferenceCounted { public: - TaskBucket(const Subspace& subspace, bool sysAccess = false, bool priorityBatch = false, bool lockAware = false); + TaskBucket(const Subspace& subspace, + AccessSystemKeys = AccessSystemKeys::FALSE, + PriorityBatch = PriorityBatch::FALSE, + LockAware = LockAware::FALSE); virtual ~TaskBucket(); void setOptions(Reference tr) { if (system_access) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (lock_aware) + if (lockAware) tr->setOption(FDBTransactionOptions::LOCK_AWARE); } @@ -191,7 +199,10 @@ public: Future doOne(Database cx, Reference futureBucket); - Future run(Database cx, Reference futureBucket, double* pollDelay, int maxConcurrentTasks); + Future run(Database cx, + Reference futureBucket, + std::shared_ptr pollDelay, + int maxConcurrentTasks); Future watchPaused(Database cx, Reference> paused); Future isEmpty(Reference tr); @@ -207,11 +218,11 @@ public: // Extend the task's timeout as if it just started and also save any parameter changes made to the task Future extendTimeout(Reference tr, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion = invalidVersion); Future extendTimeout(Database cx, Reference task, - bool updateParams, + UpdateParams updateParams, Version newTimeoutVersion = invalidVersion) { return map(runRYWTransaction(cx, [=](Reference tr) { @@ -250,7 +261,7 @@ public: bool getSystemAccess() const { return system_access; } - bool getLockAware() const { return lock_aware; } + bool getLockAware() const { return lockAware; } Key getPauseKey() const { return pauseKey; } @@ -293,20 +304,20 @@ private: uint32_t timeout; bool system_access; bool priority_batch; - bool lock_aware; + bool lockAware; }; class TaskFuture; class FutureBucket : public ReferenceCounted { public: - FutureBucket(const Subspace& subspace, bool sysAccess = false, bool lockAware = false); + FutureBucket(const Subspace& subspace, AccessSystemKeys = AccessSystemKeys::FALSE, LockAware = LockAware::FALSE); virtual ~FutureBucket(); void setOptions(Reference tr) { if (system_access) tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - if (lock_aware) + if (lockAware) tr->setOption(FDBTransactionOptions::LOCK_AWARE); } @@ -324,7 +335,7 @@ public: Reference unpack(Key key); bool isSystemAccess() const { return system_access; }; - bool isLockAware() const { return lock_aware; }; + bool isLockAware() const { return lockAware; }; private: friend class TaskFuture; @@ -333,7 +344,7 @@ private: Subspace prefix; bool system_access; - bool lock_aware; + bool lockAware; }; class TaskFuture : public ReferenceCounted { diff --git a/fdbclient/ThreadSafeTransaction.cpp b/fdbclient/ThreadSafeTransaction.cpp index 5e01474712..90716ce4f8 100644 --- a/fdbclient/ThreadSafeTransaction.cpp +++ b/fdbclient/ThreadSafeTransaction.cpp @@ -122,7 +122,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) [db, connFile, apiVersion]() { try { Database::createDatabase( - Reference(connFile), apiVersion, false, LocalityData(), db) + Reference(connFile), apiVersion, IsInternal::FALSE, LocalityData(), db) .extractPtr(); } catch (Error& e) { new (db) DatabaseContext(e); @@ -192,7 +192,7 @@ ThreadFuture> ThreadSafeTransaction::get(const KeyRef& key, bool ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, k, snapshot]() -> Future> { tr->checkDeferredError(); - return tr->get(k, snapshot); + return tr->get(k, Snapshot{ snapshot }); }); } @@ -202,7 +202,7 @@ ThreadFuture ThreadSafeTransaction::getKey(const KeySelectorRef& key, bool ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, k, snapshot]() -> Future { tr->checkDeferredError(); - return tr->getKey(k, snapshot); + return tr->getKey(k, Snapshot{ snapshot }); }); } @@ -238,7 +238,7 @@ ThreadFuture ThreadSafeTransaction::getRange(const KeySelectorRef& ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, b, e, limit, snapshot, reverse]() -> Future { tr->checkDeferredError(); - return tr->getRange(b, e, limit, snapshot, reverse); + return tr->getRange(b, e, limit, Snapshot{ snapshot }, Reverse{ reverse }); }); } @@ -253,7 +253,7 @@ ThreadFuture ThreadSafeTransaction::getRange(const KeySelectorRef& ISingleThreadTransaction* tr = this->tr; return onMainThread([tr, b, e, limits, snapshot, reverse]() -> Future { tr->checkDeferredError(); - return tr->getRange(b, e, limits, snapshot, reverse); + return tr->getRange(b, e, limits, Snapshot{ snapshot }, Reverse{ reverse }); }); } diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp index 3f413cd47d..b0853cf5e8 100644 --- a/fdbmonitor/fdbmonitor.cpp +++ b/fdbmonitor/fdbmonitor.cpp @@ -836,7 +836,7 @@ void load_conf(const char* confpath, uid_t& uid, gid_t& gid, sigset_t* mask, fdb for (auto i : id_pid) { if (!loadedConf || ini.GetSectionSize(id_command[i.first]->ssection.c_str()) == -1) { - /* Server on this port no longer configured; deconfigure it and kill it if required */ + /* Process no longer configured; deconfigure it and kill it if required */ log_msg(SevInfo, "Deconfigured %s\n", id_command[i.first]->ssection.c_str()); id_command[i.first]->deconfigured = true; diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp new file mode 100644 index 0000000000..588c6e5cc3 --- /dev/null +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -0,0 +1,274 @@ +/* + * AsyncFileEncrypted.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbrpc/AsyncFileEncrypted.h" +#include "flow/StreamCipher.h" +#include "flow/UnitTest.h" +#include "flow/xxhash.h" +#include "flow/actorcompiler.h" // must be last include + +class AsyncFileEncryptedImpl { +public: + // Determine the initialization for the first block of a file based on a hash of + // the filename. + static auto getFirstBlockIV(const std::string& filename) { + StreamCipher::IV iv; + auto salt = basename(filename); + auto pos = salt.find('.'); + salt = salt.substr(0, pos); + auto hash = XXH3_128bits(salt.c_str(), salt.size()); + auto high = reinterpret_cast(&hash.high64); + auto low = reinterpret_cast(&hash.low64); + std::copy(high, high + 8, &iv[0]); + std::copy(low, low + 6, &iv[8]); + iv[14] = iv[15] = 0; // last 16 bits identify block + return iv; + } + + // Read a single block of size ENCRYPTION_BLOCK_SIZE bytes, and decrypt. + ACTOR static Future> readBlock(AsyncFileEncrypted* self, uint16_t block) { + state Arena arena; + state unsigned char* encrypted = new (arena) unsigned char[FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE]; + int bytes = wait( + self->file->read(encrypted, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * block)); + DecryptionStreamCipher decryptor(StreamCipher::Key::getKey(), self->getIV(block)); + auto decrypted = decryptor.decrypt(encrypted, bytes, arena); + return Standalone(decrypted, arena); + } + + ACTOR static Future read(AsyncFileEncrypted* self, void* data, int length, int offset) { + state const uint16_t firstBlock = offset / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE; + state const uint16_t lastBlock = (offset + length - 1) / FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE; + state uint16_t block; + state unsigned char* output = reinterpret_cast(data); + state int bytesRead = 0; + ASSERT(self->mode == AsyncFileEncrypted::Mode::READ_ONLY); + for (block = firstBlock; block <= lastBlock; ++block) { + state StringRef plaintext; + + auto cachedBlock = self->readBuffers.get(block); + if (cachedBlock.present()) { + plaintext = cachedBlock.get(); + } else { + Standalone _plaintext = wait(readBlock(self, block)); + self->readBuffers.insert(block, _plaintext); + plaintext = _plaintext; + } + auto start = (block == firstBlock) ? plaintext.begin() + (offset % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) + : plaintext.begin(); + auto end = (block == lastBlock) + ? plaintext.begin() + ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) + : plaintext.end(); + if ((offset + length) % FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE == 0) { + end = plaintext.end(); + } + std::copy(start, end, output); + output += (end - start); + bytesRead += (end - start); + } + return bytesRead; + } + + ACTOR static Future write(AsyncFileEncrypted* self, void const* data, int length, int64_t offset) { + ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY); + // All writes must append to the end of the file: + ASSERT_EQ(offset, self->currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE + self->offsetInBlock); + state unsigned char const* input = reinterpret_cast(data); + while (length > 0) { + const auto chunkSize = std::min(length, FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE - self->offsetInBlock); + Arena arena; + auto encrypted = self->encryptor->encrypt(input, chunkSize, arena); + std::copy(encrypted.begin(), encrypted.end(), &self->writeBuffer[self->offsetInBlock]); + offset += encrypted.size(); + self->offsetInBlock += chunkSize; + length -= chunkSize; + input += chunkSize; + if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) { + wait(self->writeLastBlockToFile()); + self->offsetInBlock = 0; + ASSERT_LT(self->currentBlock, std::numeric_limits::max()); + ++self->currentBlock; + self->encryptor = std::make_unique(StreamCipher::Key::getKey(), + self->getIV(self->currentBlock)); + } + } + return Void(); + } + + ACTOR static Future sync(AsyncFileEncrypted* self) { + ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY); + wait(self->writeLastBlockToFile()); + wait(self->file->sync()); + return Void(); + } + + ACTOR static Future zeroRange(AsyncFileEncrypted* self, int64_t offset, int64_t length) { + ASSERT(self->mode == AsyncFileEncrypted::Mode::APPEND_ONLY); + // TODO: Could optimize this + Arena arena; + auto zeroes = new (arena) unsigned char[length]; + memset(zeroes, 0, length); + wait(self->write(zeroes, length, offset)); + return Void(); + } +}; + +AsyncFileEncrypted::AsyncFileEncrypted(Reference file, Mode mode) + : file(file), mode(mode), currentBlock(0), readBuffers(FLOW_KNOBS->MAX_DECRYPTED_BLOCKS) { + firstBlockIV = AsyncFileEncryptedImpl::getFirstBlockIV(file->getFilename()); + if (mode == Mode::APPEND_ONLY) { + encryptor = std::make_unique(StreamCipher::Key::getKey(), getIV(currentBlock)); + writeBuffer = std::vector(FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE, 0); + } +} + +void AsyncFileEncrypted::addref() { + ReferenceCounted::addref(); +} + +void AsyncFileEncrypted::delref() { + ReferenceCounted::delref(); +} + +Future AsyncFileEncrypted::read(void* data, int length, int64_t offset) { + return AsyncFileEncryptedImpl::read(this, data, length, offset); +} + +Future AsyncFileEncrypted::write(void const* data, int length, int64_t offset) { + return AsyncFileEncryptedImpl::write(this, data, length, offset); +} + +Future AsyncFileEncrypted::zeroRange(int64_t offset, int64_t length) { + return AsyncFileEncryptedImpl::zeroRange(this, offset, length); +} + +Future AsyncFileEncrypted::truncate(int64_t size) { + ASSERT(mode == Mode::APPEND_ONLY); + return file->truncate(size); +} + +Future AsyncFileEncrypted::sync() { + ASSERT(mode == Mode::APPEND_ONLY); + return AsyncFileEncryptedImpl::sync(this); +} + +Future AsyncFileEncrypted::flush() { + ASSERT(mode == Mode::APPEND_ONLY); + return Void(); +} + +Future AsyncFileEncrypted::size() const { + ASSERT(mode == Mode::READ_ONLY); + return file->size(); +} + +std::string AsyncFileEncrypted::getFilename() const { + return file->getFilename(); +} + +Future AsyncFileEncrypted::readZeroCopy(void** data, int* length, int64_t offset) { + throw io_error(); + return Void(); +} + +void AsyncFileEncrypted::releaseZeroCopy(void* data, int length, int64_t offset) { + throw io_error(); +} + +int64_t AsyncFileEncrypted::debugFD() const { + return file->debugFD(); +} + +StreamCipher::IV AsyncFileEncrypted::getIV(uint16_t block) const { + auto iv = firstBlockIV; + iv[14] = block / 256; + iv[15] = block % 256; + return iv; +} + +Future AsyncFileEncrypted::writeLastBlockToFile() { + return file->write(&writeBuffer[0], offsetInBlock, currentBlock * FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE); +} + +size_t AsyncFileEncrypted::RandomCache::evict() { + ASSERT_EQ(vec.size(), maxSize); + auto index = deterministicRandom()->randomInt(0, maxSize); + hashMap.erase(vec[index]); + return index; +} + +AsyncFileEncrypted::RandomCache::RandomCache(size_t maxSize) : maxSize(maxSize) { + vec.reserve(maxSize); +} + +void AsyncFileEncrypted::RandomCache::insert(uint16_t block, const Standalone& value) { + auto [_, found] = hashMap.insert({ block, value }); + if (found) { + return; + } else if (vec.size() < maxSize) { + vec.push_back(block); + } else { + auto index = evict(); + vec[index] = block; + } +} + +Optional> AsyncFileEncrypted::RandomCache::get(uint16_t block) const { + auto it = hashMap.find(block); + if (it == hashMap.end()) { + return {}; + } else { + return it->second; + } +} + +// This test writes random data into an encrypted file in random increments, +// then reads this data back from the file in random increments, then confirms that +// the bytes read match the bytes written. +TEST_CASE("fdbrpc/AsyncFileEncrypted") { + state const int bytes = FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE * deterministicRandom()->randomInt(0, 1000); + state std::vector writeBuffer(bytes, 0); + generateRandomData(&writeBuffer.front(), bytes); + state std::vector readBuffer(bytes, 0); + ASSERT(g_network->isSimulated()); + StreamCipher::Key::initializeRandomTestKey(); + int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | + IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED | + IAsyncFile::OPEN_NO_AIO; + state Reference file = + wait(IAsyncFileSystem::filesystem()->open(joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600)); + state int bytesWritten = 0; + while (bytesWritten < bytes) { + chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten); + wait(file->write(&writeBuffer[bytesWritten], chunkSize, bytesWritten)); + bytesWritten += chunkSize; + } + wait(file->sync()); + state int bytesRead = 0; + state int chunkSize; + while (bytesRead < bytes) { + chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead); + int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead)); + ASSERT_EQ(bytesReadInChunk, chunkSize); + bytesRead += bytesReadInChunk; + } + ASSERT(writeBuffer == readBuffer); + return Void(); +} diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h new file mode 100644 index 0000000000..ed5693de29 --- /dev/null +++ b/fdbrpc/AsyncFileEncrypted.h @@ -0,0 +1,81 @@ +/* + * AsyncFileEncrypted.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "fdbrpc/IAsyncFile.h" +#include "flow/FastRef.h" +#include "flow/flow.h" +#include "flow/IRandom.h" +#include "flow/StreamCipher.h" + +#include + +/* + * Append-only file encrypted using AES-128-GCM. + * */ +class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted { +public: + enum class Mode { APPEND_ONLY, READ_ONLY }; + +private: + Reference file; + StreamCipher::IV firstBlockIV; + StreamCipher::IV getIV(uint16_t block) const; + Mode mode; + Future writeLastBlockToFile(); + friend class AsyncFileEncryptedImpl; + + // Reading: + class RandomCache { + size_t maxSize; + std::vector vec; + std::unordered_map> hashMap; + size_t evict(); + + public: + RandomCache(size_t maxSize); + void insert(uint16_t block, const Standalone& value); + Optional> get(uint16_t block) const; + } readBuffers; + + // Writing (append only): + std::unique_ptr encryptor; + uint16_t currentBlock{ 0 }; + int offsetInBlock{ 0 }; + std::vector writeBuffer; + Future initialize(); + +public: + AsyncFileEncrypted(Reference, Mode); + void addref() override; + void delref() override; + Future read(void* data, int length, int64_t offset) override; + Future write(void const* data, int length, int64_t offset) override; + Future zeroRange(int64_t offset, int64_t length) override; + Future truncate(int64_t size) override; + Future sync() override; + Future flush() override; + Future size() const override; + std::string getFilename() const override; + Future readZeroCopy(void** data, int* length, int64_t offset) override; + void releaseZeroCopy(void* data, int length, int64_t offset) override; + int64_t debugFD() const override; +}; diff --git a/fdbrpc/CMakeLists.txt b/fdbrpc/CMakeLists.txt index 5dfd7e411a..026ca36972 100644 --- a/fdbrpc/CMakeLists.txt +++ b/fdbrpc/CMakeLists.txt @@ -33,6 +33,13 @@ set(FDBRPC_SRCS TraceFileIO.cpp TSSComparison.h) +if(WITH_TLS AND NOT WIN32) + set(FDBRPC_SRCS + ${FDBRPC_SRCS} + AsyncFileEncrypted.h + AsyncFileEncrypted.actor.cpp) +endif() + set(COMPILE_EIO OFF) if(NOT WIN32) diff --git a/fdbrpc/FlowTests.actor.cpp b/fdbrpc/FlowTests.actor.cpp index f6df678b6d..d3cf206c8f 100644 --- a/fdbrpc/FlowTests.actor.cpp +++ b/fdbrpc/FlowTests.actor.cpp @@ -262,8 +262,8 @@ struct YieldMockNetwork final : INetwork, ReferenceCounted { return baseNetwork->onMainThread(std::move(signal), taskID); } bool isOnMainThread() const override { return baseNetwork->isOnMainThread(); } - THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg) override { - return baseNetwork->startThread(func, arg); + THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg, int stackSize, const char* name) override { + return baseNetwork->startThread(func, arg, stackSize, name); } Future> open(std::string filename, int64_t flags, int64_t mode) { return IAsyncFileSystem::filesystem()->open(filename, flags, mode); diff --git a/fdbrpc/IAsyncFile.h b/fdbrpc/IAsyncFile.h index ed703514c6..2659b75cad 100644 --- a/fdbrpc/IAsyncFile.h +++ b/fdbrpc/IAsyncFile.h @@ -53,7 +53,8 @@ public: OPEN_LARGE_PAGES = 0x100000, OPEN_NO_AIO = 0x200000, // Don't use AsyncFileKAIO or similar implementations that rely on filesystem support for AIO - OPEN_CACHED_READ_ONLY = 0x400000 // AsyncFileCached opens files read/write even if you specify read only + OPEN_CACHED_READ_ONLY = 0x400000, // AsyncFileCached opens files read/write even if you specify read only + OPEN_ENCRYPTED = 0x800000 // File is encrypted using AES-128-GCM (must be either read-only or write-only) }; virtual void addref() = 0; diff --git a/fdbrpc/LoadBalance.actor.cpp b/fdbrpc/LoadBalance.actor.cpp index e9f4409c9f..8934a36357 100644 --- a/fdbrpc/LoadBalance.actor.cpp +++ b/fdbrpc/LoadBalance.actor.cpp @@ -18,9 +18,13 @@ * limitations under the License. */ +#include "fdbrpc/LoadBalance.actor.h" #include "flow/flow.h" #include "flow/actorcompiler.h" // This must be the last #include. +FDB_DEFINE_BOOLEAN_PARAM(AtMostOnce); +FDB_DEFINE_BOOLEAN_PARAM(TriedAllOptions); + // Throwing all_alternatives_failed will cause the client to issue a GetKeyLocationRequest to the proxy, so this actor // attempts to limit the number of these errors thrown by a single client to prevent it from saturating the proxies with // these requests @@ -49,4 +53,4 @@ ACTOR Future allAlternativesFailedDelay(Future okFuture) { when(wait(::delayJittered(delay))) { throw all_alternatives_failed(); } } return Void(); -} \ No newline at end of file +} diff --git a/fdbrpc/LoadBalance.actor.h b/fdbrpc/LoadBalance.actor.h index cf40173a07..7aeb491416 100644 --- a/fdbrpc/LoadBalance.actor.h +++ b/fdbrpc/LoadBalance.actor.h @@ -28,6 +28,7 @@ #elif !defined(FLOW_LOADBALANCE_ACTOR_H) #define FLOW_LOADBALANCE_ACTOR_H +#include "flow/BooleanParam.h" #include "flow/flow.h" #include "flow/Knobs.h" @@ -147,6 +148,7 @@ Future tssComparison(Req req, ? SevWarnAlways : SevError, TSS_mismatchTraceName(req)); + mismatchEvent.setMaxEventLength(FLOW_KNOBS->TSS_LARGE_TRACE_SIZE); mismatchEvent.detail("TSSID", tssData.tssId); if (FLOW_KNOBS->LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS && ssTeam->size() > 1) { @@ -238,6 +240,9 @@ Future tssComparison(Req req, return Void(); } +FDB_DECLARE_BOOLEAN_PARAM(AtMostOnce); +FDB_DECLARE_BOOLEAN_PARAM(TriedAllOptions); + // Stores state for a request made by the load balancer template struct RequestData : NonCopyable { @@ -245,7 +250,7 @@ struct RequestData : NonCopyable { Future response; Reference modelHolder; - bool triedAllOptions = false; + TriedAllOptions triedAllOptions{ false }; bool requestStarted = false; // true once the request has been sent to an alternative bool requestProcessed = false; // true once a response has been received and handled by checkAndProcessResult @@ -284,7 +289,7 @@ struct RequestData : NonCopyable { // Initializes the request state and starts it, possibly after a backoff delay void startRequest( double backoff, - bool triedAllOptions, + TriedAllOptions triedAllOptions, RequestStream const* stream, Request& request, QueueModel* model, @@ -320,8 +325,8 @@ struct RequestData : NonCopyable { // A return value with an error means that the error should be thrown back to original caller static ErrorOr checkAndProcessResultImpl(Reply const& result, Reference modelHolder, - bool atMostOnce, - bool triedAllOptions) { + AtMostOnce atMostOnce, + TriedAllOptions triedAllOptions) { ASSERT(modelHolder); Optional loadBalancedReply; @@ -377,7 +382,7 @@ struct RequestData : NonCopyable { // A return value of true means that the request completed successfully // A return value of false means that the request failed but should be retried // In the event of a non-retryable failure, an error is thrown indicating the failure - bool checkAndProcessResult(bool atMostOnce) { + bool checkAndProcessResult(AtMostOnce atMostOnce) { ASSERT(response.isReady()); requestProcessed = true; @@ -412,9 +417,9 @@ struct RequestData : NonCopyable { // We need to process the lagging request in order to update the queue model Reference holderCapture = std::move(modelHolder); - bool triedAllOptionsCapture = triedAllOptions; + auto triedAllOptionsCapture = triedAllOptions; Future updateModel = map(response, [holderCapture, triedAllOptionsCapture](Reply result) { - checkAndProcessResultImpl(result, holderCapture, false, triedAllOptionsCapture); + checkAndProcessResultImpl(result, holderCapture, AtMostOnce::FALSE, triedAllOptionsCapture); return Void(); }); model->addActor.send(updateModel); @@ -441,7 +446,8 @@ Future loadBalance( RequestStream Interface::*channel, Request request = Request(), TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint, - bool atMostOnce = false, // if true, throws request_maybe_delivered() instead of retrying automatically + AtMostOnce atMostOnce = + AtMostOnce::FALSE, // if true, throws request_maybe_delivered() instead of retrying automatically QueueModel* model = nullptr) { state RequestData firstRequestData; @@ -453,6 +459,8 @@ Future loadBalance( state Promise requestFinished; state double startTime = now(); + state TriedAllOptions triedAllOptions = TriedAllOptions::FALSE; + setReplyPriority(request, taskID); if (!alternatives) return Never(); @@ -556,7 +564,6 @@ Future loadBalance( state int numAttempts = 0; state double backoff = 0; - state bool triedAllOptions = false; // Issue requests to selected servers. loop { if (now() - startTime > (g_network->isSimulated() ? 30.0 : 600.0)) { @@ -595,7 +602,7 @@ Future loadBalance( break; nextAlt = (nextAlt + 1) % alternatives->size(); if (nextAlt == startAlt) - triedAllOptions = true; + triedAllOptions = TriedAllOptions::TRUE; stream = nullptr; } @@ -702,7 +709,7 @@ Future loadBalance( nextAlt = (nextAlt + 1) % alternatives->size(); if (nextAlt == startAlt) - triedAllOptions = true; + triedAllOptions = TriedAllOptions::TRUE; resetReply(request, taskID); secondDelay = Never(); } @@ -724,7 +731,7 @@ Future basicLoadBalance(Reference> al RequestStream Interface::*channel, Request request = Request(), TaskPriority taskID = TaskPriority::DefaultPromiseEndpoint, - bool atMostOnce = false) { + AtMostOnce atMostOnce = AtMostOnce::FALSE) { setReplyPriority(request, taskID); if (!alternatives) return Never(); diff --git a/fdbrpc/Net2FileSystem.cpp b/fdbrpc/Net2FileSystem.cpp index 71a7d784a1..a2a8874bed 100644 --- a/fdbrpc/Net2FileSystem.cpp +++ b/fdbrpc/Net2FileSystem.cpp @@ -32,6 +32,9 @@ #include "fdbrpc/AsyncFileCached.actor.h" #include "fdbrpc/AsyncFileEIO.actor.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) +#include "fdbrpc/AsyncFileEncrypted.h" +#endif #include "fdbrpc/AsyncFileWinASIO.actor.h" #include "fdbrpc/AsyncFileKAIO.actor.h" #include "flow/AsioReactor.h" @@ -76,6 +79,14 @@ Future> Net2FileSystem::open(const std::string& file static_cast((void*)g_network->global(INetwork::enASIOService))); if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) + if (flags & IAsyncFile::OPEN_ENCRYPTED) + f = map(f, [flags](Reference r) { + auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY + : AsyncFileEncrypted::Mode::READ_ONLY; + return Reference(new AsyncFileEncrypted(r, mode)); + }); +#endif return f; } diff --git a/fdbrpc/TSSComparison.h b/fdbrpc/TSSComparison.h index 650355e696..af5080af6f 100644 --- a/fdbrpc/TSSComparison.h +++ b/fdbrpc/TSSComparison.h @@ -41,6 +41,7 @@ struct DetailedTSSMismatch { struct TSSMetrics : ReferenceCounted, NonCopyable { CounterCollection cc; Counter requests; + Counter streamComparisons; Counter ssErrors; Counter tssErrors; Counter tssTimeouts; @@ -99,9 +100,10 @@ struct TSSMetrics : ReferenceCounted, NonCopyable { } TSSMetrics() - : cc("TSSClientMetrics"), requests("Requests", cc), ssErrors("SSErrors", cc), tssErrors("TSSErrors", cc), - tssTimeouts("TSSTimeouts", cc), mismatches("Mismatches", cc), SSgetValueLatency(1000), SSgetKeyLatency(1000), - SSgetKeyValuesLatency(1000), TSSgetValueLatency(1000), TSSgetKeyLatency(1000), TSSgetKeyValuesLatency(1000) {} + : cc("TSSClientMetrics"), requests("Requests", cc), streamComparisons("StreamComparisons", cc), + ssErrors("SSErrors", cc), tssErrors("TSSErrors", cc), tssTimeouts("TSSTimeouts", cc), + mismatches("Mismatches", cc), SSgetValueLatency(1000), SSgetKeyLatency(1000), SSgetKeyValuesLatency(1000), + TSSgetValueLatency(1000), TSSgetKeyLatency(1000), TSSgetKeyValuesLatency(1000) {} }; template diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h index 81c3bbd3b5..403d8d4dc2 100644 --- a/fdbrpc/fdbrpc.h +++ b/fdbrpc/fdbrpc.h @@ -533,6 +533,8 @@ public: } } + void reset() { *this = ReplyPromiseStream(); } + private: NetNotifiedQueueWithAcknowledgements* queue; SAV* errors; diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index ee735b963a..0121cc7451 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -33,6 +33,9 @@ #include "flow/Util.h" #include "fdbrpc/IAsyncFile.h" #include "fdbrpc/AsyncFileCached.actor.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) +#include "fdbrpc/AsyncFileEncrypted.h" +#endif #include "fdbrpc/AsyncFileNonDurable.actor.h" #include "flow/crc32c.h" #include "fdbrpc/TraceFileIO.h" @@ -1004,9 +1007,9 @@ public: THREAD_RETURN; } - THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg) override { + THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg, int stackSize, const char* name) override { SimThreadArgs* simArgs = new SimThreadArgs(func, arg); - return ::startThread(simStartThread, simArgs); + return ::startThread(simStartThread, simArgs, stackSize, name); } void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) override { @@ -1947,6 +1950,7 @@ public: g_clogging.clogRecvFor(ip, seconds); } void clogPair(const IPAddress& from, const IPAddress& to, double seconds) override { + TraceEvent("CloggingPair").detail("From", from).detail("To", to).detail("Seconds", seconds); g_clogging.clogPairFor(from, to, seconds); } std::vector getAllProcesses() const override { @@ -2473,6 +2477,14 @@ Future> Sim2FileSystem::open(const std::string& file f = AsyncFileDetachable::open(f); if (FLOW_KNOBS->PAGE_WRITE_CHECKSUM_HISTORY > 0) f = map(f, [=](Reference r) { return Reference(new AsyncFileWriteChecker(r)); }); +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) + if (flags & IAsyncFile::OPEN_ENCRYPTED) + f = map(f, [flags](Reference r) { + auto mode = flags & IAsyncFile::OPEN_READWRITE ? AsyncFileEncrypted::Mode::APPEND_ONLY + : AsyncFileEncrypted::Mode::READ_ONLY; + return Reference(new AsyncFileEncrypted(r, mode)); + }); +#endif return f; } else return AsyncFileCached::open(filename, flags, mode); diff --git a/fdbserver/BackupWorker.actor.cpp b/fdbserver/BackupWorker.actor.cpp index b89f6d89f2..8a1f2c952a 100644 --- a/fdbserver/BackupWorker.actor.cpp +++ b/fdbserver/BackupWorker.actor.cpp @@ -243,7 +243,7 @@ struct BackupData { minKnownCommittedVersion(invalidVersion), savedVersion(req.startVersion - 1), popVersion(req.startVersion - 1), cc("BackupWorker", myId.toString()), pulledVersion(0), paused(false), lock(new FlowLock(SERVER_KNOBS->BACKUP_LOCK_BYTES)) { - cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE); specialCounter(cc, "SavedVersion", [this]() { return this->savedVersion; }); specialCounter(cc, "MinKnownCommittedVersion", [this]() { return this->minKnownCommittedVersion; }); diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 0f7d5dc860..485761ebd7 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -146,7 +146,7 @@ set(FDBSERVER_SRCS workloads/BackgroundSelectors.actor.cpp workloads/BackupCorrectness.actor.cpp workloads/BackupAndParallelRestoreCorrectness.actor.cpp - workloads/ParallelRestore.actor.cpp + workloads/ClogSingleConnection.actor.cpp workloads/BackupToBlob.actor.cpp workloads/BackupToDBAbort.actor.cpp workloads/BackupToDBCorrectness.actor.cpp @@ -197,6 +197,7 @@ set(FDBSERVER_SRCS workloads/MemoryKeyValueStore.h workloads/MemoryLifetime.actor.cpp workloads/MetricLogging.actor.cpp + workloads/ParallelRestore.actor.cpp workloads/Performance.actor.cpp workloads/Ping.actor.cpp workloads/PopulateTPCC.actor.cpp diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 142409deca..8b0376c418 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -133,9 +133,9 @@ public: serverInfo(new AsyncVar()), db(DatabaseContext::create(clientInfo, Future(), LocalityData(), - true, + EnableLocalityLoadBalance::TRUE, TaskPriority::DefaultEndpoint, - true)) // SOMEDAY: Locality! + LockAware::TRUE)) // SOMEDAY: Locality! {} void setDistributor(const DataDistributorInterface& interf) { @@ -289,6 +289,7 @@ public: for (auto& it : id_worker) { auto fitness = it.second.details.processClass.machineClassFitness(ProcessClass::Storage); if (workerAvailable(it.second, false) && !conf.isExcludedServer(it.second.details.interf.addresses()) && + !isExcludedDegradedServer(it.second.details.interf.addresses()) && fitness != ProcessClass::NeverAssign && (!dcId.present() || it.second.details.interf.locality.dcId() == dcId.get())) { fitness_workers[fitness].push_back(it.second.details); @@ -529,6 +530,16 @@ public: dcIds); continue; } + if (isExcludedDegradedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "complex", + "Worker server is excluded from the cluster due to degradation", + worker_details, + fitness, + dcIds); + continue; + } if (fitness == ProcessClass::NeverAssign) { logWorkerUnavailable( SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); @@ -764,6 +775,16 @@ public: dcIds); continue; } + if (isExcludedDegradedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "simple", + "Worker server is excluded from the cluster due to degradation", + worker_details, + fitness, + dcIds); + continue; + } if (fitness == ProcessClass::NeverAssign) { logWorkerUnavailable( SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); @@ -897,6 +918,16 @@ public: dcIds); continue; } + if (isExcludedDegradedServer(worker_details.interf.addresses())) { + logWorkerUnavailable(SevInfo, + id, + "deprecated", + "Worker server is excluded from the cluster due to degradation", + worker_details, + fitness, + dcIds); + continue; + } if (fitness == ProcessClass::NeverAssign) { logWorkerUnavailable( SevDebug, id, "complex", "Worker's fitness is NeverAssign", worker_details, fitness, dcIds); @@ -1312,7 +1343,8 @@ public: for (auto& it : id_worker) { auto fitness = it.second.details.processClass.machineClassFitness(role); - if (conf.isExcludedServer(it.second.details.interf.addresses())) { + if (conf.isExcludedServer(it.second.details.interf.addresses()) || + isExcludedDegradedServer(it.second.details.interf.addresses())) { fitness = std::max(fitness, ProcessClass::ExcludeFit); } if (workerAvailable(it.second, checkStable) && fitness < unacceptableFitness && @@ -1359,6 +1391,7 @@ public: auto fitness = it.second.details.processClass.machineClassFitness(role); if (workerAvailable(it.second, checkStable) && !conf.isExcludedServer(it.second.details.interf.addresses()) && + !isExcludedDegradedServer(it.second.details.interf.addresses()) && it.second.details.interf.locality.dcId() == dcId && (!minWorker.present() || (it.second.details.interf.id() != minWorker.get().worker.interf.id() && @@ -1493,7 +1526,9 @@ public: bool checkStable = false) { std::set>> result; for (auto& it : id_worker) - if (workerAvailable(it.second, checkStable) && !conf.isExcludedServer(it.second.details.interf.addresses())) + if (workerAvailable(it.second, checkStable) && + !conf.isExcludedServer(it.second.details.interf.addresses()) && + !isExcludedDegradedServer(it.second.details.interf.addresses())) result.insert(it.second.details.interf.locality.dcId()); return result; } @@ -1543,6 +1578,20 @@ public: return result; } + // Given datacenter ID, returns the primary and remote regions. + std::pair getPrimaryAndRemoteRegion(const std::vector& regions, Key dcId) { + RegionInfo region; + RegionInfo remoteRegion; + for (const auto& r : regions) { + if (r.dcId == dcId) { + region = r; + } else { + remoteRegion = r; + } + } + return std::make_pair(region, remoteRegion); + } + ErrorOr findWorkersForConfigurationFromDC(RecruitFromConfigurationRequest const& req, Optional dcId) { RecruitFromConfigurationReply result; @@ -1555,15 +1604,7 @@ public: primaryDC.insert(dcId); result.dcId = dcId; - RegionInfo region; - RegionInfo remoteRegion; - for (auto& r : req.configuration.regions) { - if (r.dcId == dcId.get()) { - region = r; - } else { - remoteRegion = r; - } - } + auto [region, remoteRegion] = getPrimaryAndRemoteRegion(req.configuration.regions, dcId.get()); if (req.recruitSeedServers) { auto primaryStorageServers = @@ -2008,67 +2049,82 @@ public: RecruitFromConfigurationReply findWorkersForConfiguration(RecruitFromConfigurationRequest const& req) { RecruitFromConfigurationReply rep = findWorkersForConfigurationDispatch(req); if (g_network->isSimulated()) { - RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req); + // FIXME: The logic to pick a satellite in a remote region is not + // deterministic and can therefore break this nondeterminism check. + // Since satellites will generally be in the primary region, + // disable the determinism check for remote region satellites. + bool remoteDCUsedAsSatellite = false; + if (req.configuration.regions.size() > 1) { + auto [region, remoteRegion] = getPrimaryAndRemoteRegion(req.configuration.regions, req.configuration.regions[0].dcId); + for (const auto& satellite : region.satellites) { + if (satellite.dcId == remoteRegion.dcId) { + remoteDCUsedAsSatellite = true; + } + } + } + if (!remoteDCUsedAsSatellite) { + RecruitFromConfigurationReply compare = findWorkersForConfigurationDispatch(req); - std::map>, int> firstUsed; - std::map>, int> secondUsed; - updateKnownIds(&firstUsed); - updateKnownIds(&secondUsed); + std::map>, int> firstUsed; + std::map>, int> secondUsed; + updateKnownIds(&firstUsed); + updateKnownIds(&secondUsed); - // auto mworker = id_worker.find(masterProcessId); - //TraceEvent("CompareAddressesMaster") - // .detail("Master", - // mworker != id_worker.end() ? mworker->second.details.interf.address() : NetworkAddress()); + // auto mworker = id_worker.find(masterProcessId); + //TraceEvent("CompareAddressesMaster") + // .detail("Master", + // mworker != id_worker.end() ? mworker->second.details.interf.address() : NetworkAddress()); - updateIdUsed(rep.tLogs, firstUsed); - updateIdUsed(compare.tLogs, secondUsed); - compareWorkers( - req.configuration, rep.tLogs, firstUsed, compare.tLogs, secondUsed, ProcessClass::TLog, "TLog"); - updateIdUsed(rep.satelliteTLogs, firstUsed); - updateIdUsed(compare.satelliteTLogs, secondUsed); - compareWorkers(req.configuration, - rep.satelliteTLogs, - firstUsed, - compare.satelliteTLogs, - secondUsed, - ProcessClass::TLog, - "Satellite"); - updateIdUsed(rep.commitProxies, firstUsed); - updateIdUsed(compare.commitProxies, secondUsed); - updateIdUsed(rep.grvProxies, firstUsed); - updateIdUsed(compare.grvProxies, secondUsed); - updateIdUsed(rep.resolvers, firstUsed); - updateIdUsed(compare.resolvers, secondUsed); - compareWorkers(req.configuration, - rep.commitProxies, - firstUsed, - compare.commitProxies, - secondUsed, - ProcessClass::CommitProxy, - "CommitProxy"); - compareWorkers(req.configuration, - rep.grvProxies, - firstUsed, - compare.grvProxies, - secondUsed, - ProcessClass::GrvProxy, - "GrvProxy"); - compareWorkers(req.configuration, - rep.resolvers, - firstUsed, - compare.resolvers, - secondUsed, - ProcessClass::Resolver, - "Resolver"); - updateIdUsed(rep.backupWorkers, firstUsed); - updateIdUsed(compare.backupWorkers, secondUsed); - compareWorkers(req.configuration, - rep.backupWorkers, - firstUsed, - compare.backupWorkers, - secondUsed, - ProcessClass::Backup, - "Backup"); + updateIdUsed(rep.tLogs, firstUsed); + updateIdUsed(compare.tLogs, secondUsed); + compareWorkers( + req.configuration, rep.tLogs, firstUsed, compare.tLogs, secondUsed, ProcessClass::TLog, "TLog"); + updateIdUsed(rep.satelliteTLogs, firstUsed); + updateIdUsed(compare.satelliteTLogs, secondUsed); + compareWorkers(req.configuration, + rep.satelliteTLogs, + firstUsed, + compare.satelliteTLogs, + secondUsed, + ProcessClass::TLog, + "Satellite"); + updateIdUsed(rep.commitProxies, firstUsed); + updateIdUsed(compare.commitProxies, secondUsed); + updateIdUsed(rep.grvProxies, firstUsed); + updateIdUsed(compare.grvProxies, secondUsed); + updateIdUsed(rep.resolvers, firstUsed); + updateIdUsed(compare.resolvers, secondUsed); + compareWorkers(req.configuration, + rep.commitProxies, + firstUsed, + compare.commitProxies, + secondUsed, + ProcessClass::CommitProxy, + "CommitProxy"); + compareWorkers(req.configuration, + rep.grvProxies, + firstUsed, + compare.grvProxies, + secondUsed, + ProcessClass::GrvProxy, + "GrvProxy"); + compareWorkers(req.configuration, + rep.resolvers, + firstUsed, + compare.resolvers, + secondUsed, + ProcessClass::Resolver, + "Resolver"); + updateIdUsed(rep.backupWorkers, firstUsed); + updateIdUsed(compare.backupWorkers, secondUsed); + compareWorkers(req.configuration, + rep.backupWorkers, + firstUsed, + compare.backupWorkers, + secondUsed, + ProcessClass::Backup, + "Backup"); + } } return rep; } @@ -2779,6 +2835,162 @@ public: } } + // Checks that if any worker or their degraded peers have recovered. If so, remove them from `workerHealth`. + void updateRecoveredWorkers() { + double currentTime = now(); + for (auto& [workerAddress, health] : workerHealth) { + for (auto it = health.degradedPeers.begin(); it != health.degradedPeers.end();) { + if (currentTime - it->second.lastRefreshTime > SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL) { + TraceEvent("WorkerPeerHealthRecovered").detail("Worker", workerAddress).detail("Peer", it->first); + health.degradedPeers.erase(it++); + } else { + ++it; + } + } + } + + for (auto it = workerHealth.begin(); it != workerHealth.end();) { + if (it->second.degradedPeers.empty()) { + TraceEvent("WorkerAllPeerHealthRecovered").detail("Worker", it->first); + workerHealth.erase(it++); + } else { + ++it; + } + } + } + + // Returns a list of servers who are experiencing degraded links. These are candidates to perform exclusion. Note + // that only one endpoint of a bad link will be included in this list. + std::unordered_set getServersWithDegradedLink() { + updateRecoveredWorkers(); + + // Build a map keyed by measured degraded peer. This map gives the info that who complains a particular server. + std::unordered_map> degradedLinkDst2Src; + double currentTime = now(); + for (const auto& [server, health] : workerHealth) { + for (const auto& [degradedPeer, times] : health.degradedPeers) { + if (currentTime - times.startTime < SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL) { + // This degraded link is not long enough to be considered as degraded. + continue; + } + degradedLinkDst2Src[degradedPeer].insert(server); + } + } + + // Sort degraded peers based on the number of workers complaining about it. + std::vector> count2DegradedPeer; + for (const auto& [degradedPeer, complainers] : degradedLinkDst2Src) { + count2DegradedPeer.push_back({ complainers.size(), degradedPeer }); + } + std::sort(count2DegradedPeer.begin(), count2DegradedPeer.end(), std::greater<>()); + + // Go through all reported degraded peers by decreasing order of the number of complainers. For a particular + // degraded peer, if a complainer has already be considered as degraded, we skip the current examine degraded + // peer since there has been one endpoint on the link between degradedPeer and complainer considered as + // degraded. This is to address the issue that both endpoints on a bad link may be considered as degraded + // server. + // + // For example, if server A is already considered as a degraded server, and A complains B, we won't add B as + // degraded since A is already considered as degraded. + std::unordered_set currentDegradedServers; + for (const auto& [complainerCount, badServer] : count2DegradedPeer) { + for (const auto& complainer : degradedLinkDst2Src[badServer]) { + if (currentDegradedServers.find(complainer) == currentDegradedServers.end()) { + currentDegradedServers.insert(badServer); + break; + } + } + } + + // For degraded server that are complained by more than SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, we + // don't know if it is a hot server, or the network is bad. We remove from the returned degraded server list. + std::unordered_set currentDegradedServersWithinLimit; + for (const auto& badServer : currentDegradedServers) { + if (degradedLinkDst2Src[badServer].size() <= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE) { + currentDegradedServersWithinLimit.insert(badServer); + } + } + return currentDegradedServersWithinLimit; + } + + // Returns true when the cluster controller should trigger a recovery due to degraded servers are used in the + // transaction system in the primary data center. + bool shouldTriggerRecoveryDueToDegradedServers() { + if (degradedServers.size() > SERVER_KNOBS->CC_MAX_EXCLUSION_DUE_TO_HEALTH) { + return false; + } + + const ServerDBInfo dbi = db.serverInfo->get(); + if (dbi.recoveryState < RecoveryState::ACCEPTING_COMMITS) { + return false; + } + + // Do not trigger recovery if the cluster controller is excluded, since the master will change + // anyways once the cluster controller is moved + if (id_worker[clusterControllerProcessId].priorityInfo.isExcluded) { + return false; + } + + if (db.config.regions.size() > 1 && db.config.regions[0].priority > db.config.regions[1].priority && + db.config.regions[0].dcId != clusterControllerDcId.get() && versionDifferenceUpdated && + datacenterVersionDifference < SERVER_KNOBS->MAX_VERSION_DIFFERENCE) { + checkRegions(db.config.regions); + } + + for (const auto& excludedServer : degradedServers) { + if (dbi.master.addresses().contains(excludedServer)) { + return true; + } + + for (auto& logSet : dbi.logSystemConfig.tLogs) { + if (!logSet.isLocal || logSet.locality == tagLocalitySatellite) { + continue; + } + for (const auto& tlog : logSet.tLogs) { + if (tlog.present() && tlog.interf().addresses().contains(excludedServer)) { + return true; + } + } + } + + for (auto& proxy : dbi.client.grvProxies) { + if (proxy.addresses().contains(excludedServer)) { + return true; + } + } + + for (auto& proxy : dbi.client.commitProxies) { + if (proxy.addresses().contains(excludedServer)) { + return true; + } + } + + for (auto& resolver : dbi.resolvers) { + if (resolver.addresses().contains(excludedServer)) { + return true; + } + } + } + + return false; + } + + int recentRecoveryCountDueToHealth() { + while (!recentHealthTriggeredRecoveryTime.empty() && + now() - recentHealthTriggeredRecoveryTime.front() > SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL) { + recentHealthTriggeredRecoveryTime.pop(); + } + return recentHealthTriggeredRecoveryTime.size(); + } + + bool isExcludedDegradedServer(const NetworkAddressList& a) { + for (const auto& server : excludedDegradedServers) { + if (a.contains(server)) + return true; + } + return false; + } + std::map>, WorkerInfo> id_worker; std::map>, ProcessClass> id_class; // contains the mapping from process id to process class from the database @@ -2828,6 +3040,12 @@ public: // TODO(zhewu): Include disk and CPU signals. }; std::unordered_map workerHealth; + std::unordered_set + degradedServers; // The servers that the cluster controller is considered as degraded. The servers in this list + // are not excluded unless they are added to `excludedDegradedServers`. + std::unordered_set + excludedDegradedServers; // The degraded servers to be excluded when assigning workers to roles. + std::queue recentHealthTriggeredRecoveryTime; CounterCollection clusterControllerMetrics; @@ -2860,7 +3078,7 @@ public: serverInfo.clusterInterface = ccInterface; serverInfo.myLocality = locality; db.serverInfo->set(serverInfo); - cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(db.serverInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE); } ~ClusterControllerData() { @@ -4499,6 +4717,58 @@ ACTOR Future dbInfoUpdater(ClusterControllerData* self) { } } +// The actor that periodically monitors the health of tracked workers. +ACTOR Future workerHealthMonitor(ClusterControllerData* self) { + loop { + try { + while (!self->goodRecruitmentTime.isReady()) { + wait(self->goodRecruitmentTime); + } + + self->degradedServers = self->getServersWithDegradedLink(); + + // Compare `self->degradedServers` with `self->excludedDegradedServers` and remove those that have + // recovered. + for (auto it = self->excludedDegradedServers.begin(); it != self->excludedDegradedServers.end();) { + if (self->degradedServers.find(*it) == self->degradedServers.end()) { + self->excludedDegradedServers.erase(it++); + } else { + ++it; + } + } + + if (!self->degradedServers.empty()) { + std::string degradedServerString; + for (const auto& server : self->degradedServers) { + degradedServerString += server.toString() + " "; + } + TraceEvent("ClusterControllerHealthMonitor").detail("DegradedServers", degradedServerString); + + // Check if the cluster controller should trigger a recovery to exclude any degraded servers from the + // transaction system. + if (self->shouldTriggerRecoveryDueToDegradedServers()) { + if (SERVER_KNOBS->CC_HEALTH_TRIGGER_RECOVERY) { + if (self->recentRecoveryCountDueToHealth() < SERVER_KNOBS->CC_MAX_HEALTH_RECOVERY_COUNT) { + self->recentHealthTriggeredRecoveryTime.push(now()); + self->excludedDegradedServers = self->degradedServers; + TraceEvent("DegradedServerDetectedAndTriggerRecovery") + .detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth()); + self->db.forceMasterFailure.trigger(); + } + } else { + self->excludedDegradedServers.clear(); + TraceEvent("DegradedServerDetectedAndSuggestRecovery"); + } + } + } + + wait(delay(SERVER_KNOBS->CC_WORKER_HEALTH_CHECKING_INTERVAL)); + } catch (Error& e) { + TraceEvent(SevWarnAlways, "ClusterControllerHealthMonitorError").error(e); + } + } +} + ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, Future leaderFail, ServerCoordinators coordinators, @@ -4539,6 +4809,10 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, self.addActor.send(traceRole(Role::CLUSTER_CONTROLLER, interf.id())); // printf("%s: I am the cluster controller\n", g_network->getLocalAddress().toString().c_str()); + if (SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) { + self.addActor.send(workerHealthMonitor(&self)); + } + loop choose { when(ErrorOr err = wait(error)) { if (err.isError()) { @@ -4610,7 +4884,7 @@ ACTOR Future clusterControllerCore(ClusterControllerFullInterface interf, clusterRegisterMaster(&self, req); } when(UpdateWorkerHealthRequest req = waitNext(interf.updateWorkerHealth.getFuture())) { - if (SERVER_KNOBS->CLUSTER_CONTROLLER_ENABLE_WORKER_HEALTH_MONITOR) { + if (SERVER_KNOBS->CC_ENABLE_WORKER_HEALTH_MONITOR) { self.updateWorkerHealth(req); } } @@ -4771,4 +5045,264 @@ TEST_CASE("/fdbserver/clustercontroller/updateWorkerHealth") { return Void(); } +TEST_CASE("/fdbserver/clustercontroller/updateRecoveredWorkers") { + // Create a testing ClusterControllerData. Most of the internal states do not matter in this test. + ClusterControllerData data(ClusterControllerFullInterface(), + LocalityData(), + ServerCoordinators(Reference(new ClusterConnectionFile()))); + NetworkAddress worker1(IPAddress(0x01010101), 1); + NetworkAddress worker2(IPAddress(0x11111111), 1); + NetworkAddress badPeer1(IPAddress(0x02020202), 1); + NetworkAddress badPeer2(IPAddress(0x03030303), 1); + + // Create following test scenario: + // worker1 -> badPeer1 active + // worker1 -> badPeer2 recovered + // worker2 -> badPeer2 recovered + data.workerHealth[worker1].degradedPeers[badPeer1] = { + now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1, now() + }; + data.workerHealth[worker1].degradedPeers[badPeer2] = { + now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1, + now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1 + }; + data.workerHealth[worker2].degradedPeers[badPeer2] = { + now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1, + now() - SERVER_KNOBS->CC_DEGRADED_LINK_EXPIRATION_INTERVAL - 1 + }; + data.updateRecoveredWorkers(); + + ASSERT_EQ(data.workerHealth.size(), 1); + ASSERT(data.workerHealth.find(worker1) != data.workerHealth.end()); + ASSERT(data.workerHealth[worker1].degradedPeers.find(badPeer1) != data.workerHealth[worker1].degradedPeers.end()); + ASSERT(data.workerHealth[worker1].degradedPeers.find(badPeer2) == data.workerHealth[worker1].degradedPeers.end()); + ASSERT(data.workerHealth.find(worker2) == data.workerHealth.end()); + + return Void(); +} + +TEST_CASE("/fdbserver/clustercontroller/getServersWithDegradedLink") { + // Create a testing ClusterControllerData. Most of the internal states do not matter in this test. + ClusterControllerData data(ClusterControllerFullInterface(), + LocalityData(), + ServerCoordinators(Reference(new ClusterConnectionFile()))); + NetworkAddress worker(IPAddress(0x01010101), 1); + NetworkAddress badPeer1(IPAddress(0x02020202), 1); + NetworkAddress badPeer2(IPAddress(0x03030303), 1); + NetworkAddress badPeer3(IPAddress(0x04040404), 1); + NetworkAddress badPeer4(IPAddress(0x05050505), 1); + + // Test that a reported degraded link should stay for sometime before being considered as a degraded link by cluster + // controller. + { + data.workerHealth[worker].degradedPeers[badPeer1] = { now(), now() }; + ASSERT(data.getServersWithDegradedLink().empty()); + data.workerHealth.clear(); + } + + // Test that when there is only one reported degraded link, getServersWithDegradedLink can return correct degraded + // server. + { + data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + auto degradedServers = data.getServersWithDegradedLink(); + ASSERT(degradedServers.size() == 1); + ASSERT(degradedServers.find(badPeer1) != degradedServers.end()); + data.workerHealth.clear(); + } + + // Test that if both A complains B and B compalins A, only one of the server will be chosen as degraded server. + { + data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + auto degradedServers = data.getServersWithDegradedLink(); + ASSERT(degradedServers.size() == 1); + ASSERT(degradedServers.find(worker) != degradedServers.end() || + degradedServers.find(badPeer1) != degradedServers.end()); + data.workerHealth.clear(); + } + + // Test that if B complains A and C complains A, A is selected as degraded server instead of B or C. + { + ASSERT(SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE < 4); + data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[worker].degradedPeers[badPeer2] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer2].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + auto degradedServers = data.getServersWithDegradedLink(); + ASSERT(degradedServers.size() == 1); + ASSERT(degradedServers.find(worker) != degradedServers.end()); + data.workerHealth.clear(); + } + + // Test that if the number of complainers exceeds the threshold, no degraded server is returned. + { + ASSERT(SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE < 4); + data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer2].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer3].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer4].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + ASSERT(data.getServersWithDegradedLink().empty()); + data.workerHealth.clear(); + } + + // Test that if the degradation is reported both ways between A and other 4 servers, no degraded server is returned. + { + ASSERT(SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE < 4); + data.workerHealth[worker].degradedPeers[badPeer1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer1].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[worker].degradedPeers[badPeer2] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer2].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[worker].degradedPeers[badPeer3] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer3].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[worker].degradedPeers[badPeer4] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer4].degradedPeers[worker] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + ASSERT(data.getServersWithDegradedLink().empty()); + data.workerHealth.clear(); + } + + return Void(); +} + +TEST_CASE("/fdbserver/clustercontroller/recentRecoveryCountDueToHealth") { + // Create a testing ClusterControllerData. Most of the internal states do not matter in this test. + ClusterControllerData data(ClusterControllerFullInterface(), + LocalityData(), + ServerCoordinators(Reference(new ClusterConnectionFile()))); + + ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 0); + + data.recentHealthTriggeredRecoveryTime.push(now() - SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL - 1); + ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 0); + + data.recentHealthTriggeredRecoveryTime.push(now() - SERVER_KNOBS->CC_TRACKING_HEALTH_RECOVERY_INTERVAL + 1); + ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 1); + + data.recentHealthTriggeredRecoveryTime.push(now()); + ASSERT_EQ(data.recentRecoveryCountDueToHealth(), 2); + + return Void(); +} + +TEST_CASE("/fdbserver/clustercontroller/shouldTriggerRecoveryDueToDegradedServers") { + // Create a testing ClusterControllerData. Most of the internal states do not matter in this test. + ClusterControllerData data(ClusterControllerFullInterface(), + LocalityData(), + ServerCoordinators(Reference(new ClusterConnectionFile()))); + NetworkAddress master(IPAddress(0x01010101), 1); + NetworkAddress tlog(IPAddress(0x02020202), 1); + NetworkAddress satelliteTlog(IPAddress(0x03030303), 1); + NetworkAddress remoteTlog(IPAddress(0x04040404), 1); + NetworkAddress logRouter(IPAddress(0x05050505), 1); + NetworkAddress backup(IPAddress(0x06060606), 1); + NetworkAddress proxy(IPAddress(0x07070707), 1); + NetworkAddress resolver(IPAddress(0x08080808), 1); + + // Create a ServerDBInfo using above addresses. + ServerDBInfo testDbInfo; + testDbInfo.master.changeCoordinators = + RequestStream(Endpoint({ master }, UID(1, 2))); + + TLogInterface localTLogInterf; + localTLogInterf.peekMessages = RequestStream(Endpoint({ tlog }, UID(1, 2))); + TLogInterface localLogRouterInterf; + localLogRouterInterf.peekMessages = RequestStream(Endpoint({ logRouter }, UID(1, 2))); + BackupInterface backupInterf; + backupInterf.waitFailure = RequestStream>(Endpoint({ backup }, UID(1, 2))); + TLogSet localTLogSet; + localTLogSet.isLocal = true; + localTLogSet.tLogs.push_back(OptionalInterface(localTLogInterf)); + localTLogSet.logRouters.push_back(OptionalInterface(localLogRouterInterf)); + localTLogSet.backupWorkers.push_back(OptionalInterface(backupInterf)); + testDbInfo.logSystemConfig.tLogs.push_back(localTLogSet); + + TLogInterface sateTLogInterf; + sateTLogInterf.peekMessages = RequestStream(Endpoint({ satelliteTlog }, UID(1, 2))); + TLogSet sateTLogSet; + sateTLogSet.isLocal = true; + sateTLogSet.locality = tagLocalitySatellite; + sateTLogSet.tLogs.push_back(OptionalInterface(sateTLogInterf)); + testDbInfo.logSystemConfig.tLogs.push_back(sateTLogSet); + + TLogInterface remoteTLogInterf; + remoteTLogInterf.peekMessages = RequestStream(Endpoint({ remoteTlog }, UID(1, 2))); + TLogSet remoteTLogSet; + remoteTLogSet.isLocal = false; + remoteTLogSet.tLogs.push_back(OptionalInterface(remoteTLogInterf)); + testDbInfo.logSystemConfig.tLogs.push_back(remoteTLogSet); + + GrvProxyInterface proxyInterf; + proxyInterf.getConsistentReadVersion = RequestStream(Endpoint({ proxy }, UID(1, 2))); + testDbInfo.client.grvProxies.push_back(proxyInterf); + + ResolverInterface resolverInterf; + resolverInterf.resolve = RequestStream(Endpoint({ resolver }, UID(1, 2))); + testDbInfo.resolvers.push_back(resolverInterf); + + testDbInfo.recoveryState = RecoveryState::ACCEPTING_COMMITS; + + // No recovery when no degraded servers. + data.db.serverInfo->set(testDbInfo); + ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers()); + + // Trigger recovery when master is degraded. + data.degradedServers.insert(master); + ASSERT(data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // Trigger recovery when primary TLog is degraded. + data.degradedServers.insert(tlog); + ASSERT(data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // No recovery when satellite Tlog is degraded. + data.degradedServers.insert(satelliteTlog); + ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // No recovery when remote tlog is degraded. + data.degradedServers.insert(remoteTlog); + ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // No recovery when log router is degraded. + data.degradedServers.insert(logRouter); + ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // No recovery when backup worker is degraded. + data.degradedServers.insert(backup); + ASSERT(!data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // Trigger recovery when proxy is degraded. + data.degradedServers.insert(proxy); + ASSERT(data.shouldTriggerRecoveryDueToDegradedServers()); + data.degradedServers.clear(); + + // Trigger recovery when resolver is degraded. + data.degradedServers.insert(resolver); + ASSERT(data.shouldTriggerRecoveryDueToDegradedServers()); + + return Void(); +} + } // namespace diff --git a/fdbserver/ConfigDatabaseUnitTests.actor.cpp b/fdbserver/ConfigDatabaseUnitTests.actor.cpp index 88b115facd..e315156da0 100644 --- a/fdbserver/ConfigDatabaseUnitTests.actor.cpp +++ b/fdbserver/ConfigDatabaseUnitTests.actor.cpp @@ -158,13 +158,13 @@ public: ReadFromLocalConfigEnvironment(std::string const& dataDir, std::string const& configPath, std::map const& manualKnobOverrides) - : dataDir(dataDir), localConfiguration(dataDir, configPath, manualKnobOverrides, IsTest::YES), consumer(Never()) { - } + : dataDir(dataDir), localConfiguration(dataDir, configPath, manualKnobOverrides, IsTest::TRUE), + consumer(Never()) {} Future setup() { return setup(this); } Future restartLocalConfig(std::string const& newConfigPath) { - localConfiguration = LocalConfiguration(dataDir, newConfigPath, {}, IsTest::YES); + localConfiguration = LocalConfiguration(dataDir, newConfigPath, {}, IsTest::TRUE); return setup(); } diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index a4553861e2..6b882bb1e1 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -656,7 +656,7 @@ struct DDTeamCollection : ReferenceCounted { int optimalTeamCount; AsyncVar zeroOptimalTeams; - bool bestTeamStuck = false; + int bestTeamKeepStuckCount = 0; bool isTssRecruiting; // If tss recruiting is waiting on a pair, don't consider DD recruiting for the purposes of QuietDB @@ -1011,12 +1011,12 @@ struct DDTeamCollection : ReferenceCounted { // Log BestTeamStuck reason when we have healthy teams but they do not have healthy free space if (randomTeams.empty() && !self->zeroHealthyTeams->get()) { - self->bestTeamStuck = true; + self->bestTeamKeepStuckCount++; if (g_network->isSimulated()) { TraceEvent(SevWarn, "GetTeamReturnEmpty").detail("HealthyTeams", self->healthyTeamCount); } } else { - self->bestTeamStuck = false; + self->bestTeamKeepStuckCount = 0; } for (int i = 0; i < randomTeams.size(); i++) { @@ -2833,7 +2833,7 @@ struct DDTeamCollection : ReferenceCounted { std::vector> moveFutures; if (this->pid2server_info.count(pid) != 0) { for (auto& info : this->pid2server_info[pid]) { - AddressExclusion addr(info->lastKnownInterface.address().ip); + AddressExclusion addr(info->lastKnownInterface.address().ip, info->lastKnownInterface.address().port); if (this->excludedServers.count(addr) && this->excludedServers.get(addr) != DDTeamCollection::Status::NONE) { continue; // don't overwrite the value set by actor trackExcludedServer @@ -3509,7 +3509,7 @@ ACTOR Future teamTracker(DDTeamCollection* self, Reference tea bool anyUndesired = false; bool anyWrongConfiguration = false; bool anyWigglingServer = false; - int serversLeft = 0; + int serversLeft = 0, serverUndesired = 0, serverWrongConf = 0, serverWiggling = 0; for (const UID& uid : team->getServerIDs()) { change.push_back(self->server_status.onChange(uid)); @@ -3519,12 +3519,15 @@ ACTOR Future teamTracker(DDTeamCollection* self, Reference tea } if (status.isUndesired) { anyUndesired = true; + serverUndesired++; } if (status.isWrongConfiguration) { anyWrongConfiguration = true; + serverWrongConf++; } if (status.isWiggling) { anyWigglingServer = true; + serverWiggling++; } } @@ -3646,6 +3649,10 @@ ACTOR Future teamTracker(DDTeamCollection* self, Reference tea team->setPriority(SERVER_KNOBS->PRIORITY_TEAM_2_LEFT); else team->setPriority(SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY); + } else if (!badTeam && anyWigglingServer && serverWiggling == serverWrongConf && + serverWiggling == serverUndesired) { + // the wrong configured and undesired server is the wiggling server + team->setPriority(SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE); } else if (badTeam || anyWrongConfiguration) { if (redundantTeam) { team->setPriority(SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT); @@ -3654,8 +3661,6 @@ ACTOR Future teamTracker(DDTeamCollection* self, Reference tea } } else if (anyUndesired) { team->setPriority(SERVER_KNOBS->PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER); - } else if (anyWigglingServer) { - team->setPriority(SERVER_KNOBS->PRIORITY_PERPETUAL_STORAGE_WIGGLE); } else { team->setPriority(SERVER_KNOBS->PRIORITY_TEAM_HEALTHY); } @@ -3972,7 +3977,7 @@ ACTOR Future perpetualStorageWiggleIterator(AsyncVar* stopSignal, wait(delayJittered(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY)); // there must not have other teams to place wiggled data takeRest = teamCollection->server_info.size() <= teamCollection->configuration.storageTeamSize || - teamCollection->machine_info.size() < teamCollection->configuration.storageTeamSize; + teamCollection->machine_info.size() < teamCollection->configuration.storageTeamSize; } wait(updateNextWigglingStoragePID(teamCollection)); } @@ -4020,10 +4025,12 @@ ACTOR Future clusterHealthCheckForPerpetualWiggle(DDTeamCollection* self, // b. healthy teams are not enough // c. the overall disk space is not enough if (count >= SERVER_KNOBS->DD_STORAGE_WIGGLE_PAUSE_THRESHOLD || self->healthyTeamCount <= *extraTeamCount || - self->bestTeamStuck) { + self->bestTeamKeepStuckCount > SERVER_KNOBS->DD_STORAGE_WIGGLE_STUCK_THRESHOLD) { // if we pause wiggle not because the reason a, increase extraTeamCount. This helps avoid oscillation // between pause and non-pause status. - if ((self->healthyTeamCount <= *extraTeamCount || self->bestTeamStuck) && !self->pauseWiggle->get()) { + if ((self->healthyTeamCount <= *extraTeamCount || + self->bestTeamKeepStuckCount > SERVER_KNOBS->DD_STORAGE_WIGGLE_PAUSE_THRESHOLD) && + !self->pauseWiggle->get()) { *extraTeamCount = std::min(*extraTeamCount + pausePenalty, (int)self->teams.size()); pausePenalty = std::min(pausePenalty * 2, (int)self->teams.size()); } @@ -4060,6 +4067,7 @@ ACTOR Future perpetualStorageWiggler(AsyncVar* stopSignal, self->includeStorageServersForWiggle(); TraceEvent("PerpetualStorageWigglePause", self->distributorId) .detail("ProcessId", pid) + .detail("BestTeamKeepStuckCount", self->bestTeamKeepStuckCount) .detail("ExtraHealthyTeamCount", extraTeamCount) .detail("HealthyTeamCount", self->healthyTeamCount) .detail("StorageCount", movingCount); @@ -4566,6 +4574,10 @@ ACTOR Future storageServerTracker( DDTeamCollection::Status worstStatus = self->excludedServers.get(worstAddr); if (worstStatus == DDTeamCollection::Status::WIGGLING && invalidWiggleServer(worstAddr, self, server)) { + TraceEvent(SevInfo, "InvalidWiggleServer", self->distributorId) + .detail("Address", worstAddr.toString()) + .detail("ProcessId", server->lastKnownInterface.locality.processId()) + .detail("ValidWigglingId", self->wigglingPid.present()); self->excludedServers.set(worstAddr, DDTeamCollection::Status::NONE); worstStatus = DDTeamCollection::Status::NONE; } @@ -4586,6 +4598,10 @@ ACTOR Future storageServerTracker( DDTeamCollection::Status testStatus = self->excludedServers.get(testAddr); if (testStatus == DDTeamCollection::Status::WIGGLING && invalidWiggleServer(testAddr, self, server)) { + TraceEvent(SevInfo, "InvalidWiggleServer", self->distributorId) + .detail("Address", testAddr.toString()) + .detail("ProcessId", server->lastKnownInterface.locality.processId()) + .detail("ValidWigglingId", self->wigglingPid.present()); self->excludedServers.set(testAddr, DDTeamCollection::Status::NONE); testStatus = DDTeamCollection::Status::NONE; } @@ -5765,7 +5781,7 @@ ACTOR Future dataDistribution(Reference self, state double lastLimited = 0; self->addActor.send(monitorBatchLimitedTime(self->dbInfo, &lastLimited)); - state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DataDistributionLaunch, true, true); + state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DataDistributionLaunch, LockAware::TRUE); cx->locationCacheSize = SERVER_KNOBS->DD_LOCATION_CACHE_SIZE; // cx->setOption( FDBDatabaseOptions::LOCATION_CACHE_SIZE, StringRef((uint8_t*) @@ -6106,7 +6122,7 @@ static std::set const& normalDataDistributorErrors() { } ACTOR Future ddSnapCreateCore(DistributorSnapRequest snapReq, Reference> db) { - state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, true, true); + state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::TRUE); state ReadYourWritesTransaction tr(cx); loop { try { @@ -6447,7 +6463,7 @@ ACTOR Future dataDistributor(DataDistributorInterface di, Reference self(new DataDistributorData(db, di.id())); state Future collection = actorCollection(self->addActor.getFuture()); state PromiseStream getShardMetricsList; - state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, true, true); + state Database cx = openDBOnServer(db, TaskPriority::DefaultDelay, LockAware::TRUE); state ActorCollection actors(false); state DDEnabledState ddEnabledState; self->addActor.send(actors.getResult()); @@ -6498,8 +6514,8 @@ ACTOR Future dataDistributor(DataDistributorInterface di, Reference testTeamCollection(int teamSize, Reference policy, int processCount) { - Database database = - DatabaseContext::create(makeReference>(), Never(), LocalityData(), false); + Database database = DatabaseContext::create( + makeReference>(), Never(), LocalityData(), EnableLocalityLoadBalance::FALSE); DatabaseConfiguration conf; conf.storageTeamSize = teamSize; @@ -6541,8 +6557,8 @@ std::unique_ptr testTeamCollection(int teamSize, std::unique_ptr testMachineTeamCollection(int teamSize, Reference policy, int processCount) { - Database database = - DatabaseContext::create(makeReference>(), Never(), LocalityData(), false); + Database database = DatabaseContext::create( + makeReference>(), Never(), LocalityData(), EnableLocalityLoadBalance::FALSE); DatabaseConfiguration conf; conf.storageTeamSize = teamSize; diff --git a/fdbserver/DataDistributionQueue.actor.cpp b/fdbserver/DataDistributionQueue.actor.cpp index ba8e0f416a..6f55c39438 100644 --- a/fdbserver/DataDistributionQueue.actor.cpp +++ b/fdbserver/DataDistributionQueue.actor.cpp @@ -941,8 +941,6 @@ struct DDQueueData { } }; -extern bool noUnseed; - // This actor relocates the specified keys to a good place. // The inFlightActor key range map stores the actor for each RelocateData ACTOR Future dataDistributionRelocator(DDQueueData* self, RelocateData rd, const DDEnabledState* ddEnabledState) { diff --git a/fdbserver/DiskQueue.actor.cpp b/fdbserver/DiskQueue.actor.cpp index 1efc6ecee6..a7a5402374 100644 --- a/fdbserver/DiskQueue.actor.cpp +++ b/fdbserver/DiskQueue.actor.cpp @@ -29,6 +29,8 @@ typedef bool (*compare_pages)(void*, void*); typedef int64_t loc_t; +FDB_DEFINE_BOOLEAN_PARAM(CheckHashes); + // 0 -> 0 // 1 -> 4k // 4k -> 4k @@ -1241,9 +1243,9 @@ private: // start and end are on the same page ASSERT(pagedData.size() == sizeof(Page)); Page* data = reinterpret_cast(const_cast(pagedData.begin())); - if (ch == CheckHashes::YES && !data->checkHash()) + if (ch && !data->checkHash()) throw io_error(); - if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload) + if (!ch && data->payloadSize > Page::maxPayload) throw io_error(); pagedData.contents() = pagedData.substr(sizeof(PageHeader) + startingOffset, endingOffset - startingOffset); return pagedData; @@ -1252,9 +1254,9 @@ private: // we don't have to double allocate in a hot, memory hungry call. uint8_t* buf = mutateString(pagedData); Page* data = reinterpret_cast(const_cast(pagedData.begin())); - if (ch == CheckHashes::YES && !data->checkHash()) + if (ch && !data->checkHash()) throw io_error(); - if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload) + if (!ch && data->payloadSize > Page::maxPayload) throw io_error(); // Only start copying from `start` in the first page. @@ -1264,9 +1266,9 @@ private: buf += length; } data++; - if (ch == CheckHashes::YES && !data->checkHash()) + if (ch && !data->checkHash()) throw io_error(); - if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload) + if (!ch && data->payloadSize > Page::maxPayload) throw io_error(); // Copy all the middle pages @@ -1277,9 +1279,9 @@ private: memmove(buf, data->payload, length); buf += length; data++; - if (ch == CheckHashes::YES && !data->checkHash()) + if (ch && !data->checkHash()) throw io_error(); - if (ch == CheckHashes::NO && data->payloadSize > Page::maxPayload) + if (!ch && data->payloadSize > Page::maxPayload) throw io_error(); } diff --git a/fdbserver/GrvProxyServer.actor.cpp b/fdbserver/GrvProxyServer.actor.cpp index b87b0e65a6..0a7614a52c 100644 --- a/fdbserver/GrvProxyServer.actor.cpp +++ b/fdbserver/GrvProxyServer.actor.cpp @@ -253,7 +253,7 @@ struct GrvProxyData { RequestStream getConsistentReadVersion, Reference> db) : dbgid(dbgid), stats(dbgid), master(master), getConsistentReadVersion(getConsistentReadVersion), - cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true)), db(db), lastStartCommit(0), + cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE)), db(db), lastStartCommit(0), lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), updateCommitRequests(0), lastCommitTime(0), minKnownCommittedVersion(invalidVersion) {} }; diff --git a/fdbserver/IDiskQueue.h b/fdbserver/IDiskQueue.h index a632ba9d60..6439d233aa 100644 --- a/fdbserver/IDiskQueue.h +++ b/fdbserver/IDiskQueue.h @@ -24,11 +24,9 @@ #include "fdbclient/FDBTypes.h" #include "fdbserver/IKeyValueStore.h" +#include "flow/BooleanParam.h" -enum class CheckHashes { - NO, - YES, -}; +FDB_DECLARE_BOOLEAN_PARAM(CheckHashes); class IDiskQueue : public IClosable { public: diff --git a/fdbserver/KeyValueStoreMemory.actor.cpp b/fdbserver/KeyValueStoreMemory.actor.cpp index 8307e86097..0008296a96 100644 --- a/fdbserver/KeyValueStoreMemory.actor.cpp +++ b/fdbserver/KeyValueStoreMemory.actor.cpp @@ -31,8 +31,6 @@ #define OP_DISK_OVERHEAD (sizeof(OpHeader) + 1) -extern bool noUnseed; - template class KeyValueStoreMemory final : public IKeyValueStore, NonCopyable { public: diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp index 2f2c77c42e..99278a7862 100644 --- a/fdbserver/KeyValueStoreRocksDB.actor.cpp +++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp @@ -282,7 +282,9 @@ struct RocksDBKeyValueStore : IKeyValueStore { a.result.send(Value(StringRef(reinterpret_cast(value.data()), std::min(value.size(), size_t(a.maxLength))))); } else { - TraceEvent(SevError, "RocksDBError").detail("Error", s.ToString()).detail("Method", "ReadValuePrefix"); + if (!s.IsNotFound()) { + TraceEvent(SevError, "RocksDBError").detail("Error", s.ToString()).detail("Method", "ReadValuePrefix"); + } a.result.send(Optional()); } } diff --git a/fdbserver/LocalConfiguration.actor.cpp b/fdbserver/LocalConfiguration.actor.cpp index 0f0fef94dd..30974a2d19 100644 --- a/fdbserver/LocalConfiguration.actor.cpp +++ b/fdbserver/LocalConfiguration.actor.cpp @@ -28,6 +28,8 @@ #include "flow/actorcompiler.h" // This must be the last #include. +FDB_DEFINE_BOOLEAN_PARAM(IsTest); + namespace { const KeyRef configPathKey = "configPath"_sr; @@ -228,11 +230,11 @@ class LocalConfigurationImpl { void updateInMemoryState(Version lastSeenVersion) { this->lastSeenVersion = lastSeenVersion; // TODO: Support randomization? - getKnobs().reset(Randomize::NO, g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO); + getKnobs().reset(Randomize::FALSE, g_network->isSimulated() ? IsSimulated::TRUE : IsSimulated::FALSE); configKnobOverrides.update(getKnobs()); manualKnobOverrides.update(getKnobs()); // Must reinitialize in order to update dependent knobs - getKnobs().initialize(Randomize::NO, g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO); + getKnobs().initialize(Randomize::FALSE, g_network->isSimulated() ? IsSimulated::TRUE : IsSimulated::FALSE); } ACTOR static Future setSnapshot(LocalConfigurationImpl* self, @@ -329,10 +331,11 @@ public: broadcasterChanges("BroadcasterChanges", cc), snapshots("Snapshots", cc), changeRequestsFetched("ChangeRequestsFetched", cc), mutations("Mutations", cc), configKnobOverrides(configPath), manualKnobOverrides(manualKnobOverrides) { - if (isTest == IsTest::YES) { - testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, - Randomize::NO, - g_network->isSimulated() ? IsSimulated::YES : IsSimulated::NO); + if (isTest) { + testKnobCollection = + IKnobCollection::create(IKnobCollection::Type::TEST, + Randomize::FALSE, + g_network->isSimulated() ? IsSimulated::TRUE : IsSimulated::FALSE); } logger = traceCounters( "LocalConfigurationMetrics", id, SERVER_KNOBS->WORKER_LOGGING_INTERVAL, &cc, "LocalConfigurationMetrics"); @@ -401,7 +404,8 @@ public: ConfigKnobOverrides configKnobOverrides; configKnobOverrides.set( {}, "knob_name_that_does_not_exist"_sr, KnobValueRef::create(ParsedKnobValue(int{ 1 }))); - auto testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::NO, IsSimulated::NO); + auto testKnobCollection = + IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::FALSE, IsSimulated::FALSE); // Should only trace and not throw an error: configKnobOverrides.update(*testKnobCollection); } @@ -409,7 +413,8 @@ public: static void testConfigKnobOverridesInvalidValue() { ConfigKnobOverrides configKnobOverrides; configKnobOverrides.set({}, "test_int"_sr, KnobValueRef::create(ParsedKnobValue("not_an_int"))); - auto testKnobCollection = IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::NO, IsSimulated::NO); + auto testKnobCollection = + IKnobCollection::create(IKnobCollection::Type::TEST, Randomize::FALSE, IsSimulated::FALSE); // Should only trace and not throw an error: configKnobOverrides.update(*testKnobCollection); } diff --git a/fdbserver/LocalConfiguration.h b/fdbserver/LocalConfiguration.h index 2eda470e46..6f9ecabc8f 100644 --- a/fdbserver/LocalConfiguration.h +++ b/fdbserver/LocalConfiguration.h @@ -29,8 +29,7 @@ #include "flow/Arena.h" #include "flow/Knobs.h" -// To be used effectively as a boolean parameter with added type safety -enum class IsTest { NO, YES }; +FDB_DECLARE_BOOLEAN_PARAM(IsTest); /* * Each worker maintains a LocalConfiguration object used to update its knob collection. @@ -52,7 +51,7 @@ public: LocalConfiguration(std::string const& dataFolder, std::string const& configPath, std::map const& manualKnobOverrides, - IsTest isTest = IsTest::NO); + IsTest = IsTest::FALSE); LocalConfiguration(LocalConfiguration&&); LocalConfiguration& operator=(LocalConfiguration&&); ~LocalConfiguration(); diff --git a/fdbserver/MetricLogger.actor.cpp b/fdbserver/MetricLogger.actor.cpp index 65a4a3ae5f..7c1ddf1b6f 100644 --- a/fdbserver/MetricLogger.actor.cpp +++ b/fdbserver/MetricLogger.actor.cpp @@ -182,7 +182,7 @@ public: // levelKey is the prefix for the entire level, no timestamp at the end ACTOR static Future>> getLastBlock_impl(ReadYourWritesTransaction* tr, Standalone levelKey) { - RangeResult results = wait(tr->getRange(normalKeys.withPrefix(levelKey), 1, true, true)); + RangeResult results = wait(tr->getRange(normalKeys.withPrefix(levelKey), 1, Snapshot::TRUE, Reverse::TRUE)); if (results.size() == 1) return results[0].value; return Optional>(); diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp index 8103314235..a120433ac1 100644 --- a/fdbserver/MoveKeys.actor.cpp +++ b/fdbserver/MoveKeys.actor.cpp @@ -1039,8 +1039,9 @@ ACTOR Future> addStorageServer(Database cx, StorageServe LocalityData::ExcludeLocalityPrefix.toString() + l.first + ":" + l.second)))); } - state Future fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY, true); - state Future fHistoryTags = tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY, true); + state Future fTags = tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::TRUE); + state Future fHistoryTags = + tr->getRange(serverTagHistoryKeys, CLIENT_KNOBS->TOO_MANY, Snapshot::TRUE); wait(success(fTagLocalities) && success(fv) && success(fTags) && success(fHistoryTags) && success(fExclProc) && success(fExclIP) && success(fFailProc) && success(fFailIP) && diff --git a/fdbserver/OldTLogServer_6_0.actor.cpp b/fdbserver/OldTLogServer_6_0.actor.cpp index 543111ede6..24c97f741c 100644 --- a/fdbserver/OldTLogServer_6_0.actor.cpp +++ b/fdbserver/OldTLogServer_6_0.actor.cpp @@ -311,7 +311,7 @@ struct TLogData : NonCopyable { targetVolatileBytes(SERVER_KNOBS->TLOG_SPILL_THRESHOLD), overheadBytesInput(0), overheadBytesDurable(0), concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopRequest(false), ignorePopDeadline(), ignorePopUid(), dataFolder(folder), toBePopped() { - cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE); } }; diff --git a/fdbserver/OldTLogServer_6_2.actor.cpp b/fdbserver/OldTLogServer_6_2.actor.cpp index f7f25868f9..68c125858f 100644 --- a/fdbserver/OldTLogServer_6_2.actor.cpp +++ b/fdbserver/OldTLogServer_6_2.actor.cpp @@ -375,7 +375,7 @@ struct TLogData : NonCopyable { peekMemoryLimiter(SERVER_KNOBS->TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES), concurrentLogRouterReads(SERVER_KNOBS->CONCURRENT_LOG_ROUTER_READS), ignorePopRequest(false), ignorePopDeadline(), ignorePopUid(), dataFolder(folder), toBePopped() { - cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE); } }; @@ -1440,6 +1440,19 @@ ACTOR Future tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen } } + uint64_t PoppedVersionLag = logData->persistentDataDurableVersion - logData->queuePoppedVersion; + if ( SERVER_KNOBS->ENABLE_DETAILED_TLOG_POP_TRACE && + (logData->queuePoppedVersion > 0) && //avoid generating massive events at beginning + (tagData->unpoppedRecovered || PoppedVersionLag >= SERVER_KNOBS->TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE)) { //when recovery or long lag + TraceEvent("TLogPopDetails", logData->logId) + .detail("Tag", tagData->tag.toString()) + .detail("UpTo", upTo) + .detail("PoppedVersionLag", PoppedVersionLag) + .detail("MinPoppedTag", logData->minPoppedTag.toString()) + .detail("QueuePoppedVersion", logData->queuePoppedVersion) + .detail("UnpoppedRecovered", tagData->unpoppedRecovered ? "True" : "False") + .detail("NothingPersistent", tagData->nothingPersistent ? "True" : "False"); + } if (upTo > logData->persistentDataDurableVersion) wait(tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop)); //TraceEvent("TLogPop", self->dbgid).detail("Tag", tag.toString()).detail("To", upTo); @@ -1744,7 +1757,7 @@ ACTOR Future tLogPeekMessages(TLogData* self, TLogPeekRequest req, Referen state std::vector>> messageReads; messageReads.reserve(commitLocations.size()); for (const auto& pair : commitLocations) { - messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::YES)); + messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::TRUE)); } commitLocations.clear(); wait(waitForAll(messageReads)); diff --git a/fdbserver/ProxyCommitData.actor.h b/fdbserver/ProxyCommitData.actor.h index 99d210bc6e..7a2960022e 100644 --- a/fdbserver/ProxyCommitData.actor.h +++ b/fdbserver/ProxyCommitData.actor.h @@ -247,7 +247,7 @@ struct ProxyCommitData { mostRecentProcessedRequestNumber(0), getConsistentReadVersion(getConsistentReadVersion), commit(commit), lastCoalesceTime(0), localCommitBatchesStarted(0), locked(false), commitBatchInterval(SERVER_KNOBS->COMMIT_TRANSACTION_BATCH_INTERVAL_MIN), firstProxy(firstProxy), - cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true)), db(db), + cx(openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE)), db(db), singleKeyMutationEvent(LiteralStringRef("SingleKeyMutation")), commitBatchesMemBytesCount(0), lastTxsPop(0), lastStartCommit(0), lastCommitLatency(SERVER_KNOBS->REQUIRED_MIN_RECOVERY_DURATION), lastCommitTime(0), lastMasterReset(now()), lastResolverReset(now()) { diff --git a/fdbserver/QuietDatabase.actor.cpp b/fdbserver/QuietDatabase.actor.cpp index fc8f23a966..47b9a9f2f3 100644 --- a/fdbserver/QuietDatabase.actor.cpp +++ b/fdbserver/QuietDatabase.actor.cpp @@ -637,7 +637,7 @@ ACTOR Future waitForQuietDatabase(Database cx, // The quiet database check (which runs at the end of every test) will always time out due to active data movement. // To get around this, quiet Database will disable the perpetual wiggle in the setup phase. - wait(setPerpetualStorageWiggle(cx, false, true)); + wait(setPerpetualStorageWiggle(cx, false, LockAware::TRUE)); // Require 3 consecutive successful quiet database checks spaced 2 second apart state int numSuccesses = 0; diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index 804e4a537e..77bda2577b 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -1409,7 +1409,7 @@ ACTOR Future configurationMonitor(RatekeeperData* self) { } ACTOR Future ratekeeper(RatekeeperInterface rkInterf, Reference> dbInfo) { - state RatekeeperData self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true)); + state RatekeeperData self(rkInterf.id(), openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE)); state Future timeout = Void(); state std::vector> tlogTrackers; state std::vector tlogInterfs; diff --git a/fdbserver/ResolverInterface.h b/fdbserver/ResolverInterface.h index 9c71e07a7a..f3ad222811 100644 --- a/fdbserver/ResolverInterface.h +++ b/fdbserver/ResolverInterface.h @@ -49,6 +49,7 @@ struct ResolverInterface { bool operator==(ResolverInterface const& r) const { return id() == r.id(); } bool operator!=(ResolverInterface const& r) const { return id() != r.id(); } NetworkAddress address() const { return resolve.getEndpoint().getPrimaryAddress(); } + NetworkAddressList addresses() const { return resolve.getEndpoint().addresses; } void initEndpoints() { metrics.getEndpoint(TaskPriority::ResolutionMetrics); split.getEndpoint(TaskPriority::ResolutionMetrics); diff --git a/fdbserver/RestoreCommon.actor.cpp b/fdbserver/RestoreCommon.actor.cpp index 5f1bea9aa4..ace015f24b 100644 --- a/fdbserver/RestoreCommon.actor.cpp +++ b/fdbserver/RestoreCommon.actor.cpp @@ -141,8 +141,8 @@ Key RestoreConfigFR::applyMutationsMapPrefix() { ACTOR Future RestoreConfigFR::getApplyVersionLag_impl(Reference tr, UID uid) { // Both of these are snapshot reads - state Future> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), true); - state Future> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), true); + state Future> beginVal = tr->get(uidPrefixKey(applyMutationsBeginRange.begin, uid), Snapshot::TRUE); + state Future> endVal = tr->get(uidPrefixKey(applyMutationsEndRange.begin, uid), Snapshot::TRUE); wait(success(beginVal) && success(endVal)); if (!beginVal.get().present() || !endVal.get().present()) diff --git a/fdbserver/RestoreWorker.actor.cpp b/fdbserver/RestoreWorker.actor.cpp index 04bbf21ee1..827a58a25d 100644 --- a/fdbserver/RestoreWorker.actor.cpp +++ b/fdbserver/RestoreWorker.actor.cpp @@ -410,7 +410,7 @@ ACTOR Future restoreWorker(Reference connFile, LocalityData locality, std::string coordFolder) { try { - Database cx = Database::createDatabase(connFile, Database::API_VERSION_LATEST, true, locality); + Database cx = Database::createDatabase(connFile, Database::API_VERSION_LATEST, IsInternal::TRUE, locality); wait(reportErrors(_restoreWorker(cx, locality), "RestoreWorker")); } catch (Error& e) { TraceEvent("FastRestoreWorker").detail("Error", e.what()); diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index 23b6dc0221..a8a152f141 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -347,8 +347,8 @@ ACTOR Future runBackup(Reference connFile) { Database cx = Database::createDatabase(connFile, -1); state FileBackupAgent fileAgent; - state double backupPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; - agentFutures.push_back(fileAgent.run(cx, &backupPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); + agentFutures.push_back(fileAgent.run( + cx, 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); while (g_simulator.backupAgents == ISimulator::BackupAgentType::BackupToFile) { wait(delay(1.0)); @@ -383,11 +383,10 @@ ACTOR Future runDr(Reference connFile) { state DatabaseBackupAgent dbAgent = DatabaseBackupAgent(cx); state DatabaseBackupAgent extraAgent = DatabaseBackupAgent(extraDB); - state double dr1PollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; - state double dr2PollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; + auto drPollDelay = 1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; - agentFutures.push_back(extraAgent.run(cx, &dr1PollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); - agentFutures.push_back(dbAgent.run(extraDB, &dr2PollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); + agentFutures.push_back(extraAgent.run(cx, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); + agentFutures.push_back(dbAgent.run(extraDB, drPollDelay, CLIENT_KNOBS->SIM_BACKUP_TASKS_PER_AGENT)); while (g_simulator.drAgents == ISimulator::BackupAgentType::BackupToDB) { wait(delay(1.0)); diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index ee4198c7b0..ac15df2d04 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -2605,10 +2605,9 @@ ACTOR Future lockedStatusFetcher(Reference* incomplete_reasons) { state JsonBuilderObject statusObj; - state Database cx = openDBOnServer(db, - TaskPriority::DefaultEndpoint, - true, - false); // Open a new database connection that isn't lock-aware + state Database cx = + openDBOnServer(db, + TaskPriority::DefaultEndpoint); // Open a new database connection that isn't lock-aware state Transaction tr(cx); state int timeoutSeconds = 5; state Future getTimeout = delay(timeoutSeconds); diff --git a/fdbserver/StorageCache.actor.cpp b/fdbserver/StorageCache.actor.cpp index d758e32bf3..888c94c3b3 100644 --- a/fdbserver/StorageCache.actor.cpp +++ b/fdbserver/StorageCache.actor.cpp @@ -251,7 +251,7 @@ public: newestAvailableVersion.insert(allKeys, invalidVersion); newestDirtyVersion.insert(allKeys, invalidVersion); addCacheRange(CacheRangeInfo::newNotAssigned(allKeys)); - cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE); } // Puts the given cacheRange into cachedRangeMap. The caller is responsible for adding cacheRanges @@ -1194,7 +1194,7 @@ ACTOR Future tryFetchRange(Database cx, try { loop { - RangeResult rep = wait(tr.getRange(begin, end, limits, true)); + RangeResult rep = wait(tr.getRange(begin, end, limits, Snapshot::TRUE)); limits.decrement(rep); if (limits.isReached() || !rep.more) { @@ -1392,7 +1392,7 @@ ACTOR Future fetchKeys(StorageCacheData* data, AddingCacheRange* cacheRang // TODO: NEELAM: what's this for? // FIXME: remove when we no longer support upgrades from 5.X if (debug_getRangeRetries >= 100) { - data->cx->enableLocalityLoadBalance = false; + data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::FALSE; } debug_getRangeRetries++; diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index cdf8dacbad..a46d22d856 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -386,7 +386,7 @@ struct TLogData : NonCopyable { commitLatencyDist(Histogram::getHistogram(LiteralStringRef("tLog"), LiteralStringRef("commit"), Histogram::Unit::microseconds)) { - cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE); } }; @@ -1166,6 +1166,19 @@ ACTOR Future tLogPopCore(TLogData* self, Tag inputTag, Version to, Referen } } + uint64_t PoppedVersionLag = logData->persistentDataDurableVersion - logData->queuePoppedVersion; + if ( SERVER_KNOBS->ENABLE_DETAILED_TLOG_POP_TRACE && + (logData->queuePoppedVersion > 0) && //avoid generating massive events at beginning + (tagData->unpoppedRecovered || PoppedVersionLag >= SERVER_KNOBS->TLOG_POPPED_VER_LAG_THRESHOLD_FOR_TLOGPOP_TRACE)) { //when recovery or long lag + TraceEvent("TLogPopDetails", logData->logId) + .detail("Tag", tagData->tag.toString()) + .detail("UpTo", upTo) + .detail("PoppedVersionLag", PoppedVersionLag) + .detail("MinPoppedTag", logData->minPoppedTag.toString()) + .detail("QueuePoppedVersion", logData->queuePoppedVersion) + .detail("UnpoppedRecovered", tagData->unpoppedRecovered ? "True" : "False") + .detail("NothingPersistent", tagData->nothingPersistent ? "True" : "False"); + } if (upTo > logData->persistentDataDurableVersion) wait(tagData->eraseMessagesBefore(upTo, self, logData, TaskPriority::TLogPop)); //TraceEvent("TLogPop", logData->logId).detail("Tag", tag.toString()).detail("To", upTo); @@ -1784,7 +1797,7 @@ ACTOR Future peekTLog(TLogData* self, state std::vector>> messageReads; messageReads.reserve(commitLocations.size()); for (const auto& pair : commitLocations) { - messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::YES)); + messageReads.push_back(self->rawPersistentQueue->read(pair.first, pair.second, CheckHashes::TRUE)); } commitLocations.clear(); wait(waitForAll(messageReads)); diff --git a/fdbserver/VersionedBTree.actor.cpp b/fdbserver/VersionedBTree.actor.cpp index 31f74084a4..19f53cb65f 100644 --- a/fdbserver/VersionedBTree.actor.cpp +++ b/fdbserver/VersionedBTree.actor.cpp @@ -6341,20 +6341,18 @@ public: // If there is a record in the tree > query then moveNext() will move to it. // If non-zero is returned then the cursor is valid and the return value is logically equivalent // to query.compare(cursor.get()) - ACTOR Future seek_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { + ACTOR Future seek_impl(BTreeCursor* self, RedwoodRecordRef query) { state RedwoodRecordRef internalPageQuery = query.withMaxPageID(); self->path.resize(1); - debug_printf( - "seek(%s, %d) start cursor = %s\n", query.toString().c_str(), prefetchBytes, self->toString().c_str()); + debug_printf("seek(%s) start cursor = %s\n", query.toString().c_str(), self->toString().c_str()); loop { auto& entry = self->path.back(); if (entry.btPage()->isLeaf()) { int cmp = entry.cursor.seek(query); self->valid = entry.cursor.valid() && !entry.cursor.isErased(); - debug_printf("seek(%s, %d) loop exit cmp=%d cursor=%s\n", + debug_printf("seek(%s) loop exit cmp=%d cursor=%s\n", query.toString().c_str(), - prefetchBytes, cmp, self->toString().c_str()); return self->valid ? cmp : 0; @@ -6365,68 +6363,97 @@ public: // to and will be updated if anything is inserted into the cleared range, so if the seek fails // or it finds an entry with a null child page then query does not exist in the BTree. if (entry.cursor.seekLessThan(internalPageQuery) && entry.cursor.get().value.present()) { - debug_printf("seek(%s, %d) loop seek success cursor=%s\n", - query.toString().c_str(), - prefetchBytes, - self->toString().c_str()); + debug_printf( + "seek(%s) loop seek success cursor=%s\n", query.toString().c_str(), self->toString().c_str()); Future f = self->pushPage(entry.cursor); - - // Prefetch siblings, at least prefetchBytes, at level 2 but without jumping to another level 2 - // sibling - if (prefetchBytes != 0 && entry.btPage()->height == 2) { - auto c = entry.cursor; - bool fwd = prefetchBytes > 0; - prefetchBytes = abs(prefetchBytes); - // While we should still preload more bytes and a move in the target direction is successful - while (prefetchBytes > 0 && (fwd ? c.moveNext() : c.movePrev())) { - // If there is a page link, preload it. - if (c.get().value.present()) { - BTreePageIDRef childPage = c.get().getChildPage(); - preLoadPage(self->pager.getPtr(), childPage); - prefetchBytes -= self->btree->m_blockSize * childPage.size(); - } - } - } - wait(f); } else { self->valid = false; - debug_printf("seek(%s, %d) loop exit cmp=0 cursor=%s\n", - query.toString().c_str(), - prefetchBytes, - self->toString().c_str()); + debug_printf( + "seek(%s) loop exit cmp=0 cursor=%s\n", query.toString().c_str(), self->toString().c_str()); return 0; } } } - Future seek(RedwoodRecordRef query, int prefetchBytes) { return seek_impl(this, query, prefetchBytes); } + Future seek(RedwoodRecordRef query) { return seek_impl(this, query); } - ACTOR Future seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { - debug_printf("seekGTE(%s, %d) start\n", query.toString().c_str(), prefetchBytes); - int cmp = wait(self->seek(query, prefetchBytes)); + ACTOR Future seekGTE_impl(BTreeCursor* self, RedwoodRecordRef query) { + debug_printf("seekGTE(%s) start\n", query.toString().c_str()); + int cmp = wait(self->seek(query)); if (cmp > 0 || (cmp == 0 && !self->isValid())) { wait(self->moveNext()); } return Void(); } - Future seekGTE(RedwoodRecordRef query, int prefetchBytes) { - return seekGTE_impl(this, query, prefetchBytes); + Future seekGTE(RedwoodRecordRef query) { return seekGTE_impl(this, query); } + + // Start fetching sibling nodes in the forward or backward direction, stopping after recordLimit or byteLimit + void prefetch(KeyRef rangeEnd, bool directionForward, int recordLimit, int byteLimit) { + // Prefetch scans level 2 so if there are less than 2 nodes in the path there is no level 2 + if (path.size() < 2) { + return; + } + + auto firstLeaf = path.back().btPage(); + + // We know the first leaf's record count, so assume they are all relevant to the query, + // even though some may not be. + int recordsRead = firstLeaf->tree()->numItems; + + // We can't know for sure how many records are in a node without reading it, so just guess + // that siblings have about the same record count as the first leaf. + int estRecordsPerPage = recordsRead; + + // Use actual KVBytes stored for the first leaf, but use node capacity for siblings below + int bytesRead = firstLeaf->kvBytes; + + // Cursor for moving through siblings. + // Note that only immediate siblings under the same parent are considered for prefetch so far. + BTreePage::BinaryTree::Cursor c = path[path.size() - 2].cursor; + + // The loop conditions are split apart into different if blocks for readability. + // While query limits are not exceeded + while (recordsRead < recordLimit && bytesRead < byteLimit) { + // If prefetching right siblings + if (directionForward) { + // If there is no right sibling or its lower boundary is greater + // or equal to than the range end then stop. + if(!c.moveNext() || c.get().key >= rangeEnd) { + break; + } + } + else { + // Prefetching left siblings + // If the current leaf lower boundary is less than or equal to the range end + // or there is no left sibling then stop + if(c.get().key <= rangeEnd || !c.movePrev()) { + break; + } + } + + // Prefetch the sibling if the link is not null + if (c.get().value.present()) { + BTreePageIDRef childPage = c.get().getChildPage(); + preLoadPage(pager.getPtr(), childPage); + recordsRead += estRecordsPerPage; + // Use sibling node capacity as an estimate of bytes read. + bytesRead += childPage.size() * this->btree->m_blockSize; + } + } } - ACTOR Future seekLT_impl(BTreeCursor* self, RedwoodRecordRef query, int prefetchBytes) { - debug_printf("seekLT(%s, %d) start\n", query.toString().c_str(), prefetchBytes); - int cmp = wait(self->seek(query, prefetchBytes)); + ACTOR Future seekLT_impl(BTreeCursor* self, RedwoodRecordRef query) { + debug_printf("seekLT(%s) start\n", query.toString().c_str()); + int cmp = wait(self->seek(query)); if (cmp <= 0) { wait(self->movePrev()); } return Void(); } - Future seekLT(RedwoodRecordRef query, int prefetchBytes) { - return seekLT_impl(this, query, -prefetchBytes); - } + Future seekLT(RedwoodRecordRef query) { return seekLT_impl(this, query); } ACTOR Future move_impl(BTreeCursor* self, bool forward) { // Try to the move cursor at the end of the path in the correct direction @@ -6511,7 +6538,8 @@ RedwoodRecordRef VersionedBTree::dbEnd(LiteralStringRef("\xff\xff\xff\xff\xff")) class KeyValueStoreRedwoodUnversioned : public IKeyValueStore { public: KeyValueStoreRedwoodUnversioned(std::string filePrefix, UID logID) - : m_filePrefix(filePrefix), m_concurrentReads(SERVER_KNOBS->REDWOOD_KVSTORE_CONCURRENT_READS) { + : m_filePrefix(filePrefix), m_concurrentReads(SERVER_KNOBS->REDWOOD_KVSTORE_CONCURRENT_READS), + prefetch(SERVER_KNOBS->REDWOOD_KVSTORE_RANGE_PREFETCH) { int pageSize = BUGGIFY ? deterministicRandom()->randomInt(1000, 4096 * 4) : SERVER_KNOBS->REDWOOD_DEFAULT_PAGE_SIZE; @@ -6619,11 +6647,13 @@ public: return result; } - // Prefetch is disabled for now pending some decent logic for deciding how much to fetch - state int prefetchBytes = 0; - if (rowLimit > 0) { - wait(cur.seekGTE(keys.begin, prefetchBytes)); + wait(cur.seekGTE(keys.begin)); + + if (self->prefetch) { + cur.prefetch(keys.end, true, rowLimit, byteLimit); + } + while (cur.isValid()) { // Read page contents without using waits BTreePage::BinaryTree::Cursor leafCursor = cur.back().cursor; @@ -6665,7 +6695,12 @@ public: wait(cur.moveNext()); } } else { - wait(cur.seekLT(keys.end, prefetchBytes)); + wait(cur.seekLT(keys.end)); + + if (self->prefetch) { + cur.prefetch(keys.begin, false, -rowLimit, byteLimit); + } + while (cur.isValid()) { // Read page contents without using waits BTreePage::BinaryTree::Cursor leafCursor = cur.back().cursor; @@ -6726,7 +6761,7 @@ public: state FlowLock::Releaser releaser(self->m_concurrentReads); ++g_redwoodMetrics.opGet; - wait(cur.seekGTE(key, 0)); + wait(cur.seekGTE(key)); if (cur.isValid() && cur.get().key == key) { // Return a Value whose arena depends on the source page arena Value v; @@ -6762,6 +6797,7 @@ private: Promise m_closed; Promise m_error; FlowLock m_concurrentReads; + bool prefetch; template inline Future catchError(Future f) { @@ -6842,12 +6878,12 @@ ACTOR Future verifyRangeBTreeCursor(VersionedBTree* btree, start.printable().c_str(), end.printable().c_str(), randomKey.toString().c_str()); - wait(success(cur.seek(randomKey, 0))); + wait(success(cur.seek(randomKey))); } debug_printf( "VerifyRange(@%" PRId64 ", %s, %s): Actual seek\n", v, start.printable().c_str(), end.printable().c_str()); - wait(cur.seekGTE(start, 0)); + wait(cur.seekGTE(start)); state Standalone> results; @@ -6947,7 +6983,7 @@ ACTOR Future verifyRangeBTreeCursor(VersionedBTree* btree, } // Now read the range from the tree in reverse order and compare to the saved results - wait(cur.seekLT(end, 0)); + wait(cur.seekLT(end)); state std::reverse_iterator r = results.rbegin(); @@ -7024,7 +7060,7 @@ ACTOR Future seekAllBTreeCursor(VersionedBTree* btree, state Optional val = i->second; debug_printf("Verifying @%" PRId64 " '%s'\n", ver, key.c_str()); state Arena arena; - wait(cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key)), 0)); + wait(cur.seekGTE(RedwoodRecordRef(KeyRef(arena, key)))); bool foundKey = cur.isValid() && cur.get().key == key; bool hasValue = foundKey && cur.get().value.present(); @@ -7149,7 +7185,7 @@ ACTOR Future randomReader(VersionedBTree* btree) { } state KeyValue kv = randomKV(10, 0); - wait(cur.seekGTE(kv.key, 0)); + wait(cur.seekGTE(kv.key)); state int c = deterministicRandom()->randomInt(0, 100); state bool direction = deterministicRandom()->coinflip(); while (cur.isValid() && c-- > 0) { @@ -8729,7 +8765,7 @@ ACTOR Future randomSeeks(VersionedBTree* btree, int count, char firstChar, wait(btree->initBTreeCursor(&cur, readVer)); while (c < count) { state Key k = randomString(20, firstChar, lastChar); - wait(cur.seekGTE(k, 0)); + wait(cur.seekGTE(k)); ++c; } double elapsed = timer() - readStart; @@ -8740,7 +8776,7 @@ ACTOR Future randomSeeks(VersionedBTree* btree, int count, char firstChar, ACTOR Future randomScans(VersionedBTree* btree, int count, int width, - int readAhead, + int prefetchBytes, char firstChar, char lastChar) { state Version readVer = btree->getLatestVersion(); @@ -8749,29 +8785,34 @@ ACTOR Future randomScans(VersionedBTree* btree, state VersionedBTree::BTreeCursor cur; wait(btree->initBTreeCursor(&cur, readVer)); - state bool adaptive = readAhead < 0; state int totalScanBytes = 0; while (c++ < count) { state Key k = randomString(20, firstChar, lastChar); - wait(cur.seekGTE(k, readAhead)); - if (adaptive) { - readAhead = totalScanBytes / c; - } + wait(cur.seekGTE(k)); state int w = width; - state bool direction = deterministicRandom()->coinflip(); + state bool directionFwd = deterministicRandom()->coinflip(); + + if (prefetchBytes > 0) { + cur.prefetch(directionFwd ? VersionedBTree::dbEnd.key : VersionedBTree::dbBegin.key, + directionFwd, + width, + prefetchBytes); + } + while (w > 0 && cur.isValid()) { totalScanBytes += cur.get().expectedSize(); - wait(success(direction ? cur.moveNext() : cur.movePrev())); + wait(success(directionFwd ? cur.moveNext() : cur.movePrev())); --w; } } double elapsed = timer() - readStart; - printf("Completed %d scans: readAhead=%d width=%d bytesRead=%d scansRate=%d/s\n", + printf("Completed %d scans: width=%d totalbytesRead=%d prefetchBytes=%d scansRate=%d scans/s %.2f MB/s\n", count, - readAhead, width, totalScanBytes, - int(count / elapsed)); + prefetchBytes, + int(count / elapsed), + double(totalScanBytes) / 1e6 / elapsed); return Void(); } @@ -8999,6 +9040,8 @@ TEST_CASE(":/redwood/performance/set") { state int concurrentScans = params.getInt("concurrentScans").orDefault(64); state int seeks = params.getInt("seeks").orDefault(1000000); state int scans = params.getInt("scans").orDefault(20000); + state int scanWidth = params.getInt("scanWidth").orDefault(50); + state int scanPrefetchBytes = params.getInt("scanPrefetchBytes").orDefault(0); state bool pagerMemoryOnly = params.getInt("pagerMemoryOnly").orDefault(0); state bool traceMetrics = params.getInt("traceMetrics").orDefault(0); @@ -9022,6 +9065,8 @@ TEST_CASE(":/redwood/performance/set") { printf("concurrentSeeks: %d\n", concurrentSeeks); printf("seeks: %d\n", seeks); printf("scans: %d\n", scans); + printf("scanWidth: %d\n", scanWidth); + printf("scanPrefetchBytes: %d\n", scanPrefetchBytes); printf("fileName: %s\n", fileName.c_str()); printf("openExisting: %d\n", openExisting); printf("insertRecords: %d\n", insertRecords); @@ -9134,9 +9179,14 @@ TEST_CASE(":/redwood/performance/set") { } if (scans > 0) { - printf("Parallel scans, count=%d, concurrency=%d, no readAhead ...\n", scans, concurrentScans); + printf("Parallel scans, concurrency=%d, scans=%d, scanWidth=%d, scanPreftchBytes=%d ...\n", + concurrentScans, + scans, + scanWidth, + scanPrefetchBytes); for (int x = 0; x < concurrentScans; ++x) { - actors.add(randomScans(btree, scans / concurrentScans, 50, 0, firstKeyChar, lastKeyChar)); + actors.add( + randomScans(btree, scans / concurrentScans, scanWidth, scanPrefetchBytes, firstKeyChar, lastKeyChar)); } wait(actors.signalAndReset()); if (!traceMetrics) { @@ -9145,7 +9195,7 @@ TEST_CASE(":/redwood/performance/set") { } if (seeks > 0) { - printf("Parallel seeks, count=%d, concurrency=%d ...\n", seeks, concurrentSeeks); + printf("Parallel seeks, concurrency=%d, seeks=%d ...\n", concurrentSeeks, seeks); for (int x = 0; x < concurrentSeeks; ++x) { actors.add(randomSeeks(btree, seeks / concurrentSeeks, firstKeyChar, lastKeyChar)); } diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index eca8495092..e642d015dd 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -31,6 +31,7 @@ #include "fdbserver/TLogInterface.h" #include "fdbserver/RatekeeperInterface.h" #include "fdbserver/ResolverInterface.h" +#include "fdbclient/ClientBooleanParams.h" #include "fdbclient/StorageServerInterface.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbclient/FDBTypes.h" @@ -832,8 +833,8 @@ struct ServerDBInfo; class Database openDBOnServer(Reference> const& db, TaskPriority taskID = TaskPriority::DefaultEndpoint, - bool enableLocalityLoadBalance = true, - bool lockAware = false); + LockAware = LockAware::FALSE, + EnableLocalityLoadBalance = EnableLocalityLoadBalance::TRUE); ACTOR Future extractClusterInterface(Reference>> a, Reference>> b); diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index ce51f7308a..0a62a0b2fd 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -199,7 +199,6 @@ extern const char* getSourceVersion(); extern void flushTraceFileVoid(); -extern bool noUnseed; extern const int MAX_CLUSTER_FILE_BYTES; #ifdef ALLOC_INSTRUMENTATION @@ -1641,8 +1640,9 @@ int main(int argc, char* argv[]) { enableBuggify(opts.buggifyEnabled, BuggifyType::General); IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::SERVER, - Randomize::YES, - role == ServerRole::Simulation ? IsSimulated::YES : IsSimulated::NO); + Randomize::TRUE, + role == ServerRole::Simulation ? IsSimulated::TRUE + : IsSimulated::FALSE); IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder)); if (role != ServerRole::Simulation) { IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit", @@ -1677,7 +1677,7 @@ int main(int argc, char* argv[]) { KnobValue::create(int64_t{ opts.memLimit })); // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs IKnobCollection::getMutableGlobalKnobCollection().initialize( - Randomize::YES, role == ServerRole::Simulation ? IsSimulated::YES : IsSimulated::NO); + Randomize::TRUE, role == ServerRole::Simulation ? IsSimulated::TRUE : IsSimulated::FALSE); // evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY); diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp index 4edcaa18cf..ea93b35f7f 100644 --- a/fdbserver/masterserver.actor.cpp +++ b/fdbserver/masterserver.actor.cpp @@ -577,7 +577,7 @@ Future sendMasterRegistration(MasterData* self, } ACTOR Future updateRegistration(Reference self, Reference logSystem) { - state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, true, true); + state Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE); state Future trigger = self->registrationTrigger.onTrigger(); state Future updateLogsKey; @@ -1965,7 +1965,7 @@ ACTOR Future masterCore(Reference self) { self->addActor.send(resolutionBalancing(self)); self->addActor.send(changeCoordinators(self)); - Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, true, true); + Database cx = openDBOnServer(self->dbInfo, TaskPriority::DefaultEndpoint, LockAware::TRUE); self->addActor.send(configurationMonitor(self, cx)); if (self->configuration.backupWorkerEnabled) { self->addActor.send(recruitBackupWorkers(self, cx)); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 85bf05e7b3..e9a23ab309 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -852,7 +852,7 @@ public: newestDirtyVersion.insert(allKeys, invalidVersion); addShard(ShardInfo::newNotAssigned(allKeys)); - cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, true, true); + cx = openDBOnServer(db, TaskPriority::DefaultEndpoint, LockAware::TRUE); } //~StorageServer() { fclose(log); } @@ -2790,7 +2790,7 @@ ACTOR Future tryGetRange(PromiseStream results, Transaction* loop { GetRangeLimits limits(GetRangeLimits::ROW_LIMIT_UNLIMITED, SERVER_KNOBS->FETCH_BLOCK_BYTES); limits.minRows = 0; - state RangeResult rep = wait(tr->getRange(begin, end, limits, true)); + state RangeResult rep = wait(tr->getRange(begin, end, limits, Snapshot::TRUE)); if (!rep.more) { rep.readThrough = keys.end; } @@ -2903,7 +2903,7 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { tr.info.taskID = TaskPriority::FetchKeys; state PromiseStream results; state Future hold = SERVER_KNOBS->FETCH_USING_STREAMING - ? tr.getRangeStream(results, keys, GetRangeLimits(), true) + ? tr.getRangeStream(results, keys, GetRangeLimits(), Snapshot::TRUE) : tryGetRange(results, &tr, keys); state Key nfk = keys.begin; @@ -2970,7 +2970,7 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { // FIXME: remove when we no longer support upgrades from 5.X if (debug_getRangeRetries >= 100) { - data->cx->enableLocalityLoadBalance = false; + data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::FALSE; TraceEvent(SevWarnAlways, "FKDisableLB").detail("FKID", fetchKeysID); } @@ -3018,7 +3018,7 @@ ACTOR Future fetchKeys(StorageServer* data, AddingShard* shard) { } // FIXME: remove when we no longer support upgrades from 5.X - data->cx->enableLocalityLoadBalance = true; + data->cx->enableLocalityLoadBalance = EnableLocalityLoadBalance::TRUE; TraceEvent(SevWarnAlways, "FKReenableLB").detail("FKID", fetchKeysID); // We have completed the fetch and write of the data, now we wait for MVCC window to pass. diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index fe4d0034ee..3dcd9ae162 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -234,6 +234,15 @@ vector getOption(VectorRef options, Key key, vector options, Key key) { + for (const auto& option : options) { + if (option.key == key) { + return true; + } + } + return false; +} + // returns unconsumed options Standalone> checkAllOptionsConsumed(VectorRef options) { static StringRef nothing = LiteralStringRef(""); @@ -607,7 +616,7 @@ ACTOR Future testerServerWorkload(WorkloadRequest work, startRole(Role::TESTER, workIface.id(), UID(), details); if (work.useDatabase) { - cx = Database::createDatabase(ccf, -1, true, locality); + cx = Database::createDatabase(ccf, -1, IsInternal::TRUE, locality); wait(delay(1.0)); } @@ -1049,8 +1058,7 @@ std::map> testSpecGlobalKey [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedStorageEngineExcludeTypes", ""); } }, { "maxTLogVersion", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedMaxTLogVersion", ""); } }, - { "disableTss", - [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableTSS", ""); } } + { "disableTss", [](const std::string& value) { TraceEvent("TestParserTest").detail("ParsedDisableTSS", ""); } } }; std::map> testSpecTestKeys = { @@ -1178,6 +1186,11 @@ std::mapphases = TestWorkload::CHECK; } }, + { "restorePerpetualWiggleSetting", + [](const std::string& value, TestSpec* spec) { + if (value == "false") + spec->restorePerpetualWiggleSetting = false; + } }, }; vector readTests(ifstream& ifs) { @@ -1468,7 +1481,7 @@ ACTOR Future runTests(Reference extractClientInfo(Reference> db Database openDBOnServer(Reference> const& db, TaskPriority taskID, - bool enableLocalityLoadBalance, - bool lockAware) { + LockAware lockAware, + EnableLocalityLoadBalance enableLocalityLoadBalance) { auto info = makeReference>(); auto cx = DatabaseContext::create(info, extractClientInfo(db, info), @@ -1215,15 +1215,15 @@ ACTOR Future workerServer(Reference connFile, if (metricsConnFile.size() > 0) { try { state Database db = - Database::createDatabase(metricsConnFile, Database::API_VERSION_LATEST, true, locality); + Database::createDatabase(metricsConnFile, Database::API_VERSION_LATEST, IsInternal::TRUE, locality); metricsLogger = runMetrics(db, KeyRef(metricsPrefix)); } catch (Error& e) { TraceEvent(SevWarnAlways, "TDMetricsBadClusterFile").error(e).detail("ConnFile", metricsConnFile); } } else { - bool lockAware = metricsPrefix.size() && metricsPrefix[0] == '\xff'; - metricsLogger = runMetrics(openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, true, lockAware), - KeyRef(metricsPrefix)); + auto lockAware = metricsPrefix.size() && metricsPrefix[0] == '\xff' ? LockAware::TRUE : LockAware::FALSE; + metricsLogger = + runMetrics(openDBOnServer(dbInfo, TaskPriority::DefaultEndpoint, lockAware), KeyRef(metricsPrefix)); } } diff --git a/fdbserver/workloads/ApiCorrectness.actor.cpp b/fdbserver/workloads/ApiCorrectness.actor.cpp index 9aec6679c5..4ad9bfc1c1 100644 --- a/fdbserver/workloads/ApiCorrectness.actor.cpp +++ b/fdbserver/workloads/ApiCorrectness.actor.cpp @@ -435,6 +435,8 @@ public: // Gets a single range of values from the database and memory stores and compares them, returning true if the // results were the same ACTOR Future runGetRange(VectorRef data, ApiCorrectnessWorkload* self) { + state Reverse reverse = deterministicRandom()->coinflip(); + // Generate a random range Key key = self->selectRandomKey(data, 0.5); Key key2 = self->selectRandomKey(data, 0.5); @@ -444,7 +446,6 @@ public: // Generate a random maximum number of results state int limit = deterministicRandom()->randomInt(0, 101); - state bool reverse = deterministicRandom()->random01() > 0.5 ? false : true; // Get the range from memory state RangeResult storeResults = self->store.getRange(KeyRangeRef(start, end), limit, reverse); @@ -480,6 +481,8 @@ public: // Gets a single range of values using key selectors from the database and memory store and compares them, returning // true if the results were the same ACTOR Future runGetRangeSelector(VectorRef data, ApiCorrectnessWorkload* self) { + state Reverse reverse = deterministicRandom()->coinflip(); + KeySelector selectors[2]; Key keys[2]; @@ -530,7 +533,6 @@ public: // Choose a random maximum number of results state int limit = deterministicRandom()->randomInt(0, 101); - state bool reverse = deterministicRandom()->random01() < 0.5 ? false : true; // Get the range from the memory store state RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), limit, reverse); diff --git a/fdbserver/workloads/ApiWorkload.actor.cpp b/fdbserver/workloads/ApiWorkload.actor.cpp index ad68076836..7d223462c9 100644 --- a/fdbserver/workloads/ApiWorkload.actor.cpp +++ b/fdbserver/workloads/ApiWorkload.actor.cpp @@ -167,14 +167,15 @@ ACTOR Future compareDatabaseToMemory(ApiWorkload* self) { loop { // Fetch a subset of the results from each of the database and the memory store and compare them - state RangeResult storeResults = self->store.getRange(KeyRangeRef(startKey, endKey), resultsPerRange, false); + state RangeResult storeResults = + self->store.getRange(KeyRangeRef(startKey, endKey), resultsPerRange, Reverse::FALSE); state Reference transaction = self->createTransaction(); state KeyRangeRef range(startKey, endKey); loop { try { - state RangeResult dbResults = wait(transaction->getRange(range, resultsPerRange, false)); + state RangeResult dbResults = wait(transaction->getRange(range, resultsPerRange, Reverse::FALSE)); // Compare results of database and memory store Version v = wait(transaction->getReadVersion()); diff --git a/fdbserver/workloads/ApiWorkload.h b/fdbserver/workloads/ApiWorkload.h index 48eaab348a..53e2eed431 100644 --- a/fdbserver/workloads/ApiWorkload.h +++ b/fdbserver/workloads/ApiWorkload.h @@ -46,10 +46,10 @@ struct TransactionWrapper : public ReferenceCounted { virtual Future> get(KeyRef& key) = 0; // Gets a range of key-value pairs from the database specified by a key range - virtual Future getRange(KeyRangeRef& keys, int limit, bool reverse) = 0; + virtual Future getRange(KeyRangeRef& keys, int limit, Reverse reverse) = 0; // Gets a range of key-value pairs from the database specified by a pair of key selectors - virtual Future getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit, bool reverse) = 0; + virtual Future getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit, Reverse reverse) = 0; // Gets the key from the database specified by a given key selector virtual Future getKey(KeySelectorRef& key) = 0; @@ -101,13 +101,13 @@ struct FlowTransactionWrapper : public TransactionWrapper { Future> get(KeyRef& key) override { return transaction.get(key); } // Gets a range of key-value pairs from the database specified by a key range - Future getRange(KeyRangeRef& keys, int limit, bool reverse) override { - return transaction.getRange(keys, limit, false, reverse); + Future getRange(KeyRangeRef& keys, int limit, Reverse reverse) override { + return transaction.getRange(keys, limit, Snapshot::FALSE, reverse); } // Gets a range of key-value pairs from the database specified by a pair of key selectors - Future getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit, bool reverse) override { - return transaction.getRange(begin, end, limit, false, reverse); + Future getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit, Reverse reverse) override { + return transaction.getRange(begin, end, limit, Snapshot::FALSE, reverse); } // Gets the key from the database specified by a given key selector @@ -161,13 +161,13 @@ struct ThreadTransactionWrapper : public TransactionWrapper { Future> get(KeyRef& key) override { return unsafeThreadFutureToFuture(transaction->get(key)); } // Gets a range of key-value pairs from the database specified by a key range - Future getRange(KeyRangeRef& keys, int limit, bool reverse) override { - return unsafeThreadFutureToFuture(transaction->getRange(keys, limit, false, reverse)); + Future getRange(KeyRangeRef& keys, int limit, Reverse reverse) override { + return unsafeThreadFutureToFuture(transaction->getRange(keys, limit, Snapshot::FALSE, reverse)); } // Gets a range of key-value pairs from the database specified by a pair of key selectors - Future getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit, bool reverse) override { - return unsafeThreadFutureToFuture(transaction->getRange(begin, end, limit, false, reverse)); + Future getRange(KeySelectorRef& begin, KeySelectorRef& end, int limit, Reverse reverse) override { + return unsafeThreadFutureToFuture(transaction->getRange(begin, end, limit, Snapshot::FALSE, reverse)); } // Gets the key from the database specified by a given key selector diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp index 33412123f2..a4100e9e9a 100644 --- a/fdbserver/workloads/AtomicRestore.actor.cpp +++ b/fdbserver/workloads/AtomicRestore.actor.cpp @@ -31,7 +31,7 @@ struct AtomicRestoreWorkload : TestWorkload { double startAfter, restoreAfter; bool fastRestore; // true: use fast restore, false: use old style restore Standalone> backupRanges; - bool usePartitionedLogs; + UsePartitionedLog usePartitionedLogs{ false }; Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix // CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix @@ -41,8 +41,8 @@ struct AtomicRestoreWorkload : TestWorkload { restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 20.0); fastRestore = getOption(options, LiteralStringRef("fastRestore"), false); backupRanges.push_back_deep(backupRanges.arena(), normalKeys); - usePartitionedLogs = getOption( - options, LiteralStringRef("usePartitionedLogs"), deterministicRandom()->random01() < 0.5 ? true : false); + usePartitionedLogs.set(getOption( + options, LiteralStringRef("usePartitionedLogs"), deterministicRandom()->random01() < 0.5 ? true : false)); addPrefix = getOption(options, LiteralStringRef("addPrefix"), LiteralStringRef("")); removePrefix = getOption(options, LiteralStringRef("removePrefix"), LiteralStringRef("")); @@ -97,7 +97,7 @@ struct AtomicRestoreWorkload : TestWorkload { deterministicRandom()->randomInt(0, 100), BackupAgentBase::getDefaultTagName(), self->backupRanges, - false, + StopWhenDone::FALSE, self->usePartitionedLogs)); } catch (Error& e) { if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) @@ -105,7 +105,7 @@ struct AtomicRestoreWorkload : TestWorkload { } TraceEvent("AtomicRestore_Wait"); - wait(success(backupAgent.waitBackup(cx, BackupAgentBase::getDefaultTagName(), false))); + wait(success(backupAgent.waitBackup(cx, BackupAgentBase::getDefaultTagName(), StopWhenDone::FALSE))); TraceEvent("AtomicRestore_BackupStart"); wait(delay(self->restoreAfter * deterministicRandom()->random01())); TraceEvent("AtomicRestore_RestoreStart"); diff --git a/fdbserver/workloads/AtomicSwitchover.actor.cpp b/fdbserver/workloads/AtomicSwitchover.actor.cpp index 4c227f4cd0..d672759817 100644 --- a/fdbserver/workloads/AtomicSwitchover.actor.cpp +++ b/fdbserver/workloads/AtomicSwitchover.actor.cpp @@ -57,10 +57,10 @@ struct AtomicSwitchoverWorkload : TestWorkload { wait(backupAgent.submitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), self->backupRanges, - false, + StopWhenDone::FALSE, StringRef(), StringRef(), - true)); + LockDB::TRUE)); TraceEvent("AS_Submit2"); } catch (Error& e) { if (e.code() != error_code_backup_duplicate) @@ -168,21 +168,21 @@ struct AtomicSwitchoverWorkload : TestWorkload { state DatabaseBackupAgent restoreTool(self->extraDB); TraceEvent("AS_Wait1"); - wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), false))); + wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::FALSE))); TraceEvent("AS_Ready1"); wait(delay(deterministicRandom()->random01() * self->switch1delay)); TraceEvent("AS_Switch1"); wait(backupAgent.atomicSwitchover( self->extraDB, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef())); TraceEvent("AS_Wait2"); - wait(success(restoreTool.waitBackup(cx, BackupAgentBase::getDefaultTag(), false))); + wait(success(restoreTool.waitBackup(cx, BackupAgentBase::getDefaultTag(), StopWhenDone::FALSE))); TraceEvent("AS_Ready2"); wait(delay(deterministicRandom()->random01() * self->switch2delay)); TraceEvent("AS_Switch2"); wait(restoreTool.atomicSwitchover( cx, BackupAgentBase::getDefaultTag(), self->backupRanges, StringRef(), StringRef())); TraceEvent("AS_Wait3"); - wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), false))); + wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::FALSE))); TraceEvent("AS_Ready3"); wait(delay(deterministicRandom()->random01() * self->stopDelay)); TraceEvent("AS_Abort"); diff --git a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp index 79bd4424df..1f636bad6b 100644 --- a/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupAndParallelRestoreCorrectness.actor.cpp @@ -40,17 +40,17 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { bool differentialBackup, performRestore, agentRequest; Standalone> backupRanges; static int backupAgentRequests; - bool locked; + LockDB locked{ false }; bool allowPauses; bool shareLogRange; - bool usePartitionedLogs; + UsePartitionedLog usePartitionedLogs{ false }; Key addPrefix, removePrefix; // Original key will be first applied removePrefix and then applied addPrefix // CAVEAT: When removePrefix is used, we must ensure every key in backup have the removePrefix std::map, Standalone> dbKVs; BackupAndParallelRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - locked = sharedRandomNumber % 2; + locked.set(sharedRandomNumber % 2); backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0); restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0); performRestore = getOption(options, LiteralStringRef("performRestore"), true); @@ -75,8 +75,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true); allowPauses = getOption(options, LiteralStringRef("allowPauses"), true); shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false); - usePartitionedLogs = getOption( - options, LiteralStringRef("usePartitionedLogs"), deterministicRandom()->random01() < 0.5 ? true : false); + usePartitionedLogs.set( + getOption(options, LiteralStringRef("usePartitionedLogs"), deterministicRandom()->coinflip())); addPrefix = getOption(options, LiteralStringRef("addPrefix"), LiteralStringRef("")); removePrefix = getOption(options, LiteralStringRef("removePrefix"), LiteralStringRef("")); @@ -179,7 +179,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { ACTOR static Future statusLoop(Database cx, std::string tag) { state FileBackupAgent agent; loop { - std::string status = wait(agent.getStatus(cx, true, tag)); + std::string status = wait(agent.getStatus(cx, ShowErrors::TRUE, tag)); puts(status.c_str()); wait(delay(2.0)); } @@ -226,7 +226,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { deterministicRandom()->randomInt(0, 100), tag.toString(), backupRanges, - stopDifferentialDelay ? false : true, + StopWhenDone{ !stopDifferentialDelay }, self->usePartitionedLogs)); } catch (Error& e) { TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag)); @@ -251,8 +251,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { // Wait until the backup is in a restorable state and get the status, URL, and UID atomically state Reference lastBackupContainer; state UID lastBackupUID; - state EBackupState resultWait = wait( - backupAgent->waitBackup(cx, backupTag.tagName, false, &lastBackupContainer, &lastBackupUID)); + state EBackupState resultWait = wait(backupAgent->waitBackup( + cx, backupTag.tagName, StopWhenDone::FALSE, &lastBackupContainer, &lastBackupUID)); TraceEvent("BARW_DoBackupWaitForRestorable", randomID) .detail("Tag", backupTag.tagName) @@ -333,11 +333,11 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { // Wait for the backup to complete TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag)); - state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), true)); + state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::TRUE)); state std::string statusText; - std::string _statusText = wait(backupAgent->getStatus(cx, 5, tag.toString())); + std::string _statusText = wait(backupAgent->getStatus(cx, ShowErrors::TRUE, tag.toString())); statusText = _statusText; // Can we validate anything about status? @@ -377,9 +377,9 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { cx, self->backupTag, KeyRef(lastBackupContainer), - true, - -1, - true, + WaitForComplete::TRUE, + ::invalidVersion, + Verbose::TRUE, normalKeys, Key(), Key(), @@ -482,8 +482,8 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { deterministicRandom()->randomInt(0, 100), self->backupTag.toString(), self->backupRanges, - true, - false); + StopWhenDone::TRUE, + UsePartitionedLog::FALSE); } catch (Error& e) { TraceEvent("BARW_SubmitBackup2Exception", randomID) .error(e) @@ -602,7 +602,7 @@ struct BackupAndParallelRestoreCorrectnessWorkload : TestWorkload { // Wait for parallel restore to finish before we can proceed TraceEvent("FastRestoreWorkload").detail("WaitForRestoreToFinish", randomID); // Do not unlock DB when restore finish because we need to transformDatabaseContents - wait(backupAgent.parallelRestoreFinish(cx, randomID, !self->hasPrefix())); + wait(backupAgent.parallelRestoreFinish(cx, randomID, UnlockDB{ !self->hasPrefix() })); TraceEvent("FastRestoreWorkload").detail("RestoreFinished", randomID); for (auto& restore : restores) { diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp index abd3609325..e629f37085 100644 --- a/fdbserver/workloads/BackupCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupCorrectness.actor.cpp @@ -21,6 +21,7 @@ #include "fdbrpc/simulator.h" #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" +#include "fdbclient/BackupContainerFileSystem.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/workloads/BulkSetup.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -37,39 +38,43 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { std::vector> skippedRestoreRanges; Standalone> restoreRanges; static int backupAgentRequests; - bool locked; + LockDB locked{ false }; bool allowPauses; bool shareLogRange; bool shouldSkipRestoreRanges; + Optional encryptionKeyFileName; BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - locked = sharedRandomNumber % 2; - backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0); - restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0); - performRestore = getOption(options, LiteralStringRef("performRestore"), true); - backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag()); - backupRangesCount = getOption(options, LiteralStringRef("backupRangesCount"), 5); - backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1); + locked.set(sharedRandomNumber % 2); + backupAfter = getOption(options, "backupAfter"_sr, 10.0); + restoreAfter = getOption(options, "restoreAfter"_sr, 35.0); + performRestore = getOption(options, "performRestore"_sr, true); + backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag()); + backupRangesCount = getOption(options, "backupRangesCount"_sr, 5); + backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1); abortAndRestartAfter = getOption(options, - LiteralStringRef("abortAndRestartAfter"), + "abortAndRestartAfter"_sr, deterministicRandom()->random01() < 0.5 ? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter : 0.0); - differentialBackup = getOption( - options, LiteralStringRef("differentialBackup"), deterministicRandom()->random01() < 0.5 ? true : false); + differentialBackup = + getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false); stopDifferentialAfter = getOption(options, - LiteralStringRef("stopDifferentialAfter"), + "stopDifferentialAfter"_sr, differentialBackup ? deterministicRandom()->random01() * (restoreAfter - std::max(abortAndRestartAfter, backupAfter)) + std::max(abortAndRestartAfter, backupAfter) : 0.0); - agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true); - allowPauses = getOption(options, LiteralStringRef("allowPauses"), true); - shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false); - restorePrefixesToInclude = getOption(options, LiteralStringRef("restorePrefixesToInclude"), std::vector()); + agentRequest = getOption(options, "simBackupAgents"_sr, true); + allowPauses = getOption(options, "allowPauses"_sr, true); + shareLogRange = getOption(options, "shareLogRange"_sr, false); + restorePrefixesToInclude = getOption(options, "restorePrefixesToInclude"_sr, std::vector()); shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false; + if (getOption(options, "encrypted"_sr, deterministicRandom()->random01() < 0.1)) { + encryptionKeyFileName = "simfdb/test_encryption_key_file"; + } TraceEvent("BARW_ClientId").detail("Id", wcx.clientId); UID randomID = nondeterministicRandom()->randomUniqueID(); @@ -77,11 +82,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { if (shareLogRange) { bool beforePrefix = sharedRandomNumber & 1; if (beforePrefix) - backupRanges.push_back_deep(backupRanges.arena(), - KeyRangeRef(normalKeys.begin, LiteralStringRef("\xfe\xff\xfe"))); + backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr)); else backupRanges.push_back_deep(backupRanges.arena(), - KeyRangeRef(strinc(LiteralStringRef("\x00\x00\x01")), normalKeys.end)); + KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end)); } else if (backupRangesCount <= 0) { backupRanges.push_back_deep(backupRanges.arena(), normalKeys); } else { @@ -168,6 +172,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { TraceEvent(SevInfo, "BARW_Param").detail("DifferentialBackup", differentialBackup); TraceEvent(SevInfo, "BARW_Param").detail("StopDifferentialAfter", stopDifferentialAfter); TraceEvent(SevInfo, "BARW_Param").detail("AgentRequest", agentRequest); + TraceEvent(SevInfo, "BARW_Param").detail("Encrypted", encryptionKeyFileName.present()); return _start(cx, this); } @@ -215,7 +220,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { ACTOR static Future statusLoop(Database cx, std::string tag) { state FileBackupAgent agent; loop { - std::string status = wait(agent.getStatus(cx, true, tag)); + std::string status = wait(agent.getStatus(cx, ShowErrors::TRUE, tag)); puts(status.c_str()); std::string statusJSON = wait(agent.getStatusJSON(cx, tag)); puts(statusJSON.c_str()); @@ -265,7 +270,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { deterministicRandom()->randomInt(0, 100), tag.toString(), backupRanges, - stopDifferentialDelay ? false : true)); + StopWhenDone{ !stopDifferentialDelay }, + UsePartitionedLog::FALSE, + IncrementalBackupOnly::FALSE, + self->encryptionKeyFileName)); } catch (Error& e) { TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag)); if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) @@ -290,8 +298,8 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { // Wait until the backup is in a restorable state and get the status, URL, and UID atomically state Reference lastBackupContainer; state UID lastBackupUID; - state EBackupState resultWait = wait( - backupAgent->waitBackup(cx, backupTag.tagName, false, &lastBackupContainer, &lastBackupUID)); + state EBackupState resultWait = wait(backupAgent->waitBackup( + cx, backupTag.tagName, StopWhenDone::FALSE, &lastBackupContainer, &lastBackupUID)); TraceEvent("BARW_DoBackupWaitForRestorable", randomID) .detail("Tag", backupTag.tagName) @@ -372,11 +380,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { // Wait for the backup to complete TraceEvent("BARW_DoBackupWaitBackup", randomID).detail("Tag", printable(tag)); - state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), true)); + state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag.toString(), StopWhenDone::TRUE)); state std::string statusText; - std::string _statusText = wait(backupAgent->getStatus(cx, 5, tag.toString())); + std::string _statusText = wait(backupAgent->getStatus(cx, ShowErrors::TRUE, tag.toString())); statusText = _statusText; // Can we validate anything about status? @@ -415,9 +423,9 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, self->backupTag, KeyRef(lastBackupContainer), - true, - -1, - true, + WaitForComplete::TRUE, + ::invalidVersion, + Verbose::TRUE, normalKeys, Key(), Key(), @@ -456,6 +464,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { BackupAndRestoreCorrectnessWorkload::backupAgentRequests++; } + if (self->encryptionKeyFileName.present()) { + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get())); + } + try { state Future startRestore = delay(self->restoreAfter); @@ -510,12 +522,12 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag)); try { extraBackup = backupAgent.submitBackup(cx, - LiteralStringRef("file://simfdb/backups/"), + "file://simfdb/backups/"_sr, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), self->backupTag.toString(), self->backupRanges, - true); + StopWhenDone::TRUE); } catch (Error& e) { TraceEvent("BARW_SubmitBackup2Exception", randomID) .error(e) @@ -581,13 +593,17 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, restoreTag, KeyRef(lastBackupContainer->getURL()), - true, + WaitForComplete::TRUE, targetVersion, - true, + Verbose::TRUE, range, Key(), Key(), - self->locked)); + self->locked, + OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly::FALSE, + ::invalidVersion, + self->encryptionKeyFileName)); } } else { multipleRangesInOneTag = true; @@ -601,12 +617,16 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { restoreTag, KeyRef(lastBackupContainer->getURL()), self->restoreRanges, - true, + WaitForComplete::TRUE, targetVersion, - true, + Verbose::TRUE, Key(), Key(), - self->locked)); + self->locked, + OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly::FALSE, + ::invalidVersion, + self->encryptionKeyFileName)); } // Sometimes kill and restart the restore @@ -627,12 +647,16 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { restoreTags[restoreIndex], KeyRef(lastBackupContainer->getURL()), self->restoreRanges, - true, - -1, - true, + WaitForComplete::TRUE, + ::invalidVersion, + Verbose::TRUE, Key(), Key(), - self->locked); + self->locked, + OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly::FALSE, + ::invalidVersion, + self->encryptionKeyFileName); } } else { for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) { @@ -651,13 +675,17 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { cx, restoreTags[restoreIndex], KeyRef(lastBackupContainer->getURL()), - true, - -1, - true, + WaitForComplete::TRUE, + ::invalidVersion, + Verbose::TRUE, self->restoreRanges[restoreIndex], Key(), Key(), - self->locked); + self->locked, + OnlyApplyMutationLogs::FALSE, + InconsistentSnapshotOnly::FALSE, + ::invalidVersion, + self->encryptionKeyFileName); } } } @@ -721,7 +749,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { .detail("TaskCount", taskCount) .detail("WaitCycles", waitCycles); printf("EndingNonZeroTasks: %ld\n", (long)taskCount); - wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); + wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef())); } loop { @@ -820,7 +848,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { } if (displaySystemKeys) { - wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); + wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef())); } TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag)); diff --git a/fdbserver/workloads/BackupToBlob.actor.cpp b/fdbserver/workloads/BackupToBlob.actor.cpp index 5b94e4d771..44cabf8549 100644 --- a/fdbserver/workloads/BackupToBlob.actor.cpp +++ b/fdbserver/workloads/BackupToBlob.actor.cpp @@ -66,7 +66,7 @@ struct BackupToBlobWorkload : TestWorkload { self->snapshotInterval, self->backupTag.toString(), backupRanges)); - EBackupState backupStatus = wait(backupAgent.waitBackup(cx, self->backupTag.toString(), true)); + EBackupState backupStatus = wait(backupAgent.waitBackup(cx, self->backupTag.toString(), StopWhenDone::TRUE)); TraceEvent("BackupToBlob_BackupStatus").detail("Status", BackupAgentBase::getStateText(backupStatus)); return Void(); } diff --git a/fdbserver/workloads/BackupToDBAbort.actor.cpp b/fdbserver/workloads/BackupToDBAbort.actor.cpp index e0ce1c8f9f..c83b38e451 100644 --- a/fdbserver/workloads/BackupToDBAbort.actor.cpp +++ b/fdbserver/workloads/BackupToDBAbort.actor.cpp @@ -56,10 +56,10 @@ struct BackupToDBAbort : TestWorkload { wait(backupAgent.submitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), self->backupRanges, - false, + StopWhenDone::FALSE, StringRef(), StringRef(), - true)); + LockDB::TRUE)); TraceEvent("BDBA_Submit2"); } catch (Error& e) { if (e.code() != error_code_backup_duplicate) @@ -80,7 +80,7 @@ struct BackupToDBAbort : TestWorkload { TraceEvent("BDBA_Start").detail("Delay", self->abortDelay); wait(delay(self->abortDelay)); TraceEvent("BDBA_Wait"); - wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), false))); + wait(success(backupAgent.waitBackup(self->extraDB, BackupAgentBase::getDefaultTag(), StopWhenDone::FALSE))); TraceEvent("BDBA_Lock"); wait(lockDatabase(cx, self->lockid)); TraceEvent("BDBA_Abort"); diff --git a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp index 6988908633..385b492a87 100644 --- a/fdbserver/workloads/BackupToDBCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupToDBCorrectness.actor.cpp @@ -38,12 +38,12 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { Standalone> backupRanges; static int drAgentRequests; Database extraDB; - bool locked; + LockDB locked{ false }; bool shareLogRange; UID destUid; BackupToDBCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { - locked = sharedRandomNumber % 2; + locked.set(sharedRandomNumber % 2); backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0); restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0); performRestore = getOption(options, LiteralStringRef("performRestore"), true); @@ -302,10 +302,10 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { wait(backupAgent->submitBackup(cx, tag, backupRanges, - stopDifferentialDelay ? false : true, + StopWhenDone{ !stopDifferentialDelay }, self->backupPrefix, StringRef(), - self->locked, + LockDB{ self->locked }, DatabaseBackupAgent::PreBackupAction::CLEAR)); } catch (Error& e) { TraceEvent("BARW_SubmitBackup1Exception", randomID).error(e); @@ -337,7 +337,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { if (BUGGIFY) { TraceEvent("BARW_DoBackupWaitForRestorable", randomID).detail("Tag", printable(tag)); // Wait until the backup is in a restorable state - state EBackupState resultWait = wait(backupAgent->waitBackup(cx, tag, false)); + state EBackupState resultWait = wait(backupAgent->waitBackup(cx, tag, StopWhenDone::FALSE)); TraceEvent("BARW_LastBackupFolder", randomID) .detail("BackupTag", printable(tag)) @@ -383,7 +383,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { UID _destUid = wait(backupAgent->getDestUid(cx, logUid)); self->destUid = _destUid; - state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag, true)); + state EBackupState statusValue = wait(backupAgent->waitBackup(cx, tag, StopWhenDone::TRUE)); wait(backupAgent->unlockBackup(cx, tag)); state std::string statusText; @@ -617,7 +617,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { extraBackup = backupAgent.submitBackup(self->extraDB, self->backupTag, self->backupRanges, - true, + StopWhenDone::TRUE, self->extraPrefix, StringRef(), self->locked, @@ -652,7 +652,7 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { wait(restoreTool.submitBackup(cx, self->restoreTag, restoreRange, - true, + StopWhenDone::TRUE, StringRef(), self->backupPrefix, self->locked, @@ -704,10 +704,10 @@ struct BackupToDBCorrectnessWorkload : TestWorkload { // not be set yet. Adding "waitForDestUID" flag to avoid the race. wait(backupAgent.abortBackup(self->extraDB, self->backupTag, - /*partial=*/false, - /*abortOldBackup=*/false, - /*dstOnly=*/false, - /*waitForDestUID*/ true)); + PartialBackup::FALSE, + AbortOldBackup::FALSE, + DstOnly::FALSE, + WaitForDestUID::TRUE)); } catch (Error& e) { TraceEvent("BARW_AbortBackupExtraException", randomID).error(e); if (e.code() != error_code_backup_unneeded) diff --git a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp index a7e3807fce..9cf029e7ed 100644 --- a/fdbserver/workloads/BackupToDBUpgrade.actor.cpp +++ b/fdbserver/workloads/BackupToDBUpgrade.actor.cpp @@ -116,7 +116,8 @@ struct BackupToDBUpgradeWorkload : TestWorkload { tr->clear(targetRange); } } - wait(backupAgent->submitBackup(tr, tag, backupRanges, false, self->backupPrefix, StringRef())); + wait(backupAgent->submitBackup( + tr, tag, backupRanges, StopWhenDone::FALSE, self->backupPrefix, StringRef())); wait(tr->commit()); break; } catch (Error& e) { @@ -132,7 +133,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload { } } - wait(success(backupAgent->waitBackup(self->extraDB, tag, false))); + wait(success(backupAgent->waitBackup(self->extraDB, tag, StopWhenDone::FALSE))); return Void(); } @@ -499,7 +500,7 @@ struct BackupToDBUpgradeWorkload : TestWorkload { try { TraceEvent("DRU_RestoreDb").detail("RestoreTag", printable(self->restoreTag)); wait(restoreTool.submitBackup( - cx, self->restoreTag, restoreRanges, true, StringRef(), self->backupPrefix)); + cx, self->restoreTag, restoreRanges, StopWhenDone::TRUE, StringRef(), self->backupPrefix)); } catch (Error& e) { TraceEvent("DRU_RestoreSubmitBackupError").error(e).detail("Tag", printable(self->restoreTag)); if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) diff --git a/fdbserver/workloads/ClogSingleConnection.actor.cpp b/fdbserver/workloads/ClogSingleConnection.actor.cpp new file mode 100644 index 0000000000..24e0b6e32f --- /dev/null +++ b/fdbserver/workloads/ClogSingleConnection.actor.cpp @@ -0,0 +1,71 @@ +/* + * ClogSingleConnection.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/NativeAPI.actor.h" +#include "fdbserver/TesterInterface.actor.h" +#include "fdbserver/workloads/workloads.actor.h" +#include "fdbrpc/simulator.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +class ClogSingleConnectionWorkload : public TestWorkload { + double delaySeconds; + Optional clogDuration; // If empty, clog forever + +public: + ClogSingleConnectionWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { + auto minDelay = getOption(options, "minDelay"_sr, 0.0); + auto maxDelay = getOption(options, "maxDelay"_sr, 10.0); + ASSERT_LE(minDelay, maxDelay); + delaySeconds = minDelay + deterministicRandom()->random01() * (maxDelay - minDelay); + if (hasOption(options, "clogDuration"_sr)) { + clogDuration = getOption(options, "clogDuration"_sr, ""); + } + } + + std::string description() const override { + return g_network->isSimulated() ? "ClogSingleConnection" : "NoClogging"; + } + Future setup(Database const& cx) override { return Void(); } + + Future start(Database const& cx) override { + if (g_network->isSimulated() && clientId == 0) { + return map(delay(delaySeconds), [this](Void _) { + clogRandomPair(); + return Void(); + }); + } else { + return Void(); + } + } + + Future check(Database const& cx) override { return true; } + + void getMetrics(std::vector& m) override {} + + void clogRandomPair() { + auto m1 = deterministicRandom()->randomChoice(g_simulator.getAllProcesses()); + auto m2 = deterministicRandom()->randomChoice(g_simulator.getAllProcesses()); + if (m1->address.ip != m2->address.ip) { + g_simulator.clogPair(m1->address.ip, m2->address.ip, clogDuration.orDefault(10000)); + } + } +}; + +WorkloadFactory ClogSingleConnectionWorkloadFactory("ClogSingleConnection"); diff --git a/fdbserver/workloads/IncrementalBackup.actor.cpp b/fdbserver/workloads/IncrementalBackup.actor.cpp index 1d9fa547af..4087e7923d 100644 --- a/fdbserver/workloads/IncrementalBackup.actor.cpp +++ b/fdbserver/workloads/IncrementalBackup.actor.cpp @@ -93,8 +93,8 @@ struct IncrementalBackupWorkload : TestWorkload { loop { // Wait for backup container to be created and avoid race condition TraceEvent("IBackupWaitContainer"); - wait(success( - self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID))); + wait(success(self->backupAgent.waitBackup( + cx, self->tag.toString(), StopWhenDone::FALSE, &backupContainer, &backupUID))); if (!backupContainer.isValid()) { TraceEvent("IBackupCheckListContainersAttempt"); state std::vector containers = @@ -150,8 +150,15 @@ struct IncrementalBackupWorkload : TestWorkload { backupRanges.push_back_deep(backupRanges.arena(), normalKeys); TraceEvent("IBackupSubmitAttempt"); try { - wait(self->backupAgent.submitBackup( - cx, self->backupDir, 0, 1e8, self->tag.toString(), backupRanges, false, false, true)); + wait(self->backupAgent.submitBackup(cx, + self->backupDir, + 0, + 1e8, + self->tag.toString(), + backupRanges, + StopWhenDone::FALSE, + UsePartitionedLog::FALSE, + IncrementalBackupOnly::TRUE)); } catch (Error& e) { TraceEvent("IBackupSubmitError").error(e); if (e.code() != error_code_backup_duplicate) { @@ -179,7 +186,8 @@ struct IncrementalBackupWorkload : TestWorkload { state Reference backupContainer; state UID backupUID; state Version beginVersion = invalidVersion; - wait(success(self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID))); + wait(success(self->backupAgent.waitBackup( + cx, self->tag.toString(), StopWhenDone::FALSE, &backupContainer, &backupUID))); if (self->checkBeginVersion) { TraceEvent("IBackupReadSystemKeys"); state Reference tr(new ReadYourWritesTransaction(cx)); @@ -221,15 +229,15 @@ struct IncrementalBackupWorkload : TestWorkload { cx, Key(self->tag.toString()), backupURL, - true, + WaitForComplete::TRUE, invalidVersion, - true, + Verbose::TRUE, normalKeys, Key(), Key(), - true, - true, - false, + LockDB::TRUE, + OnlyApplyMutationLogs::TRUE, + InconsistentSnapshotOnly::FALSE, beginVersion))); TraceEvent("IBackupRestoreSuccess"); } diff --git a/fdbserver/workloads/Mako.actor.cpp b/fdbserver/workloads/Mako.actor.cpp index 1457814395..e5e267f3b5 100644 --- a/fdbserver/workloads/Mako.actor.cpp +++ b/fdbserver/workloads/Mako.actor.cpp @@ -463,18 +463,18 @@ struct MakoWorkload : TestWorkload { if (i == OP_GETREADVERSION) { wait(logLatency(tr.getReadVersion(), &self->opLatencies[i])); } else if (i == OP_GET) { - wait(logLatency(tr.get(rkey, false), &self->opLatencies[i])); + wait(logLatency(tr.get(rkey, Snapshot::FALSE), &self->opLatencies[i])); } else if (i == OP_GETRANGE) { - wait(logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, false), + wait(logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, Snapshot::FALSE), &self->opLatencies[i])); } else if (i == OP_SGET) { - wait(logLatency(tr.get(rkey, true), &self->opLatencies[i])); + wait(logLatency(tr.get(rkey, Snapshot::TRUE), &self->opLatencies[i])); } else if (i == OP_SGETRANGE) { // do snapshot get range here - wait(logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, true), + wait(logLatency(tr.getRange(rkeyRangeRef, CLIENT_KNOBS->TOO_MANY, Snapshot::TRUE), &self->opLatencies[i])); } else if (i == OP_UPDATE) { - wait(logLatency(tr.get(rkey, false), &self->opLatencies[OP_GET])); + wait(logLatency(tr.get(rkey, Snapshot::FALSE), &self->opLatencies[OP_GET])); if (self->latencyForLocalOperation) { double opBegin = timer(); tr.set(rkey, rval); diff --git a/fdbserver/workloads/MemoryKeyValueStore.cpp b/fdbserver/workloads/MemoryKeyValueStore.cpp index 1c951ec1a6..ac966f7621 100644 --- a/fdbserver/workloads/MemoryKeyValueStore.cpp +++ b/fdbserver/workloads/MemoryKeyValueStore.cpp @@ -78,7 +78,7 @@ Key MemoryKeyValueStore::getKey(KeySelectorRef selector) const { } // Gets a range of key-value pairs, returning a maximum of results -RangeResult MemoryKeyValueStore::getRange(KeyRangeRef range, int limit, bool reverse) const { +RangeResult MemoryKeyValueStore::getRange(KeyRangeRef range, int limit, Reverse reverse) const { RangeResult results; if (!reverse) { std::map::const_iterator mapItr = store.lower_bound(range.begin); diff --git a/fdbserver/workloads/MemoryKeyValueStore.h b/fdbserver/workloads/MemoryKeyValueStore.h index bd8318c509..c90aa8f1a4 100644 --- a/fdbserver/workloads/MemoryKeyValueStore.h +++ b/fdbserver/workloads/MemoryKeyValueStore.h @@ -38,7 +38,7 @@ public: Key getKey(KeySelectorRef selector) const; // Gets a range of key-value pairs, returning a maximum of results - RangeResult getRange(KeyRangeRef range, int limit, bool reverse) const; + RangeResult getRange(KeyRangeRef range, int limit, Reverse reverse) const; // Stores a key-value pair in the database void set(KeyRef key, ValueRef value); diff --git a/fdbserver/workloads/MemoryLifetime.actor.cpp b/fdbserver/workloads/MemoryLifetime.actor.cpp index 47a2d7e65a..bd8c5685ca 100644 --- a/fdbserver/workloads/MemoryLifetime.actor.cpp +++ b/fdbserver/workloads/MemoryLifetime.actor.cpp @@ -70,30 +70,32 @@ struct MemoryLifetime : KVWorkload { ACTOR Future _start(Database cx, MemoryLifetime* self) { state double startTime = now(); state ReadYourWritesTransaction tr(cx); + state Reverse reverse = Reverse::FALSE; + state Snapshot snapshot = Snapshot::FALSE; loop { try { int op = deterministicRandom()->randomInt(0, 4); if (op == 0) { - state bool getRange_isReverse = deterministicRandom()->random01() < 0.5; + reverse.set(deterministicRandom()->coinflip()); state Key getRange_startKey = self->getRandomKey(); state KeyRange getRange_queryRange = - getRange_isReverse ? KeyRangeRef(normalKeys.begin, keyAfter(getRange_startKey)) - : KeyRangeRef(getRange_startKey, normalKeys.end); + reverse ? KeyRangeRef(normalKeys.begin, keyAfter(getRange_startKey)) + : KeyRangeRef(getRange_startKey, normalKeys.end); state bool getRange_randomStart = deterministicRandom()->random01(); state Value getRange_newValue = self->randomValue(); - state bool getRange_isSnapshot = deterministicRandom()->random01() < 0.5; + snapshot.set(deterministicRandom()->coinflip()); - //TraceEvent("MemoryLifetimeCheck").detail("IsReverse", getRange_isReverse).detail("StartKey", printable(getRange_startKey)).detail("RandomStart", getRange_randomStart).detail("NewValue", getRange_newValue.size()).detail("IsSnapshot", getRange_isSnapshot); + //TraceEvent("MemoryLifetimeCheck").detail("IsReverse", reverse).detail("StartKey", printable(getRange_startKey)).detail("RandomStart", getRange_randomStart).detail("NewValue", getRange_newValue.size()).detail("IsSnapshot", snapshot); if (getRange_randomStart) tr.set(getRange_startKey, getRange_newValue); - state RangeResult getRange_res1 = wait(tr.getRange( - getRange_queryRange, GetRangeLimits(4000), getRange_isSnapshot, getRange_isReverse)); + state RangeResult getRange_res1 = + wait(tr.getRange(getRange_queryRange, GetRangeLimits(4000), snapshot, reverse)); tr = ReadYourWritesTransaction(cx); wait(delay(0.01)); if (getRange_randomStart) tr.set(getRange_startKey, getRange_newValue); - RangeResult getRange_res2 = wait(tr.getRange( - getRange_queryRange, GetRangeLimits(4000), getRange_isSnapshot, getRange_isReverse)); + RangeResult getRange_res2 = + wait(tr.getRange(getRange_queryRange, GetRangeLimits(4000), snapshot, reverse)); ASSERT(getRange_res1.size() == getRange_res2.size()); for (int i = 0; i < getRange_res1.size(); i++) { if (getRange_res1[i].key != getRange_res2[i].key) { @@ -121,31 +123,31 @@ struct MemoryLifetime : KVWorkload { state Key get_startKey = self->getRandomKey(); state bool get_randomStart = deterministicRandom()->random01(); state Value get_newValue = self->randomValue(); - state bool get_isSnapshot = deterministicRandom()->random01() < 0.5; + snapshot.set(deterministicRandom()->coinflip()); if (get_randomStart) tr.set(get_startKey, get_newValue); - state Optional get_res1 = wait(tr.get(get_startKey, get_isSnapshot)); + state Optional get_res1 = wait(tr.get(get_startKey, snapshot)); tr = ReadYourWritesTransaction(cx); wait(delay(0.01)); if (get_randomStart) tr.set(get_startKey, get_newValue); - Optional get_res2 = wait(tr.get(get_startKey, get_isSnapshot)); + Optional get_res2 = wait(tr.get(get_startKey, snapshot)); ASSERT(get_res1 == get_res2); } else if (op == 2) { state KeySelector getKey_selector = self->getRandomKeySelector(); state bool getKey_randomStart = deterministicRandom()->random01(); state Value getKey_newValue = self->randomValue(); - state bool getKey_isSnapshot = deterministicRandom()->random01() < 0.5; + snapshot.set(deterministicRandom()->coinflip()); if (getKey_randomStart) tr.set(getKey_selector.getKey(), getKey_newValue); - state Key getKey_res1 = wait(tr.getKey(getKey_selector, getKey_isSnapshot)); + state Key getKey_res1 = wait(tr.getKey(getKey_selector, snapshot)); tr = ReadYourWritesTransaction(cx); wait(delay(0.01)); if (getKey_randomStart) tr.set(getKey_selector.getKey(), getKey_newValue); - Key getKey_res2 = wait(tr.getKey(getKey_selector, getKey_isSnapshot)); + Key getKey_res2 = wait(tr.getKey(getKey_selector, snapshot)); ASSERT(getKey_res1 == getKey_res2); } else if (op == 3) { state Key getAddress_startKey = self->getRandomKey(); diff --git a/fdbserver/workloads/PopulateTPCC.actor.cpp b/fdbserver/workloads/PopulateTPCC.actor.cpp index ccaff977b9..40e7a2b283 100644 --- a/fdbserver/workloads/PopulateTPCC.actor.cpp +++ b/fdbserver/workloads/PopulateTPCC.actor.cpp @@ -174,7 +174,7 @@ struct PopulateTPCC : TestWorkload { item.i_data = self->dataString(item.arena); BinaryWriter w(IncludeVersion()); serializer(w, item); - tr.set(item.key(), w.toValue(), false); + tr.set(item.key(), w.toValue(), AddConflictRange::FALSE); } wait(tr.commit()); break; @@ -242,11 +242,11 @@ struct PopulateTPCC : TestWorkload { { BinaryWriter w(IncludeVersion()); serializer(w, c); - tr.set(c.key(), w.toValue(), false); + tr.set(c.key(), w.toValue(), AddConflictRange::FALSE); } { // Write index - tr.set(c.indexLastKey(), c.key(), false); + tr.set(c.indexLastKey(), c.key(), AddConflictRange::FALSE); } { BinaryWriter w(IncludeVersion()); @@ -255,7 +255,7 @@ struct PopulateTPCC : TestWorkload { BinaryWriter kW(Unversioned()); serializer(kW, k); auto key = kW.toValue().withPrefix(LiteralStringRef("History/")); - tr.set(key, w.toValue(), false); + tr.set(key, w.toValue(), AddConflictRange::FALSE); } } try { @@ -315,11 +315,11 @@ struct PopulateTPCC : TestWorkload { ol.ol_dist_info = self->aString(ol.arena, 24, 24); BinaryWriter w(IncludeVersion()); serializer(w, ol); - tr.set(ol.key(), w.toValue(), false); + tr.set(ol.key(), w.toValue(), AddConflictRange::FALSE); } BinaryWriter w(IncludeVersion()); serializer(w, o); - tr.set(o.key(), w.toValue(), false); + tr.set(o.key(), w.toValue(), AddConflictRange::FALSE); } try { wait(tr.commit()); @@ -346,7 +346,7 @@ struct PopulateTPCC : TestWorkload { no.no_w_id = w_id; BinaryWriter w(IncludeVersion()); serializer(w, no); - tr.set(no.key(), w.toValue(), false); + tr.set(no.key(), w.toValue(), AddConflictRange::FALSE); } try { wait(tr.commit()); @@ -381,7 +381,7 @@ struct PopulateTPCC : TestWorkload { d.d_next_o_id = 3000; BinaryWriter w(IncludeVersion()); serializer(w, d); - tr.set(d.key(), w.toValue(), false); + tr.set(d.key(), w.toValue(), AddConflictRange::FALSE); try { wait(tr.commit()); wait(populateCustomers(self, cx, w_id, d_id)); @@ -426,7 +426,7 @@ struct PopulateTPCC : TestWorkload { s.s_data = self->dataString(s.arena); BinaryWriter w(IncludeVersion()); serializer(w, s); - tr.set(s.key(), w.toValue(), false); + tr.set(s.key(), w.toValue(), AddConflictRange::FALSE); } try { wait(tr.commit()); @@ -458,7 +458,7 @@ struct PopulateTPCC : TestWorkload { w.w_ytd = 300000; BinaryWriter writer(IncludeVersion()); serializer(writer, w); - tr.set(w.key(), writer.toValue(), false); + tr.set(w.key(), writer.toValue(), AddConflictRange::FALSE); wait(tr.commit()); break; } catch (Error& e) { diff --git a/fdbserver/workloads/QueuePush.actor.cpp b/fdbserver/workloads/QueuePush.actor.cpp index 1649246140..8e8b9f564d 100644 --- a/fdbserver/workloads/QueuePush.actor.cpp +++ b/fdbserver/workloads/QueuePush.actor.cpp @@ -115,12 +115,12 @@ struct QueuePushWorkload : TestWorkload { state Key lastKey; if (self->forward) { - Key _lastKey = wait(tr.getKey(lastLessThan(self->endingKey), true)); + Key _lastKey = wait(tr.getKey(lastLessThan(self->endingKey), Snapshot::TRUE)); lastKey = _lastKey; if (lastKey == StringRef()) lastKey = self->startingKey; } else { - Key _lastKey = wait(tr.getKey(firstGreaterThan(self->startingKey), true)); + Key _lastKey = wait(tr.getKey(firstGreaterThan(self->startingKey), Snapshot::TRUE)); lastKey = _lastKey; if (!normalKeys.contains(lastKey)) lastKey = self->endingKey; diff --git a/fdbserver/workloads/RandomSelector.actor.cpp b/fdbserver/workloads/RandomSelector.actor.cpp index 584c63450c..8c16aa516c 100644 --- a/fdbserver/workloads/RandomSelector.actor.cpp +++ b/fdbserver/workloads/RandomSelector.actor.cpp @@ -103,7 +103,7 @@ struct RandomSelectorWorkload : TestWorkload { state int offsetB; state int randomLimit; state int randomByteLimit; - state bool reverse; + state Reverse reverse = Reverse::FALSE; state Error error; clientID = format("%08d", self->clientId); @@ -438,7 +438,7 @@ struct RandomSelectorWorkload : TestWorkload { randomLimit = deterministicRandom()->randomInt(0, 2 * self->maxOffset + self->maxKeySpace); randomByteLimit = deterministicRandom()->randomInt(0, (self->maxOffset + self->maxKeySpace) * 512); - reverse = deterministicRandom()->random01() > 0.5 ? false : true; + reverse.set(deterministicRandom()->coinflip()); //TraceEvent("RYOWgetRange").detail("KeyA", myKeyA).detail("KeyB", myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB).detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("RandomLimit",randomLimit).detail("RandomByteLimit", randomByteLimit).detail("Reverse", reverse); @@ -447,7 +447,7 @@ struct RandomSelectorWorkload : TestWorkload { wait(trRYOW.getRange(KeySelectorRef(StringRef(clientID + "b/" + myKeyA), onEqualA, offsetA), KeySelectorRef(StringRef(clientID + "b/" + myKeyB), onEqualB, offsetB), randomLimit, - false, + Snapshot::FALSE, reverse)); getRangeTest1 = getRangeTest; @@ -457,7 +457,7 @@ struct RandomSelectorWorkload : TestWorkload { tr.getRange(KeySelectorRef(StringRef(clientID + "d/" + myKeyA), onEqualA, offsetA), KeySelectorRef(StringRef(clientID + "d/" + myKeyB), onEqualB, offsetB), randomLimit, - false, + Snapshot::FALSE, reverse)); bool fail = false; diff --git a/fdbserver/workloads/RestoreBackup.actor.cpp b/fdbserver/workloads/RestoreBackup.actor.cpp index 67df2eab61..3240f7ed66 100644 --- a/fdbserver/workloads/RestoreBackup.actor.cpp +++ b/fdbserver/workloads/RestoreBackup.actor.cpp @@ -34,13 +34,13 @@ struct RestoreBackupWorkload final : TestWorkload { Standalone backupDir; Standalone tag; double delayFor; - bool stopWhenDone; + StopWhenDone stopWhenDone{ false }; RestoreBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/")); tag = getOption(options, LiteralStringRef("tag"), LiteralStringRef("default")); delayFor = getOption(options, LiteralStringRef("delayFor"), 10.0); - stopWhenDone = getOption(options, LiteralStringRef("stopWhenDone"), false); + stopWhenDone.set(getOption(options, LiteralStringRef("stopWhenDone"), false)); } static constexpr const char* DESCRIPTION = "RestoreBackup"; @@ -110,8 +110,13 @@ struct RestoreBackupWorkload final : TestWorkload { wait(delay(self->delayFor)); wait(waitOnBackup(self, cx)); wait(clearDatabase(cx)); - wait(success(self->backupAgent.restore( - cx, cx, self->tag, Key(self->backupContainer->getURL()), true, ::invalidVersion, true))); + wait(success(self->backupAgent.restore(cx, + cx, + self->tag, + Key(self->backupContainer->getURL()), + WaitForComplete::TRUE, + ::invalidVersion, + Verbose::TRUE))); return Void(); } diff --git a/fdbserver/workloads/RestoreFromBlob.actor.cpp b/fdbserver/workloads/RestoreFromBlob.actor.cpp index 57c18e7c07..01ebbede76 100644 --- a/fdbserver/workloads/RestoreFromBlob.actor.cpp +++ b/fdbserver/workloads/RestoreFromBlob.actor.cpp @@ -30,7 +30,7 @@ struct RestoreFromBlobWorkload : TestWorkload { double restoreAfter; Key backupTag; Standalone backupURL; - bool waitForComplete; + WaitForComplete waitForComplete{ false }; static constexpr const char* DESCRIPTION = "RestoreFromBlob"; @@ -44,7 +44,7 @@ struct RestoreFromBlobWorkload : TestWorkload { auto secretKeyEnvVar = getOption(options, LiteralStringRef("secretKeyVar"), LiteralStringRef("BLOB_SECRET_KEY")).toString(); bool provideKeys = getOption(options, LiteralStringRef("provideKeys"), false); - waitForComplete = getOption(options, LiteralStringRef("waitForComplete"), true); + waitForComplete.set(getOption(options, LiteralStringRef("waitForComplete"), true)); if (provideKeys) { updateBackupURL(backupURLString, accessKeyEnvVar, "", secretKeyEnvVar, ""); } diff --git a/fdbserver/workloads/RyowCorrectness.actor.cpp b/fdbserver/workloads/RyowCorrectness.actor.cpp index fc0e0fad59..817422dbf1 100644 --- a/fdbserver/workloads/RyowCorrectness.actor.cpp +++ b/fdbserver/workloads/RyowCorrectness.actor.cpp @@ -49,7 +49,7 @@ struct Operation { Value value; int limit; - bool reverse = false; + Reverse reverse{ Reverse::FALSE }; }; // A workload which executes random sequences of operations on RYOW transactions and confirms the results @@ -112,7 +112,7 @@ struct RyowCorrectnessWorkload : ApiWorkload { info.beginKey = selectRandomKey(data, .8); info.endKey = selectRandomKey(data, .8); info.limit = deterministicRandom()->randomInt(0, 1000); - info.reverse = (bool)deterministicRandom()->randomInt(0, 2); + info.reverse.set(deterministicRandom()->coinflip()); if (info.beginKey > info.endKey) std::swap(info.beginKey, info.endKey); @@ -123,7 +123,7 @@ struct RyowCorrectnessWorkload : ApiWorkload { info.beginSelector = generateKeySelector(data, 1000); info.endSelector = generateKeySelector(data, 1000); info.limit = deterministicRandom()->randomInt(0, 1000); - info.reverse = (bool)deterministicRandom()->randomInt(0, 2); + info.reverse.set(deterministicRandom()->coinflip()); break; } diff --git a/fdbserver/workloads/SelectorCorrectness.actor.cpp b/fdbserver/workloads/SelectorCorrectness.actor.cpp index 2faecd35b4..a848f50bb9 100644 --- a/fdbserver/workloads/SelectorCorrectness.actor.cpp +++ b/fdbserver/workloads/SelectorCorrectness.actor.cpp @@ -109,7 +109,7 @@ struct SelectorCorrectnessWorkload : TestWorkload { state int offsetA; state int offsetB; state Standalone maxKey; - state bool reverse; + state Reverse reverse = Reverse::FALSE; maxKey = Standalone(format("%010d", self->maxKeySpace + 1)); @@ -166,7 +166,7 @@ struct SelectorCorrectnessWorkload : TestWorkload { onEqualB = deterministicRandom()->randomInt(0, 2) != 0; offsetA = 1; //-1*deterministicRandom()->randomInt( 0, self->maxOffset ); offsetB = deterministicRandom()->randomInt(1, self->maxOffset); - reverse = deterministicRandom()->random01() > 0.5 ? false : true; + reverse.set(deterministicRandom()->coinflip()); //TraceEvent("RYOWgetRange").detail("KeyA", myKeyA).detail("KeyB", myKeyB).detail("OnEqualA",onEqualA).detail("OnEqualB",onEqualB).detail("OffsetA",offsetA).detail("OffsetB",offsetB).detail("Direction",direction); state int expectedSize = @@ -180,7 +180,7 @@ struct SelectorCorrectnessWorkload : TestWorkload { wait(trRYOW.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA), KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB), 2 * (self->maxKeySpace + self->maxOffset), - false, + Snapshot::FALSE, reverse)); int trueSize = 0; @@ -208,7 +208,7 @@ struct SelectorCorrectnessWorkload : TestWorkload { wait(tr.getRange(KeySelectorRef(StringRef(myKeyA), onEqualA, offsetA), KeySelectorRef(StringRef(myKeyB), onEqualB, offsetB), 2 * (self->maxKeySpace + self->maxOffset), - false, + Snapshot::FALSE, reverse)); int trueSize = 0; diff --git a/fdbserver/workloads/Serializability.actor.cpp b/fdbserver/workloads/Serializability.actor.cpp index 32444f2092..d774fdcf10 100644 --- a/fdbserver/workloads/Serializability.actor.cpp +++ b/fdbserver/workloads/Serializability.actor.cpp @@ -40,18 +40,18 @@ struct SerializabilityWorkload : TestWorkload { KeySelector begin; KeySelector end; int limit; - bool snapshot; - bool reverse; + Snapshot snapshot{ Snapshot::FALSE }; + Reverse reverse{ Reverse::FALSE }; }; struct GetKeyOperation { KeySelector key; - bool snapshot; + Snapshot snapshot{ Snapshot::FALSE }; }; struct GetOperation { Key key; - bool snapshot; + Snapshot snapshot{ Snapshot::FALSE }; }; struct TransactionOperation { @@ -138,20 +138,20 @@ struct SerializabilityWorkload : TestWorkload { if (operationType == 0) { GetKeyOperation getKey; getKey.key = getRandomKeySelector(); - getKey.snapshot = deterministicRandom()->random01() < 0.5; + getKey.snapshot.set(deterministicRandom()->coinflip()); op.getKeyOp = getKey; } else if (operationType == 1) { GetRangeOperation getRange; getRange.begin = getRandomKeySelector(); getRange.end = getRandomKeySelector(); getRange.limit = deterministicRandom()->randomInt(0, 1 << deterministicRandom()->randomInt(1, 10)); - getRange.reverse = deterministicRandom()->random01() < 0.5; - getRange.snapshot = deterministicRandom()->random01() < 0.5; + getRange.reverse.set(deterministicRandom()->coinflip()); + getRange.snapshot.set(deterministicRandom()->coinflip()); op.getRangeOp = getRange; } else if (operationType == 2) { GetOperation getOp; getOp.key = getRandomKey(); - getOp.snapshot = deterministicRandom()->random01() < 0.5; + getOp.snapshot.set(deterministicRandom()->coinflip()); op.getOp = getOp; } else if (operationType == 3) { KeyRange range = getRandomRange(maxClearSize); diff --git a/fdbserver/workloads/SnapTest.actor.cpp b/fdbserver/workloads/SnapTest.actor.cpp index 050162cabd..ce779d2c76 100644 --- a/fdbserver/workloads/SnapTest.actor.cpp +++ b/fdbserver/workloads/SnapTest.actor.cpp @@ -183,7 +183,7 @@ public: // workload functions Key key1Ref(Key1); std::string Val1 = std::to_string(id); Value val1Ref(Val1); - tr.set(key1Ref, val1Ref, false); + tr.set(key1Ref, val1Ref, AddConflictRange::FALSE); } wait(tr.commit()); break; diff --git a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp index c531037470..151313d249 100644 --- a/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp +++ b/fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp @@ -126,13 +126,14 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { ACTOR Future getRangeCallActor(Database cx, SpecialKeySpaceCorrectnessWorkload* self) { state double lastTime = now(); + state Reverse reverse = Reverse::FALSE; loop { wait(poisson(&lastTime, 1.0 / self->transactionsPerSecond)); - state bool reverse = deterministicRandom()->coinflip(); + reverse.set(deterministicRandom()->coinflip()); state GetRangeLimits limit = self->randomLimits(); state KeySelector begin = self->randomKeySelector(); state KeySelector end = self->randomKeySelector(); - auto correctResultFuture = self->ryw->getRange(begin, end, limit, false, reverse); + auto correctResultFuture = self->ryw->getRange(begin, end, limit, Snapshot::FALSE, reverse); ASSERT(correctResultFuture.isReady()); auto correctResult = correctResultFuture.getValue(); auto testResultFuture = cx->specialKeySpace->getRange(self->ryw.getPtr(), begin, end, limit, reverse); @@ -173,7 +174,7 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { self->ryw->clear(rkr); } // use the same key selectors again to test consistency of ryw - auto correctRywResultFuture = self->ryw->getRange(begin, end, limit, false, reverse); + auto correctRywResultFuture = self->ryw->getRange(begin, end, limit, Snapshot::FALSE, reverse); ASSERT(correctRywResultFuture.isReady()); auto correctRywResult = correctRywResultFuture.getValue(); auto testRywResultFuture = cx->specialKeySpace->getRange(self->ryw.getPtr(), begin, end, limit, reverse); @@ -547,13 +548,13 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { if (begin.getKey() < end.getKey()) break; } - bool reverse = deterministicRandom()->coinflip(); + Reverse reverse{ deterministicRandom()->coinflip() }; - auto correctResultFuture = referenceTx->getRange(begin, end, limit, false, reverse); + auto correctResultFuture = referenceTx->getRange(begin, end, limit, Snapshot::FALSE, reverse); ASSERT(correctResultFuture.isReady()); begin.setKey(begin.getKey().withPrefix(prefix, begin.arena())); end.setKey(end.getKey().withPrefix(prefix, begin.arena())); - auto testResultFuture = tx->getRange(begin, end, limit, false, reverse); + auto testResultFuture = tx->getRange(begin, end, limit, Snapshot::FALSE, reverse); ASSERT(testResultFuture.isReady()); auto correct_iter = correctResultFuture.get().begin(); auto test_iter = testResultFuture.get().begin(); @@ -631,10 +632,10 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload { { tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); for (const std::string& option : SpecialKeySpace::getManagementApiOptionsSet()) { - tx->set(LiteralStringRef("options/") - .withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin) - .withSuffix(option), - ValueRef()); + tx->set( + "options/"_sr.withPrefix(SpecialKeySpace::getModuleRange(SpecialKeySpace::MODULE::MANAGEMENT).begin) + .withSuffix(option), + ValueRef()); } RangeResult result = wait(tx->getRange( KeyRangeRef(LiteralStringRef("options/"), LiteralStringRef("options0")) diff --git a/fdbserver/workloads/SubmitBackup.actor.cpp b/fdbserver/workloads/SubmitBackup.actor.cpp index 6dbc58abf8..8a45431b44 100644 --- a/fdbserver/workloads/SubmitBackup.actor.cpp +++ b/fdbserver/workloads/SubmitBackup.actor.cpp @@ -35,8 +35,8 @@ struct SubmitBackupWorkload final : TestWorkload { double delayFor; int initSnapshotInterval; int snapshotInterval; - bool stopWhenDone; - bool incremental; + StopWhenDone stopWhenDone{ false }; + IncrementalBackupOnly incremental{ false }; SubmitBackupWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { backupDir = getOption(options, LiteralStringRef("backupDir"), LiteralStringRef("file://simfdb/backups/")); @@ -44,8 +44,8 @@ struct SubmitBackupWorkload final : TestWorkload { delayFor = getOption(options, LiteralStringRef("delayFor"), 10.0); initSnapshotInterval = getOption(options, LiteralStringRef("initSnapshotInterval"), 0); snapshotInterval = getOption(options, LiteralStringRef("snapshotInterval"), 1e8); - stopWhenDone = getOption(options, LiteralStringRef("stopWhenDone"), true); - incremental = getOption(options, LiteralStringRef("incremental"), false); + stopWhenDone.set(getOption(options, LiteralStringRef("stopWhenDone"), true)); + incremental.set(getOption(options, LiteralStringRef("incremental"), false)); } static constexpr const char* DESCRIPTION = "SubmitBackup"; @@ -62,7 +62,7 @@ struct SubmitBackupWorkload final : TestWorkload { self->tag.toString(), backupRanges, self->stopWhenDone, - false, + UsePartitionedLog::FALSE, self->incremental)); } catch (Error& e) { TraceEvent("BackupSubmitError").error(e); diff --git a/fdbserver/workloads/TPCC.actor.cpp b/fdbserver/workloads/TPCC.actor.cpp index d8e40c2266..80ad1adcdd 100644 --- a/fdbserver/workloads/TPCC.actor.cpp +++ b/fdbserver/workloads/TPCC.actor.cpp @@ -468,7 +468,7 @@ struct TPCC : TestWorkload { order.o_w_id = customer.c_w_id; order.o_d_id = customer.c_d_id; order.o_c_id = customer.c_id; - RangeResult range = wait(tr.getRange(order.keyRange(1), 1, false, true)); + RangeResult range = wait(tr.getRange(order.keyRange(1), 1, Snapshot::FALSE, Reverse::TRUE)); ASSERT(range.size() > 0); { BinaryReader r(range[0].value, IncludeVersion()); diff --git a/fdbserver/workloads/ThreadSafety.actor.cpp b/fdbserver/workloads/ThreadSafety.actor.cpp index a3dee7b77d..a04f9293c0 100644 --- a/fdbserver/workloads/ThreadSafety.actor.cpp +++ b/fdbserver/workloads/ThreadSafety.actor.cpp @@ -100,8 +100,6 @@ private: } }; -extern bool noUnseed; - // A workload which uses the thread safe API from multiple threads struct ThreadSafetyWorkload : TestWorkload { int threadsPerClient; diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 16dca1e64f..482a89ff13 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -28,6 +28,9 @@ void forceLinkFlowTests(); void forceLinkVersionedMapTests(); void forceLinkMemcpyTests(); void forceLinkMemcpyPerfTests(); +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) +void forceLinkStreamCipherTests(); +#endif void forceLinkParallelStreamTests(); void forceLinkSimExternalConnectionTests(); void forceLinkIThreadPoolTests(); @@ -37,6 +40,7 @@ struct UnitTestWorkload : TestWorkload { std::string testPattern; int testRunLimit; UnitTestParameters testParams; + bool cleanupAfterTests; PerfIntCounter testsAvailable, testsExecuted, testsFailed; PerfDoubleCounter totalWallTime, totalSimTime; @@ -46,9 +50,14 @@ struct UnitTestWorkload : TestWorkload { testsFailed("Test Cases Failed"), totalWallTime("Total wall clock time (s)"), totalSimTime("Total flow time (s)") { enabled = !clientId; // only do this on the "first" client - testPattern = getOption(options, LiteralStringRef("testsMatching"), Value()).toString(); - testRunLimit = getOption(options, LiteralStringRef("maxTestCases"), -1); - testParams.setDataDir(getOption(options, LiteralStringRef("dataDir"), "simfdb/unittests/"_sr).toString()); + testPattern = getOption(options, "testsMatching"_sr, Value()).toString(); + testRunLimit = getOption(options, "maxTestCases"_sr, -1); + if (g_network->isSimulated()) { + testParams.setDataDir(getOption(options, "dataDir"_sr, "simfdb/unittests/"_sr).toString()); + } else { + testParams.setDataDir(getOption(options, "dataDir"_sr, "/private/tmp/"_sr).toString()); + } + cleanupAfterTests = getOption(options, "cleanupAfterTests"_sr, true); // Consume all remaining options as testParams which the unit test can access for (auto& kv : options) { @@ -63,6 +72,9 @@ struct UnitTestWorkload : TestWorkload { forceLinkVersionedMapTests(); forceLinkMemcpyTests(); forceLinkMemcpyPerfTests(); +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) + forceLinkStreamCipherTests(); +#endif forceLinkParallelStreamTests(); forceLinkSimExternalConnectionTests(); forceLinkIThreadPoolTests(); @@ -117,7 +129,9 @@ struct UnitTestWorkload : TestWorkload { ++self->testsFailed; result = e; } - platform::eraseDirectoryRecursive(self->testParams.getDataDir()); + if (self->cleanupAfterTests) { + platform::eraseDirectoryRecursive(self->testParams.getDataDir()); + } ++self->testsExecuted; double wallTime = timer() - start_timer; double simTime = now() - start_now; diff --git a/fdbserver/workloads/Unreadable.actor.cpp b/fdbserver/workloads/Unreadable.actor.cpp index b1261b9c84..420d5e1e38 100644 --- a/fdbserver/workloads/Unreadable.actor.cpp +++ b/fdbserver/workloads/Unreadable.actor.cpp @@ -132,7 +132,7 @@ struct UnreadableWorkload : TestWorkload { KeySelectorRef const& _end, bool isUnreadable, int limit, - bool reverse) { + Reverse reverse) { /* for (auto it : setMap) { @@ -296,6 +296,8 @@ struct UnreadableWorkload : TestWorkload { ACTOR Future _start(Database cx, UnreadableWorkload* self) { state int testCount = 0; + state Reverse reverse = Reverse::FALSE; + state Snapshot snapshot = Snapshot::FALSE; for (; testCount < 100; testCount++) { //TraceEvent("RYWT_Start").detail("TestCount", testCount); state ReadYourWritesTransaction tr(cx); @@ -308,9 +310,7 @@ struct UnreadableWorkload : TestWorkload { state KeyRangeRef range; state KeyRef key; state ValueRef value; - state bool reverse; state int limit; - state bool snapshot; state KeySelectorRef begin; state KeySelectorRef end; state bool bypassUnreadable = deterministicRandom()->coinflip(); @@ -358,8 +358,8 @@ struct UnreadableWorkload : TestWorkload { //TraceEvent("RYWT_SetVersionstampKey").detail("Range", printable(range)); } else if (r == 16) { range = RandomTestImpl::getRandomRange(arena); - snapshot = deterministicRandom()->random01() < 0.05; - reverse = deterministicRandom()->random01() < 0.5; + snapshot.set(deterministicRandom()->random01() < 0.05); + reverse.set(deterministicRandom()->coinflip()); if (snapshot) tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE); @@ -393,8 +393,8 @@ struct UnreadableWorkload : TestWorkload { begin = RandomTestImpl::getRandomKeySelector(arena); end = RandomTestImpl::getRandomKeySelector(arena); limit = deterministicRandom()->randomInt(1, 100); // maximum number of results to return from the db - snapshot = deterministicRandom()->random01() < 0.05; - reverse = deterministicRandom()->random01() < 0.5; + snapshot.set(deterministicRandom()->random01() < 0.05); + reverse.set(deterministicRandom()->coinflip()); if (snapshot) tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE); @@ -438,7 +438,7 @@ struct UnreadableWorkload : TestWorkload { } } else if (r == 18) { key = RandomTestImpl::getRandomKey(arena); - snapshot = deterministicRandom()->random01() < 0.05; + snapshot.set(deterministicRandom()->random01() < 0.05); if (snapshot) tr.setOption(FDBTransactionOptions::SNAPSHOT_RYW_DISABLE); diff --git a/fdbserver/workloads/WriteBandwidth.actor.cpp b/fdbserver/workloads/WriteBandwidth.actor.cpp index eaf177d8f0..a5f9f13661 100644 --- a/fdbserver/workloads/WriteBandwidth.actor.cpp +++ b/fdbserver/workloads/WriteBandwidth.actor.cpp @@ -122,7 +122,7 @@ struct WriteBandwidthWorkload : KVWorkload { keyAfter(self->keyForIndex(startIdx + self->keysPerTransaction - 1, false)))); for (int i = 0; i < self->keysPerTransaction; i++) - tr.set(self->keyForIndex(startIdx + i, false), self->randomValue(), false); + tr.set(self->keyForIndex(startIdx + i, false), self->randomValue(), AddConflictRange::FALSE); start = now(); wait(tr.commit()); diff --git a/fdbserver/workloads/WriteDuringRead.actor.cpp b/fdbserver/workloads/WriteDuringRead.actor.cpp index 63a5b2eb78..c938989d49 100644 --- a/fdbserver/workloads/WriteDuringRead.actor.cpp +++ b/fdbserver/workloads/WriteDuringRead.actor.cpp @@ -148,7 +148,7 @@ struct WriteDuringReadWorkload : TestWorkload { ACTOR Future getKeyAndCompare(ReadYourWritesTransaction* tr, KeySelector key, - bool snapshot, + Snapshot snapshot, bool readYourWritesDisabled, bool snapshotRYWDisabled, WriteDuringReadWorkload* self, @@ -193,7 +193,7 @@ struct WriteDuringReadWorkload : TestWorkload { KeySelector begin, KeySelector end, GetRangeLimits limit, - bool reverse) { + Reverse reverse) { Key beginKey = memoryGetKey(db, begin); Key endKey = memoryGetKey(db, end); //TraceEvent("WDRGetRange").detail("Begin", printable(beginKey)).detail("End", printable(endKey)); @@ -223,8 +223,8 @@ struct WriteDuringReadWorkload : TestWorkload { KeySelector begin, KeySelector end, GetRangeLimits limit, - bool snapshot, - bool reverse, + Snapshot snapshot, + Reverse reverse, bool readYourWritesDisabled, bool snapshotRYWDisabled, WriteDuringReadWorkload* self, @@ -390,7 +390,7 @@ struct WriteDuringReadWorkload : TestWorkload { ACTOR Future getAndCompare(ReadYourWritesTransaction* tr, Key key, - bool snapshot, + Snapshot snapshot, bool readYourWritesDisabled, bool snapshotRYWDisabled, WriteDuringReadWorkload* self, @@ -798,7 +798,7 @@ ACTOR Future randomTransaction(Database cx, WriteDuringReadWorkload* self, if (operationType == 0 && !disableGetKey) { operations.push_back(self->getKeyAndCompare(&tr, self->getRandomKeySelector(), - deterministicRandom()->random01() < 0.5, + Snapshot{ deterministicRandom()->coinflip() }, readYourWritesDisabled, snapshotRYWDisabled, self, @@ -809,8 +809,8 @@ ACTOR Future randomTransaction(Database cx, WriteDuringReadWorkload* self, self->getRandomKeySelector(), self->getRandomKeySelector(), self->getRandomLimits(), - deterministicRandom()->random01() < 0.5, - deterministicRandom()->random01() < 0.5, + Snapshot{ deterministicRandom()->coinflip() }, + Reverse{ deterministicRandom()->coinflip() }, readYourWritesDisabled, snapshotRYWDisabled, self, @@ -819,7 +819,7 @@ ACTOR Future randomTransaction(Database cx, WriteDuringReadWorkload* self, } else if (operationType == 2 && !disableGet) { operations.push_back(self->getAndCompare(&tr, self->getRandomKey(), - deterministicRandom()->random01() > 0.5, + Snapshot{ deterministicRandom()->coinflip() }, readYourWritesDisabled, snapshotRYWDisabled, self, diff --git a/fdbserver/workloads/workloads.actor.h b/fdbserver/workloads/workloads.actor.h index fa3fb62571..d47b981409 100644 --- a/fdbserver/workloads/workloads.actor.h +++ b/fdbserver/workloads/workloads.actor.h @@ -43,6 +43,7 @@ bool getOption(VectorRef options, Key key, bool defaultValue); vector getOption(VectorRef options, Key key, vector defaultValue); // comma-separated strings +bool hasOption(VectorRef options, Key key); struct WorkloadContext { Standalone> options; @@ -53,9 +54,7 @@ struct WorkloadContext { WorkloadContext(); WorkloadContext(const WorkloadContext&); ~WorkloadContext(); - -private: - void operator=(const WorkloadContext&); + WorkloadContext& operator=(const WorkloadContext&) = delete; }; struct TestWorkload : NonCopyable, WorkloadContext { diff --git a/flow/Arena.h b/flow/Arena.h index 2e1cc36801..d752baedf0 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -237,12 +237,12 @@ public: } template - Optional map(std::function f) const { - if (present()) { - return Optional(f(get())); - } else { - return Optional(); - } + Optional map(std::function f) const& { + return present() ? Optional(f(get())) : Optional(); + } + template + Optional map(std::function f) && { + return present() ? Optional(f(std::move(*this).get())) : Optional(); } bool present() const { return impl.has_value(); } @@ -258,7 +258,14 @@ public: UNSTOPPABLE_ASSERT(impl.has_value()); return std::move(impl.value()); } - T orDefault(T const& default_value) const { return impl.value_or(default_value); } + template + T orDefault(U&& defaultValue) const& { + return impl.value_or(std::forward(defaultValue)); + } + template + T orDefault(U&& defaultValue) && { + return std::move(impl).value_or(std::forward(defaultValue)); + } // Spaceship operator. Treats not-present as less-than present. int compare(Optional const& rhs) const { diff --git a/flow/BooleanParam.h b/flow/BooleanParam.h new file mode 100644 index 0000000000..b0b7ffee54 --- /dev/null +++ b/flow/BooleanParam.h @@ -0,0 +1,46 @@ +/* + * Arena.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "flow/Trace.h" + +#define FDB_DECLARE_BOOLEAN_PARAM(ParamName) \ + class ParamName { \ + bool value; \ + \ + public: \ + explicit constexpr ParamName(bool value) : value(value) {} \ + constexpr operator bool() const { return value; } \ + static ParamName const TRUE, FALSE; \ + constexpr void set(bool value) { this->value = value; } \ + }; \ + template <> \ + struct Traceable : std::true_type { \ + static std::string toString(ParamName const& value) { return Traceable::toString(value); } \ + } + +#define FDB_DEFINE_BOOLEAN_PARAM(ParamName) \ + ParamName const ParamName::TRUE = ParamName(true); \ + ParamName const ParamName::FALSE = ParamName(false) + +#define FDB_BOOLEAN_PARAM(ParamName) \ + FDB_DECLARE_BOOLEAN_PARAM(ParamName); \ + FDB_DEFINE_BOOLEAN_PARAM(ParamName) diff --git a/flow/CMakeLists.txt b/flow/CMakeLists.txt index 543425b1f5..bc8763f35f 100644 --- a/flow/CMakeLists.txt +++ b/flow/CMakeLists.txt @@ -6,6 +6,7 @@ set(FLOW_SRCS Arena.cpp Arena.h AsioReactor.h + BooleanParam.h CompressedInt.actor.cpp CompressedInt.h Deque.cpp @@ -96,6 +97,13 @@ set(FLOW_SRCS xxhash.c xxhash.h) +if(WITH_TLS AND NOT WIN32) + set(FLOW_SRCS + ${FLOW_SRCS} + StreamCipher.cpp + StreamCipher.h) +endif() + add_library(stacktrace stacktrace.amalgamation.cpp stacktrace.h) if (USE_ASAN) target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER) @@ -133,8 +141,7 @@ target_link_libraries(flow PRIVATE ${FLOW_LIBS}) if(USE_VALGRIND) target_link_libraries(flow PUBLIC Valgrind) endif() -# TODO(atn34) Re-enable TLS for OPEN_FOR_IDE build once #2201 is resolved -if(NOT WITH_TLS OR OPEN_FOR_IDE) +if(NOT WITH_TLS) target_compile_definitions(flow PUBLIC TLS_DISABLED) else() target_link_libraries(flow PUBLIC OpenSSL::SSL) diff --git a/flow/IThreadPool.cpp b/flow/IThreadPool.cpp index 888f848a02..256465f689 100644 --- a/flow/IThreadPool.cpp +++ b/flow/IThreadPool.cpp @@ -109,7 +109,7 @@ public: } void addThread(IThreadPoolReceiver* userData, const char* name) override { threads.push_back(new Thread(this, userData)); - threads.back()->handle = startThread(start, threads.back(), stackSize, name); + threads.back()->handle = g_network->startThread(start, threads.back(), stackSize, name); } void post(PThreadAction action) override { ios.post(ActionWrapper(action)); } }; diff --git a/flow/IThreadPoolTest.actor.cpp b/flow/IThreadPoolTest.actor.cpp index c49f5c00f1..a8d04b2e48 100644 --- a/flow/IThreadPoolTest.actor.cpp +++ b/flow/IThreadPoolTest.actor.cpp @@ -35,7 +35,9 @@ struct ThreadNameReceiver : IThreadPoolReceiver { } }; -TEST_CASE("noSim/IThreadPool/NamedThread") { +TEST_CASE("/flow/IThreadPool/NamedThread") { + noUnseed = true; + state Reference pool = createGenericThreadPool(); pool->addThread(new ThreadNameReceiver(), "thread-foo"); diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 7ceeb95801..a466377842 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -18,24 +18,26 @@ * limitations under the License. */ -#include "flow/Knobs.h" #include "flow/flow.h" +#include "flow/Knobs.h" +#include "flow/BooleanParam.h" #include #include +FDB_BOOLEAN_PARAM(IsSimulated); +FDB_BOOLEAN_PARAM(Randomize); + FlowKnobs::FlowKnobs(Randomize randomize, IsSimulated isSimulated) { initialize(randomize, isSimulated); } -FlowKnobs bootstrapGlobalFlowKnobs(Randomize::NO, IsSimulated::NO); +FlowKnobs bootstrapGlobalFlowKnobs(Randomize::FALSE, IsSimulated::FALSE); FlowKnobs const* FLOW_KNOBS = &bootstrapGlobalFlowKnobs; #define init(knob, value) initKnob(knob, value, #knob) // clang-format off -void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) { - bool const randomize = _randomize == Randomize::YES; - bool const isSimulated = _isSimulated == IsSimulated::YES; +void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { init( AUTOMATIC_TRACE_DUMP, 1 ); init( PREVENT_FAST_SPIN_DELAY, .01 ); init( CACHE_REFRESH_INTERVAL_WHEN_ALL_ALTERNATIVES_FAILED, 1.0 ); @@ -129,6 +131,10 @@ void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) { init( EIO_MAX_PARALLELISM, 4 ); init( EIO_USE_ODIRECT, 0 ); + //AsyncFileEncrypted + init( ENCRYPTION_BLOCK_SIZE, 4096 ); + init( MAX_DECRYPTED_BLOCKS, 10 ); + //AsyncFileKAIO init( MAX_OUTSTANDING, 64 ); init( MIN_SUBMIT, 10 ); @@ -239,6 +245,7 @@ void FlowKnobs::initialize(Randomize _randomize, IsSimulated _isSimulated) { init( LOAD_BALANCE_TSS_TIMEOUT, 5.0 ); init( LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS, true ); if( randomize && BUGGIFY ) LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS = false; // Whether the client should validate the SS teams all agree on TSS mismatch init( LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL, false ); if( randomize && BUGGIFY ) LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL = true; // If true, saves the full details of the mismatch in a trace event. If false, saves them in the DB and the trace event references the DB row. + init( TSS_LARGE_TRACE_SIZE, 50000 ); // Health Monitor init( FAILURE_DETECTION_DELAY, 4.0 ); if( randomize && BUGGIFY ) FAILURE_DETECTION_DELAY = 1.0; diff --git a/flow/Knobs.h b/flow/Knobs.h index ef4fdcf2af..2c7ce9cc56 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -18,8 +18,9 @@ * limitations under the License. */ -#ifndef FLOW_KNOBS_H -#define FLOW_KNOBS_H +#ifndef __FLOW_KNOBS_H__ +#define __FLOW_KNOBS_H__ + #pragma once #include "flow/Platform.h" @@ -36,10 +37,6 @@ struct NoKnobFound {}; using ParsedKnobValue = std::variant; -// To be used as effectively boolean parameters with added type safety -enum class IsSimulated { NO, YES }; -enum class Randomize { NO, YES }; - class Knobs { protected: Knobs() = default; @@ -168,6 +165,10 @@ public: int EIO_MAX_PARALLELISM; int EIO_USE_ODIRECT; + // AsyncFileEncrypted + int ENCRYPTION_BLOCK_SIZE; + int MAX_DECRYPTED_BLOCKS; + // AsyncFileKAIO int MAX_OUTSTANDING; int MIN_SUBMIT; @@ -280,14 +281,15 @@ public: double LOAD_BALANCE_TSS_TIMEOUT; bool LOAD_BALANCE_TSS_MISMATCH_VERIFY_SS; bool LOAD_BALANCE_TSS_MISMATCH_TRACE_FULL; + int TSS_LARGE_TRACE_SIZE; // Health Monitor int FAILURE_DETECTION_DELAY; bool HEALTH_MONITOR_MARK_FAILED_UNSTABLE_CONNECTIONS; int HEALTH_MONITOR_CLIENT_REQUEST_INTERVAL_SECS; int HEALTH_MONITOR_CONNECTION_MAX_CLOSED; - FlowKnobs(Randomize, IsSimulated); - void initialize(Randomize, IsSimulated); + FlowKnobs(class Randomize, class IsSimulated); + void initialize(class Randomize, class IsSimulated); }; // Flow knobs are needed before the knob collections are available, so a global FlowKnobs object is used to bootstrap diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index c3b35f1203..44572113d4 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -184,7 +184,7 @@ public: } bool isSimulated() const override { return false; } - THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg) override; + THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg, int stackSize, const char* name) override; void getDiskBytes(std::string const& directory, int64_t& free, int64_t& total) override; bool isAddressOnThisHost(NetworkAddress const& addr) const override; @@ -1513,7 +1513,7 @@ void Net2::run() { double newTaskBegin = timer_monotonic(); if (check_yield(TaskPriority::Max, tscNow)) { checkForSlowTask(tscBegin, tscNow, newTaskBegin - taskBegin, currentTaskID); - taskBegin = newTaskBegin; + taskBegin = newTaskBegin; FDB_TRACE_PROBE(run_loop_yield); ++countYields; break; @@ -1765,8 +1765,8 @@ void Net2::onMainThread(Promise&& signal, TaskPriority taskID) { } } -THREAD_HANDLE Net2::startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg) { - return ::startThread(func, arg); +THREAD_HANDLE Net2::startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg, int stackSize, const char* name) { + return ::startThread(func, arg, stackSize, name); } Future> Net2::connect(NetworkAddress toAddr, const std::string& host) { diff --git a/flow/Platform.actor.cpp b/flow/Platform.actor.cpp index 087acf6e86..bf19de35db 100644 --- a/flow/Platform.actor.cpp +++ b/flow/Platform.actor.cpp @@ -30,6 +30,9 @@ #include "flow/Platform.actor.h" #include "flow/Arena.h" +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) +#include "flow/StreamCipher.h" +#endif #include "flow/Trace.h" #include "flow/Error.h" @@ -3420,6 +3423,10 @@ void crashHandler(int sig) { bool error = (sig != SIGUSR2); +#if (!defined(TLS_DISABLED) && !defined(_WIN32)) + StreamCipher::cleanup(); +#endif + fflush(stdout); { TraceEvent te(error ? SevError : SevInfo, error ? "Crash" : "ProcessTerminated"); diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp new file mode 100644 index 0000000000..922054299b --- /dev/null +++ b/flow/StreamCipher.cpp @@ -0,0 +1,192 @@ +/* + * StreamCipher.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flow/StreamCipher.h" +#include "flow/UnitTest.h" + +std::unordered_set StreamCipher::ctxs; +std::unique_ptr StreamCipher::Key::globalKey; + +StreamCipher::StreamCipher() : ctx(EVP_CIPHER_CTX_new()) { + ctxs.insert(ctx); +} + +StreamCipher::~StreamCipher() { + EVP_CIPHER_CTX_free(ctx); + ctxs.erase(ctx); +} + +EVP_CIPHER_CTX* StreamCipher::getCtx() { + return ctx; +} + +void StreamCipher::cleanup() noexcept { + Key::cleanup(); + for (auto ctx : ctxs) { + EVP_CIPHER_CTX_free(ctx); + } +} + +void StreamCipher::Key::initializeKey(RawKeyType&& arr) { + if (globalKey) { + ASSERT(globalKey->arr == arr); + } + globalKey = std::make_unique(ConstructorTag{}); + globalKey->arr = std::move(arr); + memset(arr.data(), 0, arr.size()); +} + +void StreamCipher::Key::initializeRandomTestKey() { + ASSERT(g_network->isSimulated()); + if (globalKey) return; + globalKey = std::make_unique(ConstructorTag{}); + generateRandomData(globalKey->arr.data(), globalKey->arr.size()); +} + +const StreamCipher::Key& StreamCipher::Key::getKey() { + ASSERT(globalKey); + return *globalKey; +} + +StreamCipher::Key::Key(Key&& rhs) : arr(std::move(rhs.arr)) { + memset(arr.data(), 0, arr.size()); +} + +StreamCipher::Key& StreamCipher::Key::operator=(Key&& rhs) { + arr = std::move(rhs.arr); + memset(arr.data(), 0, arr.size()); + return *this; +} + +StreamCipher::Key::~Key() { + memset(arr.data(), 0, arr.size()); +} + +void StreamCipher::Key::cleanup() noexcept { + globalKey.reset(); +} + +EncryptionStreamCipher::EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) { + EVP_EncryptInit_ex(cipher.getCtx(), EVP_aes_128_gcm(), nullptr, nullptr, nullptr); + EVP_CIPHER_CTX_ctrl(cipher.getCtx(), EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); + EVP_EncryptInit_ex(cipher.getCtx(), nullptr, nullptr, key.data(), iv.data()); +} + +StringRef EncryptionStreamCipher::encrypt(unsigned char const* plaintext, int len, Arena& arena) { + auto ciphertext = new (arena) unsigned char[len + AES_BLOCK_SIZE]; + int bytes{ 0 }; + EVP_EncryptUpdate(cipher.getCtx(), ciphertext, &bytes, plaintext, len); + return StringRef(ciphertext, bytes); +} + +StringRef EncryptionStreamCipher::finish(Arena& arena) { + auto ciphertext = new (arena) unsigned char[AES_BLOCK_SIZE]; + int bytes{ 0 }; + EVP_EncryptFinal_ex(cipher.getCtx(), ciphertext, &bytes); + return StringRef(ciphertext, bytes); +} + +DecryptionStreamCipher::DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv) { + EVP_DecryptInit_ex(cipher.getCtx(), EVP_aes_128_gcm(), nullptr, nullptr, nullptr); + EVP_CIPHER_CTX_ctrl(cipher.getCtx(), EVP_CTRL_AEAD_SET_IVLEN, iv.size(), nullptr); + EVP_DecryptInit_ex(cipher.getCtx(), nullptr, nullptr, key.data(), iv.data()); +} + +StringRef DecryptionStreamCipher::decrypt(unsigned char const* ciphertext, int len, Arena& arena) { + auto plaintext = new (arena) unsigned char[len]; + int bytesDecrypted{ 0 }; + EVP_DecryptUpdate(cipher.getCtx(), plaintext, &bytesDecrypted, ciphertext, len); + int finalBlockBytes{ 0 }; + EVP_DecryptFinal_ex(cipher.getCtx(), plaintext + bytesDecrypted, &finalBlockBytes); + return StringRef(plaintext, bytesDecrypted + finalBlockBytes); +} + +StringRef DecryptionStreamCipher::finish(Arena& arena) { + auto plaintext = new (arena) unsigned char[AES_BLOCK_SIZE]; + int finalBlockBytes{ 0 }; + EVP_DecryptFinal_ex(cipher.getCtx(), plaintext, &finalBlockBytes); + return StringRef(plaintext, finalBlockBytes); +} + +// Only used to link unit tests +void forceLinkStreamCipherTests() {} + +// Tests both encryption and decryption of random data +// using the StreamCipher class +TEST_CASE("flow/StreamCipher") { + StreamCipher::Key::initializeRandomTestKey(); + const auto& key = StreamCipher::Key::getKey(); + + StreamCipher::IV iv; + generateRandomData(iv.data(), iv.size()); + + Arena arena; + std::vector plaintext(deterministicRandom()->randomInt(0, 10001)); + generateRandomData(&plaintext.front(), plaintext.size()); + std::vector ciphertext(plaintext.size() + AES_BLOCK_SIZE); + std::vector decryptedtext(plaintext.size() + AES_BLOCK_SIZE); + + TraceEvent("StreamCipherTestStart") + .detail("PlaintextSize", plaintext.size()) + .detail("AESBlockSize", AES_BLOCK_SIZE); + { + EncryptionStreamCipher encryptor(key, iv); + int index = 0; + int encryptedOffset = 0; + while (index < plaintext.size()) { + const auto chunkSize = std::min(deterministicRandom()->randomInt(1, 101), plaintext.size() - index); + const auto encrypted = encryptor.encrypt(&plaintext[index], chunkSize, arena); + TraceEvent("StreamCipherTestEcryptedChunk") + .detail("EncryptedSize", encrypted.size()) + .detail("EncryptedOffset", encryptedOffset) + .detail("Index", index); + std::copy(encrypted.begin(), encrypted.end(), &ciphertext[encryptedOffset]); + encryptedOffset += encrypted.size(); + index += chunkSize; + } + const auto encrypted = encryptor.finish(arena); + std::copy(encrypted.begin(), encrypted.end(), &ciphertext[encryptedOffset]); + ciphertext.resize(encryptedOffset + encrypted.size()); + } + + { + DecryptionStreamCipher decryptor(key, iv); + int index = 0; + int decryptedOffset = 0; + while (index < plaintext.size()) { + const auto chunkSize = std::min(deterministicRandom()->randomInt(1, 101), plaintext.size() - index); + const auto decrypted = decryptor.decrypt(&ciphertext[index], chunkSize, arena); + TraceEvent("StreamCipherTestDecryptedChunk") + .detail("DecryptedSize", decrypted.size()) + .detail("DecryptedOffset", decryptedOffset) + .detail("Index", index); + std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]); + decryptedOffset += decrypted.size(); + index += chunkSize; + } + const auto decrypted = decryptor.finish(arena); + std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]); + ASSERT_EQ(decryptedOffset + decrypted.size(), plaintext.size()); + decryptedtext.resize(decryptedOffset + decrypted.size()); + } + + ASSERT(plaintext == decryptedtext); + return Void(); +} diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h new file mode 100644 index 0000000000..57c2e0e436 --- /dev/null +++ b/flow/StreamCipher.h @@ -0,0 +1,80 @@ +/* + * StreamCipher.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/flow.h" + +#include +#include +#include +#include +#include + +// Wrapper class for openssl implementation of AES-128-GCM +// encryption/decryption +class StreamCipher final : NonCopyable { + static std::unordered_set ctxs; + EVP_CIPHER_CTX* ctx; + +public: + StreamCipher(); + ~StreamCipher(); + EVP_CIPHER_CTX* getCtx(); + class Key : NonCopyable { + std::array arr; + static std::unique_ptr globalKey; + struct ConstructorTag {}; + + public: + using RawKeyType = decltype(arr); + Key(ConstructorTag) {} + Key(Key&&); + Key& operator=(Key&&); + ~Key(); + unsigned char const* data() const { return arr.data(); } + static void initializeKey(RawKeyType&&); + static void initializeRandomTestKey(); + static const Key& getKey(); + static void cleanup() noexcept; + }; + static void cleanup() noexcept; + using IV = std::array; +}; + +class EncryptionStreamCipher final : NonCopyable, public ReferenceCounted { + StreamCipher cipher; + +public: + EncryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv); + StringRef encrypt(unsigned char const* plaintext, int len, Arena&); + StringRef finish(Arena&); +}; + +class DecryptionStreamCipher final : NonCopyable, public ReferenceCounted { + StreamCipher cipher; + +public: + DecryptionStreamCipher(const StreamCipher::Key& key, const StreamCipher::IV& iv); + StringRef decrypt(unsigned char const* ciphertext, int len, Arena&); + StringRef finish(Arena&); +}; diff --git a/flow/Trace.h b/flow/Trace.h index 47bf46dd09..57f4911d8c 100644 --- a/flow/Trace.h +++ b/flow/Trace.h @@ -320,11 +320,11 @@ struct TraceableStringImpl : std::true_type { std::string result; result.reserve(size - nonPrintables + (nonPrintables * 4) + numBackslashes); for (auto iter = TraceableString::begin(value); !TraceableString::atEnd(value, iter); ++iter) { - if (isPrintable(*iter)) { + if (*iter == '\\') { + result.push_back('\\'); + result.push_back('\\'); + } else if (isPrintable(*iter)) { result.push_back(*iter); - } else if (*iter == '\\') { - result.push_back('\\'); - result.push_back('\\'); } else { const uint8_t byte = *iter; result.push_back('\\'); diff --git a/flow/UnitTest.h b/flow/UnitTest.h index 50af315d4b..45247778fe 100644 --- a/flow/UnitTest.h +++ b/flow/UnitTest.h @@ -99,6 +99,9 @@ struct UnitTestCollection { extern UnitTestCollection g_unittests; +// Set this to `true` to disable RNG state checking after simulation runs. + extern bool noUnseed; + #define APPEND(a, b) a##b // FILE_UNIQUE_NAME(basename) expands to a name like basename456 if on line 456 diff --git a/flow/error_definitions.h b/flow/error_definitions.h index 0fd42b3ac0..8ffb54f290 100755 --- a/flow/error_definitions.h +++ b/flow/error_definitions.h @@ -227,6 +227,7 @@ ERROR( restore_destination_not_empty, 2370, "Attempted to restore into a non-emp ERROR( restore_duplicate_uid, 2371, "Attempted to restore using a UID that had been used for an aborted restore") ERROR( task_invalid_version, 2381, "Invalid task version") ERROR( task_interrupted, 2382, "Task execution stopped due to timeout, abort, or completion by another worker") +ERROR( invalid_encryption_key_file, 2383, "The provided encryption key file has invalid contents" ) ERROR( key_not_found, 2400, "Expected key is missing") ERROR( json_malformed, 2401, "JSON string was malformed") diff --git a/flow/flow.h b/flow/flow.h index 6c5f380b52..b598f82987 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -133,22 +133,21 @@ class Never {}; template class ErrorOr : public ComposedIdentifier { + std::variant value; + public: ErrorOr() : ErrorOr(default_error_or()) {} - ErrorOr(Error const& error) : error(error) { memset(&value, 0, sizeof(value)); } - ErrorOr(const ErrorOr& o) : error(o.error) { - if (present()) - new (&value) T(o.get()); - } + ErrorOr(Error const& error) : value(std::in_place_type, error) {} template - ErrorOr(const U& t) : error() { - new (&value) T(t); - } + ErrorOr(U const& t) : value(std::in_place_type, t) {} - ErrorOr(Arena& a, const ErrorOr& o) : error(o.error) { - if (present()) - new (&value) T(a, o.get()); + ErrorOr(Arena& a, ErrorOr const& o) { + if (o.present()) { + value = std::variant(std::in_place_type, a, o.get()); + } else { + value = std::variant(std::in_place_type, o.getError()); + } } int expectedSize() const { return present() ? get().expectedSize() : 0; } @@ -158,69 +157,67 @@ public: } template - ErrorOr map(std::function f) const { - if (present()) { - return ErrorOr(f(get())); - } else { - return ErrorOr(error); - } + ErrorOr map(std::function f) const& { + return present() ? ErrorOr(f(get())) : ErrorOr(getError()); + } + template + ErrorOr map(std::function f) && { + return present() ? ErrorOr(f(std::move(*this).get())) : ErrorOr(getError()); } - ~ErrorOr() { - if (present()) - ((T*)&value)->~T(); - } - - ErrorOr& operator=(ErrorOr const& o) { - if (present()) { - ((T*)&value)->~T(); - } - if (o.present()) { - new (&value) T(o.get()); - } - error = o.error; - return *this; - } - - bool present() const { return error.code() == invalid_error_code; } - T& get() { + bool present() const { return std::holds_alternative(value); } + T& get() & { UNSTOPPABLE_ASSERT(present()); - return *(T*)&value; + return std::get(value); } - T const& get() const { + T const& get() const& { UNSTOPPABLE_ASSERT(present()); - return *(T const*)&value; + return std::get(value); } - T orDefault(T const& default_value) const { - if (present()) - return get(); - else - return default_value; + T&& get() && { + UNSTOPPABLE_ASSERT(present()); + return std::get(std::move(value)); + } + template + T orDefault(U&& defaultValue) const& { + return present() ? get() : std::forward(defaultValue); + } + template + T orDefault(U&& defaultValue) && { + return present() ? std::move(*this).get() : std::forward(defaultValue); } - template - void serialize(Ar& ar) { - // SOMEDAY: specialize for space efficiency? - serializer(ar, error); - if (present()) { - if (Ar::isDeserializing) - new (&value) T(); - serializer(ar, *(T*)&value); - } - } - - bool isError() const { return error.code() != invalid_error_code; } - bool isError(int code) const { return error.code() == code; } - const Error& getError() const { + bool isError() const { return std::holds_alternative(value); } + bool isError(int code) const { return isError() && getError().code() == code; } + Error const& getError() const { ASSERT(isError()); - return error; + return std::get(value); } - -private: - typename std::aligned_storage::type value; - Error error; }; +template +void load(Archive& ar, ErrorOr& value) { + Error error; + ar >> error; + if (error.code() != invalid_error_code) { + T t; + ar >> t; + value = ErrorOr(t); + } else { + value = ErrorOr(error); + } +} + +template +void save(Archive& ar, ErrorOr const& value) { + if (value.present()) { + ar << Error{}; // invalid error code + ar << value.get(); + } else { + ar << value.getError(); + } +} + template struct union_like_traits> : std::true_type { using Member = ErrorOr; diff --git a/flow/network.h b/flow/network.h index 00f430fb86..d12ba1a0e8 100644 --- a/flow/network.h +++ b/flow/network.h @@ -347,7 +347,8 @@ struct NetworkMetrics { std::unordered_map activeTrackers; double lastRunLoopBusyness; // network thread busyness (measured every 5s by default) - std::atomic networkBusyness; // network thread busyness which is returned to the the client (measured every 1s by default) + std::atomic + networkBusyness; // network thread busyness which is returned to the the client (measured every 1s by default) // starvation trackers which keeps track of different task priorities std::vector starvationTrackers; @@ -536,7 +537,10 @@ public: virtual void onMainThread(Promise&& signal, TaskPriority taskID) = 0; // Executes signal.send(Void()) on a/the thread belonging to this network - virtual THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), void* arg) = 0; + virtual THREAD_HANDLE startThread(THREAD_FUNC_RETURN (*func)(void*), + void* arg, + int stackSize = 0, + const char* name = nullptr) = 0; // Starts a thread and returns a handle to it virtual void run() = 0; diff --git a/flowbench/BenchEncrypt.cpp b/flowbench/BenchEncrypt.cpp new file mode 100644 index 0000000000..54834bc829 --- /dev/null +++ b/flowbench/BenchEncrypt.cpp @@ -0,0 +1,77 @@ +/* + * BenchEncrypt.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "benchmark/benchmark.h" + +#include "flow/StreamCipher.h" +#include "flowbench/GlobalData.h" + +static StreamCipher::IV getRandomIV() { + StreamCipher::IV iv; + generateRandomData(iv.data(), iv.size()); + return iv; +} + +static inline Standalone encrypt(const StreamCipher::Key& key, const StreamCipher::IV& iv, + unsigned char const* data, size_t len) { + EncryptionStreamCipher encryptor(key, iv); + Arena arena; + auto encrypted = encryptor.encrypt(data, len, arena); + return Standalone(encrypted, arena); +} + +static void bench_encrypt(benchmark::State& state) { + auto bytes = state.range(0); + auto chunks = state.range(1); + auto chunkSize = bytes / chunks; + StreamCipher::Key::initializeRandomTestKey(); + const auto& key = StreamCipher::Key::getKey(); + auto iv = getRandomIV(); + auto data = getKey(bytes); + while (state.KeepRunning()) { + for (int chunk = 0; chunk < chunks; ++chunk) { + benchmark::DoNotOptimize(encrypt(key, iv, data.begin() + chunk * chunkSize, chunkSize)); + } + } + state.SetBytesProcessed(bytes * static_cast(state.iterations())); +} + +static void bench_decrypt(benchmark::State& state) { + auto bytes = state.range(0); + auto chunks = state.range(1); + auto chunkSize = bytes / chunks; + StreamCipher::Key::initializeRandomTestKey(); + const auto& key = StreamCipher::Key::getKey(); + auto iv = getRandomIV(); + auto data = getKey(bytes); + auto encrypted = encrypt(key, iv, data.begin(), data.size()); + while (state.KeepRunning()) { + Arena arena; + DecryptionStreamCipher decryptor(key, iv); + for (int chunk = 0; chunk < chunks; ++chunk) { + benchmark::DoNotOptimize( + Standalone(decryptor.decrypt(encrypted.begin() + chunk * chunkSize, chunkSize, arena))); + } + } + state.SetBytesProcessed(bytes * static_cast(state.iterations())); +} + +BENCHMARK(bench_encrypt)->Ranges({ { 1 << 12, 1 << 20 }, { 1, 1 << 12 } }); +BENCHMARK(bench_decrypt)->Ranges({ { 1 << 12, 1 << 20 }, { 1, 1 << 12 } }); diff --git a/flowbench/CMakeLists.txt b/flowbench/CMakeLists.txt index d1f10037ae..8caad0ce02 100644 --- a/flowbench/CMakeLists.txt +++ b/flowbench/CMakeLists.txt @@ -11,6 +11,12 @@ set(FLOWBENCH_SRCS GlobalData.h GlobalData.cpp) +if(WITH_TLS AND NOT WIN32) + set(FLOWBENCH_SRCS + ${FLOWBENCH_SRCS} + BenchEncrypt.cpp) +endif() + project (flowbench) # include the configurations from benchmark.cmake configure_file(benchmark.cmake googlebenchmark-download/CMakeLists.txt) diff --git a/packaging/msi/CMakeLists.txt b/packaging/msi/CMakeLists.txt index 44a262c10c..7b5a1a1922 100644 --- a/packaging/msi/CMakeLists.txt +++ b/packaging/msi/CMakeLists.txt @@ -16,7 +16,7 @@ if(WIX_FOUND) -DVERSION=${CMAKE_PROJECT_VERSION} -DVERSION_NAME=${FDB_VERSION} -P ${CMAKE_CURRENT_SOURCE_DIR}/generate_wxs.cmake - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/FDBInstaller.wxs ${CMAKE_CURRENT_SOURCE_DIR}/generate_wsx.cmake + DEPENDS ${CMAKE_SOURCE_DIR}/packaging/msi/FDBInstaller.wxs ${CMAKE_SOURCE_DIR}/packaging/msi/generate_wxs.cmake COMMENT "Generate WIX file") add_custom_target(wix_file DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/FDBInstaller.wxs) add_custom_command( diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs index 7489ae2272..1f23a98278 100644 --- a/packaging/msi/FDBInstaller.wxs +++ b/packaging/msi/FDBInstaller.wxs @@ -134,13 +134,13 @@ - + - + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 913b39413b..1feb231389 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -134,6 +134,8 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/LocalRatekeeper.toml) add_fdb_test(TEST_FILES fast/LongStackWriteDuringRead.toml) add_fdb_test(TEST_FILES fast/LowLatency.toml) + # TODO: Fix failures and reenable this test: + add_fdb_test(TEST_FILES fast/LowLatencySingleClog.toml IGNORE) add_fdb_test(TEST_FILES fast/MemoryLifetime.toml) add_fdb_test(TEST_FILES fast/MoveKeysCycle.toml) add_fdb_test(TEST_FILES fast/ProtocolVersion.toml) diff --git a/tests/TestRunner/tmp_cluster.py b/tests/TestRunner/tmp_cluster.py index c34fc6a85e..f8ae4ef813 100755 --- a/tests/TestRunner/tmp_cluster.py +++ b/tests/TestRunner/tmp_cluster.py @@ -11,7 +11,7 @@ from random import choice from pathlib import Path class TempCluster: - def __init__(self, build_dir: str): + def __init__(self, build_dir: str,port: str = None): self.build_dir = Path(build_dir).resolve() assert self.build_dir.exists(), "{} does not exist".format(build_dir) assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir) @@ -22,7 +22,8 @@ class TempCluster: self.cluster = LocalCluster(tmp_dir, self.build_dir.joinpath('bin', 'fdbserver'), self.build_dir.joinpath('bin', 'fdbmonitor'), - self.build_dir.joinpath('bin', 'fdbcli')) + self.build_dir.joinpath('bin', 'fdbcli'), + port = port) self.log = self.cluster.log self.etc = self.cluster.etc self.data = self.cluster.data @@ -37,6 +38,10 @@ class TempCluster: self.cluster.__exit__(xc_type, exc_value, traceback) shutil.rmtree(self.tmp_dir) + def close(self): + self.cluster.__exit__(None,None,None) + shutil.rmtree(self.tmp_dir) + if __name__ == '__main__': parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter, diff --git a/tests/TestRunner/tmp_multi_cluster.py b/tests/TestRunner/tmp_multi_cluster.py new file mode 100755 index 0000000000..2edcef29de --- /dev/null +++ b/tests/TestRunner/tmp_multi_cluster.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# +# tmp_multi_cluster.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import subprocess +import sys +import shutil +from pathlib import Path +from argparse import ArgumentParser, RawDescriptionHelpFormatter +from tmp_cluster import TempCluster + + +if __name__ == '__main__': + parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter,description=""" + This script automatically configures N temporary local clusters on the machine and then + calls a command while these clusters are running. As soon as the command returns, all + configured clusters are killed and all generated data is deleted. + + The purpose of this is to support testing a set of integration tests using multiple clusters + (i.e. using the Multi-threaded client). + """) + parser.add_argument('--build-dir','-b',metavar='BUILD_DIRECTORY',help='FDB build director',required=True) + parser.add_argument('--clusters','-c',metavar='NUM_CLUSTERS',type=int,help='The number of clusters to run',required=True) + parser.add_argument('cmd', metavar='COMMAND',nargs='+',help='The command to run') + args = parser.parse_args() + errcode = 1 + + #spawn all the clusters + base_dir = args.build_dir + num_clusters = args.clusters + + build_dir=Path(base_dir) + bin_dir=build_dir.joinpath('bin') + + clusters = [] + for c in range(1,num_clusters+1): + # now start the cluster up + local_c = TempCluster(args.build_dir, port="{}501".format(c)) + + local_c.__enter__() + clusters.append(local_c) + + # all clusters should be running now, so run the subcommand + # TODO (bfines): pass through the proper ENV commands so that the client can find everything + cluster_paths = ';'.join([str(cluster.etc.joinpath('fdb.cluster')) for cluster in clusters]) + print(cluster_paths) + env = dict(**os.environ) + env['FDB_CLUSTERS'] = env.get('FDB_CLUSTERS',cluster_paths) + errcode = subprocess.run(args.cmd,stdout=sys.stdout,stderr=sys.stderr,env=env).returncode + + # shutdown all the running clusters + for tc in clusters: + tc.close() + + sys.exit(errcode) + \ No newline at end of file diff --git a/tests/fast/LowLatencySingleClog.toml b/tests/fast/LowLatencySingleClog.toml new file mode 100644 index 0000000000..cbfe6682e3 --- /dev/null +++ b/tests/fast/LowLatencySingleClog.toml @@ -0,0 +1,20 @@ +[configuration] +buggify = false +minimumReplication = 2 + +[[test]] +testTitle = 'Clogged' +connectionFailuresDisableDuration = 100000 + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 1000.0 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = 'LowLatency' + testDuration = 30.0 + + [[test.workload]] + testName = 'ClogSingleConnection' diff --git a/versions.target b/versions.target deleted file mode 100644 index 7672f81d95..0000000000 --- a/versions.target +++ /dev/null @@ -1,7 +0,0 @@ - - - - 7.1.0 - 7.1 - -