diff --git a/.gitignore b/.gitignore index 5fc9981a4f..cd575e5e0e 100644 --- a/.gitignore +++ b/.gitignore @@ -95,3 +95,6 @@ flow/coveragetool/obj .DS_Store temp/ /versions.target +/compile_commands.json +/.ccls-cache +.clangd/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 844d302637..81af95a68f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # limitations under the License. cmake_minimum_required(VERSION 3.13) project(foundationdb - VERSION 6.3.10 + VERSION 6.3.11 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 3ed3d93700..13e371cc01 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -94,6 +94,7 @@ void fdb_flow_test() { g_network->run(); } +// FDB object used by bindings namespace FDB { class DatabaseImpl : public Database, NonCopyable { public: diff --git a/build/Dockerfile b/build/Dockerfile index b4cf28ebc0..4d656118a6 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,17 +1,28 @@ +ARG IMAGE_TAG=0.1.24 FROM centos:6 +# Clean yum cache, disable default Base repo and enable Vault +RUN yum clean all &&\ + sed -i -e 's/gpgcheck=1/enabled=0/g' /etc/yum.repos.d/CentOS-Base.repo &&\ + sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo &&\ + sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo &&\ + sed -i -e "s/6\.0/$(cut -d\ -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo &&\ + yum install -y yum-utils &&\ + yum-config-manager --enable rhel-server-rhscl-7-rpms &&\ + yum -y install centos-release-scl-rh epel-release \ + http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\ + sed -i -e 's/#baseurl=/baseurl=/g' -e 's/mirror.centos.org/vault.centos.org/g' \ + -e 's/mirrorlist=/#mirrorlist=/g' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo &&\ + yum clean all + # Install dependencies for developer tools, bindings,\ # documentation, actorcompiler, and packaging tools\ -RUN yum install -y yum-utils &&\ - yum-config-manager --enable rhel-server-rhscl-7-rpms &&\ - yum -y install centos-release-scl epel-release \ - http://opensource.wandisco.com/centos/6/git/x86_64/wandisco-git-release-6-1.noarch.rpm &&\ - yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \ - devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \ - devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \ - rh-python36-python-devel rh-ruby24 golang python27 rpm-build \ - mono-core debbuild python-pip dos2unix valgrind-devel ccache \ - distcc wget git lz4 lz4-devel lz4-static &&\ +RUN yum -y install devtoolset-8-8.1-1.el6 java-1.8.0-openjdk-devel \ + devtoolset-8-gcc-8.3.1 devtoolset-8-gcc-c++-8.3.1 \ + devtoolset-8-libubsan-devel devtoolset-8-libasan-devel devtoolset-8-valgrind-devel \ + rh-python36-python-devel rh-ruby24 golang python27 rpm-build \ + mono-core debbuild python-pip dos2unix valgrind-devel ccache \ + distcc wget libxslt git lz4 lz4-devel lz4-static &&\ pip install boto3==1.1.1 USER root @@ -19,17 +30,35 @@ USER root RUN adduser --comment '' fdb && chown fdb /opt # wget of bintray without forcing UTF-8 encoding results in 403 Forbidden +# Old versions of FDB need boost 1.67 RUN cd /opt/ &&\ curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 &&\ echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt &&\ sha256sum -c boost-sha-67.txt &&\ tar -xjf boost_1_67_0.tar.bz2 &&\ - rm -rf boost_1_67_0.tar.bz2 boost-sha-67.txt boost_1_67_0/libs &&\ + rm -rf boost_1_67_0.tar.bz2 boost-sha-67.txt boost_1_67_0/libs + +# install Boost 1.72 +# wget of bintray without forcing UTF-8 encoding results in 403 Forbidden +RUN cd /tmp/ &&\ curl -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 &&\ echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt &&\ sha256sum -c boost-sha-72.txt &&\ tar -xjf boost_1_72_0.tar.bz2 &&\ - rm -rf boost_1_72_0.tar.bz2 boost-sha-72.txt boost_1_72_0/libs + cd boost_1_72_0 &&\ + scl enable devtoolset-8 -- ./bootstrap.sh --with-libraries=context &&\ + scl enable devtoolset-8 -- ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ + rm -rf boost_1_72_0.tar.bz2 boost-sha-72.txt boost_1_72_0 + +# jemalloc (needed for FDB after 6.3) +RUN cd /tmp/ &&\ + curl -L https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 &&\ + echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt &&\ + sha256sum -c jemalloc-sha.txt &&\ + tar --no-same-owner --no-same-permissions -xjf jemalloc-5.2.1.tar.bz2 &&\ + cd jemalloc-5.2.1 &&\ + scl enable devtoolset-8 -- ./configure --enable-static --disable-cxx &&\ + scl enable devtoolset-8 -- make install # install cmake RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o /tmp/cmake.tar.gz &&\ @@ -41,6 +70,8 @@ RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.1 # install Ninja RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\ + echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > /tmp/ninja-sha.txt &&\ + sha256sum -c /tmp/ninja-sha.txt &&\ unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\ cd .. && rm -rf ninja-1.9.0 ninja.zip @@ -53,17 +84,59 @@ RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o o ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\ cd /tmp/ && rm -rf /tmp/openssl-1.1.1h /tmp/openssl.tar.gz +# Install toml11 + RUN cd /tmp && curl -L https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz > toml.tar.gz &&\ + echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt &&\ + sha256sum -c toml-sha256.txt &&\ + tar xf toml.tar.gz && rm -rf build && mkdir build && cd build && scl enable devtoolset-8 -- cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml11-3.4.0 &&\ + scl enable devtoolset-8 -- cmake --build . --target install && cd / && rm -rf tmp/build && rm -rf tmp/toml11-3.4.0 + RUN cd /opt/ && curl -L https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz &&\ echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt &&\ sha256sum -c rocksdb-sha.txt && tar xf rocksdb.tar.gz && rm -rf rocksdb.tar.gz rocksdb-sha.txt +RUN cd /opt/ && curl -L https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o ctest2junit.xsl + +# Setting this environment variable switches from OpenSSL to BoringSSL +ENV OPENSSL_ROOT_DIR=/opt/boringssl + +# install BoringSSL: TODO: They don't seem to have releases(?) I picked today's master SHA. +RUN cd /opt &&\ + git clone https://boringssl.googlesource.com/boringssl &&\ + cd boringssl &&\ + git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\ + mkdir build + +# ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS +# also, enable -fPIC. +# this is moderately uglier than creating a patchfile, but easier to maintain. +RUN cd /opt/boringssl &&\ + for f in crypto/fipsmodule/rand/fork_detect_test.cc \ + include/openssl/bn.h \ + ssl/test/bssl_shim.cc ; do \ + perl -p -i -e 's/#include /#define __STDC_FORMAT_MACROS 1\n#include /g;' $f ; \ + done &&\ + perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\ + git diff + +RUN cd /opt/boringssl/build &&\ + scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\ + scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\ + ./ssl/ssl_test &&\ + mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib + # Localize time zone ARG TIMEZONEINFO=America/Los_Angeles RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime -LABEL version=0.1.19 -ENV DOCKER_IMAGEVER=0.1.19 +LABEL version=${IMAGE_TAG} +ENV DOCKER_IMAGEVER=${IMAGE_TAG} ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ + +ENV CCACHE_NOHASHDIR=true +ENV CCACHE_UMASK=0000 +ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches" + CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash diff --git a/build/Dockerfile.devel b/build/Dockerfile.devel index 33c6d03e36..d9b56e639e 100644 --- a/build/Dockerfile.devel +++ b/build/Dockerfile.devel @@ -1,4 +1,6 @@ -FROM foundationdb/foundationdb-build:0.1.19 +ARG IMAGE_TAG=0.1.24 +ARG IMAGE_VERSION=0.11.15 +FROM foundationdb/foundationdb-build:${IMAGE_TAG} USER root @@ -50,8 +52,8 @@ RUN cp -iv /usr/local/bin/clang++ /usr/local/bin/clang++.deref &&\ ldconfig &&\ rm -rf /mnt/artifacts -LABEL version=0.11.10 -ENV DOCKER_IMAGEVER=0.11.10 +LABEL version=${IMAGE_VERSION} +ENV DOCKER_IMAGEVER=${IMAGE_VERSION} ENV CLANGCC=/usr/local/bin/clang.de8a65ef ENV CLANGCXX=/usr/local/bin/clang++.de8a65ef @@ -63,8 +65,5 @@ ENV CC=/usr/local/bin/clang.de8a65ef ENV CXX=/usr/local/bin/clang++.de8a65ef ENV USE_LD=LLD ENV USE_LIBCXX=1 -ENV CCACHE_NOHASHDIR=true -ENV CCACHE_UMASK=0000 -ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches" CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml index d0942880a4..28d865abda 100644 --- a/build/docker-compose.yaml +++ b/build/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: common: &common - image: foundationdb/foundationdb-build:0.1.19 + image: foundationdb/foundationdb-build:0.1.24 build-setup: &build-setup <<: *common @@ -60,7 +60,7 @@ services: snapshot-cmake: &snapshot-cmake <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' prb-cmake: <<: *snapshot-cmake @@ -68,7 +68,7 @@ services: snapshot-bindings-cmake: &snapshot-bindings-cmake <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "bindings/all"' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "bindings/all"' prb-bindings-cmake: <<: *snapshot-bindings-cmake @@ -84,7 +84,7 @@ services: snapshot-ctest: &snapshot-ctest <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure' prb-ctest: <<: *snapshot-ctest @@ -92,7 +92,7 @@ services: snapshot-correctness: &snapshot-correctness <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' prb-correctness: <<: *snapshot-correctness diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index bb952bc9d6..1a831c5d0b 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -157,7 +157,7 @@ endif() # - OUT_DIR the directory where files will be staged # - CONTEXT the type of correctness package being built (e.g. 'valgrind correctness') function(stage_correctness_package) - set(oneValueArgs OUT_DIR CONTEXT) + set(oneValueArgs OUT_DIR CONTEXT OUT_FILES) cmake_parse_arguments(STAGE "" "${oneValueArgs}" "" "${ARGN}") file(MAKE_DIRECTORY ${STAGE_OUT_DIR}/bin) string(LENGTH "${CMAKE_SOURCE_DIR}/tests/" base_length) @@ -200,6 +200,10 @@ function(stage_correctness_package) endforeach() endforeach() list(APPEND package_files ${STAGE_OUT_DIR}/bin/fdbserver + ${STAGE_OUT_DIR}/bin/coverage.fdbserver.xml + ${STAGE_OUT_DIR}/bin/coverage.fdbclient.xml + ${STAGE_OUT_DIR}/bin/coverage.fdbrpc.xml + ${STAGE_OUT_DIR}/bin/coverage.flow.xml ${STAGE_OUT_DIR}/bin/TestHarness.exe ${STAGE_OUT_DIR}/bin/TraceLogHelper.dll ${STAGE_OUT_DIR}/CMakeCache.txt @@ -208,17 +212,27 @@ function(stage_correctness_package) OUTPUT ${package_files} DEPENDS ${CMAKE_BINARY_DIR}/CMakeCache.txt ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbclient.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbrpc.xml + ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/CMakeCache.txt ${STAGE_OUT_DIR} COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/bin/coverage.fdbserver.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbclient.xml + ${CMAKE_BINARY_DIR}/lib/coverage.fdbrpc.xml + ${CMAKE_BINARY_DIR}/lib/coverage.flow.xml ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll ${STAGE_OUT_DIR}/bin COMMENT "Copying files for ${STAGE_CONTEXT} package" ) list(APPEND package_files ${test_files} ${external_files}) - set(package_files ${package_files} PARENT_SCOPE) + if(STAGE_OUT_FILES) + set(${STAGE_OUT_FILES} ${package_files} PARENT_SCOPE) + endif() endfunction() function(create_correctness_package) @@ -226,7 +240,7 @@ function(create_correctness_package) return() endif() set(out_dir "${CMAKE_BINARY_DIR}/correctness") - stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "correctness") + stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "correctness" OUT_FILES package_files) set(tar_file ${CMAKE_BINARY_DIR}/packages/correctness-${CMAKE_PROJECT_VERSION}.tar.gz) add_custom_command( OUTPUT ${tar_file} @@ -253,7 +267,7 @@ function(create_valgrind_correctness_package) endif() if(USE_VALGRIND) set(out_dir "${CMAKE_BINARY_DIR}/valgrind_correctness") - stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "valgrind correctness") + stage_correctness_package(OUT_DIR ${out_dir} CONTEXT "valgrind correctness" OUT_FILES package_files) set(tar_file ${CMAKE_BINARY_DIR}/packages/valgrind-${CMAKE_PROJECT_VERSION}.tar.gz) add_custom_command( OUTPUT ${tar_file} diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 270fbc9f9c..541bc1cd85 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -68,6 +68,8 @@ if (NOT OPEN_FOR_IDE) endif() if(WIN32) add_definitions(-DUSE_USEFIBERS) + add_definitions(-DBOOST_USE_WINDOWS_H) + add_definitions(-DWIN32_LEAN_AND_MEAN) else() add_definitions(-DUSE_UCONTEXT) endif() @@ -107,7 +109,7 @@ if(WIN32) string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") endif() add_compile_options(/W0 /EHsc /bigobj $<$:/Zi> /MP /FC /Gm-) - add_compile_definitions(_WIN32_WINNT=${WINDOWS_TARGET} WINVER=${WINDOWS_TARGET} NTDDI_VERSION=0x05020000 BOOST_ALL_NO_LIB) + add_compile_definitions(_WIN32_WINNT=${WINDOWS_TARGET} WINVER=${WINDOWS_TARGET} NTDDI_VERSION=0x05020000 BOOST_ALL_NO_LIB NOMINMAX) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") else() @@ -259,7 +261,6 @@ else() -Wno-unused-function -Wno-unused-local-typedef -Wno-unused-parameter - -Wno-unused-value -Wno-self-assign ) if (USE_CCACHE) diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake index 40a36dfd03..ec26c07dd8 100644 --- a/cmake/FDBComponents.cmake +++ b/cmake/FDBComponents.cmake @@ -11,6 +11,7 @@ endif() ################################################################################ # SSL ################################################################################ + include(CheckSymbolExists) set(DISABLE_TLS OFF CACHE BOOL "Don't try to find OpenSSL and always build without TLS support") @@ -21,13 +22,13 @@ else() find_package(OpenSSL) if(OPENSSL_FOUND) set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) + set(WITH_TLS ON) + add_compile_options(-DHAVE_OPENSSL) check_symbol_exists("OPENSSL_INIT_NO_ATEXIT" "openssl/crypto.h" OPENSSL_HAS_NO_ATEXIT) if(OPENSSL_HAS_NO_ATEXIT) - set(WITH_TLS ON) - add_compile_options(-DHAVE_OPENSSL) + add_compile_options(-DHAVE_OPENSSL_INIT_NO_AT_EXIT) else() - message(WARNING "An OpenSSL version was found, but it doesn't support OPENSSL_INIT_NO_ATEXIT - Will compile without TLS Support") - set(WITH_TLS OFF) + message(STATUS "Found OpenSSL without OPENSSL_INIT_NO_ATEXIT: assuming BoringSSL") endif() else() message(STATUS "OpenSSL was not found - Will compile without TLS Support") diff --git a/contrib/TestHarness/CMakeLists.txt b/contrib/TestHarness/CMakeLists.txt index 3ffc23d509..4eecd82e04 100644 --- a/contrib/TestHarness/CMakeLists.txt +++ b/contrib/TestHarness/CMakeLists.txt @@ -3,7 +3,7 @@ set(SRCS Properties/AssemblyInfo.cs) set(TEST_HARNESS_REFERENCES - "-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml,${TraceLogHelperDll}") + "-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml,System.Runtime.Serialization,${TraceLogHelperDll}") set(out_file ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe) diff --git a/contrib/TestHarness/Program.cs.cmake b/contrib/TestHarness/Program.cs.cmake index acb774e796..c90d542f9b 100644 --- a/contrib/TestHarness/Program.cs.cmake +++ b/contrib/TestHarness/Program.cs.cmake @@ -1,1553 +1,1600 @@ -/* - * Program.cs - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Xml.Linq; -using System.IO; -using System.Diagnostics; -using System.ComponentModel; -using System.Runtime.InteropServices; -using System.Xml; - -namespace SummarizeTest -{ - static class Program - { - static Random random; - const int killSeconds = 60 * 30; - const int maxWarnings = 10; - const int maxStderrBytes = 1000; - const double unseedRatio = 0.05; - const double buggifyOnRatio = 0.8; - static string BINARY = "fdbserver" + (IsRunningOnMono() ? "" : ".exe"); - static string PLUGIN = "FDBLibTLS." + (IsRunningOnMono() ? "so" : "dll"); - static string OS_NAME = IsRunningOnMono() ? "linux" : "win"; - - static int Main(string[] args) - { - bool traceToStdout = false; - try - { - byte[] seed = new byte[4]; - new System.Security.Cryptography.RNGCryptoServiceProvider().GetBytes(seed); - random = new Random(new BinaryReader(new MemoryStream(seed)).ReadInt32()); - - if (args.Length < 1) - return UsageMessage(); - - if (args[0] == "summarize") - { - if (args.Length < 3) - return UsageMessage(); - string valgrindFileName = args.Length >= 4 ? args[3] : null; - string externalError = args.Length >= 5 ? args[4] : ""; - traceToStdout = args.Length == 6 && args[5] == "true"; - - int unseed; - bool retryableError; - //SOMEDAY: This only works if a run generated just one trace file. We should change the summarize command to take multiple trace files - return Summarize(new string[]{ args[1] }, args[2], null, null, null, null, null, null, valgrindFileName, -1, out unseed, out retryableError, true, - traceToStdout: traceToStdout, externalError: externalError); - } - else if (args[0] == "remote") - { - if (args.Length < 6) - return UsageMessage(); - - return Remote(args[1], args[2], double.Parse(args[3]), int.Parse(args[4]), args[5], args.Length == 7 ? args[6] : "default"); - } - else if (args[0] == "run") - { - try - { - if (args.Length < 6) - return UsageMessage(); - - string runDir = null; - if(args[1] != "temp") runDir = args[1]; - - bool useValgind = args.Length > 6 && args[6].ToLower() == "true"; - int maxTries = (args.Length > 7) ? int.Parse(args[7]) : 3; - return Run(args[2], args[3], args[4], args[5], null, runDir, null, useValgind, maxTries); - } - catch(Exception e) - { - var xout = new XElement("TestHarnessError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorMessage", e.Message)); - - AppendXmlMessageToSummary(args[5], xout); - throw; - } - } - else if (args[0] == "replay") - { - if (args.Length != 6) - return UsageMessage(); - - string runDir = null; - if (args[1] != "temp") runDir = args[1]; - - return Replay(args[2], args[3], args[4], args[5], runDir); - } - else if (args[0] == "auto") - { - if (args.Length < 4) - return UsageMessage(); - - string runDir = null; - if (args[1] != "temp") runDir = args[1]; - - string cacheDir = Path.Combine(runDir, "test_harness_cache"); - bool useValgrind = args.Length > 4 && args[4].ToLower() == "true"; - int maxTries = (args.Length > 5) ? int.Parse(args[5]) : 3; - - return Auto(args[2], Path.Combine(runDir, "runs"), args[3], cacheDir, useValgrind, maxTries); - } - else if (args[0] == "extract-errors") - { - if (args.Length != 3) - return UsageMessage(); - - return ExtractErrors(args[1], args[2]); - } - else if (args[0] == "joshua-run") - { - traceToStdout = true; - - try - { - string oldBinaryFolder = (args.Length > 1) ? args[1] : Path.Combine("/opt", "joshua", "global_data", "oldBinaries"); - bool useValgrind = args.Length > 2 && args[2].ToLower() == "true"; - int maxTries = (args.Length > 3) ? int.Parse(args[3]) : 3; - return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN)); - } - catch(Exception e) - { - var xout = new XElement("TestHarnessError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorMessage", e.ToString())); - - AppendXmlMessageToSummary("summary.xml", xout, true); - throw; - } - } - else if (args[0] == "version") - { - return VersionInfo(); - } - - return UsageMessage(); - } - catch (Exception e) - { - if (!traceToStdout) - { - Console.WriteLine("Error:"); - Console.WriteLine(e.ToString()); - } - - return 100; - } - } - - static T Choice(this Random random, IList items) - { - return items[ random.Next(items.Count) ]; - } - - static bool IsRunningOnMono() - { - return Type.GetType("Mono.Runtime") != null; - } - - static int Replay(string fdbserverName, string tlsPluginFile, string inputSummaryFileName, string outputSummaryFileName, string runDir) - { - using (var summaryFileIn = System.IO.File.Open(inputSummaryFileName, - System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) - { - foreach (var test in Magnesium.XmlParser.Parse(summaryFileIn, inputSummaryFileName).OfType()) - { - if (test is Magnesium.Test) continue; // Only process the test plans - int unseed; - bool retryableError; - RunTest(fdbserverName, tlsPluginFile, outputSummaryFileName, null, test.randomSeed, test.Buggify, test.TestFile, runDir, test.TestUID, -1, out unseed, out retryableError, true, false); - } - } - return 0; - } - - // Ver format - x.x.x (eg, 5.0.1, 4.2.11) - // Returns true is ver1 is greater than or equal to ver2 - static bool versionGreaterThanOrEqual(string ver1, string ver2) - { - string[] tokens1 = ver1.Split('.'); - string[] tokens2 = ver2.Split('.'); - if (tokens1.Length != tokens2.Length || tokens2.Length != 3) - { - throw new ArgumentException("Invalid Version Format Version1: " + ver1 + " Version2: " + ver2); - } - int[] version1 = Array.ConvertAll(tokens1, int.Parse); - int[] version2 = Array.ConvertAll(tokens2, int.Parse); - return ((System.Collections.IStructuralComparable)version1).CompareTo(version2, System.Collections.Generic.Comparer.Default) >= 0; - } - - static bool versionLessThan(string ver1, string ver2) { - return !versionGreaterThanOrEqual(ver1, ver2); - } - - static string getFdbserverVersion(string fdbserverName) { - using (var process = new System.Diagnostics.Process()) - { - process.StartInfo.UseShellExecute = false; - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.FileName = fdbserverName; - process.StartInfo.Arguments = "--version"; - process.StartInfo.RedirectStandardError = true; - - process.Start(); - var output = process.StandardOutput.ReadToEnd(); - // If the process finished successfully, we call the parameterless WaitForExit to ensure that output buffers get flushed - process.WaitForExit(); - - var match = Regex.Match(output, @"v(\d+\.\d+\.\d+)"); - if (match.Groups.Count < 1) return ""; - return match.Groups[1].Value; - } - } - - static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "") - { - int seed = random.Next(1000000000); - bool buggify = random.NextDouble() < buggifyOnRatio; - string testFile = null; - string testDir = ""; - string oldServerName = ""; - - if (Directory.Exists(testFolder)) - { - int poolSize = 0; - if( Directory.Exists(Path.Combine(testFolder, "rare")) ) poolSize += 1; - if( Directory.Exists(Path.Combine(testFolder, "slow")) ) poolSize += 5; - if( Directory.Exists(Path.Combine(testFolder, "fast")) ) poolSize += 14; - if( Directory.Exists(Path.Combine(testFolder, "restarting")) ) poolSize += 1; - - if( poolSize == 0 ) { - Console.WriteLine("Passed folder ({0}) did not have a fast, slow, rare, or restarting sub-folder", testFolder); - return 1; - } - int selection = random.Next(poolSize); - int selectionWindow = 0; - - if( Directory.Exists(Path.Combine(testFolder, "rare")) ) selectionWindow += 1; - if (selection < selectionWindow) - testDir = Path.Combine(testFolder, "rare"); - else - { - if (Directory.Exists(Path.Combine(testFolder, "restarting"))) selectionWindow += 1; - if (selection < selectionWindow) - testDir = Path.Combine(testFolder, "restarting"); - else - { - if (Directory.Exists(Path.Combine(testFolder, "slow"))) selectionWindow += 5; - if (selection < selectionWindow) - testDir = Path.Combine(testFolder, "slow"); - else - testDir = Path.Combine(testFolder, "fast"); - } - } - string[] files = Directory.GetFiles(testDir, "*", SearchOption.AllDirectories); - string[] uniqueFiles; - if (testDir.EndsWith("restarting")) - { - ISet uniqueFileSet = new HashSet(); - foreach(string file in files) { - uniqueFileSet.Add(file.Substring(0, file.LastIndexOf("-"))); // all restarting tests end with -1.txt or -2.txt - } - uniqueFiles = uniqueFileSet.ToArray(); - testFile = random.Choice(uniqueFiles); - string oldBinaryVersionLowerBound = "0.0.0"; - string lastFolderName = Path.GetFileName(Path.GetDirectoryName(testFile)); - if (lastFolderName.Contains("from_") || lastFolderName.Contains("to_")) // Only perform upgrade/downgrade tests from certain versions - { - oldBinaryVersionLowerBound = lastFolderName.Split('_').Last(); - } - string oldBinaryVersionUpperBound = getFdbserverVersion(fdbserverName); - string[] currentBinary = { fdbserverName }; - IEnumerable oldBinaries = Array.FindAll( - Directory.GetFiles(oldBinaryFolder), - x => versionGreaterThanOrEqual(Path.GetFileName(x).Split('-').Last(), oldBinaryVersionLowerBound) - && versionLessThan(Path.GetFileName(x).Split('-').Last(), oldBinaryVersionUpperBound)); - oldBinaries = oldBinaries.Concat(currentBinary); - oldServerName = random.Choice(oldBinaries.ToList()); - } - else - { - uniqueFiles = Directory.GetFiles(testDir); - testFile = random.Choice(uniqueFiles); - } - } - else if (File.Exists(testFolder)) - testFile = testFolder; - else - { - Console.WriteLine("Passed path ({0}) was not a folder or file", testFolder); - return 1; - } - - int result = 0; - bool unseedCheck = random.NextDouble() < unseedRatio; - for (int i = 0; i < maxTries; ++i) - { - bool logOnRetryableError = i == maxTries - 1; - bool retryableError = false; - - if (testDir.EndsWith("restarting")) - { - bool isDowngrade = Path.GetFileName(Path.GetDirectoryName(testFile)).Contains("to_"); - string firstServerName = isDowngrade ? fdbserverName : oldServerName; - string secondServerName = isDowngrade ? oldServerName : fdbserverName; - int expectedUnseed = -1; - int unseed; - string uid = Guid.NewGuid().ToString(); - bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0"); - result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile + "-1.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout); - if (result == 0) - { - result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile + "-2.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout); - } - } - else - { - int expectedUnseed = -1; - if (!useValgrind && unseedCheck) - { - result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout); - } - - if (!retryableError) - { - int unseed; - result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout); - } - } - - if (!retryableError) - { - return result; - } - } - - return result; - } - - private static int RunTest(string fdbserverName, string tlsPluginFile, string summaryFileName, string errorFileName, int seed, - bool buggify, string testFile, string runDir, string uid, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, bool useValgrind, bool restarting = false, - bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false) - { - unseed = -1; - - retryableError = false; - string tempPath = Path.Combine(runDir != null ? Path.GetFullPath(runDir) : Path.GetTempPath(), uid); - string oldDir = Directory.GetCurrentDirectory(); - - try - { - fdbserverName = Path.GetFullPath(fdbserverName); - tlsPluginFile = (tlsPluginFile.Length > 0) ? Path.GetFullPath(tlsPluginFile) : ""; - testFile = Path.GetFullPath(testFile); - var ok = 0; - - if (summaryFileName != null) - summaryFileName = Path.GetFullPath(summaryFileName); - - Directory.CreateDirectory(tempPath); - Directory.SetCurrentDirectory(tempPath); - - if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, oldBinaryName); - - string valgrindOutputFile = null; - using (var process = new System.Diagnostics.Process()) - { - ErrorOutputListener errorListener = new ErrorOutputListener(); - process.StartInfo.UseShellExecute = false; - if (tlsPluginFile.Length > 0) { - process.StartInfo.EnvironmentVariables["FDB_TLS_PLUGIN"] = tlsPluginFile; - } - process.StartInfo.RedirectStandardOutput = true; - var args = ""; - if (willRestart && oldBinaryName.EndsWith("alpha6")) - { - args = string.Format("-Rs 1000000000 -r simulation {0} -s {1} -f \"{2}\" -b {3} --tls_plugin={4} --crash", - IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginFile); - } - else - { - args = string.Format("-Rs 1GB -r simulation {0} -s {1} -f \"{2}\" -b {3} --tls_plugin={4} --crash", - IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginFile); - } - if (restarting) args = args + " --restarting"; - if (useValgrind && !willRestart) - { - valgrindOutputFile = string.Format("valgrind-{0}.xml", seed); - process.StartInfo.FileName = "valgrind"; - // Add extra debug directory, if environment variables is defined - string valgrindDbgDir = System.Environment.GetEnvironmentVariable("FDB_VALGRIND_DBGPATH"); - process.StartInfo.Arguments = (string.IsNullOrEmpty(valgrindDbgDir)) ? "" : string.Format("--extra-debuginfo-path={0} ", valgrindDbgDir); - - process.StartInfo.Arguments += - string.Format("--xml=yes --xml-file={0} -q {1} {2}", valgrindOutputFile, fdbserverName, args); - } - else - { - process.StartInfo.FileName = fdbserverName; - process.StartInfo.Arguments = args; - } - process.StartInfo.RedirectStandardError = true; - process.ErrorDataReceived += new DataReceivedEventHandler(errorListener.handleData); - - if (!traceToStdout) - { - Console.WriteLine("Executing {0} {1} in {2} (buggify: {3}, valgrind: {4})", - process.StartInfo.FileName, process.StartInfo.Arguments, tempPath, - buggify ? "on" : "off", - useValgrind ? "on" : "off"); - } - - process.Start(); - - // SOMEDAY: Do we want to actually do anything with standard output or error? - // Using standarderror requires async read, see http://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx - //process.StandardOutput.ReadToEnd(); - OutputCopier copier = new OutputCopier(process.StandardOutput); - Thread consoleThread = new Thread(new ThreadStart(copier.copyOutput)); - consoleThread.Start(); - - MemoryChecker memChecker = new MemoryChecker(process); - Thread memCheckThread = new Thread(new ThreadStart(memChecker.monitorMemory)); - memCheckThread.Start(); - - process.BeginErrorReadLine(); - - Stopwatch stopwatch = new Stopwatch(); - stopwatch.Start(); - - var ms = 1000 * killSeconds; - if (useValgrind) - ms *= 20; - bool killed = !process.WaitForExit(ms); // Wait "killSeconds" seconds here - - stopwatch.Stop(); - - if (!killed) - process.WaitForExit(); // If the process finished successfully, we call the parameterless WaitForExit to ensure that output buffers get flushed - else - { - // It seems that WaitForExit sometimes returns false well before the timeout has elapsed - // We won't report that as an error, but just in case the process didn't actually finish we attempt to - // kill it here - if (killed && ms - stopwatch.ElapsedMilliseconds > 10000) - killed = false; - - if (!traceToStdout) - { - Console.WriteLine("Warning: Killing process after {0} seconds...", killSeconds); - } - - try - { - process.Kill(); - } - catch (Exception e) - { - if (!traceToStdout) - { - Console.WriteLine("Warning: Process.Kill returned {0}", e); - } - } - - int waitSeconds = 60 * 2; - process.WaitForExit(1000 * waitSeconds); - if (!process.HasExited) - { - if(!traceToStdout) - { - Console.WriteLine("Warning: Unable to kill process after {0} seconds...", waitSeconds); - } - - var xout = new XElement("UnableToKillProcess", - new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways)); - - AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); - return 104; - } - } - if (!traceToStdout) - { - Console.WriteLine("Exit code is {0}", process.ExitCode); - } - memCheckThread.Join(); - consoleThread.Join(); - - var traceFiles = Directory.GetFiles(tempPath, "trace*.xml"); - if (traceFiles.Length == 0) - { - if (!traceToStdout) - { - Console.WriteLine("Warning: No trace file was generated, summary will not be affected."); - } - - var xout = new XElement((useValgrind ? "NoTraceFileGeneratedLibPos" : "NoTraceFileGenerated"), - new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), - new XAttribute("Plugin", tlsPluginFile), - new XAttribute("MachineName", System.Environment.MachineName)); - - AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); - ok = useValgrind ? 0 : 103; - } - else - { - var result = Summarize(traceFiles, summaryFileName, errorFileName, killed, errorListener.Errors, - process.ExitCode, memChecker.MaxMem, uid, - valgrindOutputFile == null ? null : Path.Combine(tempPath, valgrindOutputFile), - expectedUnseed, out unseed, out retryableError, logOnRetryableError, willRestart, restarting, oldBinaryName, traceToStdout); - - if (result != 0) - { - ok = result; - } - } - - if (willRestart) foreach (var f in traceFiles) File.Delete(f); - - process.Close(); - } - if (!traceToStdout) - { - Console.WriteLine("Done (unseed {0})", unseed); - } - - return ok; - } - catch (Exception e) - { - if (!traceToStdout) - { - Console.WriteLine("Error in Run:"); - Console.WriteLine(e); - } - - var xout = new XElement("TestHarnessRunError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorMessage", e.Message)); - - AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); - return 101; - } - finally - { - Directory.SetCurrentDirectory(oldDir); - if (!willRestart) Directory.Delete(tempPath, true); - } - } - - class OutputCopier - { - private StreamReader reader; - - public OutputCopier(StreamReader streamReader) - { - this.reader = streamReader; - } - - public void copyOutput() - { - //Console.WriteLine(" Beginning console copy..."); - while (!reader.EndOfStream) - { - reader.ReadLine(); - //Console.WriteLine(" : " + s); - } - } - } - - private class ErrorOutputListener - { - public List Errors { get; set; } - int maxErrorLength; - int maxErrors; - - public ErrorOutputListener(int maxErrorLength = 100, int maxErrors = 10) - { - Errors = new List(); - this.maxErrorLength = maxErrorLength; - this.maxErrors = maxErrors; - } - - public bool hasError = false; - public void handleData(object sendingProcess, DataReceivedEventArgs errLine) - { - if(!String.IsNullOrEmpty(errLine.Data)) - { - hasError = true; - if(Errors.Count < maxErrors) - Errors.Add(errLine.Data.Substring(0, Math.Min(maxErrorLength, errLine.Data.Length))); - } - } - } - - class MemoryChecker - { - //private System.Diagnostics.Process process; - private long maxMem; - private Process process; - - public long MaxMem - { - get { return maxMem; } - set { maxMem = value; } - } - - public MemoryChecker(System.Diagnostics.Process process) - { - this.process = process; - this.maxMem = 0; - } - - public void monitorMemory() - { - while (true) - { - try - { - process.Refresh(); - if (process.HasExited) - return; - long mem = process.PrivateMemorySize64; - MaxMem = Math.Max(MaxMem, mem); - //Console.WriteLine(string.Format("Process used {0} bytes", MaxMem)); - Thread.Sleep(1000); - } - catch - { - return; - } - } - } - } - - static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, string oldBinary="") - { - var xout = new XElement("TestPlan", - new XAttribute("TestUID", uid), - new XAttribute("RandomSeed", randomSeed), - new XAttribute("TestFile", testFileName), - new XAttribute("BuggifyEnabled", buggify ? "1" : "0"), - new XAttribute("DeterminismCheck", testDeterminism ? "1" : "0"), - new XAttribute("OldBinary", Path.GetFileName(oldBinary))); - AppendToSummary(summaryFileName, xout); - } - - // Parses the valgrind XML file and returns a list of "what" tags for each error. - // All errors for which the "kind" tag starts with "Leak" are ignored - static string[] ParseValgrindOutput(string valgrindOutputFileName, bool traceToStdout) - { - if (!traceToStdout) - { - Console.WriteLine("Reading vXML file: " + valgrindOutputFileName); - } - - ISet whats = new HashSet(); - XElement xdoc = XDocument.Load(valgrindOutputFileName).Element("valgrindoutput"); - foreach(var elem in xdoc.Elements()) { - if (elem.Name != "error") - continue; - string kind = elem.Element("kind").Value; - if(kind.StartsWith("Leak")) - continue; - whats.Add(elem.Element("what").Value); - } - return whats.ToArray(); - } - - static int Summarize(string[] traceFiles, string summaryFileName, - string errorFileName, bool? killed, List outputErrors, int? exitCode, long? peakMemory, - string uid, string valgrindOutputFileName, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, - bool willRestart = false, bool restarted = false, string oldBinaryName = "", bool traceToStdout = false, string externalError = "") - { - unseed = -1; - retryableError = false; - List errorList = new List(); - var xout = new XElement("Test"); - if (uid != null) - xout.Add(new XAttribute("TestUID", uid)); - bool ok = false; - string testFile = "Unknown"; - int testsPassed = 0, testCount = -1, warnings = 0, errors = 0; - bool testBeginFound = false, testEndFound = false, error = false; - string firstRetryableError = ""; - int stderrSeverity = (int)Magnesium.Severity.SevError; - - Dictionary, Magnesium.Severity> severityMap = new Dictionary, Magnesium.Severity>(); - Dictionary, bool> codeCoverage = new Dictionary, bool>(); - - foreach (var traceFileName in traceFiles) - { - if(!traceToStdout) { - Console.WriteLine("Summarizing {0}", traceFileName); - } - using (var traceFile = System.IO.File.Open(traceFileName, - System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) - { - try - { - foreach (var ev in Magnesium.XmlParser.Parse(traceFile, traceFileName)) - { - Magnesium.Severity newSeverity; - if (severityMap.TryGetValue(new KeyValuePair(ev.Type, ev.Severity), out newSeverity)) - ev.Severity = newSeverity; - if (ev.Severity >= Magnesium.Severity.SevWarnAlways && ev.DDetails.ContainsKey("ErrorIsInjectedFault")) - ev.Severity = Magnesium.Severity.SevWarn; - - if (ev.Type == "ProgramStart" && !testBeginFound - // Just in case the first ProgramStart seen is a Simulated one, ignore it - && (!ev.DDetails.ContainsKey("Simulated") || ev.Details.Simulated != "1")) - { - xout.Add( - new XAttribute("RandomSeed", ev.Details.RandomSeed), - new XAttribute("SourceVersion", ev.Details.SourceVersion), - new XAttribute("Time", ev.Details.ActualTime), - new XAttribute("BuggifyEnabled", ev.Details.BuggifyEnabled), - new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"), - new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); - testBeginFound = true; - } - if (ev.Type == "Simulation") - { - xout.Add( - new XAttribute("TestFile", ev.Details.TestFile)); - testFile = ev.Details.TestFile.Substring(ev.Details.TestFile.IndexOf("tests")); - } - if (ev.Type == "ActualRun") - { - xout.Add( - new XAttribute("TestFile", ev.Details.RunID)); - } - if (ev.Type == "ElapsedTime" && !testEndFound) - { - testEndFound = true; - unseed = int.Parse(ev.Details.RandomUnseed); - if (expectedUnseed != -1 && expectedUnseed != unseed) - { - Magnesium.Severity severity; - if (!severityMap.TryGetValue(new KeyValuePair("UnseedMismatch", Magnesium.Severity.SevError), out severity)) - severity = Magnesium.Severity.SevError; - - if (severity >= Magnesium.Severity.SevWarnAlways) - { - xout.Add(new XElement("UnseedMismatch", - new XAttribute("Unseed", unseed), - new XAttribute("ExpectedUnseed", expectedUnseed), - new XAttribute("Severity", (int)severity))); - if ( severity == Magnesium.Severity.SevError ) { - error = true; - errorList.Add("UnseedMismatch"); - } - } - } - xout.Add( - new XAttribute("SimElapsedTime", ev.Details.SimTime), - new XAttribute("RealElapsedTime", ev.Details.RealTime), - new XAttribute("RandomUnseed", ev.Details.RandomUnseed)); - } - if (ev.Severity == Magnesium.Severity.SevWarnAlways) - { - if (warnings < maxWarnings) - { - xout.Add(new XElement(ev.Type, - new XAttribute("Severity", (int)ev.Severity), - ev.DDetails - //.Where(kv => true) - .Select(kv => new XAttribute(kv.Key, kv.Value)))); - } - warnings++; - } - if (ev.Severity >= Magnesium.Severity.SevError) - { - string errorString = ev.FormatTestError(true); - if (errorString.Contains("platform_error")) - { - if (!retryableError) - { - firstRetryableError = errorString; - } - retryableError = true; - } - if (errors < maxWarnings) - { - xout.Add(new XElement(ev.Type, - new XAttribute("Severity", (int)ev.Severity), - ev.DDetails - //.Where(kv => true) - .Select(kv => new XAttribute(kv.Key, kv.Value)))); - errorList.Add(errorString); - } - errors++; - error = true; - } - if (ev.Type == "CodeCoverage" && !willRestart) - { - bool covered = true; - if(ev.DDetails.ContainsKey("Covered")) - { - covered = int.Parse(ev.Details.Covered) != 0; - } - - var key = new Tuple(ev.Details.File, ev.Details.Line); - if (covered || !codeCoverage.ContainsKey(key)) - { - codeCoverage[key] = covered; - } - } - if (ev.Type == "FaultInjected" || (ev.Type == "BuggifySection" && ev.Details.Activated == "1")) - { - xout.Add(new XElement(ev.Type, new XAttribute("File", ev.Details.File), new XAttribute("Line", ev.Details.Line))); - } - if (ev.Type == "TestsExpectedToPass") - testCount = int.Parse(ev.Details.Count); - if (ev.Type == "TestResults" && ev.Details.Passed == "1") - testsPassed++; - if (ev.Type == "RemapEventSeverity") - severityMap[new KeyValuePair(ev.Details.TargetEvent, (Magnesium.Severity)int.Parse(ev.Details.OriginalSeverity))] = (Magnesium.Severity)int.Parse(ev.Details.NewSeverity); - if (ev.Type == "StderrSeverity") - stderrSeverity = int.Parse(ev.Details.NewSeverity); - } - - } - catch (Exception e) - { - if (!traceToStdout) - { - Console.WriteLine("Error summarizing {0}: {1}", traceFileName, e); - } - - error = true; - xout.Add(new XElement("SummarizationError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorMessage", e.Message))); - errorList.Add("SummarizationError " + e.Message); - break; - } - } - } - - if (externalError.Length > 0) { - xout.Add(new XElement(externalError, new XAttribute("Severity", (int)Magnesium.Severity.SevError))); - } - - foreach(var kv in codeCoverage) - { - var element = new XElement("CodeCoverage", new XAttribute("File", kv.Key.Item1), new XAttribute("Line", kv.Key.Item2)); - if(!kv.Value) - { - element.Add(new XAttribute("Covered", "0")); - } - - xout.Add(element); - } - - if (warnings > maxWarnings) - { - //error = true; - xout.Add(new XElement("WarningLimitExceeded", - new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), - new XAttribute("WarningCount", warnings))); - } - if (errors > maxWarnings) - { - error = true; - xout.Add(new XElement("ErrorLimitExceeded", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorCount", errors))); - errorList.Add("ErrorLimitExceeded"); - } - if (killed == true) - { - if (!retryableError) - { - firstRetryableError = "ExternalTimeout"; - } - - retryableError = true; - error = true; - xout.Add(new XElement("ExternalTimeout", new XAttribute("Severity", (int)Magnesium.Severity.SevError))); - } - if (outputErrors != null) - { - int stderrBytes = 0; - foreach (string err in outputErrors) - { - if (stderrSeverity == (int)Magnesium.Severity.SevError) - { - error = true; - } - - int remainingBytes = maxStderrBytes - stderrBytes; - if (remainingBytes > 0) - { - string outErr = (err.Length > remainingBytes) ? err.Substring(remainingBytes) + "..." : err; - - xout.Add(new XElement("StdErrOutput", - new XAttribute("Severity", stderrSeverity), - new XAttribute("Output", outErr))); - } - - stderrBytes += err.Length; - } - - if (stderrBytes > maxStderrBytes) - { - xout.Add(new XElement("StdErrOutputTruncated", - new XAttribute("Severity", stderrSeverity), - new XAttribute("BytesRemaining", stderrBytes - maxStderrBytes))); - } - } - if (exitCode.HasValue && exitCode != 0) - { - error = true; - xout.Add(new XElement("ExitCode", new XAttribute("Code", exitCode.Value), new XAttribute("Severity", (int)Magnesium.Severity.SevError))); - errorList.Add(string.Format("ExitCode 0x{0:x}", exitCode.Value)); - } - if (!testEndFound && !willRestart) - { - // We didn't terminate the test, but it didn't reach the end? - error = true; - xout.Add(new XElement("TestUnexpectedlyNotFinished"), new XAttribute("Severity", (int)Magnesium.Severity.SevError)); - errorList.Add("TestUnexpectedlyNotFinished"); - } - ok = testsPassed == testCount && testsPassed > 0 && !error; - xout.Add( - new XAttribute("Passed", testsPassed), - new XAttribute("Failed", testCount - testsPassed)); - if (peakMemory.HasValue) - xout.Add(new XAttribute("PeakMemory", peakMemory.Value)); - - if (valgrindOutputFileName != null && valgrindOutputFileName.Length > 0) - { - try - { - // If there are any errors reported "ok" will be set to false - var whats = ParseValgrindOutput(valgrindOutputFileName, traceToStdout); - foreach (var what in whats) - { - xout.Add(new XElement("ValgrindError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("What", what))); - ok = false; - error = true; - } - } - catch (Exception e) - { - if (!traceToStdout) - { - Console.WriteLine(e); - } - - error = true; - xout.Add(new XElement("ValgrindParseError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorMessage", e.Message))); - errorList.Add("Failed to parse valgrind output: " + e.Message); - } - } - - if (retryableError && !logOnRetryableError) - { - xout = new XElement("Test", xout.Attributes()); - xout.Add(new XElement("RetryingError", - new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), - new XAttribute("What", firstRetryableError))); - } - else - { - xout.Add(new XAttribute("OK", ok || willRestart)); - } - - AppendToSummary(summaryFileName, xout, traceToStdout); - - if ((!retryableError || logOnRetryableError) && errorFileName != null && (errorList.Count > 0 || !ok) && !willRestart) - { - var errorText = string.Join("\n\t", errorList - .Concat((!ok && errorList.Count == 0) ? new string[] { "Failed with no explanation" } : new string[] { }) - .Distinct() - .ToArray()); - AppendToFile(errorFileName, string.Format("Test {0} failed with:\n\t{1}\n", testFile, errorText)); - } - if (!error) { - return 0; - } - else { - return 102; - } - } - - static int ExtractErrors(string summaryFileName, string errorSummaryFileName) - { - Console.WriteLine("Extracting from {0}", summaryFileName); - List xout = new List(); - var coverage = new Dictionary,Tuple>(); - using (var traceFile = System.IO.File.Open(summaryFileName, - System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) - { - try - { - var events = Magnesium.XmlParser.Parse(traceFile, summaryFileName, true); - events = Magnesium.TraceLogUtil.IdentifyFailedTestPlans(events); - foreach (var ev in events) - { - Magnesium.Test t = ev as Magnesium.Test; - if (t != null) - { - foreach (var tev in t.events) - { - if (tev.Type == "CodeCoverage" || tev.Type == "FaultInjected") - { - var keyTuple = Tuple.Create(tev.Details.File, int.Parse(tev.Details.Line)); - if (coverage.ContainsKey(keyTuple)) - { - var old = coverage[keyTuple]; - coverage[keyTuple] = Tuple.Create(old.Item1 + 1, old.Item2 + (t.ok ? 0 : 1)); - } - else - { - coverage[keyTuple] = Tuple.Create(1, (t.ok ? 0 : 1)); - } - } - } - if (!t.ok) - { - if (t.original != null) - { - foreach (var c in t.original.Elements("CodeCoverage")) - c.Remove(); - foreach (var f in t.original.Elements("FaultInjected")) - f.Remove(); - - xout.Add(t.original); - } - else - { - xout.Add(new XElement("Test", - new XAttribute("Type", t.Type), - new XAttribute("Time", t.Time), - new XAttribute("Machine", t.Machine), - new XAttribute("TestUID", t.TestUID), - new XAttribute("TestFile", t.TestFile), - new XAttribute("randomSeed", t.randomSeed), - new XAttribute("Buggify", t.Buggify), - new XAttribute("DeterminismCheck", t.DeterminismCheck), - new XAttribute("OldBinary", t.OldBinary), - new XElement("TestNotSummarized", - new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways) - ) - ) - ); - } - } - } - } - } - catch (Exception e) - { - Console.WriteLine("Error summarizing {0}: {1}", summaryFileName, e); - xout.Add(new XElement("SummarizationError", - new XAttribute("Severity", (int)Magnesium.Severity.SevError), - new XAttribute("ErrorMessage", e.Message))); - //failedTests.Add("SummarizationError " + e.Message); - } - } - - foreach (var e in coverage) - { - xout.Add(new XElement("Event", - new XAttribute("Type", "CoverageSummary"), - new XAttribute("Time", 0), - new XAttribute("Machine", ""), - new XAttribute("File", e.Key.Item1), - new XAttribute("Line", e.Key.Item2), - new XAttribute("Covered", e.Value.Item1), - new XAttribute("Failed", e.Value.Item2))); - } - - AppendToErrorSummary(errorSummaryFileName, xout); - return 0; - } - - private static void AppendToErrorSummary(string summaryFileName, List elements) - { - if (summaryFileName == null) - return; - takeLock(summaryFileName); - try { - foreach (XElement e in elements) - AppendToSummary(summaryFileName, e, false, false); - } - finally - { - releaseLock(summaryFileName); - } - } - - private static void AppendToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, bool shouldLock = true) - { - if (traceToStdout) - { - using (var wr = System.Xml.XmlWriter.Create(Console.OpenStandardOutput(), new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true, Encoding = new System.Text.UTF8Encoding(false) })) - xout.WriteTo(wr); - Console.WriteLine(); - return; - } - - if (summaryFileName == null) - return; - if (shouldLock) - takeLock(summaryFileName); - try - { - - using (var f = System.IO.File.Open(summaryFileName, System.IO.FileMode.Append, System.IO.FileAccess.Write)) - { - if (f.Length == 0) - { - byte[] bytes = Encoding.UTF8.GetBytes(""); - f.Write(bytes, 0, bytes.Length); - } - using (var wr = System.Xml.XmlWriter.Create(f, new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true })) - xout.Save(wr); - var endl = Encoding.UTF8.GetBytes(Environment.NewLine); - f.Write(endl, 0, endl.Length); - } - } - finally - { - if (shouldLock) - releaseLock(summaryFileName); - } - } - private static void AppendXmlMessageToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, string testFile = null, - int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null) - { - var test = new XElement("Test", xout); - if(testFile != null) - test.Add(new XAttribute("TestFile", testFile)); - if(seed != null) - test.Add(new XAttribute("RandomSeed", seed)); - if(buggify != null) - test.Add(new XAttribute("BuggifyEnabled", buggify.Value ? "1" : "0")); - if(determinismCheck != null) - test.Add(new XAttribute("DeterminismCheck", determinismCheck.Value ? "1" : "0")); - if(oldBinaryName != null) - test.Add(new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); - - test.Add(xout); - AppendToSummary(summaryFileName, test, traceToStdout); - } - - private static void AppendToFile(string fileName, string content) - { - if (fileName == null) - return; - takeLock(fileName); - try - { - using (var f = System.IO.File.Open(fileName, System.IO.FileMode.Append, System.IO.FileAccess.Write)) - { - var endl = Encoding.UTF8.GetBytes(content); - f.Write(endl, 0, endl.Length); - } - } - finally - { - releaseLock(fileName); - } - } - - static int Remote(string queue, string fdbRoot, double addHours, int testCount, string testTypes, string userScope) - { - queue = Path.GetFullPath(queue); - fdbRoot = Path.GetFullPath(fdbRoot); - var output = Path.Combine(queue, "archive"); - var now = DateTime.Now; - string date = string.Format("{0}-{1:00}-{2:00}-{3:00}-{4:00}", - now.Year, now.Month, now.Day, now.Hour, now.Minute); - - if (!Directory.Exists(queue)) - Directory.CreateDirectory(queue); - - int maxCount = 0; - foreach (var f in Directory.GetFiles(queue, String.Format("{0}-*.xml", date))) - { - var count = Int32.Parse(f.Split('-')[5]); - maxCount = Math.Max(maxCount, count); - } - maxCount++; - - var suffix = String.Format("{0}-{1}-{2}", Environment.UserName, userScope, OS_NAME); - foreach (var f in Directory.GetFiles(queue, "*" + suffix + ".xml")) - File.Delete(f); - - string label = String.Format("{0}-{1}-{2}", date, maxCount, suffix); - - var testStaging = Path.Combine(output, label); - Directory.CreateDirectory(testStaging); - File.Create(Path.Combine(testStaging, "errors.txt")); - - var release = Path.Combine(fdbRoot, "bin"); - if(!IsRunningOnMono()) - release = Path.Combine(release, "Release"); - - File.Copy(Path.Combine(release, BINARY), Path.Combine(testStaging, BINARY)); - File.Copy(Path.Combine(fdbRoot, "tls-plugins", PLUGIN), Path.Combine(testStaging, PLUGIN)); - - if (IsRunningOnMono()) - File.Copy(Path.Combine(release, BINARY + ".debug"), Path.Combine(testStaging, BINARY + ".debug")); - - //using (var f = System.IO.File.Open( - // Path.Combine(testStaging, "summary.xml"), - // System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.Delete)) - //{ - // byte[] bytes = Encoding.UTF8.GetBytes(""); - // f.Write(bytes, 0, bytes.Length); - //} - - var coverageFiles = Directory.GetFiles(release, "coverage*.xml"); - foreach (var coverage in coverageFiles) - File.Copy(coverage, Path.Combine(testStaging, Path.GetFileName(coverage))); - - Directory.CreateDirectory(Path.Combine(testStaging, "tests")); - - if (testTypes == "fast" || testTypes == "all") - CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "fast")), new DirectoryInfo(Path.Combine(testStaging, "tests", "fast"))); - if (testTypes == "restarting" || testTypes == "all") - { - CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "restarting")), new DirectoryInfo(Path.Combine(testStaging, "tests", "restarting"))); - //CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "oldBinaries")), new DirectoryInfo(Path.Combine(testStaging, "tests", "oldBinaries"))); - } - if (testTypes == "all") - { - CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "slow")), new DirectoryInfo(Path.Combine(testStaging, "tests", "slow"))); - CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "rare")), new DirectoryInfo(Path.Combine(testStaging, "tests", "rare"))); - } - - - if (testTypes != "fast" && testTypes != "all" && testTypes != "restarting") - { - FileInfo file = new FileInfo(testTypes); - Directory.CreateDirectory(Path.Combine(testStaging, "tests", file.Directory.Name)); - file.CopyTo(Path.Combine(testStaging, "tests", file.Directory.Name, file.Name), true); - } - - var e = - new XElement("TestDefinition", - new XElement("Duration", - new XAttribute("Hours", addHours)), - new XElement("TestCount", testCount)); - new XDocument(e).Save(Path.Combine(queue, label + ".xml")); - File.Copy(Path.Combine(queue, label + ".xml"), Path.Combine(testStaging, label + ".xml")); - - var summaryFile = Path.Combine(testStaging, "summary.xml"); - Console.WriteLine(label); - - if (!IsRunningOnMono()) - { - using (var mProcess = new System.Diagnostics.Process()) - { - mProcess.StartInfo.UseShellExecute = false; - mProcess.StartInfo.RedirectStandardOutput = true; - mProcess.StartInfo.FileName = "Magnesium.exe"; - mProcess.StartInfo.Arguments = "Summary " + summaryFile; - mProcess.Start(); - } - } - - return 0; - } - - public static void CopyAll(DirectoryInfo source, DirectoryInfo target, Func predicate = null) - { - // Check if the target directory exists, if not, create it. - if (!Directory.Exists(target.FullName)) - Directory.CreateDirectory(target.FullName); - - // Copy each file into it's new directory. - foreach (FileInfo fi in source.GetFiles()) - if (predicate == null || predicate(fi)) - fi.CopyTo(Path.Combine(target.ToString(), fi.Name), true); - - // Copy each subdirectory using recursion. - foreach (DirectoryInfo diSourceSubDir in source.GetDirectories()) - CopyAll(diSourceSubDir, target.CreateSubdirectory(diSourceSubDir.Name), predicate); - } - - static int Auto(string queueDirectory, string runDir, string shareDir, string cacheDir, bool useValgrind, int maxTries) - { - try - { - queueDirectory = Path.GetFullPath(queueDirectory); - while (true) - { - Test test = getTest(queueDirectory, runDir, shareDir, cacheDir); - Console.WriteLine("Running test {0}", test.label); - - // run test - test.run(useValgrind, maxTries); - } - } - catch (Exception e) - { - Console.WriteLine("Error: {0}", e); - return 100; - } - } - - class Test - { - public DateTime? testEnd; - public string queueDirectory; - public string label; - public string runDir; - public string inputDir; - public string outputDir; - public string oldBinaryDir; - public int testCount; - - public Test(string queueDirectory, string label, string runDir, string shareDir, string cacheDir) - { - this.queueDirectory = queueDirectory; - this.label = label; - this.runDir = runDir; - var specFile = Path.Combine(queueDirectory, label + ".xml"); - var testDef = XDocument.Load(specFile).Element("TestDefinition"); - var testDuration = double.Parse(testDef.Element("Duration").Attribute("Hours").Value); - var testBegin = File.GetCreationTime(specFile); - testEnd = testDuration < 0 ? (DateTime?)null : testBegin.AddHours(testDuration); - - testCount = int.Parse(testDef.Element("TestCount").Value); - outputDir = Path.Combine(queueDirectory, "archive", label); - Directory.CreateDirectory(outputDir); - - string oldBinarySourceDir = Path.Combine(shareDir, "oldBinaries"); - - if (cacheDir != null) - { - inputDir = Path.Combine(cacheDir, "archive", label); - Directory.CreateDirectory(Path.Combine(cacheDir, "archive")); - - if (!Directory.Exists(inputDir)) - { - takeLock(inputDir); - if (!Directory.Exists(inputDir)) - { - string tmpDir = Path.Combine(cacheDir, "archive.part", label + "." + Path.GetRandomFileName() + ".part"); - Directory.CreateDirectory(tmpDir); - CopyAll(new DirectoryInfo(outputDir), new DirectoryInfo(tmpDir), (FileInfo file) => - file.Name != "fdbserver.debug" && - !file.Name.StartsWith("summary-") && - file.Name != "errors.txt" - ); - Directory.Move(tmpDir, inputDir); - } - releaseLock(inputDir); - } - - oldBinaryDir = Path.Combine(cacheDir, "oldBinaries"); - - Directory.CreateDirectory(oldBinaryDir); - foreach (FileInfo fi in new DirectoryInfo(oldBinarySourceDir).GetFiles()) - { - var targetName = Path.Combine(oldBinaryDir, fi.Name); - if (!File.Exists(targetName) || fi.LastWriteTimeUtc != File.GetLastWriteTimeUtc(targetName)) - { - fi.CopyTo(targetName, true); - File.SetLastWriteTimeUtc(targetName, fi.LastWriteTimeUtc); - } - } - - foreach (FileInfo fi in new DirectoryInfo(oldBinaryDir).GetFiles()) - { - var targetName = Path.Combine(oldBinarySourceDir, fi.Name); - if (!File.Exists(targetName)) - fi.Delete(); - } - } - else - { - inputDir = outputDir; - oldBinaryDir = oldBinarySourceDir; - } - //Console.WriteLine("TestEnd {0}, now {1}, duration {2}, done {3}", testEnd, DateTime.Now, testDuration, done()); - } - public bool done() - { - return testEnd.HasValue && System.DateTime.Now > testEnd.Value; - } - public void run(bool useValgrind, int maxTries) - { - Run(Path.Combine(inputDir, BINARY), - Path.Combine(inputDir, PLUGIN), - Path.Combine(inputDir, "tests"), - Path.Combine(outputDir, "summary-" + Environment.MachineName + ".xml"), - Path.Combine(outputDir, "errors.txt"), - runDir, - oldBinaryDir, - useValgrind, - maxTries); - } - - public void finalize() - { - try - { - Console.WriteLine("Deleting: {0}", Path.Combine(queueDirectory, label + ".xml")); - File.Delete(Path.Combine(queueDirectory, label + ".xml")); - } - catch (Exception e) - { - Console.WriteLine("Error deleting queue folder: {0}", e.Message); - } - } - } - - static Test getTest(string parent, string runDir, string shareDir, string cacheDir) - { - while (true) - { - var testFiles = Directory.GetFiles(parent, String.Format("*{0}.xml", OS_NAME)); - // (if no tests, wait, try again) - if (testFiles.Length != 0) - { - /*try - {*/ - var testFile = testFiles[random.Next(testFiles.Length)]; - var test = new Test(parent, Path.GetFileNameWithoutExtension(testFile), runDir, shareDir, cacheDir); - if ((test.testCount < 0 || UpdateTestTotals(Path.Combine(test.outputDir, "testCount"), test.testCount)) && !test.done()) - return test; - else - test.finalize(); - /*} - catch (Exception) - { - // retry opening a test - }*/ - } - System.Threading.Thread.Sleep(1000); - } - } - - static void takeLock(string targetFile) - { - // Console.WriteLine("Attempting to take lock on {0}", targetFile); - string lockFile = targetFile + ".lock"; - while (true) - { - try - { - using (var f = System.IO.File.Open(lockFile, System.IO.FileMode.CreateNew)) - { - return; - } - } - catch (System.IO.IOException e) - { - Console.WriteLine("Waiting for file lock: {0}", e.Message); - System.Threading.Thread.Sleep(250); - } - } - } - - static void releaseLock(string targetFile) - { - File.Delete(targetFile + ".lock"); - } - - private static bool UpdateTestTotals(string countFileName, int desiredTestCount) - { - takeLock(countFileName); - try - { - using (var f = System.IO.File.Open(countFileName, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite)) - { - int currentCount = 0; - byte[] b; - - if (f.Length != 0) - { - b = new byte[f.Length]; - f.Read(b, 0, b.Length); - currentCount = int.Parse(Encoding.UTF8.GetString(b)); - } - - if (currentCount >= desiredTestCount) - return false; - - f.SetLength(0); - b = Encoding.UTF8.GetBytes(String.Format("{0}", currentCount + 1)); - f.Write(b, 0, b.Length); - - return true; - } - } - finally - { - releaseLock(countFileName); - } - } - - private static int VersionInfo() - { - Console.WriteLine("Version: 1.02"); - - Console.WriteLine("FDB Project Ver: " + "${CMAKE_PROJECT_VERSION}"); - Console.WriteLine("FDB Version: " + "${CMAKE_PROJECT_VERSION_MAJOR}" + "." + "${CMAKE_PROJECT_VERSION_MINOR}"); - Console.WriteLine("Source Version: " + "${CURRENT_GIT_VERSION}"); - return 1; - } - - private static int UsageMessage() - { - Console.WriteLine("Usage:"); - Console.WriteLine(" TestHarness run [temp/runDir] [fdbserver[.exe]] [TLSplugin] [testfolder] [summary.xml] "); - Console.WriteLine(" TestHarness summarize [trace.xml] [summary.xml] "); - Console.WriteLine(" TestHarness replay [temp/runDir] [fdbserver[.exe]] [TLSplugin] [summary-in.xml] [summary-out.xml]"); - Console.WriteLine(" TestHarness auto [temp/runDir] [directory] [shareDir] "); - Console.WriteLine(" TestHarness remote [queue folder] [root foundation folder] [duration in hours] [amount of tests] [all/fast/] [scope]"); - Console.WriteLine(" TestHarness extract-errors [summary-file] [error-summary-file]"); - Console.WriteLine(" TestHarness joshua-run "); - VersionInfo(); - return 1; - } - } -} +/* + * Program.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Xml.Linq; +using System.IO; +using System.Diagnostics; +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Xml; +using System.Runtime.Serialization.Json; + +namespace SummarizeTest +{ + static class Program + { + static Random random; + const int killSeconds = 60 * 30; + const int maxWarnings = 10; + const int maxStderrBytes = 1000; + const double unseedRatio = 0.05; + const double buggifyOnRatio = 0.8; + static string BINARY = "fdbserver" + (IsRunningOnMono() ? "" : ".exe"); + static string PLUGIN = "FDBLibTLS." + (IsRunningOnMono() ? "so" : "dll"); + static string OS_NAME = IsRunningOnMono() ? "linux" : "win"; + + static int Main(string[] args) + { + bool traceToStdout = false; + try + { + byte[] seed = new byte[4]; + new System.Security.Cryptography.RNGCryptoServiceProvider().GetBytes(seed); + random = new Random(new BinaryReader(new MemoryStream(seed)).ReadInt32()); + + if (args.Length < 1) + return UsageMessage(); + + if (args[0] == "summarize") + { + if (args.Length < 3) + return UsageMessage(); + string valgrindFileName = args.Length >= 4 ? args[3] : null; + string externalError = args.Length >= 5 ? args[4] : ""; + traceToStdout = args.Length == 6 && args[5] == "true"; + + int unseed; + bool retryableError; + //SOMEDAY: This only works if a run generated just one trace file. We should change the summarize command to take multiple trace files + return Summarize(new string[]{ args[1] }, args[2], null, null, null, null, null, null, valgrindFileName, -1, out unseed, out retryableError, true, + traceToStdout: traceToStdout, externalError: externalError); + } + else if (args[0] == "remote") + { + if (args.Length < 6) + return UsageMessage(); + + return Remote(args[1], args[2], double.Parse(args[3]), int.Parse(args[4]), args[5], args.Length == 7 ? args[6] : "default"); + } + else if (args[0] == "run") + { + try + { + if (args.Length < 6) + return UsageMessage(); + + string runDir = null; + if(args[1] != "temp") runDir = args[1]; + + bool useValgind = args.Length > 6 && args[6].ToLower() == "true"; + int maxTries = (args.Length > 7) ? int.Parse(args[7]) : 3; + return Run(args[2], args[3], args[4], args[5], null, runDir, null, useValgind, maxTries); + } + catch(Exception e) + { + var xout = new XElement("TestHarnessError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message)); + + AppendXmlMessageToSummary(args[5], xout); + throw; + } + } + else if (args[0] == "replay") + { + if (args.Length != 6) + return UsageMessage(); + + string runDir = null; + if (args[1] != "temp") runDir = args[1]; + + return Replay(args[2], args[3], args[4], args[5], runDir); + } + else if (args[0] == "auto") + { + if (args.Length < 4) + return UsageMessage(); + + string runDir = null; + if (args[1] != "temp") runDir = args[1]; + + string cacheDir = Path.Combine(runDir, "test_harness_cache"); + bool useValgrind = args.Length > 4 && args[4].ToLower() == "true"; + int maxTries = (args.Length > 5) ? int.Parse(args[5]) : 3; + + return Auto(args[2], Path.Combine(runDir, "runs"), args[3], cacheDir, useValgrind, maxTries); + } + else if (args[0] == "extract-errors") + { + if (args.Length != 3) + return UsageMessage(); + + return ExtractErrors(args[1], args[2]); + } + else if (args[0] == "joshua-run") + { + traceToStdout = true; + + try + { + string oldBinaryFolder = (args.Length > 1) ? args[1] : Path.Combine("/opt", "joshua", "global_data", "oldBinaries"); + bool useValgrind = args.Length > 2 && args[2].ToLower() == "true"; + int maxTries = (args.Length > 3) ? int.Parse(args[3]) : 3; + return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN)); + } + catch(Exception e) + { + var xout = new XElement("TestHarnessError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.ToString())); + + AppendXmlMessageToSummary("summary.xml", xout, true); + throw; + } + } + else if (args[0] == "version") + { + return VersionInfo(); + } + + return UsageMessage(); + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Error:"); + Console.WriteLine(e.ToString()); + } + + return 100; + } + } + + static T Choice(this Random random, IList items) + { + return items[ random.Next(items.Count) ]; + } + + static bool IsRunningOnMono() + { + return Type.GetType("Mono.Runtime") != null; + } + + static int Replay(string fdbserverName, string tlsPluginFile, string inputSummaryFileName, string outputSummaryFileName, string runDir) + { + using (var summaryFileIn = System.IO.File.Open(inputSummaryFileName, + System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) + { + foreach (var test in Magnesium.XmlParser.Parse(summaryFileIn, inputSummaryFileName).OfType()) + { + if (test is Magnesium.Test) continue; // Only process the test plans + int unseed; + bool retryableError; + RunTest(fdbserverName, tlsPluginFile, outputSummaryFileName, null, test.randomSeed, test.Buggify, test.TestFile, runDir, test.TestUID, -1, out unseed, out retryableError, true, false); + } + } + return 0; + } + + // Ver format - x.x.x (eg, 5.0.1, 4.2.11) + // Returns true is ver1 is greater than or equal to ver2 + static bool versionGreaterThanOrEqual(string ver1, string ver2) + { + string[] tokens1 = ver1.Split('.'); + string[] tokens2 = ver2.Split('.'); + if (tokens1.Length != tokens2.Length || tokens2.Length != 3) + { + throw new ArgumentException("Invalid Version Format Version1: " + ver1 + " Version2: " + ver2); + } + int[] version1 = Array.ConvertAll(tokens1, int.Parse); + int[] version2 = Array.ConvertAll(tokens2, int.Parse); + return ((System.Collections.IStructuralComparable)version1).CompareTo(version2, System.Collections.Generic.Comparer.Default) >= 0; + } + + static bool versionLessThan(string ver1, string ver2) { + return !versionGreaterThanOrEqual(ver1, ver2); + } + + static string getFdbserverVersion(string fdbserverName) { + using (var process = new System.Diagnostics.Process()) + { + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.FileName = fdbserverName; + process.StartInfo.Arguments = "--version"; + process.StartInfo.RedirectStandardError = true; + + process.Start(); + var output = process.StandardOutput.ReadToEnd(); + // If the process finished successfully, we call the parameterless WaitForExit to ensure that output buffers get flushed + process.WaitForExit(); + + var match = Regex.Match(output, @"v(\d+\.\d+\.\d+)"); + if (match.Groups.Count < 1) return ""; + return match.Groups[1].Value; + } + } + + static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "") + { + int seed = random.Next(1000000000); + bool buggify = random.NextDouble() < buggifyOnRatio; + string testFile = null; + string testDir = ""; + string oldServerName = ""; + + if (Directory.Exists(testFolder)) + { + int poolSize = 0; + if( Directory.Exists(Path.Combine(testFolder, "rare")) ) poolSize += 1; + if( Directory.Exists(Path.Combine(testFolder, "slow")) ) poolSize += 5; + if( Directory.Exists(Path.Combine(testFolder, "fast")) ) poolSize += 14; + if( Directory.Exists(Path.Combine(testFolder, "restarting")) ) poolSize += 1; + + if( poolSize == 0 ) { + Console.WriteLine("Passed folder ({0}) did not have a fast, slow, rare, or restarting sub-folder", testFolder); + return 1; + } + int selection = random.Next(poolSize); + int selectionWindow = 0; + + if( Directory.Exists(Path.Combine(testFolder, "rare")) ) selectionWindow += 1; + if (selection < selectionWindow) + testDir = Path.Combine(testFolder, "rare"); + else + { + if (Directory.Exists(Path.Combine(testFolder, "restarting"))) selectionWindow += 1; + if (selection < selectionWindow) + testDir = Path.Combine(testFolder, "restarting"); + else + { + if (Directory.Exists(Path.Combine(testFolder, "slow"))) selectionWindow += 5; + if (selection < selectionWindow) + testDir = Path.Combine(testFolder, "slow"); + else + testDir = Path.Combine(testFolder, "fast"); + } + } + string[] files = Directory.GetFiles(testDir, "*", SearchOption.AllDirectories); + string[] uniqueFiles; + if (testDir.EndsWith("restarting")) + { + ISet uniqueFileSet = new HashSet(); + foreach(string file in files) { + uniqueFileSet.Add(file.Substring(0, file.LastIndexOf("-"))); // all restarting tests end with -1.txt or -2.txt + } + uniqueFiles = uniqueFileSet.ToArray(); + testFile = random.Choice(uniqueFiles); + // The on-disk format changed in 4.0.0, and 5.x can't load files from 3.x. + string oldBinaryVersionLowerBound = "4.0.0"; + string lastFolderName = Path.GetFileName(Path.GetDirectoryName(testFile)); + if (lastFolderName.Contains("from_") || lastFolderName.Contains("to_")) // Only perform upgrade/downgrade tests from certain versions + { + oldBinaryVersionLowerBound = lastFolderName.Split('_').Last(); + } + string oldBinaryVersionUpperBound = getFdbserverVersion(fdbserverName); + if (versionGreaterThanOrEqual("4.0.0", oldBinaryVersionUpperBound)) { + // If the binary under test is from 3.x, then allow upgrade tests from 3.x binaries. + oldBinaryVersionLowerBound = "0.0.0"; + } + string[] currentBinary = { fdbserverName }; + IEnumerable oldBinaries = Array.FindAll( + Directory.GetFiles(oldBinaryFolder), + x => versionGreaterThanOrEqual(Path.GetFileName(x).Split('-').Last(), oldBinaryVersionLowerBound) + && versionLessThan(Path.GetFileName(x).Split('-').Last(), oldBinaryVersionUpperBound)); + oldBinaries = oldBinaries.Concat(currentBinary); + oldServerName = random.Choice(oldBinaries.ToList()); + } + else + { + uniqueFiles = Directory.GetFiles(testDir); + testFile = random.Choice(uniqueFiles); + } + } + else if (File.Exists(testFolder)) + testFile = testFolder; + else + { + Console.WriteLine("Passed path ({0}) was not a folder or file", testFolder); + return 1; + } + + int result = 0; + bool unseedCheck = random.NextDouble() < unseedRatio; + for (int i = 0; i < maxTries; ++i) + { + bool logOnRetryableError = i == maxTries - 1; + bool retryableError = false; + + if (testDir.EndsWith("restarting")) + { + bool isDowngrade = Path.GetFileName(Path.GetDirectoryName(testFile)).Contains("to_"); + string firstServerName = isDowngrade ? fdbserverName : oldServerName; + string secondServerName = isDowngrade ? oldServerName : fdbserverName; + int expectedUnseed = -1; + int unseed; + string uid = Guid.NewGuid().ToString(); + bool useNewPlugin = (oldServerName == fdbserverName) || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0"); + bool useToml = File.Exists(testFile + "-1.toml"); + string testFile1 = useToml ? testFile + "-1.toml" : testFile + "-1.txt"; + result = RunTest(firstServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile1, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout); + if (result == 0) + { + string testFile2 = useToml ? testFile + "-2.toml" : testFile + "-2.txt"; + result = RunTest(secondServerName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile2, runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout); + } + } + else + { + int expectedUnseed = -1; + if (!useValgrind && unseedCheck) + { + result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout); + } + + if (!retryableError) + { + int unseed; + result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout); + } + } + + if (!retryableError) + { + return result; + } + } + + return result; + } + + private static int RunTest(string fdbserverName, string tlsPluginFile, string summaryFileName, string errorFileName, int seed, + bool buggify, string testFile, string runDir, string uid, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, bool useValgrind, bool restarting = false, + bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false) + { + unseed = -1; + + retryableError = false; + string tempPath = Path.Combine(runDir != null ? Path.GetFullPath(runDir) : Path.GetTempPath(), uid); + string oldDir = Directory.GetCurrentDirectory(); + + try + { + fdbserverName = Path.GetFullPath(fdbserverName); + tlsPluginFile = (tlsPluginFile.Length > 0) ? Path.GetFullPath(tlsPluginFile) : ""; + testFile = Path.GetFullPath(testFile); + var ok = 0; + + if (summaryFileName != null) + summaryFileName = Path.GetFullPath(summaryFileName); + + Directory.CreateDirectory(tempPath); + Directory.SetCurrentDirectory(tempPath); + + if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, oldBinaryName); + + string valgrindOutputFile = null; + using (var process = new System.Diagnostics.Process()) + { + ErrorOutputListener errorListener = new ErrorOutputListener(); + process.StartInfo.UseShellExecute = false; + string tlsPluginArg = ""; + if (tlsPluginFile.Length > 0) { + process.StartInfo.EnvironmentVariables["FDB_TLS_PLUGIN"] = tlsPluginFile; + tlsPluginArg = "--tls_plugin=" + tlsPluginFile; + } + process.StartInfo.RedirectStandardOutput = true; + var args = ""; + if (willRestart && oldBinaryName.EndsWith("alpha6")) + { + args = string.Format("-Rs 1000000000 -r simulation {0} -s {1} -f \"{2}\" -b {3} {4} --crash", + IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg); + } + else + { + args = string.Format("-Rs 1GB -r simulation {0} -s {1} -f \"{2}\" -b {3} {4} --crash", + IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginArg); + } + if (restarting) args = args + " --restarting"; + if (useValgrind && !willRestart) + { + valgrindOutputFile = string.Format("valgrind-{0}.xml", seed); + process.StartInfo.FileName = "valgrind"; + // Add extra debug directory, if environment variables is defined + string valgrindDbgDir = System.Environment.GetEnvironmentVariable("FDB_VALGRIND_DBGPATH"); + process.StartInfo.Arguments = (string.IsNullOrEmpty(valgrindDbgDir)) ? "" : string.Format("--extra-debuginfo-path={0} ", valgrindDbgDir); + + process.StartInfo.Arguments += + string.Format("--xml=yes --xml-file={0} -q {1} {2}", valgrindOutputFile, fdbserverName, args); + } + else + { + process.StartInfo.FileName = fdbserverName; + process.StartInfo.Arguments = args; + } + process.StartInfo.RedirectStandardError = true; + process.ErrorDataReceived += new DataReceivedEventHandler(errorListener.handleData); + + if (!traceToStdout) + { + Console.WriteLine("Executing {0} {1} in {2} (buggify: {3}, valgrind: {4})", + process.StartInfo.FileName, process.StartInfo.Arguments, tempPath, + buggify ? "on" : "off", + useValgrind ? "on" : "off"); + } + + process.Start(); + + // SOMEDAY: Do we want to actually do anything with standard output or error? + // Using standarderror requires async read, see http://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx + //process.StandardOutput.ReadToEnd(); + OutputCopier copier = new OutputCopier(process.StandardOutput); + Thread consoleThread = new Thread(new ThreadStart(copier.copyOutput)); + consoleThread.Start(); + + MemoryChecker memChecker = new MemoryChecker(process); + Thread memCheckThread = new Thread(new ThreadStart(memChecker.monitorMemory)); + memCheckThread.Start(); + + process.BeginErrorReadLine(); + + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + + var ms = 1000 * killSeconds; + if (useValgrind) + ms *= 20; + bool killed = !process.WaitForExit(ms); // Wait "killSeconds" seconds here + + stopwatch.Stop(); + + if (!killed) + process.WaitForExit(); // If the process finished successfully, we call the parameterless WaitForExit to ensure that output buffers get flushed + else + { + // It seems that WaitForExit sometimes returns false well before the timeout has elapsed + // We won't report that as an error, but just in case the process didn't actually finish we attempt to + // kill it here + if (killed && ms - stopwatch.ElapsedMilliseconds > 10000) + killed = false; + + if (!traceToStdout) + { + Console.WriteLine("Warning: Killing process after {0} seconds...", killSeconds); + } + + try + { + process.Kill(); + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Warning: Process.Kill returned {0}", e); + } + } + + int waitSeconds = 60 * 2; + process.WaitForExit(1000 * waitSeconds); + if (!process.HasExited) + { + if(!traceToStdout) + { + Console.WriteLine("Warning: Unable to kill process after {0} seconds...", waitSeconds); + } + + var xout = new XElement("UnableToKillProcess", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways)); + + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + return 104; + } + } + if (!traceToStdout) + { + Console.WriteLine("Exit code is {0}", process.ExitCode); + } + memCheckThread.Join(); + consoleThread.Join(); + + var traceFiles = Directory.GetFiles(tempPath, "trace*.*").Where(s => s.EndsWith(".xml") || s.EndsWith(".json")).ToArray(); + if (traceFiles.Length == 0) + { + if (!traceToStdout) + { + Console.WriteLine("Warning: No trace file was generated, summary will not be affected."); + } + + var xout = new XElement((useValgrind ? "NoTraceFileGeneratedLibPos" : "NoTraceFileGenerated"), + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), + new XAttribute("Plugin", tlsPluginFile), + new XAttribute("MachineName", System.Environment.MachineName)); + + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + ok = useValgrind ? 0 : 103; + } + else + { + var result = Summarize(traceFiles, summaryFileName, errorFileName, killed, errorListener.Errors, + process.ExitCode, memChecker.MaxMem, uid, + valgrindOutputFile == null ? null : Path.Combine(tempPath, valgrindOutputFile), + expectedUnseed, out unseed, out retryableError, logOnRetryableError, willRestart, restarting, oldBinaryName, traceToStdout); + + if (result != 0) + { + ok = result; + } + } + + if (willRestart) foreach (var f in traceFiles) File.Delete(f); + + process.Close(); + } + if (!traceToStdout) + { + Console.WriteLine("Done (unseed {0})", unseed); + } + + return ok; + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Error in Run:"); + Console.WriteLine(e); + } + + var xout = new XElement("TestHarnessRunError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message)); + + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + return 101; + } + finally + { + Directory.SetCurrentDirectory(oldDir); + if (!willRestart) Directory.Delete(tempPath, true); + } + } + + class OutputCopier + { + private StreamReader reader; + + public OutputCopier(StreamReader streamReader) + { + this.reader = streamReader; + } + + public void copyOutput() + { + //Console.WriteLine(" Beginning console copy..."); + while (!reader.EndOfStream) + { + reader.ReadLine(); + //Console.WriteLine(" : " + s); + } + } + } + + private class ErrorOutputListener + { + public List Errors { get; set; } + int maxErrorLength; + int maxErrors; + bool errorsExceeded; + + public ErrorOutputListener(int maxErrorLength = 1000, int maxErrors = 10) + { + Errors = new List(); + this.maxErrorLength = maxErrorLength; + this.maxErrors = maxErrors; + this.errorsExceeded = false; + } + + public bool hasError = false; + public void handleData(object sendingProcess, DataReceivedEventArgs errLine) + { + if(!String.IsNullOrEmpty(errLine.Data)) + { + hasError = true; + if(Errors.Count < maxErrors) { + if(errLine.Data.Length > maxErrorLength) { + Errors.Add(errLine.Data.Substring(0, maxErrorLength) + "..."); + } + else { + Errors.Add(errLine.Data); + } + } + else if(!errorsExceeded) { + Errors.Add("TestHarness error limit exceeded"); + errorsExceeded = true; + } + } + } + } + + class MemoryChecker + { + //private System.Diagnostics.Process process; + private long maxMem; + private Process process; + + public long MaxMem + { + get { return maxMem; } + set { maxMem = value; } + } + + public MemoryChecker(System.Diagnostics.Process process) + { + this.process = process; + this.maxMem = 0; + } + + public void monitorMemory() + { + while (true) + { + try + { + process.Refresh(); + if (process.HasExited) + return; + long mem = process.PrivateMemorySize64; + MaxMem = Math.Max(MaxMem, mem); + //Console.WriteLine(string.Format("Process used {0} bytes", MaxMem)); + Thread.Sleep(1000); + } + catch + { + return; + } + } + } + } + + static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, string oldBinary="") + { + var xout = new XElement("TestPlan", + new XAttribute("TestUID", uid), + new XAttribute("RandomSeed", randomSeed), + new XAttribute("TestFile", testFileName), + new XAttribute("BuggifyEnabled", buggify ? "1" : "0"), + new XAttribute("DeterminismCheck", testDeterminism ? "1" : "0"), + new XAttribute("OldBinary", Path.GetFileName(oldBinary))); + AppendToSummary(summaryFileName, xout); + } + + // Parses the valgrind XML file and returns a list of "what" tags for each error. + // All errors for which the "kind" tag starts with "Leak" are ignored + static string[] ParseValgrindOutput(string valgrindOutputFileName, bool traceToStdout) + { + if (!traceToStdout) + { + Console.WriteLine("Reading vXML file: " + valgrindOutputFileName); + } + + ISet whats = new HashSet(); + XElement xdoc = XDocument.Load(valgrindOutputFileName).Element("valgrindoutput"); + foreach(var elem in xdoc.Elements()) { + if (elem.Name != "error") + continue; + string kind = elem.Element("kind").Value; + if(kind.StartsWith("Leak")) + continue; + whats.Add(elem.Element("what").Value); + } + return whats.ToArray(); + } + + delegate IEnumerable parseDelegate(System.IO.Stream stream, string file, + bool keepOriginalElement = false, double startTime = -1, double endTime = Double.MaxValue, + double samplingFactor = 1.0); + + static int Summarize(string[] traceFiles, string summaryFileName, + string errorFileName, bool? killed, List outputErrors, int? exitCode, long? peakMemory, + string uid, string valgrindOutputFileName, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, + bool willRestart = false, bool restarted = false, string oldBinaryName = "", bool traceToStdout = false, string externalError = "") + { + unseed = -1; + retryableError = false; + List errorList = new List(); + var xout = new XElement("Test"); + if (uid != null) + xout.Add(new XAttribute("TestUID", uid)); + bool ok = false; + string testFile = "Unknown"; + int testsPassed = 0, testCount = -1, warnings = 0, errors = 0; + bool testBeginFound = false, testEndFound = false, error = false; + string firstRetryableError = ""; + int stderrSeverity = (int)Magnesium.Severity.SevError; + + Dictionary, Magnesium.Severity> severityMap = new Dictionary, Magnesium.Severity>(); + Dictionary, bool> codeCoverage = new Dictionary, bool>(); + + foreach (var traceFileName in traceFiles) + { + if(!traceToStdout) { + Console.WriteLine("Summarizing {0}", traceFileName); + } + using (var traceFile = System.IO.File.Open(traceFileName, + System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) + { + try + { + parseDelegate parse; + if (traceFileName.EndsWith(".json")) + parse = Magnesium.JsonParser.Parse; + else + parse = Magnesium.XmlParser.Parse; + foreach (var ev in parse(traceFile, traceFileName)) + { + Magnesium.Severity newSeverity; + if (severityMap.TryGetValue(new KeyValuePair(ev.Type, ev.Severity), out newSeverity)) + ev.Severity = newSeverity; + if (ev.Severity >= Magnesium.Severity.SevWarnAlways && ev.DDetails.ContainsKey("ErrorIsInjectedFault")) + ev.Severity = Magnesium.Severity.SevWarn; + + if (ev.Type == "ProgramStart" && !testBeginFound + // Just in case the first ProgramStart seen is a Simulated one, ignore it + && (!ev.DDetails.ContainsKey("Simulated") || ev.Details.Simulated != "1")) + { + xout.Add( + new XAttribute("RandomSeed", ev.Details.RandomSeed), + new XAttribute("SourceVersion", ev.Details.SourceVersion), + new XAttribute("Time", ev.Details.ActualTime), + new XAttribute("BuggifyEnabled", ev.Details.BuggifyEnabled), + new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"), + new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); + testBeginFound = true; + } + if (ev.Type == "Simulation") + { + xout.Add( + new XAttribute("TestFile", ev.Details.TestFile)); + testFile = ev.Details.TestFile.Substring(ev.Details.TestFile.IndexOf("tests")); + } + if (ev.Type == "ActualRun") + { + xout.Add( + new XAttribute("TestFile", ev.Details.RunID)); + } + if (ev.Type == "ElapsedTime" && !testEndFound) + { + testEndFound = true; + unseed = int.Parse(ev.Details.RandomUnseed); + if (expectedUnseed != -1 && expectedUnseed != unseed) + { + Magnesium.Severity severity; + if (!severityMap.TryGetValue(new KeyValuePair("UnseedMismatch", Magnesium.Severity.SevError), out severity)) + severity = Magnesium.Severity.SevError; + + if (severity >= Magnesium.Severity.SevWarnAlways) + { + xout.Add(new XElement("UnseedMismatch", + new XAttribute("Unseed", unseed), + new XAttribute("ExpectedUnseed", expectedUnseed), + new XAttribute("Severity", (int)severity))); + if ( severity == Magnesium.Severity.SevError ) { + error = true; + errorList.Add("UnseedMismatch"); + } + } + } + xout.Add( + new XAttribute("SimElapsedTime", ev.Details.SimTime), + new XAttribute("RealElapsedTime", ev.Details.RealTime), + new XAttribute("RandomUnseed", ev.Details.RandomUnseed)); + } + if (ev.Severity == Magnesium.Severity.SevWarnAlways) + { + if (warnings < maxWarnings) + { + xout.Add(new XElement(ev.Type, + new XAttribute("Severity", (int)ev.Severity), + ev.DDetails + //.Where(kv => true) + .Select(kv => new XAttribute(kv.Key, kv.Value)))); + } + warnings++; + } + if (ev.Severity >= Magnesium.Severity.SevError) + { + string errorString = ev.FormatTestError(true); + if (errorString.Contains("platform_error")) + { + if (!retryableError) + { + firstRetryableError = errorString; + } + retryableError = true; + } + if (errors < maxWarnings) + { + xout.Add(new XElement(ev.Type, + new XAttribute("Severity", (int)ev.Severity), + ev.DDetails + //.Where(kv => true) + .Select(kv => new XAttribute(kv.Key, kv.Value)))); + errorList.Add(errorString); + } + errors++; + error = true; + } + if (ev.Type == "CodeCoverage" && !willRestart) + { + bool covered = true; + if(ev.DDetails.ContainsKey("Covered")) + { + covered = int.Parse(ev.Details.Covered) != 0; + } + + var key = new Tuple(ev.Details.File, ev.Details.Line); + if (covered || !codeCoverage.ContainsKey(key)) + { + codeCoverage[key] = covered; + } + } + if (ev.Type == "FaultInjected" || (ev.Type == "BuggifySection" && ev.Details.Activated == "1")) + { + xout.Add(new XElement(ev.Type, new XAttribute("File", ev.Details.File), new XAttribute("Line", ev.Details.Line))); + } + if (ev.Type == "TestsExpectedToPass") + testCount = int.Parse(ev.Details.Count); + if (ev.Type == "TestResults" && ev.Details.Passed == "1") + testsPassed++; + if (ev.Type == "RemapEventSeverity") + severityMap[new KeyValuePair(ev.Details.TargetEvent, (Magnesium.Severity)int.Parse(ev.Details.OriginalSeverity))] = (Magnesium.Severity)int.Parse(ev.Details.NewSeverity); + if (ev.Type == "StderrSeverity") + stderrSeverity = int.Parse(ev.Details.NewSeverity); + } + + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Error summarizing {0}: {1}", traceFileName, e); + } + + error = true; + xout.Add(new XElement("SummarizationError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message))); + errorList.Add("SummarizationError " + e.Message); + break; + } + } + } + + if (externalError.Length > 0) { + xout.Add(new XElement(externalError, new XAttribute("Severity", (int)Magnesium.Severity.SevError))); + } + + foreach(var kv in codeCoverage) + { + var element = new XElement("CodeCoverage", new XAttribute("File", kv.Key.Item1), new XAttribute("Line", kv.Key.Item2)); + if(!kv.Value) + { + element.Add(new XAttribute("Covered", "0")); + } + + xout.Add(element); + } + + if (warnings > maxWarnings) + { + //error = true; + xout.Add(new XElement("WarningLimitExceeded", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), + new XAttribute("WarningCount", warnings))); + } + if (errors > maxWarnings) + { + error = true; + xout.Add(new XElement("ErrorLimitExceeded", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorCount", errors))); + errorList.Add("ErrorLimitExceeded"); + } + if (killed == true) + { + if (!retryableError) + { + firstRetryableError = "ExternalTimeout"; + } + + retryableError = true; + error = true; + xout.Add(new XElement("ExternalTimeout", new XAttribute("Severity", (int)Magnesium.Severity.SevError))); + } + if (outputErrors != null) + { + int stderrBytes = 0; + foreach (string err in outputErrors) + { + if (stderrSeverity == (int)Magnesium.Severity.SevError) + { + error = true; + } + + int remainingBytes = maxStderrBytes - stderrBytes; + if (remainingBytes > 0) + { + string outErr = (err.Length > remainingBytes) ? err.Substring(remainingBytes) + "..." : err; + + xout.Add(new XElement("StdErrOutput", + new XAttribute("Severity", stderrSeverity), + new XAttribute("Output", outErr))); + } + + stderrBytes += err.Length; + } + + if (stderrBytes > maxStderrBytes) + { + xout.Add(new XElement("StdErrOutputTruncated", + new XAttribute("Severity", stderrSeverity), + new XAttribute("BytesRemaining", stderrBytes - maxStderrBytes))); + } + } + if (exitCode.HasValue && exitCode != 0) + { + error = true; + xout.Add(new XElement("ExitCode", new XAttribute("Code", exitCode.Value), new XAttribute("Severity", (int)Magnesium.Severity.SevError))); + errorList.Add(string.Format("ExitCode 0x{0:x}", exitCode.Value)); + } + if (!testEndFound && !willRestart) + { + // We didn't terminate the test, but it didn't reach the end? + error = true; + xout.Add(new XElement("TestUnexpectedlyNotFinished"), new XAttribute("Severity", (int)Magnesium.Severity.SevError)); + errorList.Add("TestUnexpectedlyNotFinished"); + } + ok = testsPassed == testCount && testsPassed > 0 && !error; + xout.Add( + new XAttribute("Passed", testsPassed), + new XAttribute("Failed", testCount - testsPassed)); + if (peakMemory.HasValue) + xout.Add(new XAttribute("PeakMemory", peakMemory.Value)); + + if (valgrindOutputFileName != null && valgrindOutputFileName.Length > 0) + { + try + { + // If there are any errors reported "ok" will be set to false + var whats = ParseValgrindOutput(valgrindOutputFileName, traceToStdout); + foreach (var what in whats) + { + xout.Add(new XElement("ValgrindError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("What", what))); + ok = false; + error = true; + } + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine(e); + } + + error = true; + xout.Add(new XElement("ValgrindParseError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message))); + errorList.Add("Failed to parse valgrind output: " + e.Message); + } + } + + if (retryableError && !logOnRetryableError) + { + xout = new XElement("Test", xout.Attributes()); + xout.Add(new XElement("RetryingError", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), + new XAttribute("What", firstRetryableError))); + } + else + { + xout.Add(new XAttribute("OK", ok || willRestart)); + } + + AppendToSummary(summaryFileName, xout, traceToStdout); + + if ((!retryableError || logOnRetryableError) && errorFileName != null && (errorList.Count > 0 || !ok) && !willRestart) + { + var errorText = string.Join("\n\t", errorList + .Concat((!ok && errorList.Count == 0) ? new string[] { "Failed with no explanation" } : new string[] { }) + .Distinct() + .ToArray()); + AppendToFile(errorFileName, string.Format("Test {0} failed with:\n\t{1}\n", testFile, errorText)); + } + if (!error) { + return 0; + } + else { + return 102; + } + } + + static int ExtractErrors(string summaryFileName, string errorSummaryFileName) + { + Console.WriteLine("Extracting from {0}", summaryFileName); + List xout = new List(); + var coverage = new Dictionary,Tuple>(); + using (var traceFile = System.IO.File.Open(summaryFileName, + System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) + { + try + { + var events = Magnesium.XmlParser.Parse(traceFile, summaryFileName, true); + events = Magnesium.TraceLogUtil.IdentifyFailedTestPlans(events); + foreach (var ev in events) + { + Magnesium.Test t = ev as Magnesium.Test; + if (t != null) + { + foreach (var tev in t.events) + { + if (tev.Type == "CodeCoverage" || tev.Type == "FaultInjected") + { + var keyTuple = Tuple.Create(tev.Details.File, int.Parse(tev.Details.Line)); + if (coverage.ContainsKey(keyTuple)) + { + var old = coverage[keyTuple]; + coverage[keyTuple] = Tuple.Create(old.Item1 + 1, old.Item2 + (t.ok ? 0 : 1)); + } + else + { + coverage[keyTuple] = Tuple.Create(1, (t.ok ? 0 : 1)); + } + } + } + if (!t.ok) + { + if (t.original != null) + { + foreach (var c in t.original.Elements("CodeCoverage")) + c.Remove(); + foreach (var f in t.original.Elements("FaultInjected")) + f.Remove(); + + xout.Add(t.original); + } + else + { + xout.Add(new XElement("Test", + new XAttribute("Type", t.Type), + new XAttribute("Time", t.Time), + new XAttribute("Machine", t.Machine), + new XAttribute("TestUID", t.TestUID), + new XAttribute("TestFile", t.TestFile), + new XAttribute("randomSeed", t.randomSeed), + new XAttribute("Buggify", t.Buggify), + new XAttribute("DeterminismCheck", t.DeterminismCheck), + new XAttribute("OldBinary", t.OldBinary), + new XElement("TestNotSummarized", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways) + ) + ) + ); + } + } + } + } + } + catch (Exception e) + { + Console.WriteLine("Error summarizing {0}: {1}", summaryFileName, e); + xout.Add(new XElement("SummarizationError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message))); + //failedTests.Add("SummarizationError " + e.Message); + } + } + + foreach (var e in coverage) + { + xout.Add(new XElement("Event", + new XAttribute("Type", "CoverageSummary"), + new XAttribute("Time", 0), + new XAttribute("Machine", ""), + new XAttribute("File", e.Key.Item1), + new XAttribute("Line", e.Key.Item2), + new XAttribute("Covered", e.Value.Item1), + new XAttribute("Failed", e.Value.Item2))); + } + + AppendToErrorSummary(errorSummaryFileName, xout); + return 0; + } + + private static void AppendToErrorSummary(string summaryFileName, List elements) + { + if (summaryFileName == null) + return; + takeLock(summaryFileName); + try { + foreach (XElement e in elements) + AppendToSummary(summaryFileName, e, false, false); + } + finally + { + releaseLock(summaryFileName); + } + } + + private static void AppendToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, bool shouldLock = true) + { + bool useXml = true; + if (summaryFileName != null && summaryFileName.EndsWith(".json")) { + useXml = false; + } + + if (traceToStdout) + { + if (useXml) { + using (var wr = System.Xml.XmlWriter.Create(Console.OpenStandardOutput(), new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true, Encoding = new System.Text.UTF8Encoding(false) })) + xout.WriteTo(wr); + } else { + using (var wr = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonWriter(Console.OpenStandardOutput())) + xout.WriteTo(wr); + } + Console.WriteLine(); + return; + } + + if (summaryFileName == null) + return; + if (shouldLock) + takeLock(summaryFileName); + try + { + using (var f = System.IO.File.Open(summaryFileName, System.IO.FileMode.Append, System.IO.FileAccess.Write)) + { + if (f.Length == 0) + { + byte[] bytes = Encoding.UTF8.GetBytes(""); + f.Write(bytes, 0, bytes.Length); + } + if (useXml) { + using (var wr = System.Xml.XmlWriter.Create(f, new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true })) + xout.Save(wr); + } else { + using (var wr = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonWriter(f)) + xout.WriteTo(wr); + } + var endl = Encoding.UTF8.GetBytes(Environment.NewLine); + f.Write(endl, 0, endl.Length); + } + } + finally + { + if (shouldLock) + releaseLock(summaryFileName); + } + } + + private static void AppendXmlMessageToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, string testFile = null, + int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null) + { + var test = new XElement("Test", xout); + if(testFile != null) + test.Add(new XAttribute("TestFile", testFile)); + if(seed != null) + test.Add(new XAttribute("RandomSeed", seed)); + if(buggify != null) + test.Add(new XAttribute("BuggifyEnabled", buggify.Value ? "1" : "0")); + if(determinismCheck != null) + test.Add(new XAttribute("DeterminismCheck", determinismCheck.Value ? "1" : "0")); + if(oldBinaryName != null) + test.Add(new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); + + test.Add(xout); + AppendToSummary(summaryFileName, test, traceToStdout); + } + + private static void AppendToFile(string fileName, string content) + { + if (fileName == null) + return; + takeLock(fileName); + try + { + using (var f = System.IO.File.Open(fileName, System.IO.FileMode.Append, System.IO.FileAccess.Write)) + { + var endl = Encoding.UTF8.GetBytes(content); + f.Write(endl, 0, endl.Length); + } + } + finally + { + releaseLock(fileName); + } + } + + static int Remote(string queue, string fdbRoot, double addHours, int testCount, string testTypes, string userScope) + { + queue = Path.GetFullPath(queue); + fdbRoot = Path.GetFullPath(fdbRoot); + var output = Path.Combine(queue, "archive"); + var now = DateTime.Now; + string date = string.Format("{0}-{1:00}-{2:00}-{3:00}-{4:00}", + now.Year, now.Month, now.Day, now.Hour, now.Minute); + + if (!Directory.Exists(queue)) + Directory.CreateDirectory(queue); + + int maxCount = 0; + foreach (var f in Directory.GetFiles(queue, String.Format("{0}-*.xml", date))) + { + var count = Int32.Parse(f.Split('-')[5]); + maxCount = Math.Max(maxCount, count); + } + maxCount++; + + var suffix = String.Format("{0}-{1}-{2}", Environment.UserName, userScope, OS_NAME); + foreach (var f in Directory.GetFiles(queue, "*" + suffix + ".xml")) + File.Delete(f); + + string label = String.Format("{0}-{1}-{2}", date, maxCount, suffix); + + var testStaging = Path.Combine(output, label); + Directory.CreateDirectory(testStaging); + File.Create(Path.Combine(testStaging, "errors.txt")); + + var release = Path.Combine(fdbRoot, "bin"); + if(!IsRunningOnMono()) + release = Path.Combine(release, "Release"); + + File.Copy(Path.Combine(release, BINARY), Path.Combine(testStaging, BINARY)); + File.Copy(Path.Combine(fdbRoot, "tls-plugins", PLUGIN), Path.Combine(testStaging, PLUGIN)); + + if (IsRunningOnMono()) + File.Copy(Path.Combine(release, BINARY + ".debug"), Path.Combine(testStaging, BINARY + ".debug")); + + //using (var f = System.IO.File.Open( + // Path.Combine(testStaging, "summary.xml"), + // System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.Delete)) + //{ + // byte[] bytes = Encoding.UTF8.GetBytes(""); + // f.Write(bytes, 0, bytes.Length); + //} + + var coverageFiles = Directory.GetFiles(release, "coverage*.xml"); + foreach (var coverage in coverageFiles) + File.Copy(coverage, Path.Combine(testStaging, Path.GetFileName(coverage))); + + Directory.CreateDirectory(Path.Combine(testStaging, "tests")); + + if (testTypes == "fast" || testTypes == "all") + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "fast")), new DirectoryInfo(Path.Combine(testStaging, "tests", "fast"))); + if (testTypes == "restarting" || testTypes == "all") + { + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "restarting")), new DirectoryInfo(Path.Combine(testStaging, "tests", "restarting"))); + //CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "oldBinaries")), new DirectoryInfo(Path.Combine(testStaging, "tests", "oldBinaries"))); + } + if (testTypes == "all") + { + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "slow")), new DirectoryInfo(Path.Combine(testStaging, "tests", "slow"))); + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "rare")), new DirectoryInfo(Path.Combine(testStaging, "tests", "rare"))); + } + + + if (testTypes != "fast" && testTypes != "all" && testTypes != "restarting") + { + FileInfo file = new FileInfo(testTypes); + Directory.CreateDirectory(Path.Combine(testStaging, "tests", file.Directory.Name)); + file.CopyTo(Path.Combine(testStaging, "tests", file.Directory.Name, file.Name), true); + } + + var e = + new XElement("TestDefinition", + new XElement("Duration", + new XAttribute("Hours", addHours)), + new XElement("TestCount", testCount)); + new XDocument(e).Save(Path.Combine(queue, label + ".xml")); + File.Copy(Path.Combine(queue, label + ".xml"), Path.Combine(testStaging, label + ".xml")); + + var summaryFile = Path.Combine(testStaging, "summary.xml"); + Console.WriteLine(label); + + if (!IsRunningOnMono()) + { + using (var mProcess = new System.Diagnostics.Process()) + { + mProcess.StartInfo.UseShellExecute = false; + mProcess.StartInfo.RedirectStandardOutput = true; + mProcess.StartInfo.FileName = "Magnesium.exe"; + mProcess.StartInfo.Arguments = "Summary " + summaryFile; + mProcess.Start(); + } + } + + return 0; + } + + public static void CopyAll(DirectoryInfo source, DirectoryInfo target, Func predicate = null) + { + // Check if the target directory exists, if not, create it. + if (!Directory.Exists(target.FullName)) + Directory.CreateDirectory(target.FullName); + + // Copy each file into it's new directory. + foreach (FileInfo fi in source.GetFiles()) + if (predicate == null || predicate(fi)) + fi.CopyTo(Path.Combine(target.ToString(), fi.Name), true); + + // Copy each subdirectory using recursion. + foreach (DirectoryInfo diSourceSubDir in source.GetDirectories()) + CopyAll(diSourceSubDir, target.CreateSubdirectory(diSourceSubDir.Name), predicate); + } + + static int Auto(string queueDirectory, string runDir, string shareDir, string cacheDir, bool useValgrind, int maxTries) + { + try + { + queueDirectory = Path.GetFullPath(queueDirectory); + while (true) + { + Test test = getTest(queueDirectory, runDir, shareDir, cacheDir); + Console.WriteLine("Running test {0}", test.label); + + // run test + test.run(useValgrind, maxTries); + } + } + catch (Exception e) + { + Console.WriteLine("Error: {0}", e); + return 100; + } + } + + class Test + { + public DateTime? testEnd; + public string queueDirectory; + public string label; + public string runDir; + public string inputDir; + public string outputDir; + public string oldBinaryDir; + public int testCount; + + public Test(string queueDirectory, string label, string runDir, string shareDir, string cacheDir) + { + this.queueDirectory = queueDirectory; + this.label = label; + this.runDir = runDir; + var specFile = Path.Combine(queueDirectory, label + ".xml"); + var testDef = XDocument.Load(specFile).Element("TestDefinition"); + var testDuration = double.Parse(testDef.Element("Duration").Attribute("Hours").Value); + var testBegin = File.GetCreationTime(specFile); + testEnd = testDuration < 0 ? (DateTime?)null : testBegin.AddHours(testDuration); + + testCount = int.Parse(testDef.Element("TestCount").Value); + outputDir = Path.Combine(queueDirectory, "archive", label); + Directory.CreateDirectory(outputDir); + + string oldBinarySourceDir = Path.Combine(shareDir, "oldBinaries"); + + if (cacheDir != null) + { + inputDir = Path.Combine(cacheDir, "archive", label); + Directory.CreateDirectory(Path.Combine(cacheDir, "archive")); + + if (!Directory.Exists(inputDir)) + { + takeLock(inputDir); + if (!Directory.Exists(inputDir)) + { + string tmpDir = Path.Combine(cacheDir, "archive.part", label + "." + Path.GetRandomFileName() + ".part"); + Directory.CreateDirectory(tmpDir); + CopyAll(new DirectoryInfo(outputDir), new DirectoryInfo(tmpDir), (FileInfo file) => + file.Name != "fdbserver.debug" && + !file.Name.StartsWith("summary-") && + file.Name != "errors.txt" + ); + Directory.Move(tmpDir, inputDir); + } + releaseLock(inputDir); + } + + oldBinaryDir = Path.Combine(cacheDir, "oldBinaries"); + + Directory.CreateDirectory(oldBinaryDir); + foreach (FileInfo fi in new DirectoryInfo(oldBinarySourceDir).GetFiles()) + { + var targetName = Path.Combine(oldBinaryDir, fi.Name); + if (!File.Exists(targetName) || fi.LastWriteTimeUtc != File.GetLastWriteTimeUtc(targetName)) + { + fi.CopyTo(targetName, true); + File.SetLastWriteTimeUtc(targetName, fi.LastWriteTimeUtc); + } + } + + foreach (FileInfo fi in new DirectoryInfo(oldBinaryDir).GetFiles()) + { + var targetName = Path.Combine(oldBinarySourceDir, fi.Name); + if (!File.Exists(targetName)) + fi.Delete(); + } + } + else + { + inputDir = outputDir; + oldBinaryDir = oldBinarySourceDir; + } + //Console.WriteLine("TestEnd {0}, now {1}, duration {2}, done {3}", testEnd, DateTime.Now, testDuration, done()); + } + public bool done() + { + return testEnd.HasValue && System.DateTime.Now > testEnd.Value; + } + public void run(bool useValgrind, int maxTries) + { + Run(Path.Combine(inputDir, BINARY), + Path.Combine(inputDir, PLUGIN), + Path.Combine(inputDir, "tests"), + Path.Combine(outputDir, "summary-" + Environment.MachineName + ".xml"), + Path.Combine(outputDir, "errors.txt"), + runDir, + oldBinaryDir, + useValgrind, + maxTries); + } + + public void finalize() + { + try + { + Console.WriteLine("Deleting: {0}", Path.Combine(queueDirectory, label + ".xml")); + File.Delete(Path.Combine(queueDirectory, label + ".xml")); + } + catch (Exception e) + { + Console.WriteLine("Error deleting queue folder: {0}", e.Message); + } + } + } + + static Test getTest(string parent, string runDir, string shareDir, string cacheDir) + { + while (true) + { + var testFiles = Directory.GetFiles(parent, String.Format("*{0}.xml", OS_NAME)); + // (if no tests, wait, try again) + if (testFiles.Length != 0) + { + /*try + {*/ + var testFile = testFiles[random.Next(testFiles.Length)]; + var test = new Test(parent, Path.GetFileNameWithoutExtension(testFile), runDir, shareDir, cacheDir); + if ((test.testCount < 0 || UpdateTestTotals(Path.Combine(test.outputDir, "testCount"), test.testCount)) && !test.done()) + return test; + else + test.finalize(); + /*} + catch (Exception) + { + // retry opening a test + }*/ + } + System.Threading.Thread.Sleep(1000); + } + } + + static void takeLock(string targetFile) + { + // Console.WriteLine("Attempting to take lock on {0}", targetFile); + string lockFile = targetFile + ".lock"; + while (true) + { + try + { + using (var f = System.IO.File.Open(lockFile, System.IO.FileMode.CreateNew)) + { + return; + } + } + catch (System.IO.IOException e) + { + Console.WriteLine("Waiting for file lock: {0}", e.Message); + System.Threading.Thread.Sleep(250); + } + } + } + + static void releaseLock(string targetFile) + { + File.Delete(targetFile + ".lock"); + } + + private static bool UpdateTestTotals(string countFileName, int desiredTestCount) + { + takeLock(countFileName); + try + { + using (var f = System.IO.File.Open(countFileName, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite)) + { + int currentCount = 0; + byte[] b; + + if (f.Length != 0) + { + b = new byte[f.Length]; + f.Read(b, 0, b.Length); + currentCount = int.Parse(Encoding.UTF8.GetString(b)); + } + + if (currentCount >= desiredTestCount) + return false; + + f.SetLength(0); + b = Encoding.UTF8.GetBytes(String.Format("{0}", currentCount + 1)); + f.Write(b, 0, b.Length); + + return true; + } + } + finally + { + releaseLock(countFileName); + } + } + + private static int VersionInfo() + { + Console.WriteLine("Version: 1.02"); + + Console.WriteLine("FDB Project Ver: " + "${CMAKE_PROJECT_VERSION}"); + Console.WriteLine("FDB Version: " + "${CMAKE_PROJECT_VERSION_MAJOR}" + "." + "${CMAKE_PROJECT_VERSION_MINOR}"); + Console.WriteLine("Source Version: " + "${CURRENT_GIT_VERSION}"); + return 1; + } + + private static int UsageMessage() + { + Console.WriteLine("Usage:"); + Console.WriteLine(" TestHarness run [temp/runDir] [fdbserver[.exe]] [TLSplugin] [testfolder] [summary.xml] "); + Console.WriteLine(" TestHarness summarize [trace.xml] [summary.xml] "); + Console.WriteLine(" TestHarness replay [temp/runDir] [fdbserver[.exe]] [TLSplugin] [summary-in.xml] [summary-out.xml]"); + Console.WriteLine(" TestHarness auto [temp/runDir] [directory] [shareDir] "); + Console.WriteLine(" TestHarness remote [queue folder] [root foundation folder] [duration in hours] [amount of tests] [all/fast/] [scope]"); + Console.WriteLine(" TestHarness extract-errors [summary-file] [error-summary-file]"); + Console.WriteLine(" TestHarness joshua-run "); + VersionInfo(); + return 1; + } + } +} diff --git a/contrib/TraceLogHelper/JsonParser.cs b/contrib/TraceLogHelper/JsonParser.cs index 996a1e0e3c..9d7272a37f 100644 --- a/contrib/TraceLogHelper/JsonParser.cs +++ b/contrib/TraceLogHelper/JsonParser.cs @@ -51,7 +51,7 @@ namespace Magnesium } catch (Exception e) { - throw new Exception(string.Format("Failed to parse {0}", root), e); + throw new Exception(string.Format("Failed to parse JSON {0}", root), e); } if (ev != null) yield return ev; } @@ -81,7 +81,7 @@ namespace Magnesium DDetails = xEvent.Elements() .Where(a=>a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "ID" && a.Name != "Severity" && (!rolledEvent || a.Name != "OriginalTime")) .ToDictionary(a=>string.Intern(a.Name.LocalName), a=>(object)a.Value), - original = keepOriginalElement ? xEvent : null, + original = keepOriginalElement ? xEvent : null }; } diff --git a/contrib/TraceLogHelper/XmlParser.cs b/contrib/TraceLogHelper/XmlParser.cs index 17b2405060..3728c58c3b 100644 --- a/contrib/TraceLogHelper/XmlParser.cs +++ b/contrib/TraceLogHelper/XmlParser.cs @@ -53,7 +53,7 @@ namespace Magnesium } catch (Exception e) { - throw new Exception(string.Format("Failed to parse {0}", xev), e); + throw new Exception(string.Format("Failed to parse XML {0}", xev), e); } if (ev != null) yield return ev; } diff --git a/documentation/sphinx/source/api-general.rst b/documentation/sphinx/source/api-general.rst index 81f981dc8b..55ecb5c25e 100644 --- a/documentation/sphinx/source/api-general.rst +++ b/documentation/sphinx/source/api-general.rst @@ -125,6 +125,17 @@ Because it can have only a single network thread, a client process may become li If you suspect that a client process's workload may be saturating the network thread, this can be confirmed by checking whether the network thread is running with high CPU usage. In the :ref:`client trace logs `, the ``ProcessMetrics`` trace event has a field for ``MainThreadCPUSeconds`` that indicates the number of seconds out of ``Elapsed`` that the network thread was busy. You can also attempt to identify a busy thread from any tool that reports the CPU activity of threads in your process. +.. note:: FoundationDB 6.3 introduced :ref:`multi-threaded client `, and can alternatively be used to scale clients. + +.. _multi-threaded-client: + +Multi-threaded Client +===================== + +FoundationDB client library can start multiple worker threads for each version of client that is loaded. Every single cluster will be serviced by one of the client threads. If the client is connected to only one cluster, exactly one thread would be active and the rest will remain idle. Hence, using this feature is useful when the client is actively using more than one cluster. + +Clients can be configured to use worker-threads by setting the ``FDBNetworkOptions::CLIENT_THREADS_PER_VERSION`` option. + .. _client-trace-logging: Client trace logging diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index 6c75b64869..84eef0d696 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -469,6 +469,11 @@ Prints a list of currently active transaction tag throttles. ``LIMIT`` - The number of throttles to print. Defaults to 100. +triggerddteaminfolog +-------------------- + +The ``triggerddteaminfolog`` command would trigger the data distributor to log very detailed teams information into trace event logs. + unlock ------ diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index 825eefab40..b6255f42ec 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.3.9.pkg `_ +* `FoundationDB-6.3.10.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.3.9-1_amd64.deb `_ -* `foundationdb-server-6.3.9-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.3.10-1_amd64.deb `_ +* `foundationdb-server-6.3.10-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.3.9-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.3.9-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.10-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.3.10-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.3.9-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.3.9-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.3.10-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.3.10-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.3.9-x64.msi `_ +* `foundationdb-6.3.10-x64.msi `_ API Language Bindings ===================== @@ -58,18 +58,18 @@ On macOS and Windows, the FoundationDB Python API bindings are installed as part If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package: -* `foundationdb-6.3.9.tar.gz `_ +* `foundationdb-6.3.10.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.3.9.gem `_ +* `fdb-6.3.10.gem `_ Java 8+ ------- -* `fdb-java-6.3.9.jar `_ -* `fdb-java-6.3.9-javadoc.jar `_ +* `fdb-java-6.3.10.jar `_ +* `fdb-java-6.3.10-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/images/FDB_multiple_txn_swimlane_diagram.png b/documentation/sphinx/source/images/FDB_multiple_txn_swimlane_diagram.png new file mode 100644 index 0000000000..910aba5524 Binary files /dev/null and b/documentation/sphinx/source/images/FDB_multiple_txn_swimlane_diagram.png differ diff --git a/documentation/sphinx/source/images/FDB_read_path.png b/documentation/sphinx/source/images/FDB_read_path.png new file mode 100644 index 0000000000..11a48718ae Binary files /dev/null and b/documentation/sphinx/source/images/FDB_read_path.png differ diff --git a/documentation/sphinx/source/images/FDB_write_path.png b/documentation/sphinx/source/images/FDB_write_path.png new file mode 100644 index 0000000000..54162ebb14 Binary files /dev/null and b/documentation/sphinx/source/images/FDB_write_path.png differ diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 294ddc560c..0262f78e12 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -93,6 +93,11 @@ "counter":0, "roughness":0.0 }, + "low_priority_queries":{ + "hz":0.0, + "counter":0, + "roughness":0.0 + }, "bytes_queried":{ "hz":0.0, "counter":0, @@ -479,6 +484,11 @@ "hz":0.0, "counter":0, "roughness":0.0 + }, + "low_priority_reads":{ // measures number of incoming low priority read requests + "hz":0.0, + "counter":0, + "roughness":0.0 }, "location_requests":{ // measures number of outgoing key server location responses "hz":0.0, @@ -536,6 +546,11 @@ "counter":0, "roughness":0.0 }, + "rejected_for_queued_too_long":{ + "hz":0.0, + "counter":0, + "roughness":0.0 + }, "committed":{ "hz":0.0, "counter":0, diff --git a/documentation/sphinx/source/read-write-path.rst b/documentation/sphinx/source/read-write-path.rst new file mode 100644 index 0000000000..c9459a03fd --- /dev/null +++ b/documentation/sphinx/source/read-write-path.rst @@ -0,0 +1,469 @@ +############################## +FDB Read and Write Path +############################## + +| Author: Meng Xu +| Reviewer: Evan Tschannen, Jingyu Zhou +| Audience: FDB developers, SREs and expert users. + +This document explains how FDB works at high level in database terms without mentioning FDB internal concepts. + +We first discuss the read path and the write path separately for a single transaction. +We then describe how the read path and write path work together for a read and write transaction. +In the last section, we illustrate how multiple outstanding write transactions are processed and *ordered* in FDB. +The processing order of multiple transactions is important because it affects the parallelism of transaction processing and the write throughput. + +The content is based on FDB 6.2 and is true for FDB 6.3. A new timestamp proxy role is introduced in post FDB 6.3, +which affects the read path. We will discuss the timestamp proxy role in the future version of this document. + +.. image:: /images/FDB_read_path.png + +Components +================= + +FDB is built on top of several key components. +The terms below are common database or distributed system terms, instead of FDB specific terms. + +**Timestamp generator.** It serves logical time, which defines happen-before relation: +An event at t1 happens before another event at t2, if t1 < t2. +The logic time is used to order events in FDB distributed systems and it is used by concurrency control to decide if two transactions have conflicts. +The logical time is the timestamp for a transaction. + +* A read-only transaction has only one timestamp which is assigned when the transaction is created; +* A read-write transaction has one timestamp at the transaction’s creation time and one timestamp at its commit time. + + +**Concurrency Control.** It decides if two transactions can be executed concurrently without violating Strict Serializable Isolation (SSI) property. +It uses the Optimistic Concurrency Control (OCC) mechanism described in [SSI] to achieve that. + +**Client.** It is a library, an FDB application uses, to access the database. +It exposes the transaction concept to applications. +Client in FDB is a *fat* client that does multiple complex operations: +(1) It calculates read and write conflict ranges for transactions; +(2) it batches a transaction's operations and send them all together at commit for better throughput; +(3) it automatically retries failed transactions. + +**Proxies.** It is a subsystem that acts like reverse proxies to serve clients’ requests. Its main purposes is: + +* Serve for read request by (1) serving the logical time to client; and (2) providing which storage server has data for a key; +* Process write transactions on behalf of clients and return the results; + +Each proxy has the system’s metadata, called transaction state store (txnStateStore). The metadata decides: +(1) which key should go to which storage servers in the storage system; +(2) which key should go to which processes in the durable queuing system; +(3) is the database locked; etc. + +The metadata on all proxies are consistent at any given timestamp. +To achieve that, when a proxy has a metadata mutation that changes the metadata at the timestamp V1, +the mutation is propagated to all proxies (through the concurrency control component), and +its effect is applied on all proxies before any proxy can process transactions after the timestamp V1. + +**Durable queuing system.** It is a queuing system for write traffic. +Its producers are proxies that send transaction mutation data for durability purpose. +Its consumers are storage systems that index data and serve read request. +The queuing system is partitioned for the key-space. +A shard (i.e., key-range) is mapped to *k* log processes in the queuing system, where *k* is the replication factor. +The mapping between shard and storage servers decides the mapping between shard and log processes. + +**Storage system.** It is a collection of storage servers (SS), each of which is a sqlite database running on a single thread. +It indexes data and serves read requests. +Each SS has an in-memory p-tree data structure that stores the past 5-second mutations and an on-disk sqlite data. +The in-memory data structure can serve multiple versions of key-values in the past 5 seconds. +Due to memory limit, the in-memory data cannot hold more than 5 seconds’ multi-version key-values, +which is the root cause why FDB’s transactions cannot be longer than 5 seconds. +The on-disk sqlite data has only the most-recent key-value. + +**Zookeeper like system.** The system solves two main problems: + +* Store the configuration of the transaction system, which includes information such as generations of queuing systems and their processes. + The system used to be zookeeper. FDB later replaced it with its own implementation. + +* Service discovery. Processes in the zookeeper-like system serve as well-known endpoints for clients to connect to the cluster. + These well-known endpoint returns the list of proxies to clients. + + + +Read path of a transaction +================================== + +Fig. 1 above shows a high-level view of the read path. An application uses FDB client library to read data. +It creates a transaction and calls its read() function. The read() operation will lead to several steps. + +* **Step 1 (Timestamp request)**: The read operation needs a timestamp. + The client initiates the timestamp request through an RPC to proxy. The request will trigger Step 2 and Step 3; + + * To improve throughput and reduce load on the server side, each client dynamically batches the timestamp requests. + A client keeps adding requests to the current batch until + *when* the number of requests in a batch exceeds a configurable threshold or + *when* the batching times out at a dynamically computed threshold. + Each batch sends only one timestamp request to proxy and all requests in the same batch share the same timestamp. + +* **Step 2 (Get latest commit version)**: When the timestamp request arrives at a proxy, + the proxy wants to get the largest commit version as the return value. + So it contacts the rest of (n-1) proxies for their latest commit versions and + uses the largest one as the return value for Step 1. + + * O(n^2) communication cost: Because each proxy needs to contact the rest of (n-1) proxies to serve clients’ timestamp request, + the communication cost is n*(n-1), where n is the number of proxies; + + * Batching: To reduce communication cost, each proxy batches clients’ timestamp requests for a configurable time period (say 1ms) and + return the same timestamp for requests in the same batch. + +* **Step 3 (Confirm proxy’s liveness)**: To prevent proxies that are no longer a part of the system (such as due to network partition) from serving requests, + each proxy contacts the queuing system for each timestamp request to confirm it is still a valid proxy + (i.e., not replaced by a newer generation proxy process). + This is based on the FDB property that at most one active queuing system is available at any given time. + + * Why do we need this step? This is to achieve consensus (i.e., external consistency). + Compared to serializable isolation, Strict Serializable Isolation (SSI) requires external consistency. + It means the timestamp received by clients cannot decrease. If we do not have step and network partition happens, + a set of old proxies that are disconnected from the rest of systems can still serve timestamp requests to clients. + These timestamps can be smaller than the new generation of proxies, which breaks the external consistency in SSI. + + * O(n * m) communication cost: To confirm a proxy’s liveness, the proxy has to contact all members in the queuing system to + ensure the queuing system is still active. This causes *m* network communication, where *m* is the number of processes in the queuing system. + A system with n proxies will have O(n * m) network communications at the step 3. In our deployment, n is typically equal to m; + + * Do FDB production clusters have this overhead? No. Our production clusters disable the external consistency by + configuring the knob ALWAYS_CAUSAL_READ_RISKY. + +* **Step 4 (Locality request)**: The client gets which storage servers have its requested keys by sending another RPC to proxy. + This step returns a set of *k* storage server interfaces, where k is the replication factor; + + * Client cache mechanism: The key location will be cached in client. + Future requests will use the cache to directly read from storage servers, + which saves a trip to proxy. If location is stale, read will return error and client will retry and refresh the cache. + +* **Step 5 (Get data request)**: The client uses the location information from step 4 to directly query keys from corresponding storage servers. + * Direct read from client’s memory: If a key’s value exists in the client’s memory, the client reads it directly from its local memory. + This happens when a client updates a key’s value and later reads it. + This optimization reduces the amount of unnecessary requests to storage servers. + + * Load balance: Each data exists on k storage servers, where k is the replication factor. + To balance the load across the k replicas, client has a load balancing algorithm to balance the number of requests to each replica. + + * Transaction succeed: If the storage server has the data at the read timestamp, the client will receive the data and return succeed. + + * Transaction too old error: If the read request’s timestamp is older than 5 seconds, + storage server may have already flushed the data from its in-memory multi-version data structure to its on-disk single-version data structure. + This means storage server does not have the data older than 5 seconds. So client will receive transaction too old error. + The client will retry with a new timestamp. + One scenario that can lead to the error is when it takes too long for a client to send the read request after it gets the timestamp. + + * Future transaction error: Each storage server pulls data in increasing order of data’s timestamp from the queuing system. + Let’s define a storage server’s timestamp as the largest timestamp of data the storage server has. + If the read request’s timestamp is larger than the storage server’s timestamp, + the storage server will reply future-transaction-error to the client. + The client will retry. One scenario that can lead to the error is when the connection between the SS and the queuing system is slow. + + * Wrong shard error: If keys in the request or result depend on data outside this storage server OR + if a large selector offset prevents all data from being read in one range read. + Client will invalidate its locality cache for the key and retry the read request at the failed key. + +Implementation of FDB read path +------------------------------------------ + +* **Step 1 (Timestamp request)**: + * Each read request tries to get a timestamp if its transaction has not got one: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L2104 + * Client batches the get-timestamp requests: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L3172 + * Dynamic batching algorithm: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L3101-L3104 + +* **Step 2 (Get latest commit version)**: Contacting (n-1) proxies for commit version: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1196 + +* **Step 3 (Confirm proxy’s liveness)**: + * We typically set our clusters’ knob ALWAYS_CAUSAL_READ_RISKY to 1 to skip this step + * Proxy confirm queuing system is alive: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1199 + * How is confirmEpochLive(..) implemented for the above item: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/TagPartitionedLogSystem.actor.cpp#L1216-L1225 + +* **Step 4 (Locality request)**: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1312-L1313 + +* **Step 5 (Get data request)**: + * Logics of handling get value request: + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1306-L1396 + * Load balance algorithm: The loadBalance() at + https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1342-L1344 + + + +Write path of a transaction +================================ + +Suppose a client has a write-only transaction. Fig. 2 below shows the write path in a non-HA cluster. +We will discuss how a transaction with both read and write works in the next section. + +.. image:: /images/FDB_write_path.png + +To simplify the explanation, the steps below do not include transaction batching on proxy, +which is a typical database technique to increase transaction throughput. + +* **Step 1 (Client buffers write mutations):** Client buffers all writes in a transaction until commit is called on the transaction. + In the rest of document, a write is also named as a mutation. + + * Client is a fat client that preprocess transactions: + (a) For atomic operations, if client knows the key value, it will convert atomic operations to set operations; + (b) For version stamp atomic operations, client adds extra bytes to key or value for the version stamp; + (c) If a key has multiple operations, client coalesces them to one operation whenever possible. + + * How client buffers mutations: + https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2345-L2361 + +* **Step 2 (Client commits the transaction):** When a client calls commit(), it performs several operations: + + * **Step 2-1**: Add extra conflict ranges that are added by user but cannot be calculated from mutations. + + * **Step 2-2**: Get a timestamp as the transaction’s start time. The timestamp does not need causal consistency because the transaction has no read. + * This request goes to one of proxies. The proxy will contact all other (n-1) proxies to get the most recent commit version as it does in read path. + The proxy does not need to contact log systems to confirm its activeness because it does not need causal consistency. + + * **Step 2-3**: Sends the transaction’s information to a proxy. Load balancer in client decides which proxy will be used to handle a transaction. + A transaction’s information includes: + + * All of its mutations; + * Read and write conflict range; + * Transaction options that control a transaction’s behavior. For example, should the transaction write when the DB is locked? + Shall the transaction uses the first proxy in the proxy list to commit? + + * Implementation: + * Transaction commit function: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2895-L2899 + * Major work of commit in client side is done at here: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2784-L2868 + * Step 2-1: Add extra conflict ranges: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2826-L2828 + * Step 2-2: getReadVersion at commit which does not need external consistency because we do not have read in the transaction: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2822-L2823 + * Step 2-3: Send transaction to a proxy via RPC: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbclient/NativeAPI.actor.cpp#L2691-L2700 + +* When a proxy receives clients’ transactions, it commits the transaction on behalf of clients with Step 3 - 9. + +* **Step 3 (Proxy gets commit timestamp)**: The proxy gets the timestamp of the transaction’s commit time from the time oracle through an RPC call. + + * To improve transaction throughput and reduce network communication overhead, + each proxy dynamically batch transactions and process transactions in batches. + A proxy keeps batching transactions until the batch time exceeds a configurable timeout value or + until the number of transactions exceed a configurable value or + until the total bytes of the batch exceeds a dynamically calculated desired size. + + * The network overhead is 1 network communication per batch of commit transactions; + + * How is the dynamically calculated batch size calculated: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1770-L1774 + * How commit transactions are batched: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L416-L486 + * How each transaction batch is handled: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L523-L1174 + * Where does proxy sends commit timestamp request to the timestamp generator: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L586-L587 + +* **Step 4 (Proxy builds transactions’ conflict ranges)**: Because the concurrency control component may have multiple processes, + each of which is responsible for resolving conflicts in a key range, + the proxy needs to build one transaction-conflict-resolution request for each concurrency control process: + For each transaction, the proxy splits its read and write conflict ranges based on concurrency control process’ responsible ranges. + The proxy will create k conflict resolution requests for each transaction, where k is the number of processes in the concurrency control component. + + * Implementation: https://github.com/apple/foundationdb/blob/4086e3a2750b776cc8bfb0f0e463fe00ac905595/fdbserver/MasterProxyServer.actor.cpp#L607-L618 + +* **Step 5 (Proxy sends conflict resolution requests to concurrency control)**: + Each concurrency control process is responsible for checking conflicts in a key range. + Each process checks if the transaction has conflicts with other transactions in its key-range. + Each process returns the conflict checking result back to the proxy. + + * What is conflict range? + * A transaction’s write conflict range includes any key and key-ranges that are modified in the transactions. + * A transaction’s read conflict range includes any key and key-ranges that are read in the transaction. + * Client can also use transaction options to add explicit read-conflict-range or write-conflict-range. + Example: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L2634-L2635 + + * **Piggy-back metadata change**. If the transaction changes database’s metadata, such as locking the database, + the change is considered as a special mutation and also checked for conflicts by the concurrency control component. + The primary difference between metadata mutation and normal mutations is that the metadata change must be propagated to all proxies + so that all proxies have a consistent view of database’s metadata. + This is achieved by piggy-backing metadata change in the reply from resolver to proxies. + + * Implementation + * Create conflict resolution requests for a batch of transactions: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L607-L618 + * Metadata mutations are sent from proxy to concurrency control processes: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L366-L369 + +* **Step 6 (Resolve conflicts among concurrent transactions)**: + Each concurrency control process checks conflicts among transactions based on the theory in [1]. + In a nutshell, it checks for read-write conflicts. Suppose two transactions operates on the same key. + If a write transaction’s time overlaps between another read-write transaction’s start time and commit time, + only one transaction can commit: the one that arrives first at all concurrency control processes will commit. + + * Implementation + * Proxy sends conflict checking request: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L626-L629 + * Concurrency control process handles the request: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/Resolver.actor.cpp#L320-L322 + +* **Step 7 (Proxy’s post resolution processing)**: + Once the proxy receives conflict-resolution replies from all concurrency control processes, it performs three steps + + * **Step 7-1 (Apply metadata effect caused by other proxies)**: As mentioned above, when a proxy changes database’s metadata, + the metadata mutations will be propagated via the concurrency control component to other proxies. + So the proxy needs to first compute and apply these metadata mutations onto the proxy’s local states. + Otherwise, the proxy will operate in a different view of database’s metadata. + + * For example, if one proxy locks the database in a committed transaction at time t1, all other proxies should have seen the lock immediately after t1. Since another proxy may have transactions in flight already at t1, the proxy must first apply the “lock“ effect before it can process its in-flight transactions. + * How metadata effect is applied in implementation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L678-L719 + + * **Step 7-2 (Determine which transactions are committed)**: Proxy combines results from all concurrency control processes. + Only if all concurrency control processes say a transaction is committed, will the transaction be considered as committed by the proxy. + + * Implementation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L721-L757 + + * **Step 7-3 (Apply metadata effect caused by this proxy)**: For each committed transaction, + this proxy applies its metadata mutations to the proxy’s local state. + + * Note: These metadata mutations are also sent to concurrency control processes and propagated to other proxies at Step 5. + This step is to apply metadata effect on its own proxy’s states. + * Implementation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L763-L777 + + * **Step 7-4 (Assign mutations to storage servers and serialize them)**: + In order to let the rest of system (the queuing system and storage system) know which process a mutation should be routed to, + the proxy needs to add tags to mutations. + The proxy serializes mutations with the same tag into the same message and sends the serialized message to the queuing system. + + * Implementation of adding tags and serializing mutations into messages: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L800-L910 + * The lines that add tags to a mutation and serialize it: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L846-L847 + + * **Step 7-5 (Duplicate and serialize mutations to backup system keyspace)**: + When backup or disaster recovery (DR) is enabled, each proxy captures mutation streams into a dedicated system keyspace. + Mutations in a transaction batch are serialized as a single mutation in a dedicated system keyspace. + + * How mutations are duplicated for backup and DR: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L912-L986 + * Note: FDB will have a new backup system that avoids duplicating mutations to the system keyspace. + Its design is similar to database’s Change Data Capture (CDC) design. The new backup system is not production-ready yet. + +* **Step 8 (Make mutation messages durable in the queuing system)**: + Proxy sends serialized mutation messages to the queuing system. + The queuing system will append the mutation to an append-only file, fsync it, and send the respnose back. + Each message has a tag, which decides which process in the queuing system the message should be sent to. + The queuing system returns to the proxy the minimum known committed version, which is the smallest commit version among all proxies. + The minimum known commit version is used when the system recovers from fault. + + * Sending messages to the queuing system is abstracted into a push() operation: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L1045 + * The minimum known committed version is called minKnownCommittedVersion. It is updated for each commit: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L1067 + +* **Step 9 (Reply to client)**: Proxy replies the transaction’s result to client. + If the transaction fails (say due to transaction conflicts), proxy sends the error message to the client. + + * Reply to clients based on different transaction’s results: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/MasterProxyServer.actor.cpp#L1117-L1138 + +* **Step 10 (Storage systems pull data from queuing system)**: + Storage system asynchronously pulls data from queuing system and indexes data for read path. + + * Each SS has a primary process (called primary tLog) in the queuing system to pull data from the SS’s data from the queuing system. + Each SS only gets in-ordered streams of mutations that are owned by the SS. + + * In failure scenario when a SS cannot reach the primary tLog, the SS will pull data from different tLogs that have part of the SS’s data. + The SS will then merge the stream of data from different tLogs. + + * Each SS does not make its pulled data durable to disk until the data becomes + at least 5 seconds older than the most recent data the SS has pulled. + This allows each SS to roll back at least 5 seconds of mutations. + + * Why do we need roll back feature for SS? This comes from an optimization used in FDB. + To make a mutation available in a SS as soon as possible, + a SS may fetch a mutation from the queuing system that has not been fully replicated. + The mutation’s transaction may be aborted in rare situations, such as + when FDB has to recover from faults and decides to throw away the last few non-fully-durable transactions. + SSes must throw away data in the aborted transactions. + + * Why does SS not make data durable until 5 seconds later? + This is because today’s SS does not support rolling back data that has already been made durable on disk. + To support roll back, SS keeps data that might be rolled back in memory. + When roll-back is needed, SS just throws away the in-memory data. This simplifies the SS implementation. + + + * Each storage process pulls data from the queuing system: https://github.com/apple/foundationdb/blob/07e354c499158630d760283aa845440cbeaaa1ca/fdbserver/storageserver.actor.cpp#L3593-L3599 + + + +Read write path of a transaction +==================================== + +This section uses an example transaction to describe how a transaction with both read and write operation works in FDB. + +Suppose application creates the following transaction, where *Future* is an object that holds an asynchronous call and +becomes ready when the async call returns, and *wait()* is a synchronous point when the code waits for futures to be ready. +The following code reads key k1 and k2 from database, increases k1’s value by 1 and write back k1’s new value into database. + +**Example Transaction** :: + + Line1: Transaction tr; + Line2: Future fv1 = tr.get(k1); + Line3: Future fv2 = tr.get(k2); + Line4: v1 = wait(fv1); + Line5: v2 = wait(fv2); + Line6: tr.set(v1+v2); + Line7: tr.commit(); + +The transaction starts with the read path: + +* When tr.get() is called, FDB client issues a timestamp request to proxies *if* the transaction has not set its start timestamp. + The logic is the Step 1 in the read path; + +* Batching timestamp requests. When another tr.get() is called, it will try to get a timestamp as well. If we let every get request to follow the Step 1 in the read path, the performance overhead (especially network communication) will be a lot. In addition, this is not necessary because a transaction has only one start timestamp. To solve this problem, client chooses to batch timestamp requests from the same transaction and only issues one timestamp request when the transaction size reaches a preconfigured threshold or when the transaction duration reaches the batching timeout threshold. + * Timestamp requests are batched: https://github.com/apple/foundationdb/blob/4086e3a2750b776cc8bfb0f0e463fe00ac905595/fdbclient/NativeAPI.actor.cpp#L3185 + * Thresholds for client to send the timestamp request: https://github.com/apple/foundationdb/blob/4086e3a2750b776cc8bfb0f0e463fe00ac905595/fdbclient/NativeAPI.actor.cpp#L3095-L3098 + +* Each read request, i.e., tr.get operation in the example, will follow the read path to get data from storage servers, except that they will share the same timestamp; + * These read requests are sent to FDB cluster in parallel. + The ordering of which read request will be ready first depends on requests’ network path and storage servers’ load. + * In the example, tr.get(k2) may return result earlier than tr.get(k1). + +* Client will likely block at the synchronization point at Line 4, until the value is returned from the cluster. + * To maximize clients’ performance, a client can issue multiple transactions concurrently. + When one transaction is blocked at the synchronization point, + the client can switch to work on the other transactions concurrently. + +* Client may or may not block at the synchronization point at Line 5. + If tr.get(k2) returns earlier than tr.get(k1), the future fv2 is already ready when the client arrives at Line 5. + +* At Line 6, client starts the write path. Because the transaction already has its start timestamp, + client does not need to request for the transaction’s start time any more and can skip the Step 2-2 in the write path. + +* At Line 7, client commits the transaction, which will trigger the operations from Step 2 in the write path. + + +A transaction can get more complex than the example above. + +* A transaction can have more writes operations between Line 6 and Line 7. + Those writes will be buffered in client’s memory, which is the Step 1 in the write path. + Only when the client calls commit(), will the rest of steps in the write path will be triggered; + +* A transaction can have reads operations between Line 6 and Line 7 as well. + +* A transaction may return commit_unknown_result, which indicate the transaction may or may not succeed. + If application simply retries the transaction, the transaction may get executed twice. + To solve this problem, the application can adds a transaction id to the transaction and + check if the transaction id exists on the commit_unknown_result error. + + + +Concurrency and ordering of multiple write transactions +======================================================================= + +FDB orders concurrent transactions in increasing order of the transactions’ commit timestamp. +The ordering is enforced in the timestamp generator, the concurrency control component and the durable queuing system. + +* When timestamp generator serves the commit timestamp request from a proxy, + the reply includes not only the commit timestamp but also the latest commit timestamp the generator has sent out. + For example, the timestamp generator just gave out the commit timestamp 50. + When the next request arrives, the generator’s timestamp is 100 and the generator replies (50, 100). + When the second request arrives and the generator’s timestamp is 200, the generator replies (100, 200). + +* When a proxy sends conflict resolution requests to concurrency control processes or durable requests to the queuing system, + each request includes both the current transaction’s commit timestamp and the previous transaction’s commit timestamp. + +* Each concurrency control process and each process in the queuing system always process requests in the strict order of the request’s commit version. + The semantics is do not process a request whose commit timestamp is V2 until the request at its previous commit timestamp V1 has been processed. + + +We use the following example and draw its swimlane diagram to illustrate how two write transactions are ordered in FDB. +The diagram with notes can be viewed at `here `_. + +.. image:: /images/FDB_multiple_txn_swimlane_diagram.png + +Reference +============ + +[SSI] Serializable Snapshot Isolation in PostgreSQL. https://arxiv.org/pdf/1208.4179.pdf diff --git a/documentation/sphinx/source/release-notes/release-notes-620.rst b/documentation/sphinx/source/release-notes/release-notes-620.rst index 3c1c902724..2e963bc7ef 100644 --- a/documentation/sphinx/source/release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/release-notes/release-notes-620.rst @@ -2,12 +2,36 @@ Release Notes ############# +6.2.31 +====== +* Fix a rare invalid memory access on data distributor when snapshotting large clusters. This is a follow up to `PR #4076 `_. `(PR #4317) `_ + +6.2.30 +====== +* A storage server which has fallen behind will deprioritize reads in order to catch up. This change causes some saturating workloads to experience high read latencies instead of high GRV latencies. `(PR #4218) `_ +* Added ``low_priority_queries`` to the ``processes.roles`` section of status to record the number of deprioritized reads on each storage server. `(PR #4218) `_ +* Added ``low_priority_reads`` to the ``workload.operations`` section of status to record the total number of deprioritized reads. `(PR #4218) `_ +* Backup to locally mounted filesystems now appends to files in large block writes, 1MB each by default. `(PR #4199) `_ +* Changed the default SSL implementation from OpenSSL to BoringSSL `(PR #4153) `_ +* SQLite now supports configurable disk write rate limiting. `(PR #4259) `_ +* If a disk operation takes more than two minutes, the system will treat the disk as failed. `(PR #4243) `_ + +6.2.29 +====== +* Fix invalid memory access on data distributor when snapshotting large clusters. `(PR #4076) `_ +* Add human-readable DateTime to trace events `(PR #4087) `_ +* Proxy rejects transaction batch that exceeds MVCC window `(PR #4113) `_ +* Add a command in fdbcli to manually trigger the detailed teams information loggings in data distribution. `(PR #4060) `_ +* Add documentation on read and write Path. `(PR #4099) `_ +* Add a histogram to expose commit batching window on Proxies. `(PR #4166) `_ +* Fix double counting of range reads in TransactionMetrics. `(PR #4130) `_ +* Add a trace event that can be used as an indicator of the load on the proxy. `(PR #4166) `_ + 6.2.28 ====== * Log detailed team collection information when median available space ratio of all teams is too low. `(PR #3912) `_ * Bug fix, blob client did not support authentication key sizes over 64 bytes. `(PR #3964) `_ - 6.2.27 ====== * For clusters with a large number of shards, avoid slow tasks in the data distributor by adding yields to the shard map destruction. `(PR #3834) `_ diff --git a/documentation/sphinx/source/release-notes/release-notes-630.rst b/documentation/sphinx/source/release-notes/release-notes-630.rst index b79db36a9c..234e263767 100644 --- a/documentation/sphinx/source/release-notes/release-notes-630.rst +++ b/documentation/sphinx/source/release-notes/release-notes-630.rst @@ -6,6 +6,12 @@ Release Notes 6.3.10 ====== +* Make fault tolerance metric calculation in HA clusters consistent with 6.2 branch. `(PR #4175) `_ +* Bug fix, stack overflow in redwood storage engine. `(PR #4161) `_ +* Bug fix, getting certain special keys fail. `(PR #4128) `_ +* Prevent slow task on TLog by yielding while processing ignored pop requests. `(PR #4112) `_ +* Support reading xxhash3 sqlite checksums. `(PR #4104) `_ +* Fix a race between submit and abort backup. `(PR #3935) `_ Packaging --------- @@ -84,6 +90,7 @@ Status * Removed fields ``worst_version_lag_storage_server`` and ``limiting_version_lag_storage_server`` from the ``cluster.qos`` section. The ``worst_data_lag_storage_server`` and ``limiting_data_lag_storage_server`` objects can be used instead. `(PR #3196) `_ * If a process is unable to flush trace logs to disk, the problem will now be reported via the output of ``status`` command inside ``fdbcli``. `(PR #2605) `_ `(PR #2820) `_ * When a configuration key is changed, it will always be included in ``status json`` output, even the value is reverted back to the default value. [6.3.5] `(PR #3610) `_ +* Added transactions.rejected_for_queued_too_long for bookkeeping the number of transactions rejected by commit proxy because its queuing time exceeds MVCC window. `(PR #4353) `_ Bindings -------- @@ -134,7 +141,7 @@ Fixes from previous versions * The 6.3.3 patch release includes all fixes from the patch release 6.2.23. :doc:`(6.2 Release Notes) ` * The 6.3.5 patch release includes all fixes from the patch releases 6.2.24 and 6.2.25. :doc:`(6.2 Release Notes) ` * The 6.3.9 patch release includes all fixes from the patch releases 6.2.26. :doc:`(6.2 Release Notes) ` -* The 6.3.10 patch release includes all fixes from the patch releases 6.2.27. :doc:`(6.2 Release Notes) ` +* The 6.3.10 patch release includes all fixes from the patch releases 6.2.27-6.2.29 :doc:`(6.2 Release Notes) ` Fixes only impacting 6.3.0+ --------------------------- diff --git a/documentation/sphinx/source/technical-overview.rst b/documentation/sphinx/source/technical-overview.rst index c5dbf29e37..f66dfb3311 100644 --- a/documentation/sphinx/source/technical-overview.rst +++ b/documentation/sphinx/source/technical-overview.rst @@ -28,6 +28,8 @@ These documents explain the engineering design of FoundationDB, with detailed in * :doc:`kv-architecture` provides a description of every major role a process in FoundationDB can fulfill. +* :doc:`read-write-path` describes how FDB read and write path works. + .. toctree:: :maxdepth: 1 :titlesonly: @@ -45,3 +47,4 @@ These documents explain the engineering design of FoundationDB, with detailed in flow testing kv-architecture + read-write-path diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 7e4c5b934f..f4a57b2b03 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -590,6 +590,9 @@ void initHelp() { CommandHelp("unlock ", "unlock the database with the provided lockUID", "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the " "user will be asked to enter a passphrase to confirm their intent."); + helpMap["triggerddteaminfolog"] = + CommandHelp("triggerddteaminfolog", "trigger the data distributor teams logging", + "Trigger the data distributor to log detailed information about its teams."); hiddenCommands.insert("expensive_data_check"); hiddenCommands.insert("datadistribution"); @@ -1741,6 +1744,23 @@ int printStatusFromJSON( std::string const& jsonFileName ) { } } +ACTOR Future triggerDDTeamInfoLog(Database db) { + state ReadYourWritesTransaction tr(db); + loop { + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + std::string v = deterministicRandom()->randomUniqueID().toString(); + tr.set(triggerDDTeamInfoPrintKey, v); + wait(tr.commit()); + printf("Triggered team info logging in data distribution.\n"); + return Void(); + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + ACTOR Future timeWarning( double when, const char* msg ) { wait( delay(when) ); fputs( msg, stderr ); @@ -3161,6 +3181,11 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } + if (tokencmp(tokens[0], "triggerddteaminfolog")) { + wait(triggerDDTeamInfoLog(db)); + continue; + } + if (tokencmp(tokens[0], "configure")) { bool err = wait(configure(db, tokens, db->getConnectionFile(), &linenoise, warn)); if (err) is_error = true; diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index aa7e56f140..f9292c8cc7 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -377,6 +377,9 @@ public: Reference futureBucket; }; +template<> inline Tuple Codec::pack(FileBackupAgent::ERestoreState const &val) { return Tuple().append(val); } +template<> inline FileBackupAgent::ERestoreState Codec::unpack(Tuple const &val) { return (FileBackupAgent::ERestoreState)val.getInt(0); } + class DatabaseBackupAgent : public BackupAgentBase { public: DatabaseBackupAgent(); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 725d6a0741..c6303a567a 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -1716,15 +1716,39 @@ public: class BackupFile : public IBackupFile, ReferenceCounted { public: - BackupFile(std::string fileName, Reference file, std::string finalFullPath) : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath) {} + BackupFile(std::string fileName, Reference file, std::string finalFullPath) + : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath), m_writeOffset(0) + { + m_buffer.reserve(m_buffer.arena(), CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK); + } Future append(const void *data, int len) { - Future r = m_file->write(data, len, m_offset); - m_offset += len; + m_buffer.append(m_buffer.arena(), (const uint8_t *)data, len); + + if(m_buffer.size() >= CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) { + return flush(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK); + } + + return Void(); + } + + Future flush(int size) { + ASSERT(size <= m_buffer.size()); + + // Keep a reference to the old buffer + Standalone> old = m_buffer; + // Make a new buffer, initialized with the excess bytes over the block size from the old buffer + m_buffer = Standalone>(old.slice(size, old.size())); + + // Write the old buffer to the underlying file and update the write offset + Future r = holdWhile(old, m_file->write(old.begin(), size, m_writeOffset)); + m_writeOffset += size; + return r; } ACTOR static Future finish_impl(Reference f) { + wait(f->flush(f->m_buffer.size())); wait(f->m_file->truncate(f->size())); // Some IAsyncFile implementations extend in whole block sizes. wait(f->m_file->sync()); std::string name = f->m_file->getFilename(); @@ -1733,6 +1757,10 @@ public: return Void(); } + int64_t size() const { + return m_buffer.size() + m_writeOffset; + } + Future finish() { return finish_impl(Reference::addRef(this)); } @@ -1742,6 +1770,8 @@ public: private: Reference m_file; + Standalone> m_buffer; + int64_t m_writeOffset; std::string m_finalFullPath; }; @@ -1874,7 +1904,7 @@ public: class BackupFile : public IBackupFile, ReferenceCounted { public: - BackupFile(std::string fileName, Reference file) : IBackupFile(fileName), m_file(file) {} + BackupFile(std::string fileName, Reference file) : IBackupFile(fileName), m_file(file), m_offset(0) {} Future append(const void *data, int len) { Future r = m_file->write(data, len, m_offset); @@ -1887,11 +1917,16 @@ public: return map(m_file->sync(), [=](Void _) { self->m_file.clear(); return Void(); }); } + int64_t size() const { + return m_offset; + } + void addref() final { return ReferenceCounted::addref(); } void delref() final { return ReferenceCounted::delref(); } private: Reference m_file; + int64_t m_offset; }; Future> writeFile(std::string path) final { diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index fdce885329..d3e33f5a68 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -40,7 +40,7 @@ Future timeKeeperVersionFromDatetime(std::string const &datetime, Datab // TODO: Move the log file and range file format encoding/decoding stuff to this file and behind interfaces. class IBackupFile { public: - IBackupFile(std::string fileName) : m_fileName(fileName), m_offset(0) {} + IBackupFile(std::string fileName) : m_fileName(fileName) {} virtual ~IBackupFile() {} // Backup files are append-only and cannot have more than 1 append outstanding at once. virtual Future append(const void *data, int len) = 0; @@ -48,16 +48,13 @@ public: inline std::string getFileName() const { return m_fileName; } - inline int64_t size() const { - return m_offset; - } + virtual int64_t size() const = 0; virtual void addref() = 0; virtual void delref() = 0; Future appendStringRefWithLen(Standalone s); protected: std::string m_fileName; - int64_t m_offset; }; // Structures for various backup components diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index 46f1d76dba..1d85c93606 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -51,9 +51,19 @@ struct RegionInfo { int32_t priority; Reference satelliteTLogPolicy; + + // Number of tLogs that should be recruited in satellite datacenters. int32_t satelliteDesiredTLogCount; + + // Total number of copies made for each mutation across all satellite tLogs in all DCs. int32_t satelliteTLogReplicationFactor; + + // Number of tLog replies we can ignore when waiting for quorum. Hence, effective quorum is + // satelliteDesiredTLogCount - satelliteTLogWriteAntiQuorum. Locality of individual tLogs is not taken + // into account. int32_t satelliteTLogWriteAntiQuorum; + + // Number of satellite datacenters for current region, as set by `satellite_redundancy_mode`. int32_t satelliteTLogUsableDcs; Reference satelliteTLogPolicyFallback; @@ -63,27 +73,32 @@ struct RegionInfo { std::vector satellites; - RegionInfo() : priority(0), satelliteDesiredTLogCount(-1), satelliteTLogReplicationFactor(0), satelliteTLogWriteAntiQuorum(0), satelliteTLogUsableDcs(1), - satelliteTLogReplicationFactorFallback(0), satelliteTLogWriteAntiQuorumFallback(0), satelliteTLogUsableDcsFallback(0) {} + RegionInfo() + : priority(0), satelliteDesiredTLogCount(-1), satelliteTLogReplicationFactor(0), satelliteTLogWriteAntiQuorum(0), + satelliteTLogUsableDcs(1), satelliteTLogReplicationFactorFallback(0), satelliteTLogWriteAntiQuorumFallback(0), + satelliteTLogUsableDcsFallback(0) {} struct sort_by_priority { - bool operator ()(RegionInfo const&a, RegionInfo const& b) const { return a.priority > b.priority; } + bool operator()(RegionInfo const& a, RegionInfo const& b) const { return a.priority > b.priority; } }; template void serialize(Ar& ar) { - serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor, satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, - satelliteTLogPolicyFallback, satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback, satelliteTLogUsableDcsFallback, satellites); + serializer(ar, dcId, priority, satelliteTLogPolicy, satelliteDesiredTLogCount, satelliteTLogReplicationFactor, + satelliteTLogWriteAntiQuorum, satelliteTLogUsableDcs, satelliteTLogPolicyFallback, + satelliteTLogReplicationFactorFallback, satelliteTLogWriteAntiQuorumFallback, + satelliteTLogUsableDcsFallback, satellites); } }; struct DatabaseConfiguration { DatabaseConfiguration(); - void applyMutation( MutationRef mutation ); - bool set( KeyRef key, ValueRef value ); // Returns true if a configuration option that requires recovery to take effect is changed - bool clear( KeyRangeRef keys ); - Optional get( KeyRef key ) const; + void applyMutation(MutationRef mutation); + bool set(KeyRef key, + ValueRef value); // Returns true if a configuration option that requires recovery to take effect is changed + bool clear(KeyRangeRef keys); + Optional get(KeyRef key) const; bool isValid() const; @@ -92,63 +107,75 @@ struct DatabaseConfiguration { std::string toString() const; StatusObject toJSON(bool noPolicies = false) const; StatusArray getRegionJSON() const; - - RegionInfo getRegion( Optional dcId ) const { - if(!dcId.present()) { + + RegionInfo getRegion(Optional dcId) const { + if (!dcId.present()) { return RegionInfo(); } - for(auto& r : regions) { - if(r.dcId == dcId.get()) { + for (auto& r : regions) { + if (r.dcId == dcId.get()) { return r; } } return RegionInfo(); } - int expectedLogSets( Optional dcId ) const { + int expectedLogSets(Optional dcId) const { int result = 1; - if(dcId.present() && getRegion(dcId.get()).satelliteTLogReplicationFactor > 0 && usableRegions > 1) { + if (dcId.present() && getRegion(dcId.get()).satelliteTLogReplicationFactor > 0 && usableRegions > 1) { result++; } - - if(usableRegions > 1) { + + if (usableRegions > 1) { result++; } return result; } + // Counts the number of DCs required including remote and satellites for current database configuraiton. int32_t minDatacentersRequired() const { int minRequired = 0; - for(auto& r : regions) { + for (auto& r : regions) { minRequired += 1 + r.satellites.size(); } return minRequired; } + int32_t minZonesRequiredPerDatacenter() const { - int minRequired = std::max( remoteTLogReplicationFactor, std::max(tLogReplicationFactor, storageTeamSize) ); - for(auto& r : regions) { - minRequired = std::max( minRequired, r.satelliteTLogReplicationFactor/std::max(1, r.satelliteTLogUsableDcs) ); + int minRequired = std::max(remoteTLogReplicationFactor, std::max(tLogReplicationFactor, storageTeamSize)); + for (auto& r : regions) { + minRequired = + std::max(minRequired, r.satelliteTLogReplicationFactor / std::max(1, r.satelliteTLogUsableDcs)); } return minRequired; } - //Killing an entire datacenter counts as killing one zone in modes that support it + // Retuns the maximum number of discrete failures a cluster can tolerate. + // In HA mode, `fullyReplicatedRegions` is set to false initially when data is being + // replicated to remote, and will be true later. `forAvailablity` is set to true + // if we want to account the number for machines that can recruit new tLogs/SS after failures. + // Killing an entire datacenter counts as killing one zone in modes that support it int32_t maxZoneFailuresTolerated(int fullyReplicatedRegions, bool forAvailability) const { int worstSatellite = regions.size() ? std::numeric_limits::max() : 0; int regionsWithNonNegativePriority = 0; - for(auto& r : regions) { - if(r.priority >= 0) { + for (auto& r : regions) { + if (r.priority >= 0) { regionsWithNonNegativePriority++; } - worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum); - if(r.satelliteTLogUsableDcsFallback > 0) { - worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - r.satelliteTLogWriteAntiQuorumFallback); + worstSatellite = + std::min(worstSatellite, r.satelliteTLogReplicationFactor - r.satelliteTLogWriteAntiQuorum); + if (r.satelliteTLogUsableDcsFallback > 0) { + worstSatellite = std::min(worstSatellite, r.satelliteTLogReplicationFactorFallback - + r.satelliteTLogWriteAntiQuorumFallback); } } - if(usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && (!forAvailability || regionsWithNonNegativePriority > 1)) { - return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, worstSatellite - 1), storageTeamSize - 1); - } else if(worstSatellite > 0) { - return std::min(tLogReplicationFactor + worstSatellite - 2 - tLogWriteAntiQuorum, storageTeamSize - 1); + if (usableRegions > 1 && fullyReplicatedRegions > 1 && worstSatellite > 0 && + (!forAvailability || regionsWithNonNegativePriority > 1)) { + return 1 + std::min(std::max(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, worstSatellite - 1), + storageTeamSize - 1); + } else if (worstSatellite > 0) { + // Primary and Satellite tLogs are synchronously replicated, hence we can lose all but 1. + return std::min(tLogReplicationFactor + worstSatellite - 1 - tLogWriteAntiQuorum, storageTeamSize - 1); } return std::min(tLogReplicationFactor - 1 - tLogWriteAntiQuorum, storageTeamSize - 1); } @@ -185,27 +212,47 @@ struct DatabaseConfiguration { // Backup Workers bool backupWorkerEnabled; - //Data centers - int32_t usableRegions; + // Data centers + int32_t usableRegions; // Number of regions which have a replica of the database. int32_t repopulateRegionAntiQuorum; std::vector regions; // Excluded servers (no state should be here) - bool isExcludedServer( NetworkAddressList ) const; + bool isExcludedServer(NetworkAddressList) const; std::set getExcludedServers() const; - int32_t getDesiredProxies() const { if(masterProxyCount == -1) return autoMasterProxyCount; return masterProxyCount; } - int32_t getDesiredResolvers() const { if(resolverCount == -1) return autoResolverCount; return resolverCount; } - int32_t getDesiredLogs() const { if(desiredTLogCount == -1) return autoDesiredTLogCount; return desiredTLogCount; } - int32_t getDesiredRemoteLogs() const { if(remoteDesiredTLogCount == -1) return getDesiredLogs(); return remoteDesiredTLogCount; } - int32_t getDesiredSatelliteLogs( Optional dcId ) const { - auto desired = getRegion(dcId).satelliteDesiredTLogCount; - if(desired == -1) return autoDesiredTLogCount; return desired; + int32_t getDesiredProxies() const { + if (masterProxyCount == -1) return autoMasterProxyCount; + return masterProxyCount; } - int32_t getRemoteTLogReplicationFactor() const { if(remoteTLogReplicationFactor == 0) return tLogReplicationFactor; return remoteTLogReplicationFactor; } - Reference getRemoteTLogPolicy() const { if(remoteTLogReplicationFactor == 0) return tLogPolicy; return remoteTLogPolicy; } + int32_t getDesiredResolvers() const { + if (resolverCount == -1) return autoResolverCount; + return resolverCount; + } + int32_t getDesiredLogs() const { + if (desiredTLogCount == -1) return autoDesiredTLogCount; + return desiredTLogCount; + } + int32_t getDesiredRemoteLogs() const { + if (remoteDesiredTLogCount == -1) return getDesiredLogs(); + return remoteDesiredTLogCount; + } + int32_t getDesiredSatelliteLogs(Optional dcId) const { - bool operator == ( DatabaseConfiguration const& rhs ) const { + auto desired = getRegion(dcId).satelliteDesiredTLogCount; + if (desired == -1) return autoDesiredTLogCount; + return desired; + } + int32_t getRemoteTLogReplicationFactor() const { + if (remoteTLogReplicationFactor == 0) return tLogReplicationFactor; + return remoteTLogReplicationFactor; + } + Reference getRemoteTLogPolicy() const { + if (remoteTLogReplicationFactor == 0) return tLogPolicy; + return remoteTLogPolicy; + } + + bool operator==(DatabaseConfiguration const& rhs) const { const_cast(this)->makeConfigurationImmutable(); const_cast(&rhs)->makeConfigurationImmutable(); return rawConfiguration == rhs.rawConfiguration; @@ -216,8 +263,7 @@ struct DatabaseConfiguration { if (!ar.isDeserializing) makeConfigurationImmutable(); serializer(ar, rawConfiguration); if (ar.isDeserializing) { - for(auto c=rawConfiguration.begin(); c!=rawConfiguration.end(); ++c) - setInternal(c->key, c->value); + for (auto c = rawConfiguration.begin(); c != rawConfiguration.end(); ++c) setInternal(c->key, c->value); setDefaultReplicationPolicy(); } } @@ -225,13 +271,13 @@ struct DatabaseConfiguration { void fromKeyValues(Standalone> rawConfig); private: - Optional< std::map > mutableConfiguration; // If present, rawConfiguration is not valid - Standalone> rawConfiguration; // sorted by key + Optional> mutableConfiguration; // If present, rawConfiguration is not valid + Standalone> rawConfiguration; // sorted by key void makeConfigurationMutable(); void makeConfigurationImmutable(); - bool setInternal( KeyRef key, ValueRef value ); + bool setInternal(KeyRef key, ValueRef value); void resetInternal(); void setDefaultReplicationPolicy(); diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 62df0225ec..12fc3f7db9 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -274,6 +274,7 @@ public: Counter transactionsCommitCompleted; Counter transactionKeyServerLocationRequests; Counter transactionKeyServerLocationRequestsCompleted; + Counter transactionStatusRequests; Counter transactionsTooOld; Counter transactionsFutureVersions; Counter transactionsNotCommitted; diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index c859408f37..6ae199f8be 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -37,16 +37,16 @@ typedef StringRef ValueRef; typedef int64_t Generation; enum { - tagLocalitySpecial = -1, + tagLocalitySpecial = -1, // tag with this locality means it is invalidTag (id=0), txsTag (id=1), or cacheTag (id=2) tagLocalityLogRouter = -2, - tagLocalityRemoteLog = -3, + tagLocalityRemoteLog = -3, // tag created by log router for remote tLogs tagLocalityUpgraded = -4, tagLocalitySatellite = -5, - tagLocalityLogRouterMapped = -6, // used by log router to pop from TLogs + tagLocalityLogRouterMapped = -6, // The pseudo tag used by log routers to pop the real LogRouter tag (i.e., -2) tagLocalityTxs = -7, tagLocalityBackup = -8, // used by backup role to pop from TLogs tagLocalityInvalid = -99 -}; //The TLog and LogRouter require these number to be as compact as possible +}; // The TLog and LogRouter require these number to be as compact as possible inline bool isPseudoLocality(int8_t locality) { return locality == tagLocalityLogRouterMapped || locality == tagLocalityBackup; @@ -54,6 +54,11 @@ inline bool isPseudoLocality(int8_t locality) { #pragma pack(push, 1) struct Tag { + // if locality > 0, + // locality decides which DC id the tLog is in; + // id decides which SS owns the tag; id <-> SS mapping is in the system keyspace: serverTagKeys. + // if locality < 0, locality decides the type of tLog set: satellite, LR, or remote tLog, etc. + // id decides which tLog in the tLog type will be used. int8_t locality; uint16_t id; @@ -180,6 +185,10 @@ std::string describe( Reference const& item ) { return item->toString(); } +static std::string describe(UID const& item) { + return item.shortString(); +} + template std::string describe( T const& item ) { return item.toString(); diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index 835d1cd4ef..8ad4846943 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -99,9 +99,6 @@ StringRef FileBackupAgent::restoreStateText(ERestoreState id) { } } -template<> Tuple Codec::pack(ERestoreState const &val) { return Tuple().append(val); } -template<> ERestoreState Codec::unpack(Tuple const &val) { return (ERestoreState)val.getInt(0); } - ACTOR Future> TagUidMap::getAll_impl(TagUidMap *tagsMap, Reference tr, bool snapshot) { state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot)); diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index e3113b8370..8dc7499b85 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -122,6 +122,7 @@ void ClientKnobs::initialize(bool randomize) { init( TASKBUCKET_MAX_TASK_KEYS, 1000 ); if( randomize && BUGGIFY ) TASKBUCKET_MAX_TASK_KEYS = 20; //Backup + init( BACKUP_LOCAL_FILE_WRITE_BLOCK, 1024*1024 ); if( randomize && BUGGIFY ) BACKUP_LOCAL_FILE_WRITE_BLOCK = 100; init( BACKUP_CONCURRENT_DELETES, 100 ); init( BACKUP_SIMULATED_LIMIT_BYTES, 1e6 ); if( randomize && BUGGIFY ) BACKUP_SIMULATED_LIMIT_BYTES = 1000; init( BACKUP_GET_RANGE_LIMIT_BYTES, 1e6 ); diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h index 0a56e09a3d..861e3f1853 100644 --- a/fdbclient/Knobs.h +++ b/fdbclient/Knobs.h @@ -118,6 +118,7 @@ public: int TASKBUCKET_MAX_TASK_KEYS; // Backup + int BACKUP_LOCAL_FILE_WRITE_BLOCK; int BACKUP_CONCURRENT_DELETES; int BACKUP_SIMULATED_LIMIT_BYTES; int BACKUP_GET_RANGE_LIMIT_BYTES; diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index 354f64e7ee..4c8f2de181 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -989,11 +989,12 @@ ACTOR Future changeQuorum( Database cx, ReferenceisSimulated()) { for(int i = 0; i < (desiredCoordinators.size()/2)+1; i++) { - auto addresses = g_simulator.getProcessByAddress(desiredCoordinators[i])->addresses; + auto process = g_simulator.getProcessByAddress(desiredCoordinators[i]); + ASSERT(process->isReliable() || process->rebooting); - g_simulator.protectedAddresses.insert(addresses.address); - if(addresses.secondaryAddress.present()) { - g_simulator.protectedAddresses.insert(addresses.secondaryAddress.get()); + g_simulator.protectedAddresses.insert(process->addresses.address); + if (process->addresses.secondaryAddress.present()) { + g_simulator.protectedAddresses.insert(process->addresses.secondaryAddress.get()); } TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace(); } @@ -1208,8 +1209,7 @@ struct AutoQuorumChange : IQuorumChange { continue; } // Exclude faulty node due to machine assassination - if (g_network->isSimulated() && g_simulator.protectedAddresses.count(worker->address) && - !g_simulator.getProcessByAddress(worker->address)->isReliable()) { + if (g_network->isSimulated() && !g_simulator.getProcessByAddress(worker->address)->isReliable()) { TraceEvent("AutoSelectCoordinators").detail("SkipUnreliableWorker", worker->address.toString()); continue; } @@ -1607,6 +1607,7 @@ ACTOR Future> checkForExcludingServers(Database cx, vec wait( delayJittered( 1.0 ) ); // SOMEDAY: watches! } catch (Error& e) { + TraceEvent("CheckForExcludingServersError").error(e); wait( tr.onError(e) ); } } @@ -2019,6 +2020,12 @@ TEST_CASE("/ManagementAPI/AutoQuorumChange/checkLocality") { data.locality.set(LiteralStringRef("machineid"), StringRef(machineId)); data.address.ip = IPAddress(i); + if (g_network->isSimulated()) { + g_simulator.newProcess("TestCoordinator", data.address.ip, data.address.port, false, 1, data.locality, + ProcessClass(ProcessClass::CoordinatorClass, ProcessClass::CommandLineSource), "", + ""); + } + workers.push_back(data); } diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 7dedd2f267..6a931e9cd2 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -18,6 +18,8 @@ * limitations under the License. */ +#include + #include "fdbclient/MultiVersionTransaction.h" #include "fdbclient/MultiVersionAssignmentVars.h" #include "fdbclient/ThreadSafeTransaction.h" @@ -275,7 +277,8 @@ void loadClientFunction(T *fp, void *lib, std::string libPath, const char *funct } } -DLApi::DLApi(std::string fdbCPath) : api(new FdbCApi()), fdbCPath(fdbCPath), networkSetup(false) {} +DLApi::DLApi(std::string fdbCPath, bool unlinkOnLoad) + : api(new FdbCApi()), fdbCPath(fdbCPath), unlinkOnLoad(unlinkOnLoad), networkSetup(false) {} void DLApi::init() { if(isLibraryLoaded(fdbCPath.c_str())) { @@ -287,6 +290,13 @@ void DLApi::init() { TraceEvent(SevError, "ErrorLoadingExternalClientLibrary").detail("LibraryPath", fdbCPath); throw platform_error(); } + if (unlinkOnLoad) { + int err = unlink(fdbCPath.c_str()); + if (err) { + TraceEvent(SevError, "ErrorUnlinkingTempClientLibraryFile").GetLastError().detail("LibraryPath", fdbCPath); + throw platform_error(); + } + } loadClientFunction(&api->selectApiVersion, lib, fdbCPath, "fdb_select_api_version_impl"); loadClientFunction(&api->getClientVersion, lib, fdbCPath, "fdb_get_client_version", headerVersion >= 410); @@ -685,7 +695,9 @@ void MultiVersionTransaction::reset() { } // MultiVersionDatabase -MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi *api, std::string clusterFilePath, Reference db, bool openConnectors) : dbState(new DatabaseState()) { +MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi* api, int threadIdx, std::string clusterFilePath, + Reference db, bool openConnectors) + : dbState(new DatabaseState()), threadIdx(threadIdx) { dbState->db = db; dbState->dbVar->set(db); @@ -701,7 +713,7 @@ MultiVersionDatabase::MultiVersionDatabase(MultiVersionApi *api, std::string clu dbState->currentClientIndex = -1; } - api->runOnExternalClients([this, clusterFilePath](Reference client) { + api->runOnExternalClients(threadIdx, [this, clusterFilePath](Reference client) { dbState->addConnection(client, clusterFilePath); }); @@ -714,7 +726,8 @@ MultiVersionDatabase::~MultiVersionDatabase() { } Reference MultiVersionDatabase::debugCreateFromExistingDatabase(Reference db) { - return Reference(new MultiVersionDatabase(MultiVersionApi::api, "", db, false)); + return Reference(new MultiVersionDatabase( + MultiVersionApi::api, 0, "", db, false)); } Reference MultiVersionDatabase::createTransaction() { @@ -841,7 +854,7 @@ void MultiVersionDatabase::DatabaseState::stateChanged() { } if(newIndex == -1) { - ASSERT(currentClientIndex == 0); // This can only happen for the local client, which we set as the current connection before we know it's connected + ASSERT_EQ(currentClientIndex, 0); // This can only happen for the local client, which we set as the current connection before we know it's connected return; } @@ -904,30 +917,39 @@ void MultiVersionDatabase::DatabaseState::cancelConnections() { // MultiVersionApi bool MultiVersionApi::apiVersionAtLeast(int minVersion) { - ASSERT(MultiVersionApi::api->apiVersion != 0); + ASSERT_NE(MultiVersionApi::api->apiVersion, 0); return MultiVersionApi::api->apiVersion >= minVersion || MultiVersionApi::api->apiVersion < 0; } +void MultiVersionApi::runOnExternalClientsAllThreads(std::function)> func, + bool runOnFailedClients) { + for (int i = 0; i < threadCount; i++) { + runOnExternalClients(i, func, runOnFailedClients); + } +} + // runOnFailedClients should be used cautiously. Some failed clients may not have successfully loaded all symbols. -void MultiVersionApi::runOnExternalClients(std::function)> func, bool runOnFailedClients) { +void MultiVersionApi::runOnExternalClients(int threadIdx, std::function)> func, + bool runOnFailedClients) { bool newFailure = false; auto c = externalClients.begin(); while(c != externalClients.end()) { + auto client = c->second[threadIdx]; try { - if(!c->second->failed || runOnFailedClients) { // TODO: Should we ignore some failures? - func(c->second); + if (!client->failed || runOnFailedClients) { // TODO: Should we ignore some failures? + func(client); } } catch(Error &e) { if(e.code() == error_code_external_client_already_loaded) { - TraceEvent(SevInfo, "ExternalClientAlreadyLoaded").error(e).detail("LibPath", c->second->libPath); + TraceEvent(SevInfo, "ExternalClientAlreadyLoaded").error(e).detail("LibPath", c->first); c = externalClients.erase(c); continue; } else { - TraceEvent(SevWarnAlways, "ExternalClientFailure").error(e).detail("LibPath", c->second->libPath); - c->second->failed = true; + TraceEvent(SevWarnAlways, "ExternalClientFailure").error(e).detail("LibPath", c->first); + client->failed = true; newFailure = true; } } @@ -989,23 +1011,25 @@ void MultiVersionApi::setCallbacksOnExternalThreads() { callbackOnMainThread = false; } - void MultiVersionApi::addExternalLibrary(std::string path) { std::string filename = basename(path); - if(filename.empty() || !fileExists(path)) { + if (filename.empty() || !fileExists(path)) { TraceEvent("ExternalClientNotFound").detail("LibraryPath", filename); throw file_not_found(); } MutexHolder holder(lock); - if(networkStartSetup) { + if (networkStartSetup) { throw invalid_option(); // SOMEDAY: it might be good to allow clients to be added after the network is setup } - if(externalClients.count(filename) == 0) { + // external libraries always run on their own thread; ensure we allocate at least one thread to run this library. + threadCount = std::max(threadCount, 1); + + if (externalClientDescriptions.count(filename) == 0) { TraceEvent("AddingExternalClient").detail("LibraryPath", filename); - externalClients[filename] = Reference(new ClientInfo(new DLApi(path), path)); + externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(path, true))); } } @@ -1014,25 +1038,92 @@ void MultiVersionApi::addExternalLibraryDirectory(std::string path) { std::vector files = platform::listFiles(path, DYNAMIC_LIB_EXT); MutexHolder holder(lock); - if(networkStartSetup) { - throw invalid_option(); // SOMEDAY: it might be good to allow clients to be added after the network is setup. For directories, we can monitor them for the addition of new files. + if (networkStartSetup) { + throw invalid_option(); // SOMEDAY: it might be good to allow clients to be added after the network is setup } + // external libraries always run on their own thread; ensure we allocate at least one thread to run this library. + threadCount = std::max(threadCount, 1); + for(auto filename : files) { std::string lib = abspath(joinPath(path, filename)); - if(externalClients.count(filename) == 0) { + if (externalClientDescriptions.count(filename) == 0) { TraceEvent("AddingExternalClient").detail("LibraryPath", filename); - externalClients[filename] = Reference(new ClientInfo(new DLApi(lib), lib)); - } + externalClientDescriptions.emplace(std::make_pair(filename, ClientDesc(lib, true))); + } } } +#if defined(__unixish__) +std::vector> MultiVersionApi::copyExternalLibraryPerThread(std::string path) { + ASSERT_GE(threadCount, 1); + // Copy library for each thread configured per version + std::vector> paths; + // It's tempting to use the so once without copying. However, we don't know + // if the thing we're about to copy is the shared object executing this code + // or not, so this optimization is unsafe. + // paths.push_back({path, false}); + for (int ii = 0; ii < threadCount; ++ii) { + std::string filename = basename(path); + + char tempName[PATH_MAX + 12]; + sprintf(tempName, "/tmp/%s-XXXXXX", filename.c_str()); + int tempFd = mkstemp(tempName); + int fd; + + if ((fd = open(path.c_str(), O_RDONLY)) == -1) { + TraceEvent("ExternalClientNotFound").detail("LibraryPath", path); + throw file_not_found(); + } + + constexpr size_t buf_sz = 4096; + char buf[buf_sz]; + while (1) { + ssize_t readCount = read(fd, buf, buf_sz); + if (readCount == 0) { + // eof + break; + } + if (readCount == -1) { + TraceEvent(SevError, "ExternalClientCopyFailedReadError").GetLastError().detail("LibraryPath", path); + throw platform_error(); + } + ssize_t written = 0; + while (written != readCount) { + ssize_t writeCount = write(tempFd, buf + written, readCount - written); + if (writeCount == -1) { + TraceEvent(SevError, "ExternalClientCopyFailedWriteError").GetLastError().detail("LibraryPath", path); + throw platform_error(); + } + written += writeCount; + } + } + + close(fd); + close(tempFd); + + paths.push_back({tempName, true}); // use + delete temporary copies of the library. + } + + return paths; +} +#else +std::vector > MultiVersionApi::copyExternalLibraryPerThread(std::string path) { + if (threadCount > 1) { + TraceEvent(SevError, "MultipleClientThreadsUnsupportedOnWindows"); + throw unsupported_operation(); + } + std::vector> paths; + paths.push_back({ path , false }); + return paths; +} +#endif void MultiVersionApi::disableLocalClient() { MutexHolder holder(lock); if(networkStartSetup || bypassMultiClientApi) { throw invalid_option(); } - + threadCount = std::max(threadCount, 1); localClientDisabled = true; } @@ -1040,13 +1131,14 @@ void MultiVersionApi::setSupportedClientVersions(Standalone versions) MutexHolder holder(lock); ASSERT(networkSetup); - // This option must be set on the main thread because it modifes structures that can be used concurrently by the main thread + // This option must be set on the main thread because it modifies structures that can be used concurrently by the + // main thread onMainThreadVoid([this, versions](){ localClient->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions); }, NULL); if(!bypassMultiClientApi) { - runOnExternalClients([versions](Reference client) { + runOnExternalClientsAllThreads([versions](Reference client) { client->api->setNetworkOption(FDBNetworkOptions::SUPPORTED_CLIENT_VERSIONS, versions); }); } @@ -1099,14 +1191,26 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option, ASSERT(!value.present() && !networkStartSetup); externalClient = true; bypassMultiClientApi = true; - } - else { + } else if (option == FDBNetworkOptions::CLIENT_THREADS_PER_VERSION) { + MutexHolder holder(lock); + validateOption(value, true, false, false); + ASSERT(!networkStartSetup); +#if defined(__unixish__) + threadCount = extractIntOption(value, 1, 1024); +#else + // multiple client threads are not supported on windows. + threadCount = extractIntOption(value, 1, 1); +#endif + if (threadCount > 1) { + disableLocalClient(); + } + } else { MutexHolder holder(lock); localClient->api->setNetworkOption(option, value); if(!bypassMultiClientApi) { if(networkSetup) { - runOnExternalClients( + runOnExternalClientsAllThreads( [option, value](Reference client) { client->api->setNetworkOption(option, value); }); } else { @@ -1128,6 +1232,24 @@ void MultiVersionApi::setupNetwork() { throw network_already_setup(); } + for (auto i : externalClientDescriptions) { + std::string path = i.second.libPath; + std::string filename = basename(path); + + // Copy external lib for each thread + if (externalClients.count(filename) == 0) { + externalClients[filename] = {}; + for (const auto& tmp : copyExternalLibraryPerThread(path)) { + TraceEvent("AddingExternalClient") + .detail("FileName", filename) + .detail("LibraryPath", path) + .detail("TempPath", tmp.first); + externalClients[filename].push_back( + Reference(new ClientInfo(new DLApi(tmp.first, tmp.second /*unlink on load*/), path))); + } + } + } + networkStartSetup = true; if(externalClients.empty()) { @@ -1145,14 +1267,14 @@ void MultiVersionApi::setupNetwork() { localClient->loadProtocolVersion(); if(!bypassMultiClientApi) { - runOnExternalClients([this](Reference client) { + runOnExternalClientsAllThreads([this](Reference client) { TraceEvent("InitializingExternalClient").detail("LibraryPath", client->libPath); client->api->selectApiVersion(apiVersion); client->loadProtocolVersion(); }); MutexHolder holder(lock); - runOnExternalClients([this, transportId](Reference client) { + runOnExternalClientsAllThreads([this, transportId](Reference client) { for(auto option : options) { client->api->setNetworkOption(option.first, option.second.castTo()); } @@ -1193,7 +1315,7 @@ void MultiVersionApi::runNetwork() { std::vector handles; if(!bypassMultiClientApi) { - runOnExternalClients([&handles](Reference client) { + runOnExternalClientsAllThreads([&handles](Reference client) { if(client->external) { handles.push_back(g_network->startThread(&runNetworkThread, client.getPtr())); } @@ -1218,9 +1340,7 @@ void MultiVersionApi::stopNetwork() { localClient->api->stopNetwork(); if(!bypassMultiClientApi) { - runOnExternalClients([](Reference client) { - client->api->stopNetwork(); - }, true); + runOnExternalClientsAllThreads([](Reference client) { client->api->stopNetwork(); }, true); } } @@ -1235,7 +1355,7 @@ void MultiVersionApi::addNetworkThreadCompletionHook(void (*hook)(void*), void * localClient->api->addNetworkThreadCompletionHook(hook, hookParameter); if(!bypassMultiClientApi) { - runOnExternalClients([hook, hookParameter](Reference client) { + runOnExternalClientsAllThreads([hook, hookParameter](Reference client) { client->api->addNetworkThreadCompletionHook(hook, hookParameter); }); } @@ -1247,22 +1367,36 @@ Reference MultiVersionApi::createDatabase(const char *clusterFilePath lock.leave(); throw network_not_setup(); } - lock.leave(); - std::string clusterFile(clusterFilePath); - if(localClientDisabled) { - return Reference(new MultiVersionDatabase(this, clusterFile, Reference())); + + if (threadCount > 1 || localClientDisabled) { + ASSERT(localClientDisabled); + ASSERT(!bypassMultiClientApi); + + int threadIdx = nextThread; + nextThread = (nextThread + 1) % threadCount; + lock.leave(); + for (auto it : externalClients) { + TraceEvent("CreatingDatabaseOnExternalClient") + .detail("LibraryPath", it.first) + .detail("Failed", it.second[threadIdx]->failed); + } + return Reference(new MultiVersionDatabase(this, threadIdx, clusterFile, Reference())); } + lock.leave(); + auto db = localClient->api->createDatabase(clusterFilePath); if(bypassMultiClientApi) { return db; } else { for(auto it : externalClients) { - TraceEvent("CreatingDatabaseOnExternalClient").detail("LibraryPath", it.second->libPath).detail("Failed", it.second->failed); + TraceEvent("CreatingDatabaseOnExternalClient") + .detail("LibraryPath", it.first) + .detail("Failed", it.second[0]->failed); } - return Reference(new MultiVersionDatabase(this, clusterFile, db)); + return Reference(new MultiVersionDatabase(this, 0, clusterFile, db)); } } @@ -1270,7 +1404,9 @@ void MultiVersionApi::updateSupportedVersions() { if(networkSetup) { Standalone> versionStr; - runOnExternalClients([&versionStr](Reference client){ + // not mutating the client, so just call on one instance of each client version. + // thread 0 always exists. + runOnExternalClients(0, [&versionStr](Reference client) { const char *ver = client->api->getClientVersion(); versionStr.append(versionStr.arena(), (uint8_t*)ver, (int)strlen(ver)); versionStr.append(versionStr.arena(), (uint8_t*)";", 1); @@ -1365,7 +1501,9 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() { envOptionsLoaded = true; } -MultiVersionApi::MultiVersionApi() : bypassMultiClientApi(false), networkStartSetup(false), networkSetup(false), callbackOnMainThread(true), externalClient(false), localClientDisabled(false), apiVersion(0), envOptionsLoaded(false) {} +MultiVersionApi::MultiVersionApi() + : bypassMultiClientApi(false), networkStartSetup(false), networkSetup(false), callbackOnMainThread(true), + externalClient(false), localClientDisabled(false), apiVersion(0), envOptionsLoaded(false), threadCount(0) {} MultiVersionApi* MultiVersionApi::api = new MultiVersionApi(); @@ -1382,7 +1520,7 @@ void ClientInfo::loadProtocolVersion() { protocolVersion = ProtocolVersion(strtoull(protocolVersionStr.c_str(), &next, 16)); ASSERT(protocolVersion.version() != 0 && protocolVersion.version() != ULLONG_MAX); - ASSERT(next == &protocolVersionStr[protocolVersionStr.length()]); + ASSERT_EQ(next, &protocolVersionStr[protocolVersionStr.length()]); } bool ClientInfo::canReplace(Reference other) const { @@ -1430,7 +1568,7 @@ TEST_CASE("/fdbclient/multiversionclient/EnvironmentVariableParsing" ) { ASSERT(false); } catch(Error &e) { - ASSERT(e.code() == error_code_invalid_option_value); + ASSERT_EQ(e.code(), error_code_invalid_option_value); } return Void(); @@ -1589,7 +1727,7 @@ ACTOR Future checkUndestroyedFutures(std::vectordebugGetReferenceCount() == 1); + ASSERT_EQ(f->debugGetReferenceCount(), 1); ASSERT(f->isReady()); f->cancel(); @@ -1673,7 +1811,7 @@ struct AbortableTest { auto newFuture = FutureInfo(abortableFuture(f.future, ThreadFuture(abort)), f.expectedValue, f.legalErrors); if(!abort->isReady() && deterministicRandom()->coinflip()) { - ASSERT(abort->status == ThreadSingleAssignmentVarBase::Unset); + ASSERT_EQ(abort->status, ThreadSingleAssignmentVarBase::Unset); newFuture.threads.push_back(g_network->startThread(setAbort, abort)); } @@ -1717,7 +1855,7 @@ private: struct DLTest { static FutureInfo createThreadFuture(FutureInfo f) { return FutureInfo(toThreadFuture(getApi(), (FdbCApi::FDBFuture*)f.future.extractPtr(), [](FdbCApi::FDBFuture *f, FdbCApi *api) { - ASSERT(((ThreadSingleAssignmentVar*)f)->debugGetReferenceCount() >= 1); + ASSERT_GE(((ThreadSingleAssignmentVar*)f)->debugGetReferenceCount(), 1); return ((ThreadSingleAssignmentVar*)f)->get(); }), f.expectedValue, f.legalErrors); } diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index c803032cc7..4a3fb0a9a3 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -189,7 +189,7 @@ private: class DLApi : public IClientApi { public: - DLApi(std::string fdbCPath); + DLApi(std::string fdbCPath, bool unlinkOnLoad = false); void selectApiVersion(int apiVersion) override; const char* getClientVersion() override; @@ -207,6 +207,7 @@ public: private: const std::string fdbCPath; const Reference api; + const bool unlinkOnLoad; int headerVersion; bool networkSetup; @@ -278,17 +279,23 @@ private: std::vector>>> persistentOptions; }; -struct ClientInfo : ThreadSafeReferenceCounted { +struct ClientDesc { + std::string const libPath; + bool const external; + + ClientDesc(std::string libPath, bool external) : libPath(libPath), external(external) {} +}; + +struct ClientInfo : ClientDesc, ThreadSafeReferenceCounted { ProtocolVersion protocolVersion; IClientApi *api; - std::string libPath; - bool external; bool failed; std::vector> threadCompletionHooks; - ClientInfo() : protocolVersion(0), api(NULL), external(false), failed(true) {} - ClientInfo(IClientApi *api) : protocolVersion(0), api(api), libPath("internal"), external(false), failed(false) {} - ClientInfo(IClientApi *api, std::string libPath) : protocolVersion(0), api(api), libPath(libPath), external(true), failed(false) {} + ClientInfo() : ClientDesc(std::string(), false), protocolVersion(0), api(NULL), failed(true) {} + ClientInfo(IClientApi* api) : ClientDesc("internal", false), protocolVersion(0), api(api), failed(false) {} + ClientInfo(IClientApi* api, std::string libPath) + : ClientDesc(libPath, true), protocolVersion(0), api(api), failed(false) {} void loadProtocolVersion(); bool canReplace(Reference other) const; @@ -298,7 +305,8 @@ class MultiVersionApi; class MultiVersionDatabase : public IDatabase, ThreadSafeReferenceCounted { public: - MultiVersionDatabase(MultiVersionApi *api, std::string clusterFilePath, Reference db, bool openConnectors=true); + MultiVersionDatabase(MultiVersionApi* api, int threadIdx, std::string clusterFilePath, Reference db, + bool openConnectors = true); ~MultiVersionDatabase(); Reference createTransaction() override; @@ -361,6 +369,7 @@ private: }; const Reference dbState; + const int threadIdx; friend class MultiVersionTransaction; }; @@ -379,7 +388,9 @@ public: static MultiVersionApi* api; Reference getLocalClient(); - void runOnExternalClients(std::function)>, bool runOnFailedClients=false); + void runOnExternalClients(int threadId, std::function)>, + bool runOnFailedClients = false); + void runOnExternalClientsAllThreads(std::function)>, bool runOnFailedClients = false); void updateSupportedVersions(); @@ -397,13 +408,17 @@ private: void setCallbacksOnExternalThreads(); void addExternalLibrary(std::string path); void addExternalLibraryDirectory(std::string path); + // Return a vector of (pathname, unlink_on_close) pairs. Makes threadCount - 1 copies of the library stored in path, + // and returns a vector of length threadCount. + std::vector> copyExternalLibraryPerThread(std::string path); void disableLocalClient(); void setSupportedClientVersions(Standalone versions); void setNetworkOptionInternal(FDBNetworkOptions::Option option, Optional value); Reference localClient; - std::map> externalClients; + std::map externalClientDescriptions; + std::map>> externalClients; bool networkStartSetup; volatile bool networkSetup; @@ -411,6 +426,9 @@ private: volatile bool externalClient; int apiVersion; + int nextThread = 0; + int threadCount; + Mutex lock; std::vector>>> options; std::map>> setEnvOptions; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 15310c901d..8a4fab2799 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -651,8 +651,7 @@ DatabaseContext::DatabaseContext(Reference(specialKeys.begin, specialKeys.end, /* test */ false)) { dbId = deterministicRandom()->randomUniqueID(); connected = clientInfo->get().proxies.size() ? Void() : clientInfo->onChange(); @@ -716,6 +716,7 @@ DatabaseContext::DatabaseContext(Reference Future> { if (ryw->getDatabase().getPtr() && ryw->getDatabase()->getConnectionFile()) { + ++ryw->getDatabase()->transactionStatusRequests; return getJSON(ryw->getDatabase()); } else { return Optional(); @@ -761,22 +762,35 @@ DatabaseContext::DatabaseContext(Reference> clientInfo, Future clientInfoMonitor, LocalityData clientLocality, bool enableLocalityLoadBalance, TaskPriority taskID, bool lockAware, int apiVersion, bool switchable) { return Database( new DatabaseContext( Reference>>(), clientInfo, clientInfoMonitor, taskID, clientLocality, enableLocalityLoadBalance, lockAware, true, apiVersion, switchable ) ); @@ -1515,6 +1529,12 @@ ACTOR Future< vector< pair> > > getKeyRangeLoca } } +// Get the SS locations for each shard in the 'keys' key-range; +// Returned vector size is the number of shards in the input keys key-range. +// Returned vector element is pairs, where +// ShardRange is the whole shard key-range, not a part of the given key range. +// Example: If query the function with key range (b, d), the returned list of pairs could be something like: +// [([a, b1), locationInfo), ([b1, c), locationInfo), ([c, d1), locationInfo)]. template Future< vector< pair> > > getKeyRangeLocations( Database const& cx, KeyRange const& keys, int limit, bool reverse, F StorageServerInterface::*member, TransactionInfo const& info ) { ASSERT (!keys.empty()); @@ -2191,7 +2211,6 @@ ACTOR Future> getRange( Database cx, ReferencetransactionPhysicalReads; - ++cx->transactionGetRangeRequests; state GetKeyValuesReply rep; try { if (CLIENT_BUGGIFY) { diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 8e80db76f1..07ea6f6a44 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -1234,6 +1234,7 @@ Future< Optional > ReadYourWritesTransaction::get( const Key& key, bool s } else { if (key == LiteralStringRef("\xff\xff/status/json")) { if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) { + ++tr.getDatabase()->transactionStatusRequests; return getJSON(tr.getDatabase()); } else { return Optional(); diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 805820bf61..1a065fb9d8 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -116,6 +116,11 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "counter":0, "roughness":0.0 }, + "low_priority_queries":{ + "hz":0.0, + "counter":0, + "roughness":0.0 + }, "bytes_queried":{ "hz":0.0, "counter":0, @@ -521,6 +526,11 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "counter":0, "roughness":0.0 }, + "low_priority_reads":{ + "hz":0.0, + "counter":0, + "roughness":0.0 + }, "location_requests":{ "hz":0.0, "counter":0, @@ -577,6 +587,11 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "counter":0, "roughness":0.0 }, + "rejected_for_queued_too_long":{ + "hz":0.0, + "counter":0, + "roughness":0.0 + }, "committed":{ "hz":0.0, "counter":0, diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index e15898695c..752b9243c6 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -115,27 +115,46 @@ ACTOR Future normalizeKeySelectorActor(SpecialKeySpace* sks, ReadYourWrite KeyRangeRef boundary, int* actualOffset, Standalone* result, Optional>* cache) { + // If offset < 1, where we need to move left, iter points to the range containing at least one smaller key + // (It's a wasting of time to walk through the range whose begin key is same as ks->key) + // (rangeContainingKeyBefore itself handles the case where ks->key == Key()) + // Otherwise, we only need to move right if offset > 1, iter points to the range containing the key + // Since boundary.end is always a key in the RangeMap, it is always safe to move right state RangeMap::Iterator iter = ks->offset < 1 ? sks->getImpls().rangeContainingKeyBefore(ks->getKey()) : sks->getImpls().rangeContaining(ks->getKey()); - while ((ks->offset < 1 && iter->begin() > boundary.begin) || (ks->offset > 1 && iter->begin() < boundary.end)) { + while ((ks->offset < 1 && iter->begin() >= boundary.begin) || (ks->offset > 1 && iter->begin() < boundary.end)) { if (iter->value() != nullptr) { wait(moveKeySelectorOverRangeActor(iter->value(), ryw, ks, cache)); } - ks->offset < 1 ? --iter : ++iter; + // Check if we can still move the iterator left + if (ks->offset < 1) { + if (iter == sks->getImpls().ranges().begin()) { + break; + } else { + --iter; + } + } else if (ks->offset > 1) { + // Always safe to move right + ++iter; + } } *actualOffset = ks->offset; - if (iter->begin() == boundary.begin || iter->begin() == boundary.end) ks->setKey(iter->begin()); if (!ks->isFirstGreaterOrEqual()) { - // The Key Selector clamps up to the legal key space TraceEvent(SevDebug, "ReadToBoundary") .detail("TerminateKey", ks->getKey()) .detail("TerminateOffset", ks->offset); - if (ks->offset < 1) + // If still not normalized after moving to the boundary, + // let key selector clamp up to the boundary + if (ks->offset < 1) { result->readToBegin = true; - else + ks->setKey(boundary.begin); + } + else { result->readThroughEnd = true; + ks->setKey(boundary.end); + } ks->offset = 1; } return Void(); diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index eb6fbf1d38..d9d03de246 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -137,7 +137,7 @@ struct StorageInfo : NonCopyable, public ReferenceCounted { }; struct ServerCacheInfo { - std::vector tags; + std::vector tags; // all tags in both primary and remote DC for the key-range std::vector> src_info; std::vector> dest_info; @@ -426,7 +426,12 @@ struct SplitMetricsRequest { // Should always be used inside a `Standalone`. struct ReadHotRangeWithMetrics { KeyRangeRef keys; + // density refers to the ratio of bytes sent(because of the read) and bytes on disk. + // For example if key range [A, B) and [B, C) respectively has byte size 100 bytes on disk. + // Key range [A,B) was read 30 times. + // The density for key range [A,C) is 30 * 100 / 200 = 15 double density; + // How many bytes of data was sent in a period of time because of read requests. double readBandwidth; ReadHotRangeWithMetrics() = default; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 799276cd66..3b8845d0aa 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -572,6 +572,8 @@ ProcessClass decodeProcessClassValue( ValueRef const& value ) { const KeyRangeRef configKeys( LiteralStringRef("\xff/conf/"), LiteralStringRef("\xff/conf0") ); const KeyRef configKeysPrefix = configKeys.begin; +const KeyRef triggerDDTeamInfoPrintKey(LiteralStringRef("\xff/triggerDDTeamInfoPrint")); + const KeyRangeRef excludedServersKeys( LiteralStringRef("\xff/conf/excluded/"), LiteralStringRef("\xff/conf/excluded0") ); const KeyRef excludedServersPrefix = excludedServersKeys.begin; const KeyRef excludedServersVersionKey = LiteralStringRef("\xff/conf/excluded"); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index d62022dccd..f9c27b64d9 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -167,6 +167,9 @@ UID decodeProcessClassKeyOld( KeyRef const& key ); extern const KeyRangeRef configKeys; extern const KeyRef configKeysPrefix; +// Change the value of this key to anything and that will trigger detailed data distribution team info log. +extern const KeyRef triggerDDTeamInfoPrintKey; + // "\xff/conf/excluded/1.2.3.4" := "" // "\xff/conf/excluded/1.2.3.4:4000" := "" // These are inside configKeysPrefix since they represent a form of configuration and they are convenient diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index d7463e5845..837a2f27c8 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -108,7 +108,10 @@ description is not currently required but encouraged. paramType="String" paramDescription="path to directory containing client libraries" description="Searches the specified path for dynamic libraries and adds them to the list of client libraries for use by the multi-version client API. Must be set before setting up the network." />