diff --git a/.gitignore b/.gitignore index 7b23facbe3..5fc9981a4f 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,11 @@ compile_commands.json flow/actorcompiler/obj flow/coveragetool/obj +# IDE indexing (commonly used tools) +/compile_commands.json +/.ccls-cache +/.clangd + # Temporary and user configuration files *~ *.orig @@ -89,5 +94,4 @@ flow/coveragetool/obj .envrc .DS_Store temp/ -/compile_commands.json -/.ccls-cache +/versions.target diff --git a/ACKNOWLEDGEMENTS b/ACKNOWLEDGEMENTS index dd99a3e1ec..c9f154657f 100644 --- a/ACKNOWLEDGEMENTS +++ b/ACKNOWLEDGEMENTS @@ -479,3 +479,41 @@ SHIBUKAWA Yoshiki (sphinxcontrib-rubydomain) THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Armon Dadgar (ART) + Copyright (c) 2012, Armon Dadgar + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the organization nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL ARMON DADGAR BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Copyright (C) 2009 The Guava Authors + + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + in compliance with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations under + the License. diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d497b3f73..00bdde8e1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ # limitations under the License. cmake_minimum_required(VERSION 3.13) project(foundationdb - VERSION 7.0.0 + VERSION 6.3.0 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) @@ -80,42 +80,10 @@ message(STATUS "Current git version ${CURRENT_GIT_VERSION}") # Version information ################################################################################ -if(NOT WIN32) - add_custom_target(version_file ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/versions.target) - execute_process( - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build/get_version.sh ${CMAKE_CURRENT_SOURCE_DIR}/versions.target - OUTPUT_VARIABLE FDB_VERSION_WNL) - execute_process( - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build/get_package_name.sh ${CMAKE_CURRENT_SOURCE_DIR}/versions.target - OUTPUT_VARIABLE FDB_PACKAGE_NAME_WNL) - string(STRIP "${FDB_VERSION_WNL}" FDB_VERSION_TARGET_FILE) - string(STRIP "${FDB_PACKAGE_NAME_WNL}" FDB_PACKAGE_NAME_TARGET_FILE) -endif() - -set(USE_VERSIONS_TARGET OFF CACHE BOOL "Use the deprecated versions.target file") -if(USE_VERSIONS_TARGET) - if (WIN32) - message(FATAL_ERROR "USE_VERSION_TARGET us not supported on Windows") - endif() - set(FDB_VERSION ${FDB_VERION_TARGET_FILE}) - set(FDB_PACKAGE_NAME ${FDB_PACKAGE_NAME_TARGET_FILE}) - set(FDB_VERSION_PLAIN ${FDB_VERSION}) -else() - set(FDB_PACKAGE_NAME "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}") - set(FDB_VERSION ${PROJECT_VERSION}) - set(FDB_VERSION_PLAIN ${FDB_VERSION}) - if(NOT WIN32) - # we need to assert that the cmake version is in sync with the target version - if(NOT (FDB_VERSION STREQUAL FDB_VERSION_TARGET_FILE)) - message(SEND_ERROR "The project version in cmake is set to ${FDB_VERSION},\ - but versions.target has it at ${FDB_VERSION_TARGET_FILE}") - endif() - if(NOT (FDB_PACKAGE_NAME STREQUAL FDB_PACKAGE_NAME_TARGET_FILE)) - message(SEND_ERROR "The package name in cmake is set to ${FDB_PACKAGE_NAME},\ - but versions.target has it set to ${FDB_PACKAGE_NAME_TARGET_FILE}") - endif() - endif() -endif() +set(FDB_PACKAGE_NAME "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}") +set(FDB_VERSION ${PROJECT_VERSION}) +set(FDB_VERSION_PLAIN ${FDB_VERSION}) +configure_file(${CMAKE_SOURCE_DIR}/versions.target.cmake ${CMAKE_SOURCE_DIR}/versions.target) message(STATUS "FDB version is ${FDB_VERSION}") message(STATUS "FDB package name is ${FDB_PACKAGE_NAME}") @@ -178,10 +146,11 @@ set(SEED "0x${SEED_}" CACHE STRING "Random seed for testing") # components ################################################################################ -include(CompileBoost) -if(WITH_TLS) - add_subdirectory(FDBLibTLS) +if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + include_directories(/usr/local/include) endif() + +include(CompileBoost) add_subdirectory(flow) add_subdirectory(fdbrpc) add_subdirectory(fdbclient) @@ -192,13 +161,15 @@ if(NOT WIN32) else() add_subdirectory(fdbservice) endif() -add_subdirectory(bindings) add_subdirectory(fdbbackup) +add_subdirectory(contrib) add_subdirectory(tests) +if(WITH_PYTHON) + add_subdirectory(bindings) +endif() if(WITH_DOCUMENTATION) add_subdirectory(documentation) endif() -add_subdirectory(contrib/monitoring) if(WIN32) add_subdirectory(packaging/msi) @@ -206,17 +177,21 @@ else() include(CPack) endif() +if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + add_link_options(-lexecinfo) +endif() + ################################################################################ # process compile commands for IDE ################################################################################ -if (CMAKE_EXPORT_COMPILE_COMMANDS) +if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON) add_custom_command( - OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py - ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json - COMMENT "Build compile commands for IDE" + OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json + COMMAND $ ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py + ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json + COMMENT "Build compile commands for IDE" ) add_custom_target(processed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json) endif() diff --git a/FDBLibTLS/CMakeLists.txt b/FDBLibTLS/CMakeLists.txt index cd22748648..62ea4d5cad 100644 --- a/FDBLibTLS/CMakeLists.txt +++ b/FDBLibTLS/CMakeLists.txt @@ -9,4 +9,4 @@ set(SRCS FDBLibTLSVerify.h) add_library(FDBLibTLS STATIC ${SRCS}) -target_link_libraries(FDBLibTLS PUBLIC LibreSSL boost_target PRIVATE flow) +target_link_libraries(FDBLibTLS PUBLIC OpenSSL::SSL boost_target PRIVATE flow) diff --git a/FDBLibTLS/FDBLibTLS.vcxproj b/FDBLibTLS/FDBLibTLS.vcxproj deleted file mode 100644 index 315ea7a37a..0000000000 --- a/FDBLibTLS/FDBLibTLS.vcxproj +++ /dev/null @@ -1,33 +0,0 @@ - - - - - Debug - X64 - - - Release - X64 - - - - - - - - - - - - - - StaticLibrary - MultiByte - v141 - - - StaticLibrary - MultiByte - v141 - - diff --git a/FDBLibTLS/FDBLibTLSPolicy.cpp b/FDBLibTLS/FDBLibTLSPolicy.cpp index d22f7d8f67..1fb9f65277 100644 --- a/FDBLibTLS/FDBLibTLSPolicy.cpp +++ b/FDBLibTLS/FDBLibTLSPolicy.cpp @@ -300,7 +300,7 @@ bool FDBLibTLSPolicy::set_verify_peers(int count, const uint8_t* verify_peers[], } Reference verify = Reference(new FDBLibTLSVerify(verifyString.substr(start))); verify_rules.push_back(verify); - } catch ( const std::runtime_error& e ) { + } catch ( const std::runtime_error& ) { verify_rules.clear(); std::string verifyString((const char*)verify_peers[i], verify_peers_len[i]); TraceEvent(SevError, "FDBLibTLSVerifyPeersParseError").detail("Config", verifyString); diff --git a/FDBLibTLS/FDBLibTLSSession.cpp b/FDBLibTLS/FDBLibTLSSession.cpp index d73f655dc9..d81dc4e509 100644 --- a/FDBLibTLS/FDBLibTLSSession.cpp +++ b/FDBLibTLS/FDBLibTLSSession.cpp @@ -347,7 +347,7 @@ bool FDBLibTLSSession::verify_peer() { if(now() - lastVerifyFailureLogged > 1.0) { for (std::string reason : verify_failure_reasons) { lastVerifyFailureLogged = now(); - TraceEvent("FDBLibTLSVerifyFailure", uid).detail("Reason", reason); + TraceEvent("FDBLibTLSVerifyFailure", uid).suppressFor(1.0).detail("Reason", reason); } } } diff --git a/FDBLibTLS/Makefile b/FDBLibTLS/Makefile deleted file mode 100644 index bc2ecbc397..0000000000 --- a/FDBLibTLS/Makefile +++ /dev/null @@ -1,109 +0,0 @@ -PROJECTPATH = $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) -PLUGINPATH = $(PROJECTPATH)/$(PLUGIN) - -CFLAGS ?= -O2 -g - -CXXFLAGS ?= -std=c++0x - -CFLAGS += -I/usr/local/include -I../flow -I../fdbrpc -LDFLAGS += -L/usr/local/lib - -LIBS += -ltls -lssl -lcrypto - -PLATFORM := $(shell uname) -ifneq ($(PLATFORM),Darwin) - PLATFORM := $(shell uname -o) -endif - -ifeq ($(PLATFORM),Cygwin) - HOST := x86_64-w64-mingw32 - CC := $(HOST)-gcc - CXX := $(HOST)-g++ - STRIP := $(HOST)-strip --strip-all - - DYEXT = dll - PLUGINPATH = $(PLUGIN) - - LIBS += -static-libstdc++ -static-libgcc - LIBS += -lws2_32 - - LINK_LDFLAGS = -shared - LINK_LDFLAGS += -Wl,-soname,$(PLUGIN) - LINK_LDFLAGS += -Wl,--version-script=FDBLibTLS.map - LINK_LDFLAGS += -Wl,-Bstatic $(LIBS) -Wl,-Bdynamic - -else ifeq ($(PLATFORM),Darwin) - CC := clang - CXX := clang++ - STRIP := strip -S -x - - CFLAGS += -fPIC - - DYEXT = dylib - - vpath %.a /usr/local/lib - .LIBPATTERNS = lib%.a lib%.dylib lib%.so - - LINK_LDFLAGS = -shared - LINK_LDFLAGS += -Wl,-exported_symbols_list,FDBLibTLS.symbols - LINK_LDFLAGS += -Wl,-dylib_install_name,$(PLUGIN) - LINK_LDFLAGS += $(LIBS) - -else ifeq ($(PLATFORM),GNU/Linux) - CC := clang - CXX := clang++ - STRIP := strip --strip-all - - CFLAGS += -fPIC - DYEXT = so - - LIBS += -static-libstdc++ -static-libgcc -lrt - - LINK_LDFLAGS = -shared - LINK_LDFLAGS += -Wl,-soname,$(PLUGIN) - LINK_LDFLAGS += -Wl,--version-script=FDBLibTLS.map - LINK_LDFLAGS += -Wl,-Bstatic $(LIBS) -Wl,-Bdynamic - -else -$(error Unknown platform $(PLATFORM)) -endif - -PLUGIN := FDBLibTLS.$(DYEXT) -OBJECTS := FDBLibTLSPlugin.o FDBLibTLSPolicy.o FDBLibTLSSession.o FDBLibTLSVerify.o -LINKLINE := $(CXXFLAGS) $(CFLAGS) $(LDFLAGS) $(OBJECTS) $(LINK_LDFLAGS) -o $(PLUGIN) - -all: $(PLUGIN) - -build-depends-linux: - apt install clang make libboost-dev - -clean: - @rm -f *.o *.d $(PLUGIN) plugin-test verify-test - @rm -rf *.dSYM - -DEPS := $(patsubst %.o,%.d,$(OBJECTS)) --include $(DEPS) - -$(OBJECTS): %.o: %.cpp Makefile - @echo "Compiling $<" - @$(CXX) $(CXXFLAGS) $(CFLAGS) $(INCLUDES) -c $< -o $@ -MD -MP - -$(PLUGIN): $(OBJECTS) Makefile - @echo "Linking $@" - @$(CXX) $(LINKLINE) - @echo "Stripping $@" - @$(STRIP) $@ - -test: test-plugin test-verify - -test-plugin: plugin-test.cpp $(PLUGIN) Makefile - @echo "Compiling plugin-test" - @$(CXX) $(CXXFLAGS) $(CFLAGS) plugin-test.cpp -ldl -o plugin-test - @echo "Running plugin-test..." - @$(PROJECTPATH)/plugin-test $(PLUGINPATH) - -test-verify: verify-test.cpp $(OBJECTS) Makefile - @echo "Compiling verify-test" - @$(CXX) $(CXXFLAGS) $(CFLAGS) $(LDFLAGS) $(OBJECTS) verify-test.cpp $(LIBS) -o verify-test - @echo "Running verify-test..." - @$(PROJECTPATH)/verify-test diff --git a/FDBLibTLS/local.mk b/FDBLibTLS/local.mk deleted file mode 100644 index 0a0618f30a..0000000000 --- a/FDBLibTLS/local.mk +++ /dev/null @@ -1,28 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -FDBLibTLS_BUILD_SOURCES += - - -FDBLibTLS_CFLAGS := -fPIC -I/usr/local/include -isystem$(BOOSTDIR) -I. -DUSE_UCONTEXT - -lib/libFDBLibTLS.a: bin/coverage.FDBLibTLS.xml diff --git a/Makefile b/Makefile deleted file mode 100644 index 79f2cb05ec..0000000000 --- a/Makefile +++ /dev/null @@ -1,248 +0,0 @@ -export -PLATFORM := $(shell uname) -ARCH := $(shell uname -m) -TOPDIR := $(shell pwd) - -# Allow custom libc++ hack for Ubuntu -ifeq ("$(wildcard /etc/centos-release)", "") - LIBSTDCPP_HACK ?= 1 -endif - -ifeq ($(ARCH),x86_64) - ARCH := x64 -else - $(error Not prepared to compile on $(ARCH)) -endif - -MONO := $(shell which mono 2>/dev/null) -ifeq ($(MONO),) - MONO := /usr/bin/mono -endif - -MCS := $(shell which mcs 2>/dev/null) -ifeq ($(MCS),) - MCS := $(shell which dmcs 2>/dev/null) -endif -ifeq ($(MCS),) - MCS := /usr/bin/mcs -endif - -CFLAGS := -Werror -Wno-error=format -fPIC -DNO_INTELLISENSE -fvisibility=hidden -DNDEBUG=1 -Wreturn-type -fno-omit-frame-pointer -ifeq ($(RELEASE),true) - CFLAGS += -DFDB_CLEAN_BUILD -endif -ifeq ($(NIGHTLY),true) - CFLAGS += -DFDB_CLEAN_BUILD -endif - -BOOST_BASENAME ?= boost_1_67_0 -ifeq ($(PLATFORM),Linux) - PLATFORM := linux - - CC ?= gcc - CXX ?= g++ - - ifneq '' '$(findstring clang++,$(CXX))' - CXXFLAGS += -Wno-undefined-var-template -Wno-unknown-warning-option -Wno-unused-command-line-argument -Wno-register -Wno-logical-op-parentheses - else - CXXFLAGS += -Wno-attributes - endif - - CXXFLAGS += -std=c++17 - - BOOST_BASEDIR ?= /opt - TLS_LIBDIR ?= /usr/local/lib - DLEXT := so - java_DLEXT := so - TARGET_LIBC_VERSION ?= 2.11 -else ifeq ($(PLATFORM),Darwin) - PLATFORM := osx - - CC := /usr/bin/clang - CXX := /usr/bin/clang - - CFLAGS += -mmacosx-version-min=10.14 -stdlib=libc++ - CXXFLAGS += -mmacosx-version-min=10.14 -std=c++17 -stdlib=libc++ -msse4.2 -Wno-undefined-var-template -Wno-unknown-warning-option - - .LIBPATTERNS := lib%.dylib lib%.a - - BOOST_BASEDIR ?= ${HOME} - TLS_LIBDIR ?= /usr/local/lib - DLEXT := dylib - java_DLEXT := jnilib -else - $(error Not prepared to compile on platform $(PLATFORM)) -endif -BOOSTDIR ?= ${BOOST_BASEDIR}/${BOOST_BASENAME} - -CCACHE := $(shell which ccache 2>/dev/null) -ifneq ($(CCACHE),) - CCACHE_CC := $(CCACHE) $(CC) - CCACHE_CXX := $(CCACHE) $(CXX) -else - CCACHE_CC := $(CC) - CCACHE_CXX := $(CXX) -endif - -# Default variables don't get pushed into the environment, but scripts in build/ -# rely on the existence of CC in the environment. -ifeq ($(origin CC), default) - CC := $(CC) -endif - -ACTORCOMPILER := bin/actorcompiler.exe - -# UNSTRIPPED := 1 - -# Normal optimization level -CFLAGS += -O2 - -# Or turn off optimization entirely -# CFLAGS += -O0 - -# Debugging symbols are a good thing (and harmless, since we keep them -# in external debug files) -CFLAGS += -g - -# valgrind-compatibile builds are enabled by uncommenting lines in valgind.mk - -# Define the TLS compilation and link variables -ifdef TLS_DISABLED -CFLAGS += -DTLS_DISABLED -FDB_TLS_LIB := -TLS_LIBS := -else -FDB_TLS_LIB := lib/libFDBLibTLS.a -TLS_LIBS += $(addprefix $(TLS_LIBDIR)/,libtls.a libssl.a libcrypto.a) -endif - -CXXFLAGS += -Wno-deprecated -DBOOST_ERROR_CODE_HEADER_ONLY -DBOOST_SYSTEM_NO_DEPRECATED -LDFLAGS := -LIBS := -STATIC_LIBS := - -# Add library search paths (that aren't -Llib) to the VPATH -VPATH += $(addprefix :,$(filter-out lib,$(patsubst -L%,%,$(filter -L%,$(LDFLAGS))))) - -CS_PROJECTS := flow/actorcompiler flow/coveragetool fdbclient/vexillographer -CPP_PROJECTS := flow fdbrpc fdbclient fdbbackup fdbserver fdbcli bindings/c bindings/java fdbmonitor bindings/flow/tester bindings/flow -ifndef TLS_DISABLED -CPP_PROJECTS += FDBLibTLS -endif -OTHER_PROJECTS := bindings/python bindings/ruby bindings/go - -CS_MK_GENERATED := $(CS_PROJECTS:=/generated.mk) -CPP_MK_GENERATED := $(CPP_PROJECTS:=/generated.mk) - -MK_GENERATED := $(CS_MK_GENERATED) $(CPP_MK_GENERATED) - -# build/valgrind.mk needs to be included before any _MK_GENERATED (which in turn includes local.mk) -MK_INCLUDE := build/scver.mk build/valgrind.mk $(CS_MK_GENERATED) $(CPP_MK_GENERATED) $(OTHER_PROJECTS:=/include.mk) build/packages.mk - -ALL_MAKEFILES := Makefile $(MK_INCLUDE) $(patsubst %/generated.mk,%/local.mk,$(MK_GENERATED)) - -TARGETS = - -.PHONY: clean all Makefiles - -default: fdbserver fdbbackup fdbcli fdb_c fdb_python fdb_python_sdist - -all: $(CS_PROJECTS) $(CPP_PROJECTS) $(OTHER_PROJECTS) - -# These are always defined and ready to use. Any target that uses them and needs them up to date -# should depend on versions.target -VERSION := $(shell cat versions.target | grep '' | sed -e 's,^[^>]*>,,' -e 's,<.*,,') -PACKAGE_NAME := $(shell cat versions.target | grep '' | sed -e 's,^[^>]*>,,' -e 's,<.*,,') - -versions.h: Makefile versions.target - @rm -f $@ -ifeq ($(RELEASE),true) - @echo "#define FDB_VT_VERSION \"$(VERSION)\"" >> $@ -else - @echo "#define FDB_VT_VERSION \"$(VERSION)-PRERELEASE\"" >> $@ -endif - @echo "#define FDB_VT_PACKAGE_NAME \"$(PACKAGE_NAME)\"" >> $@ - -bindings: fdb_c fdb_python fdb_ruby fdb_java fdb_flow fdb_flow_tester fdb_go fdb_go_tester fdb_c_tests - -Makefiles: $(MK_GENERATED) - -$(CS_MK_GENERATED): build/csprojtom4.py build/csproj.mk Makefile - @echo "Creating $@" - @python build/csprojtom4.py $(@D)/*.csproj | m4 -DGENDIR="$(@D)" -DGENNAME=`basename $(@D)/*.csproj .csproj` - build/csproj.mk > $(@D)/generated.mk - -$(CPP_MK_GENERATED): build/vcxprojtom4.py build/vcxproj.mk Makefile - @echo "Creating $@" - @python build/vcxprojtom4.py $(@D)/*.vcxproj | m4 -DGENDIR="$(@D)" -DGENNAME=`basename $(@D)/*.vcxproj .vcxproj` - build/vcxproj.mk > $(@D)/generated.mk - -DEPSDIR := .deps -OBJDIR := .objs -CMDDIR := .cmds - -COMPILE_COMMANDS_JSONS := $(addprefix $(CMDDIR)/,$(addsuffix /compile_commands.json,${CPP_PROJECTS})) -compile_commands.json: build/concatinate_jsons.py ${COMPILE_COMMANDS_JSONS} - @build/concatinate_jsons.py ${COMPILE_COMMANDS_JSONS} - -include $(MK_INCLUDE) - -clean: $(CLEAN_TARGETS) docpreview_clean - @echo "Cleaning toplevel" - @rm -rf $(OBJDIR) - @rm -rf $(DEPSDIR) - @rm -rf lib/ - @rm -rf bin/coverage.*.xml - @rm -rf $(CMDDIR) compile_commands.json - @find . -name "*.g.cpp" -exec rm -f {} \; -or -name "*.g.h" -exec rm -f {} \; - -targets: - @echo "Available targets:" - @for i in $(sort $(TARGETS)); do echo " $$i" ; done - @echo "Append _clean to clean specific target." - -lib/libstdc++.a: $(shell $(CC) -print-file-name=libstdc++_pic.a) - @echo "Frobnicating $@" - @mkdir -p lib - @rm -rf .libstdc++ - @mkdir .libstdc++ - @(cd .libstdc++ && ar x $<) - @for i in .libstdc++/*.o ; do \ - nm $$i | grep -q \@ || continue ; \ - nm $$i | awk '$$3 ~ /@@/ { COPY = $$3; sub(/@@.*/, "", COPY); print $$3, COPY; }' > .libstdc++/replacements ; \ - objcopy --redefine-syms=.libstdc++/replacements $$i $$i.new && mv $$i.new $$i ; \ - rm .libstdc++/replacements ; \ - nm $$i | awk '$$3 ~ /@/ { print $$3; }' > .libstdc++/deletes ; \ - objcopy --strip-symbols=.libstdc++/deletes $$i $$i.new && mv $$i.new $$i ; \ - rm .libstdc++/deletes ; \ - done - @ar rcs $@ .libstdc++/*.o - @rm -r .libstdc++ - - -docpreview: javadoc - @echo "Generating docpreview" - @TARGETS= $(MAKE) -C documentation docpreview - -docpreview_clean: - @echo "Cleaning docpreview" - @CLEAN_TARGETS= $(MAKE) -C documentation -s --no-print-directory docpreview_clean - -packages/foundationdb-docs-$(VERSION).tar.gz: FORCE javadoc - @echo "Packaging documentation" - @TARGETS= $(MAKE) -C documentation docpackage - @mkdir -p packages - @rm -f packages/foundationdb-docs-$(VERSION).tar.gz - @cp documentation/sphinx/.dist/foundationdb-docs-$(VERSION).tar.gz packages/foundationdb-docs-$(VERSION).tar.gz - -docpackage: packages/foundationdb-docs-$(VERSION).tar.gz - -FORCE: - -.SECONDEXPANSION: - -bin/coverage.%.xml: bin/coveragetool.exe $$(%_ALL_SOURCES) - @echo "Creating $@" - @$(MONO) bin/coveragetool.exe $@ $(filter-out $<,$^) >/dev/null - -$(CPP_MK_GENERATED): $$(@D)/*.vcxproj - -$(CS_MK_GENERATED): $$(@D)/*.csproj diff --git a/README.md b/README.md index edbec64cda..e27dca73fc 100755 --- a/README.md +++ b/README.md @@ -33,6 +33,10 @@ CMake-based build system. Both of them should currently work for most users, and CMake should be the preferred choice as it will eventually become the only build system available. +If compiling for local development, please set `-DUSE_WERROR=ON` in +cmake. Our CI compiles with `-Werror` on, so this way you'll find out about +compiler warnings that break the build earlier. + ## CMake To build with CMake, generally the following is required (works on Linux and @@ -47,8 +51,8 @@ Mac OS - for Windows see below): 1. Create a build directory (you can have the build directory anywhere you like): `mkdir build` 1. `cd build` -1. `cmake -DBOOST_ROOT= ` -1. `make` +1. `cmake -GNinja -DBOOST_ROOT= ` +1. `ninja` CMake will try to find its dependencies. However, for LibreSSL this can be often problematic (especially if OpenSSL is installed as well). For that we recommend @@ -57,7 +61,7 @@ LibreSSL is installed under `/usr/local/libressl-2.8.3`, you should call cmake l this: ``` -cmake -DLibreSSL_ROOT=/usr/local/libressl-2.8.3/ ../foundationdb +cmake -GNinja -DLibreSSL_ROOT=/usr/local/libressl-2.8.3/ ../foundationdb ``` FoundationDB will build just fine without LibreSSL, however, the resulting @@ -119,6 +123,37 @@ cmake -G Xcode -DOPEN_FOR_IDE=ON You should create a second build-directory which you will use for building (probably with make or ninja) and debugging. +#### FreeBSD + +1. Check out this repo on your server. +1. Install compile-time dependencies from ports. +1. (Optional) Use tmpfs & ccache for significantly faster repeat builds +1. (Optional) Install a [JDK](https://www.freshports.org/java/openjdk8/) + for Java Bindings. FoundationDB currently builds with Java 8. +1. Navigate to the directory where you checked out the foundationdb + repo. +1. Build from source. + + ```shell + sudo pkg install -r FreeBSD \ + shells/bash devel/cmake devel/ninja devel/ccache \ + lang/mono lang/python3 \ + devel/boost-libs devel/libeio \ + security/openssl + mkdir .build && cd .build + cmake -G Ninja \ + -DUSE_CCACHE=on \ + -DDISABLE_TLS=off \ + -DUSE_DTRACE=off \ + .. + ninja -j 10 + # run fast tests + ctest -L fast + # run all tests + ctest --output-on-failure -v + ``` + + ### Linux There are no special requirements for Linux. A docker image can be pulled from @@ -129,31 +164,31 @@ If you want to create a package you have to tell cmake what platform it is for. And then you can build by simply calling `cpack`. So for debian, call: ``` -cmake -DINSTALL_LAYOUT=DEB -make -cpack +cmake -GNinja +ninja +cpack -G DEB ``` For RPM simply replace `DEB` with `RPM`. ### MacOS -The build under MacOS will work the same way as on Linux. To get LibreSSL and boost you -can use [Homebrew](https://brew.sh/). LibreSSL will not be installed in -`/usr/local` instead it will stay in `/usr/local/Cellar`. So the cmake command -will look something like this: +The build under MacOS will work the same way as on Linux. To get LibreSSL, +boost, and ninja you can use [Homebrew](https://brew.sh/). LibreSSL will not be +installed in `/usr/local` instead it will stay in `/usr/local/Cellar`. So the +cmake command will look something like this: ```sh -cmake -DLibreSSL_ROOT=/usr/local/Cellar/libressl/2.8.3 +cmake -GNinja -DLibreSSL_ROOT=/usr/local/Cellar/libressl/2.8.3 ``` To generate a installable package, you have to call CMake with the corresponding arguments and then use cpack to generate the package: ```sh -cmake -DINSTALL_LAYOUT=OSX -make -cpack +cmake -GNinja +ninja +cpack -G productbuild ``` ### Windows @@ -202,37 +237,3 @@ will automatically find it and build with TLS support. If you installed WIX before running `cmake` you should find the `FDBInstaller.msi` in your build directory under `packaging/msi`. -## Makefile (Deprecated - all users should transition to using cmake) - -#### MacOS - -1. Check out this repo on your Mac. -1. Install the Xcode command-line tools. -1. Download version 1.67.0 of [Boost](https://sourceforge.net/projects/boost/files/boost/1.67.0/). -1. Set the `BOOSTDIR` environment variable to the location containing this boost installation. -1. Install [Mono](http://www.mono-project.com/download/stable/). -1. Install a [JDK](http://www.oracle.com/technetwork/java/javase/downloads/index.html). FoundationDB currently builds with Java 8. -1. Navigate to the directory where you checked out the foundationdb repo. -1. Run `make`. - -#### Linux - -1. Install [Docker](https://www.docker.com/). -1. Check out the foundationdb repo. -1. Run the docker image interactively with [Docker Run](https://docs.docker.com/engine/reference/run/#general-form), and with the directory containing the foundationdb repo mounted via [Docker Mounts](https://docs.docker.com/storage/volumes/). - - ```shell - docker run -it -v '/local/dir/path/foundationdb:/docker/dir/path/foundationdb' foundationdb/foundationdb-build:latest - ``` - -1. Run `$ scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash` within the running container. This enables a more modern compiler, which is required to build FoundationDB. -1. Navigate to the container's mounted directory which contains the foundationdb repo. - - ```shell - cd /docker/dir/path/foundationdb - ``` - -1. Run `make`. - -This will build the fdbserver binary and the python bindings. If you want to build our other bindings, you will need to install a runtime for the language whose binding you want to build. Each binding has an `.mk` file which provides specific targets for that binding. - diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index b84a57100e..e363695ac2 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -13,3 +13,6 @@ endif() if(WITH_RUBY) add_subdirectory(ruby) endif() +if(NOT WIN32 AND NOT OPEN_FOR_IDE) + package_bindingtester() +endif() diff --git a/bindings/bindingtester/__init__.py b/bindings/bindingtester/__init__.py index 75454625c0..0adababb92 100644 --- a/bindings/bindingtester/__init__.py +++ b/bindings/bindingtester/__init__.py @@ -26,7 +26,7 @@ sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', import util -FDB_API_VERSION = 620 +FDB_API_VERSION = 630 LOGGING = { 'version': 1, diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py index 185e2582c1..6feed3b283 100755 --- a/bindings/bindingtester/bindingtester.py +++ b/bindings/bindingtester/bindingtester.py @@ -157,7 +157,7 @@ def choose_api_version(selected_api_version, tester_min_version, tester_max_vers api_version = min_version elif random.random() < 0.9: api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430, - 440, 450, 460, 500, 510, 520, 600, 610, 620] if v >= min_version and v <= max_version]) + 440, 450, 460, 500, 510, 520, 600, 610, 620, 630] if v >= min_version and v <= max_version]) else: api_version = random.randint(min_version, max_version) @@ -199,7 +199,7 @@ class TestRunner(object): raise Exception('Not all testers support concurrency') # Test types should be intersection of all tester supported types - self.args.types = reduce(lambda t1, t2: filter(t1.__contains__, t2), map(lambda tester: tester.types, self.testers)) + self.args.types = list(reduce(lambda t1, t2: filter(t1.__contains__, t2), map(lambda tester: tester.types, self.testers))) self.args.no_directory_snapshot_ops = self.args.no_directory_snapshot_ops or any([not tester.directory_snapshot_ops_enabled for tester in self.testers]) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index 2c5211a3df..ee82663411 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -20,7 +20,7 @@ import os -MAX_API_VERSION = 620 +MAX_API_VERSION = 630 COMMON_TYPES = ['null', 'bytes', 'string', 'int', 'uuid', 'bool', 'float', 'double', 'tuple'] ALL_TYPES = COMMON_TYPES + ['versionstamp'] diff --git a/bindings/bindingtester/run_binding_tester.sh b/bindings/bindingtester/run_binding_tester.sh index 0382707fa5..06c3f0a710 100644 --- a/bindings/bindingtester/run_binding_tester.sh +++ b/bindings/bindingtester/run_binding_tester.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ###################################################### # # FoundationDB Binding Test Script @@ -25,7 +25,8 @@ BREAKONERROR="${BREAKONERROR:-0}" RUNSCRIPTS="${RUNSCRIPTS:-1}" RUNTESTS="${RUNTESTS:-1}" RANDOMTEST="${RANDOMTEST:-0}" -BINDINGTESTS="${BINDINGTESTS:-python python3 java java_async ruby go flow}" +# BINDINGTESTS="${BINDINGTESTS:-python python3 java java_async ruby go flow}" +BINDINGTESTS="${BINDINGTESTS:-python python3 java java_async go flow}" LOGLEVEL="${LOGLEVEL:-INFO}" _BINDINGTESTS=(${BINDINGTESTS}) DISABLEDTESTS=() diff --git a/bindings/bindingtester/run_tester_loop.sh b/bindings/bindingtester/run_tester_loop.sh index d78915a489..2bdcee44b3 100755 --- a/bindings/bindingtester/run_tester_loop.sh +++ b/bindings/bindingtester/run_tester_loop.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash LOGGING_LEVEL=WARNING diff --git a/bindings/bindingtester/spec/bindingApiTester.md b/bindings/bindingtester/spec/bindingApiTester.md index 2f67c9dc1f..af21147fd1 100644 --- a/bindings/bindingtester/spec/bindingApiTester.md +++ b/bindings/bindingtester/spec/bindingApiTester.md @@ -164,6 +164,13 @@ futures must apply the following rules to the result: database using the get() method. May optionally push a future onto the stack. +#### GET_ESTIMATED_RANGE_SIZE + + Pops the top two items off of the stack as BEGIN_KEY and END_KEY to + construct a key range. Then call the `getEstimatedRangeSize` API of + the language binding. Make sure the API returns without error. Finally + push the string "GOT_ESTIMATED_RANGE_SIZE" onto the stack. + #### GET_KEY (_SNAPSHOT, _DATABASE) Pops the top four items off of the stack as KEY, OR_EQUAL, OFFSET, PREFIX diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 9d84fb43af..5e8d2d66a2 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -157,6 +157,7 @@ class ApiTest(Test): read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] txn_sizes = ['GET_APPROXIMATE_SIZE'] + storage_metrics = ['GET_ESTIMATED_RANGE_SIZE'] op_choices += reads op_choices += mutations @@ -170,6 +171,7 @@ class ApiTest(Test): op_choices += write_conflicts op_choices += resets op_choices += txn_sizes + op_choices += storage_metrics idempotent_atomic_ops = ['BIT_AND', 'BIT_OR', 'MAX', 'MIN', 'BYTE_MIN', 'BYTE_MAX'] atomic_ops = idempotent_atomic_ops + ['ADD', 'BIT_XOR', 'APPEND_IF_FITS'] @@ -536,6 +538,21 @@ class ApiTest(Test): instructions.push_args(d) instructions.append(op) self.add_strings(1) + elif op == 'GET_ESTIMATED_RANGE_SIZE': + # Protect against inverted range and identical keys + key1 = self.workspace.pack(self.random.random_tuple(1)) + key2 = self.workspace.pack(self.random.random_tuple(1)) + + while key1 == key2: + key1 = self.workspace.pack(self.random.random_tuple(1)) + key2 = self.workspace.pack(self.random.random_tuple(1)) + + if key1 > key2: + key1, key2 = key2, key1 + + instructions.push_args(key1, key2) + instructions.append(op) + self.add_strings(1) else: assert False, 'Unknown operation: ' + op diff --git a/bindings/bindingtester/tests/directory.py b/bindings/bindingtester/tests/directory.py index 8b3c56fae7..e6a51d0869 100644 --- a/bindings/bindingtester/tests/directory.py +++ b/bindings/bindingtester/tests/directory.py @@ -52,12 +52,12 @@ class DirectoryTest(Test): self.dir_list.append(child) self.dir_index = directory_util.DEFAULT_DIRECTORY_INDEX - def generate_layer(self): + def generate_layer(self, allow_partition=True): if random.random() < 0.7: return b'' else: choice = random.randint(0, 3) - if choice == 0: + if choice == 0 and allow_partition: return b'partition' elif choice == 1: return b'test_layer' @@ -184,7 +184,9 @@ class DirectoryTest(Test): test_util.blocking_commit(instructions) path = generate_path() - op_args = test_util.with_length(path) + (self.generate_layer(),) + # Partitions that use the high-contention allocator can result in non-determinism if they fail to commit, + # so we disallow them in comparison tests + op_args = test_util.with_length(path) + (self.generate_layer(allow_partition=args.concurrency>1),) directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) if not op.endswith('_DATABASE') and args.concurrency == 1: diff --git a/bindings/bindingtester/tests/scripted.py b/bindings/bindingtester/tests/scripted.py index b7d65e347d..60b1959864 100644 --- a/bindings/bindingtester/tests/scripted.py +++ b/bindings/bindingtester/tests/scripted.py @@ -34,7 +34,7 @@ fdb.api_version(FDB_API_VERSION) class ScriptedTest(Test): - TEST_API_VERSION = 620 + TEST_API_VERSION = 630 def __init__(self, subspace): super(ScriptedTest, self).__init__(subspace, ScriptedTest.TEST_API_VERSION, ScriptedTest.TEST_API_VERSION) diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index c80dc44b3f..9d05990e0c 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -38,6 +38,21 @@ else() endif() add_dependencies(fdb_c fdb_c_generated fdb_c_options) target_link_libraries(fdb_c PUBLIC $) +if(APPLE) + set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols) + add_custom_command(OUTPUT ${symbols} + COMMAND $ ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py + ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h + ${symbols} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py ${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h + COMMENT "Generate exported_symbols_list") + add_custom_target(exported_symbols_list DEPENDS ${symbols}) + add_dependencies(fdb_c exported_symbols_list) + target_link_options(fdb_c PRIVATE "LINKER:-no_weak_exports,-exported_symbols_list,${symbols}") +elseif(WIN32) +else() + target_link_options(fdb_c PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map,-z,nodelete") +endif() target_include_directories(fdb_c PUBLIC $ $ diff --git a/bindings/c/ThreadCleanup.cpp b/bindings/c/ThreadCleanup.cpp index 20b49cf8e5..966e38b800 100644 --- a/bindings/c/ThreadCleanup.cpp +++ b/bindings/c/ThreadCleanup.cpp @@ -34,6 +34,10 @@ BOOL WINAPI DllMain( HINSTANCE dll, DWORD reason, LPVOID reserved ) { #elif defined( __unixish__ ) +#ifdef __INTEL_COMPILER +#pragma warning ( disable:2415 ) +#endif + static pthread_key_t threadDestructorKey; static void threadDestructor(void*) { @@ -57,4 +61,4 @@ static int threadDestructorKeyInit = initThreadDestructorKey(); #else #error Port me! -#endif \ No newline at end of file +#endif diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index 356c3225d5..12eb749aeb 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #define FDB_INCLUDE_LEGACY_TYPES #include "fdbclient/MultiVersionTransaction.h" @@ -44,8 +44,9 @@ int g_api_version = 0; // Legacy (pre API version 610) #define CLUSTER(c) ((char*)c) -/* - * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi instance (e.g. from ThreadSafeApi) +/* + * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi + * instance (e.g. from ThreadSafeApi) */ #define API ((IClientApi*)MultiVersionApi::api) @@ -74,12 +75,10 @@ fdb_bool_t fdb_error_predicate( int predicate_test, fdb_error_t code ) { code == error_code_cluster_version_changed; } if(predicate_test == FDBErrorPredicates::RETRYABLE_NOT_COMMITTED) { - return code == error_code_not_committed || - code == error_code_transaction_too_old || - code == error_code_future_version || - code == error_code_database_locked || - code == error_code_proxy_memory_limit_exceeded || - code == error_code_process_behind; + return code == error_code_not_committed || code == error_code_transaction_too_old || + code == error_code_future_version || code == error_code_database_locked || + code == error_code_proxy_memory_limit_exceeded || code == error_code_batch_transaction_throttled || + code == error_code_process_behind; } return false; } @@ -628,6 +627,13 @@ fdb_error_t fdb_transaction_add_conflict_range( FDBTransaction*tr, uint8_t const } +extern "C" DLLEXPORT +FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length ) { + KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length)); + return (FDBFuture*)(TXN(tr)->getEstimatedRangeSizeBytes(range).extractPtr()); +} + #include "fdb_c_function_pointers.g.h" #define FDB_API_CHANGED(func, ver) if (header_version < ver) fdb_api_ptr_##func = (void*)&(func##_v##ver##_PREV); else if (fdb_api_ptr_##func == (void*)&fdb_api_ptr_unimpl) fdb_api_ptr_##func = (void*)&(func##_impl); diff --git a/bindings/c/fdb_c.vcxproj b/bindings/c/fdb_c.vcxproj deleted file mode 100644 index 61322f8c23..0000000000 --- a/bindings/c/fdb_c.vcxproj +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - FDB_CLEAN_BUILD;%(PreprocessorDefinitions) - - - - Debug - x64 - - - Release - x64 - - - - - - - - - - - - - - - - - - {CACB2C8E-3E55-4309-A411-2A9C56C6C1CB} - c - - - - DynamicLibrary - true - MultiByte - v141 - - - DynamicLibrary - false - true - MultiByte - v141 - - - - - - - - - - - - - - - - - -FOR /F "tokens=1" %%i in ('hg.exe id') do copy /Y "$(TargetPath)" "$(TargetPath)-%%i" - - - - ..\..\;C:\Program Files\boost_1_67_0;$(IncludePath) - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - - - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - ..\..\;C:\Program Files\boost_1_67_0;$(IncludePath) - - - - Level3 - Disabled - TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - MultiThreadedDebug - /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - true - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;%(AdditionalDependencies) - - - - - Level3 - MaxSpeed - true - true - TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - MultiThreaded - /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - true - true - true - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;%(AdditionalDependencies) - - - - _MASM;ClCompile - - - - c:\Python27\python.exe "$(ProjectDir)/generate_asm.py" windows "$(ProjectDir)/fdb_c.cpp" "$(ProjectDir)/fdb_c.g.asm" "$(ProjectDir)/fdb_c_function_pointers.g.h" - Generating API trampolines - $(ProjectDir)/fdb_c_function_pointers.g.h;$(ProjectDir)/fdb_c.g.asm - $(ProjectDir)/fdb_c.cpp;$(ProjectDir)/generate_asm.py - - - - - - - diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index 5459952cf7..a930434819 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -28,10 +28,10 @@ #endif #if !defined(FDB_API_VERSION) -#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 620) +#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 630) #elif FDB_API_VERSION < 13 #error API version no longer supported (upgrade to 13) -#elif FDB_API_VERSION > 620 +#elif FDB_API_VERSION > 630 #error Requested API version requires a newer version of this header #endif @@ -91,12 +91,21 @@ extern "C" { DLLEXPORT WARN_UNUSED_RESULT fdb_error_t fdb_add_network_thread_completion_hook(void (*hook)(void*), void *hook_parameter); #pragma pack(push, 4) +#if FDB_API_VERSION >= 630 + typedef struct keyvalue { + const uint8_t* key; + int key_length; + const uint8_t* value; + int value_length; + } FDBKeyValue; +#else typedef struct keyvalue { const void* key; int key_length; const void* value; int value_length; } FDBKeyValue; +#endif #pragma pack(pop) DLLEXPORT void fdb_future_cancel( FDBFuture* f ); @@ -247,6 +256,10 @@ extern "C" { int end_key_name_length, FDBConflictRangeType type); + DLLEXPORT WARN_UNUSED_RESULT FDBFuture* + fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); + #define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0 #define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0 #define FDB_KEYSEL_FIRST_GREATER_THAN(k, l) k, l, 1, 1 diff --git a/bindings/c/generate_asm.py b/bindings/c/generate_asm.py index cf06ef207e..284a6e6824 100755 --- a/bindings/c/generate_asm.py +++ b/bindings/c/generate_asm.py @@ -61,7 +61,7 @@ def write_windows_asm(asmfile, functions): def write_unix_asm(asmfile, functions, prefix): asmfile.write(".intel_syntax noprefix\n") - if platform == "linux": + if platform == "linux" or platform == "freebsd": asmfile.write("\n.data\n") for f in functions: asmfile.write("\t.extern fdb_api_ptr_%s\n" % f) diff --git a/bindings/c/local.mk b/bindings/c/local.mk deleted file mode 100644 index df2859c6a3..0000000000 --- a/bindings/c/local.mk +++ /dev/null @@ -1,113 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdb_c_CFLAGS := $(fdbclient_CFLAGS) -fdb_c_LDFLAGS := $(fdbrpc_LDFLAGS) -fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(FDB_TLS_LIB) -fdb_c_STATIC_LIBS := $(TLS_LIBS) -fdb_c_tests_LIBS := -Llib -lfdb_c -lstdc++ -fdb_c_tests_HEADERS := -Ibindings/c - -CLEAN_TARGETS += fdb_c_tests_clean - -ifeq ($(PLATFORM),linux) - fdb_c_LDFLAGS += -Wl,--version-script=bindings/c/fdb_c.map -static-libgcc -Wl,-z,nodelete -lm -lpthread -lrt -ldl - # Link our custom libstdc++ statically in Ubuntu, if hacking - ifeq ("$(wildcard /etc/centos-release)", "") - ifeq ($(LIBSTDCPP_HACK),1) - fdb_c_LIBS += lib/libstdc++.a - endif - # Link stdc++ statically in Centos, if not hacking - else - fdb_c_STATIC_LIBS += -static-libstdc++ - endif - fdb_c_tests_LIBS += -lpthread -endif - -ifeq ($(PLATFORM),osx) - fdb_c_LDFLAGS += -lc++ -Xlinker -exported_symbols_list -Xlinker bindings/c/fdb_c.symbols - fdb_c_tests_LIBS += -lpthread - - lib/libfdb_c.dylib: bindings/c/fdb_c.symbols - - bindings/c/fdb_c.symbols: bindings/c/foundationdb/fdb_c.h $(ALL_MAKEFILES) - @awk '{sub(/^[ \t]+/, "");} /^#/ {next;} /DLLEXPORT\ .*[^ ]\(/ {sub(/\(.*/, ""); print "_" $$NF; next;} /DLLEXPORT/ { DLLEXPORT=1; next;} DLLEXPORT==1 {sub(/\(.*/, ""); print "_" $$0; DLLEXPORT=0}' $< | sort | uniq > $@ - - fdb_c_clean: fdb_c_symbols_clean - - fdb_c_symbols_clean: - @rm -f bindings/c/fdb_c.symbols - - fdb_javac_release: lib/libfdb_c.$(DLEXT) - mkdir -p lib - rm -f lib/libfdb_c.$(java_DLEXT)-* - cp lib/libfdb_c.$(DLEXT) lib/libfdb_c.$(DLEXT)-$(VERSION_ID) - cp lib/libfdb_c.$(DLEXT)-debug lib/libfdb_c.$(DLEXT)-debug-$(VERSION_ID) - - fdb_javac_release_clean: - rm -f lib/libfdb_c.$(DLEXT)-* - rm -f lib/libfdb_c.$(javac_DLEXT)-* - - # OS X needs to put its java lib in packages - packages: fdb_javac_lib_package - - fdb_javac_lib_package: lib/libfdb_c.dylib - mkdir -p packages - cp lib/libfdb_c.$(DLEXT) packages/libfdb_c.$(DLEXT)-$(VERSION_ID) - cp lib/libfdb_c.$(DLEXT)-debug packages/libfdb_c.$(DLEXT)-debug-$(VERSION_ID) -endif - -fdb_c_GENERATED_SOURCES += bindings/c/foundationdb/fdb_c_options.g.h bindings/c/fdb_c.g.S bindings/c/fdb_c_function_pointers.g.h - -bindings/c/%.g.S bindings/c/%_function_pointers.g.h: bindings/c/%.cpp bindings/c/generate_asm.py $(ALL_MAKEFILES) - @echo "Scanning $<" - @bindings/c/generate_asm.py $(PLATFORM) bindings/c/fdb_c.cpp bindings/c/fdb_c.g.S bindings/c/fdb_c_function_pointers.g.h - -.PRECIOUS: bindings/c/fdb_c_function_pointers.g.h - -fdb_c_BUILD_SOURCES += bindings/c/fdb_c.g.S - -bindings/c/foundationdb/fdb_c_options.g.h: bin/vexillographer.exe fdbclient/vexillographer/fdb.options $(ALL_MAKEFILES) - @echo "Building $@" - @$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options c $@ - -bin/fdb_c_performance_test: bindings/c/test/performance_test.c bindings/c/test/test.h fdb_c - @echo "Compiling fdb_c_performance_test" - @$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/performance_test.c $(fdb_c_tests_LIBS) - -bin/fdb_c_ryw_benchmark: bindings/c/test/ryw_benchmark.c bindings/c/test/test.h fdb_c - @echo "Compiling fdb_c_ryw_benchmark" - @$(CC) $(CFLAGS) $(fdb_c_tests_HEADERS) -o $@ bindings/c/test/ryw_benchmark.c $(fdb_c_tests_LIBS) - -packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz: bin/fdb_c_performance_test bin/fdb_c_ryw_benchmark - @echo "Packaging $@" - @rm -rf packages/fdb-c-tests-$(VERSION)-$(PLATFORM) - @mkdir -p packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin - @cp bin/fdb_c_performance_test packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin - @cp bin/fdb_c_ryw_benchmark packages/fdb-c-tests-$(VERSION)-$(PLATFORM)/bin - @tar -C packages -czvf $@ fdb-c-tests-$(VERSION)-$(PLATFORM) > /dev/null - @rm -rf packages/fdb-c-tests-$(VERSION)-$(PLATFORM) - -fdb_c_tests: packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz - -fdb_c_tests_clean: - @rm -f packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz diff --git a/bindings/c/symbolify.py b/bindings/c/symbolify.py new file mode 100644 index 0000000000..55d8dc81fd --- /dev/null +++ b/bindings/c/symbolify.py @@ -0,0 +1,10 @@ +if __name__ == '__main__': + import re + import sys + r = re.compile('DLLEXPORT[^(]*(fdb_[^(]*)[(]') + (fdb_c_h, symbols_file) = sys.argv[1:] + with open(fdb_c_h, 'r') as f: + symbols = sorted(set('_' + m.group(1) for m in r.finditer(f.read()))) + with open(symbols_file, 'w') as f: + f.write('\n'.join(symbols)) + f.write('\n') diff --git a/bindings/c/test/mako/mako.c b/bindings/c/test/mako/mako.c old mode 100755 new mode 100644 index d6f4d041e3..26db163691 --- a/bindings/c/test/mako/mako.c +++ b/bindings/c/test/mako/mako.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -8,10 +9,12 @@ #include #include #include -#include #if defined(__linux__) #include +#elif defined(__FreeBSD__) +#include +#define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST #elif defined(__APPLE__) #include #define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC @@ -19,1931 +22,1898 @@ #include #endif +#include "fdbclient/zipf.h" #include "mako.h" #include "utils.h" -#include "fdbclient/zipf.h" /* global variables */ -FILE *printme; /* descriptor used for default messages */ -FILE *annoyme; /* descriptor used for annoying messages */ -FILE *debugme; /* descriptor used for debug messages */ +FILE* printme; /* descriptor used for default messages */ +FILE* annoyme; /* descriptor used for annoying messages */ +FILE* debugme; /* descriptor used for debug messages */ -#define check_fdb_error(_e) \ - do { \ - if (_e) { \ - fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, \ - fdb_get_error(_e)); \ - goto failExit; \ - } \ - } while (0) +#define check_fdb_error(_e) \ + do { \ + if (_e) { \ + fprintf(stderr, "ERROR: Failed at %s:%d (%s)\n", __FILE__, __LINE__, fdb_get_error(_e)); \ + goto failExit; \ + } \ + } while (0) -#define fdb_block_wait(_f) \ - do { \ - if ((fdb_future_block_until_ready(_f)) != 0) { \ - fprintf(stderr, "ERROR: fdb_future_block_until_ready failed at %s:%d\n", \ - __FILE__, __LINE__); \ - goto failExit; \ - } \ - } while (0) +#define fdb_block_wait(_f) \ + do { \ + if ((fdb_future_block_until_ready(_f)) != 0) { \ + fprintf(stderr, "ERROR: fdb_future_block_until_ready failed at %s:%d\n", __FILE__, __LINE__); \ + goto failExit; \ + } \ + } while (0) -#define fdb_wait_and_handle_error(_func, _f, _t) \ - do { \ - int err = wait_future(_f); \ - if (err) { \ - int err2; \ - if ((err != 1020 /* not_committed */) && \ - (err != 1021 /* commit_unknown_result */)) { \ - fprintf(stderr, "ERROR: Error %s (%d) occured at %s\n", \ - #_func, err, fdb_get_error(err)); \ - } else { \ - fprintf(annoyme, "ERROR: Error %s (%d) occured at %s\n", \ - #_func, err, fdb_get_error(err)); \ - } \ - fdb_future_destroy(_f); \ - _f = fdb_transaction_on_error(_t, err); \ - /* this will return the original error for non-retryable errors */ \ - err2 = wait_future(_f); \ - fdb_future_destroy(_f); \ - if (err2) { \ - /* unretryable error */ \ - fprintf(stderr, \ - "ERROR: fdb_transaction_on_error returned %d at %s:%d\n", \ - err2, __FILE__, __LINE__); \ - fdb_transaction_reset(_t); \ - /* TODO: if we adda retry limit in the future, \ - * handle the conflict stats properly. \ - */ \ - return FDB_ERROR_ABORT; \ - } \ - if (err == 1020 /* not_committed */) { \ - return FDB_ERROR_CONFLICT; \ - } \ - return FDB_ERROR_RETRY; \ - } \ - } while (0) +#define fdb_wait_and_handle_error(_func, _f, _t) \ + do { \ + int err = wait_future(_f); \ + if (err) { \ + int err2; \ + if ((err != 1020 /* not_committed */) && (err != 1021 /* commit_unknown_result */)) { \ + fprintf(stderr, "ERROR: Error %s (%d) occured at %s\n", #_func, err, fdb_get_error(err)); \ + } else { \ + fprintf(annoyme, "ERROR: Error %s (%d) occured at %s\n", #_func, err, fdb_get_error(err)); \ + } \ + fdb_future_destroy(_f); \ + _f = fdb_transaction_on_error(_t, err); \ + /* this will return the original error for non-retryable errors */ \ + err2 = wait_future(_f); \ + fdb_future_destroy(_f); \ + if (err2) { \ + /* unretryable error */ \ + fprintf(stderr, "ERROR: fdb_transaction_on_error returned %d at %s:%d\n", err2, __FILE__, __LINE__); \ + fdb_transaction_reset(_t); \ + /* TODO: if we adda retry limit in the future, \ + * handle the conflict stats properly. \ + */ \ + return FDB_ERROR_ABORT; \ + } \ + if (err == 1020 /* not_committed */) { \ + return FDB_ERROR_CONFLICT; \ + } \ + return FDB_ERROR_RETRY; \ + } \ + } while (0) +fdb_error_t wait_future(FDBFuture* f) { + fdb_error_t err; -fdb_error_t wait_future(FDBFuture *f) { - fdb_error_t err; - - err = fdb_future_block_until_ready(f); - if (err) { - return err; /* error from fdb_future_block_until_ready() */ - } - return fdb_future_get_error(f); + err = fdb_future_block_until_ready(f); + if (err) { + return err; /* error from fdb_future_block_until_ready() */ + } + return fdb_future_get_error(f); } +int commit_transaction(FDBTransaction* transaction) { + FDBFuture* f; -int commit_transaction(FDBTransaction *transaction) { - FDBFuture *f; + f = fdb_transaction_commit(transaction); + fdb_wait_and_handle_error(commit_transaction, f, transaction); - f = fdb_transaction_commit(transaction); - fdb_wait_and_handle_error(commit_transaction, f, transaction); - - return FDB_SUCCESS; + return FDB_SUCCESS; } +void update_op_lat_stats(struct timespec* start, struct timespec* end, int op, mako_stats_t* stats) { + uint64_t latencyus; -void update_op_lat_stats(struct timespec *start, struct timespec *end, int op, - mako_stats_t *stats) { - uint64_t latencyus; - - latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) - - ((uint64_t)start->tv_sec * 1000000000 + start->tv_nsec)) / - 1000; - stats->latency_samples[op]++; - stats->latency_us_total[op] += latencyus; - if (latencyus < stats->latency_us_min[op]) { - stats->latency_us_min[op] = latencyus; - } - if (latencyus > stats->latency_us_max[op]) { - stats->latency_us_max[op] = latencyus; - } + latencyus = (((uint64_t)end->tv_sec * 1000000000 + end->tv_nsec) - + ((uint64_t)start->tv_sec * 1000000000 + start->tv_nsec)) / + 1000; + stats->latency_samples[op]++; + stats->latency_us_total[op] += latencyus; + if (latencyus < stats->latency_us_min[op]) { + stats->latency_us_min[op] = latencyus; + } + if (latencyus > stats->latency_us_max[op]) { + stats->latency_us_max[op] = latencyus; + } } - /* FDB network thread */ -void *fdb_network_thread(void *args) { - fdb_error_t err; +void* fdb_network_thread(void* args) { + fdb_error_t err; - fprintf(debugme, "DEBUG: fdb_network_thread started\n"); + fprintf(debugme, "DEBUG: fdb_network_thread started\n"); - err = fdb_run_network(); - if (err) { - fprintf(stderr, "ERROR: fdb_run_network: %s\n", fdb_get_error(err)); - } + err = fdb_run_network(); + if (err) { + fprintf(stderr, "ERROR: fdb_run_network: %s\n", fdb_get_error(err)); + } - return 0; + return 0; } - /* cleanup database */ -int cleanup(FDBTransaction *transaction, mako_args_t *args) { - struct timespec timer_start, timer_end; - char beginstr[7]; - char endstr[7]; +int cleanup(FDBTransaction* transaction, mako_args_t* args) { + struct timespec timer_start, timer_end; + char beginstr[7]; + char endstr[7]; - strncpy(beginstr, "mako", 4); - beginstr[4] = 0x00; - strncpy(endstr, "mako", 4); - endstr[4] = 0xff; - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); - fdb_transaction_clear_range(transaction, (uint8_t *)beginstr, 5, - (uint8_t *)endstr, 5); - if (commit_transaction(transaction) != FDB_SUCCESS) - goto failExit; + strncpy(beginstr, "mako", 4); + beginstr[4] = 0x00; + strncpy(endstr, "mako", 4); + endstr[4] = 0xff; + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); + fdb_transaction_clear_range(transaction, (uint8_t*)beginstr, 5, (uint8_t*)endstr, 5); + if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; - fdb_transaction_reset(transaction); - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); - fprintf(printme, "INFO: Clear range: %6.3f sec\n", - ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + - timer_end.tv_nsec - timer_start.tv_nsec) / - 1000000000); - return 0; + fdb_transaction_reset(transaction); + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_end); + fprintf(printme, "INFO: Clear range: %6.3f sec\n", + ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / + 1000000000); + return 0; failExit: - fprintf(stderr, "ERROR: FDB failure in cleanup()\n"); - return -1; + fprintf(stderr, "ERROR: FDB failure in cleanup()\n"); + return -1; } - /* populate database */ -int populate(FDBTransaction *transaction, mako_args_t *args, int worker_id, - int thread_id, int thread_tps, mako_stats_t *stats) { - int i; - struct timespec timer_start, timer_end; - struct timespec timer_prev, timer_now; /* for throttling */ - struct timespec timer_per_xact_start, timer_per_xact_end; - char *keystr; - char *valstr; +int populate(FDBTransaction* transaction, mako_args_t* args, int worker_id, int thread_id, int thread_tps, + mako_stats_t* stats) { + int i; + struct timespec timer_start, timer_end; + struct timespec timer_prev, timer_now; /* for throttling */ + struct timespec timer_per_xact_start, timer_per_xact_end; + char* keystr; + char* valstr; - int begin = insert_begin(args->rows, worker_id, thread_id, - args->num_processes, args->num_threads); - int end = insert_end(args->rows, worker_id, thread_id, args->num_processes, - args->num_threads); - int xacts = 0; + int begin = insert_begin(args->rows, worker_id, thread_id, args->num_processes, args->num_threads); + int end = insert_end(args->rows, worker_id, thread_id, args->num_processes, args->num_threads); + int xacts = 0; + int tracetimer = 0; - keystr = (char *)malloc(sizeof(char) * args->key_length + 1); - if (!keystr) - return -1; - valstr = (char *)malloc(sizeof(char) * args->value_length + 1); - if (!valstr) { - free(keystr); - return -1; - } + keystr = (char*)malloc(sizeof(char) * args->key_length + 1); + if (!keystr) return -1; + valstr = (char*)malloc(sizeof(char) * args->value_length + 1); + if (!valstr) { + free(keystr); + return -1; + } - clock_gettime(CLOCK_MONOTONIC, &timer_start); - timer_prev.tv_sec = timer_start.tv_sec; - timer_prev.tv_nsec = timer_start.tv_nsec; - timer_per_xact_start.tv_sec = timer_start.tv_sec; - timer_per_xact_start.tv_nsec = timer_start.tv_nsec; + clock_gettime(CLOCK_MONOTONIC, &timer_start); + timer_prev.tv_sec = timer_start.tv_sec; + timer_prev.tv_nsec = timer_start.tv_nsec; + timer_per_xact_start.tv_sec = timer_start.tv_sec; + timer_per_xact_start.tv_nsec = timer_start.tv_nsec; - for (i = begin; i <= end; i++) { + for (i = begin; i <= end; i++) { - if ((thread_tps > 0) && (xacts >= thread_tps)) { - /* throttling is on */ + /* sequential keys */ + genkey(keystr, i, args->rows, args->key_length + 1); + /* random values */ + randstr(valstr, args->value_length + 1); - throttle: - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || - ((timer_now.tv_sec == timer_prev.tv_sec + 1) && - (timer_now.tv_nsec > timer_prev.tv_nsec))) { - /* more than 1 second passed, no need to throttle */ - xacts = 0; - timer_prev.tv_sec = timer_now.tv_sec; - timer_prev.tv_nsec = timer_now.tv_nsec; - } else { - /* 1 second not passed, throttle */ - usleep(1000); /* sleep for 1ms */ - goto throttle; - } - } /* throttle */ + if (((thread_tps > 0) && (xacts >= thread_tps)) /* throttle */ || (args->txntrace) /* txn tracing */) { - /* sequential keys */ - genkey(keystr, i, args->rows, args->key_length + 1); - /* random values */ - randstr(valstr, args->value_length + 1); + throttle: + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || + ((timer_now.tv_sec == timer_prev.tv_sec + 1) && (timer_now.tv_nsec > timer_prev.tv_nsec))) { + /* more than 1 second passed, no need to throttle */ + xacts = 0; + timer_prev.tv_sec = timer_now.tv_sec; + timer_prev.tv_nsec = timer_now.tv_nsec; - /* insert (SET) */ - fdb_transaction_set(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)valstr, strlen(valstr)); - stats->ops[OP_INSERT]++; + /* enable transaction tracing */ + if (args->txntrace) { + tracetimer++; + if (tracetimer == args->txntrace) { + fdb_error_t err; + tracetimer = 0; + fprintf(debugme, "DEBUG: txn tracing %s\n", keystr); + err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, + (uint8_t*)keystr, strlen(keystr)); + if (err) { + fprintf( + stderr, + "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER): %s\n", + fdb_get_error(err)); + } + err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); + if (err) { + fprintf(stderr, "ERROR: fdb_transaction_set_option(FDB_TR_OPTION_LOG_TRANSACTION): %s\n", + fdb_get_error(err)); + } + } + } + } else { + if (thread_tps > 0) { + /* 1 second not passed, throttle */ + usleep(1000); /* sleep for 1ms */ + goto throttle; + } + } + } /* throttle or txntrace */ - /* commit every 100 inserts (default) */ - if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) { + /* insert (SET) */ + fdb_transaction_set(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)valstr, strlen(valstr)); + stats->ops[OP_INSERT]++; - if (commit_transaction(transaction) != FDB_SUCCESS) - goto failExit; + /* commit every 100 inserts (default) */ + if (i % args->txnspec.ops[OP_INSERT][OP_COUNT] == 0) { - /* xact latency stats */ - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, - stats); - stats->ops[OP_COMMIT]++; - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); + if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; - fdb_transaction_reset(transaction); - stats->xacts++; - xacts++; /* for throttling */ - } - } + /* xact latency stats */ + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + stats->ops[OP_COMMIT]++; + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); - if (commit_transaction(transaction) != FDB_SUCCESS) - goto failExit; + fdb_transaction_reset(transaction); + stats->xacts++; + xacts++; /* for throttling */ + } + } - /* xact latency stats */ - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + if (commit_transaction(transaction) != FDB_SUCCESS) goto failExit; - clock_gettime(CLOCK_MONOTONIC, &timer_end); - stats->xacts++; + /* xact latency stats */ + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); - fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, - end, - ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + - timer_end.tv_nsec - timer_start.tv_nsec) / - 1000000000); + clock_gettime(CLOCK_MONOTONIC, &timer_end); + stats->xacts++; - free(keystr); - free(valstr); - return 0; + fprintf(debugme, "DEBUG: Populated %d rows (%d-%d): %6.3f sec\n", end - begin, begin, end, + ((timer_end.tv_sec - timer_start.tv_sec) * 1000000000.0 + timer_end.tv_nsec - timer_start.tv_nsec) / + 1000000000); + + free(keystr); + free(valstr); + return 0; failExit: - if (keystr) - free(keystr); - if (valstr) - free(valstr); - fprintf(stderr, "ERROR: FDB failure in populate()\n"); - return -1; + if (keystr) free(keystr); + if (valstr) free(valstr); + fprintf(stderr, "ERROR: FDB failure in populate()\n"); + return -1; } +int64_t run_op_getreadversion(FDBTransaction* transaction, int64_t* rv) { + FDBFuture* f; + fdb_error_t err; -int64_t run_op_getreadversion(FDBTransaction *transaction, int64_t *rv) { - FDBFuture *f; - fdb_error_t err; + *rv = 0; - *rv = 0; - - f = fdb_transaction_get_read_version(transaction); - fdb_wait_and_handle_error(fdb_transaction_get_read_version, f, transaction); + f = fdb_transaction_get_read_version(transaction); + fdb_wait_and_handle_error(fdb_transaction_get_read_version, f, transaction); #if FDB_API_VERSION < 620 - err = fdb_future_get_version(f, rv); + err = fdb_future_get_version(f, rv); #else - err = fdb_future_get_int64(f, rv); + err = fdb_future_get_int64(f, rv); #endif - fdb_future_destroy(f); - if (err) { + fdb_future_destroy(f); + if (err) { #if FDB_API_VERSION < 620 - fprintf(stderr, "ERROR: fdb_future_get_version: %s\n", fdb_get_error(err)); + fprintf(stderr, "ERROR: fdb_future_get_version: %s\n", fdb_get_error(err)); #else - fprintf(stderr, "ERROR: fdb_future_get_int64: %s\n", fdb_get_error(err)); + fprintf(stderr, "ERROR: fdb_future_get_int64: %s\n", fdb_get_error(err)); #endif - return FDB_ERROR_RETRY; - } + return FDB_ERROR_RETRY; + } - /* fail if rv not properly set */ - if (!*rv) { - return FDB_ERROR_RETRY; - } - return FDB_SUCCESS; + /* fail if rv not properly set */ + if (!*rv) { + return FDB_ERROR_RETRY; + } + return FDB_SUCCESS; } +int run_op_get(FDBTransaction* transaction, char* keystr, char* valstr, int snapshot) { + FDBFuture* f; + int out_present; + char* val; + int vallen; + fdb_error_t err; -int run_op_get(FDBTransaction *transaction, char *keystr, char *valstr, - int snapshot) { - FDBFuture *f; - int out_present; - char *val; - int vallen; - fdb_error_t err; + f = fdb_transaction_get(transaction, (uint8_t*)keystr, strlen(keystr), snapshot); + fdb_wait_and_handle_error(fdb_transaction_get, f, transaction); - f = fdb_transaction_get(transaction, (uint8_t *)keystr, strlen(keystr), - snapshot); - fdb_wait_and_handle_error(fdb_transaction_get, f, transaction); - - err = fdb_future_get_value(f, &out_present, (const uint8_t **)&val, &vallen); - fdb_future_destroy(f); - if (err || !out_present) { - /* error or value not present */ - return FDB_ERROR_RETRY; - } - strncpy(valstr, val, vallen); - valstr[vallen] = '\0'; - return FDB_SUCCESS; + err = fdb_future_get_value(f, &out_present, (const uint8_t**)&val, &vallen); + fdb_future_destroy(f); + if (err || !out_present) { + /* error or value not present */ + return FDB_ERROR_RETRY; + } + strncpy(valstr, val, vallen); + valstr[vallen] = '\0'; + return FDB_SUCCESS; } +int run_op_getrange(FDBTransaction* transaction, char* keystr, char* keystr2, char* valstr, int snapshot, int reverse) { + FDBFuture* f; + fdb_error_t err; + FDBKeyValue const* out_kv; + int out_count; + int out_more; -int run_op_getrange(FDBTransaction *transaction, char *keystr, char *keystr2, - char *valstr, int snapshot, int reverse) { - FDBFuture *f; - fdb_error_t err; - FDBKeyValue const *out_kv; - int out_count; - int out_more; + f = fdb_transaction_get_range(transaction, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)keystr, strlen(keystr)), + FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t*)keystr2, strlen(keystr2)) + 1, 0 /* limit */, + 0 /* target_bytes */, FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */, + 0 /* iteration */, snapshot, reverse /* reverse */); + fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction); - f = fdb_transaction_get_range( - transaction, - FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t *)keystr, strlen(keystr)), - FDB_KEYSEL_LAST_LESS_OR_EQUAL((uint8_t *)keystr2, strlen(keystr2)) + 1, - 0 /* limit */, 0 /* target_bytes */, - FDB_STREAMING_MODE_WANT_ALL /* FDBStreamingMode */, 0 /* iteration */, - snapshot, reverse /* reverse */); - fdb_wait_and_handle_error(fdb_transaction_get_range, f, transaction); - - err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); - if (err) { - fprintf(stderr, "ERROR: fdb_future_get_keyvalue_array: %s\n", - fdb_get_error(err)); - fdb_future_destroy(f); - return FDB_ERROR_RETRY; - } - fdb_future_destroy(f); - return FDB_SUCCESS; + err = fdb_future_get_keyvalue_array(f, &out_kv, &out_count, &out_more); + if (err) { + fprintf(stderr, "ERROR: fdb_future_get_keyvalue_array: %s\n", fdb_get_error(err)); + fdb_future_destroy(f); + return FDB_ERROR_RETRY; + } + fdb_future_destroy(f); + return FDB_SUCCESS; } - /* Update -- GET and SET the same key */ -int run_op_update(FDBTransaction *transaction, char *keystr, char *valstr) { - FDBFuture *f; - int out_present; - char *val; - int vallen; - fdb_error_t err; +int run_op_update(FDBTransaction* transaction, char* keystr, char* valstr) { + FDBFuture* f; + int out_present; + char* val; + int vallen; + fdb_error_t err; - /* GET first */ - f = fdb_transaction_get(transaction, (uint8_t *)keystr, strlen(keystr), 0); - fdb_wait_and_handle_error(fdb_transaction_get, f, transaction); + /* GET first */ + f = fdb_transaction_get(transaction, (uint8_t*)keystr, strlen(keystr), 0); + fdb_wait_and_handle_error(fdb_transaction_get, f, transaction); - err = fdb_future_get_value(f, &out_present, (const uint8_t **)&val, &vallen); - fdb_future_destroy(f); - if (err || !out_present) { - /* error or value not present */ - return FDB_ERROR_RETRY; - } + err = fdb_future_get_value(f, &out_present, (const uint8_t**)&val, &vallen); + fdb_future_destroy(f); + if (err || !out_present) { + /* error or value not present */ + return FDB_ERROR_RETRY; + } - /* Update Value (SET) */ - fdb_transaction_set(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)valstr, strlen(valstr)); - return FDB_SUCCESS; + /* Update Value (SET) */ + fdb_transaction_set(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)valstr, strlen(valstr)); + return FDB_SUCCESS; } - -int run_op_insert(FDBTransaction *transaction, char *keystr, char *valstr) { - fdb_transaction_set(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)valstr, strlen(valstr)); - return FDB_SUCCESS; +int run_op_insert(FDBTransaction* transaction, char* keystr, char* valstr) { + fdb_transaction_set(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)valstr, strlen(valstr)); + return FDB_SUCCESS; } - -int run_op_clear(FDBTransaction *transaction, char *keystr) { - fdb_transaction_clear(transaction, (uint8_t *)keystr, strlen(keystr)); - return FDB_SUCCESS; +int run_op_clear(FDBTransaction* transaction, char* keystr) { + fdb_transaction_clear(transaction, (uint8_t*)keystr, strlen(keystr)); + return FDB_SUCCESS; } - -int run_op_clearrange(FDBTransaction *transaction, char *keystr, - char *keystr2) { - fdb_transaction_clear_range(transaction, (uint8_t *)keystr, strlen(keystr), - (uint8_t *)keystr2, strlen(keystr2)); - return FDB_SUCCESS; +int run_op_clearrange(FDBTransaction* transaction, char* keystr, char* keystr2) { + fdb_transaction_clear_range(transaction, (uint8_t*)keystr, strlen(keystr), (uint8_t*)keystr2, strlen(keystr2)); + return FDB_SUCCESS; } - /* run one transaction */ -int run_one_transaction(FDBTransaction *transaction, mako_args_t *args, - mako_stats_t *stats, char *keystr, char *keystr2, - char *valstr) { - int i; - int count; - int rc; - struct timespec timer_start, timer_end; - struct timespec timer_per_xact_start, timer_per_xact_end; - int docommit = 0; - int keynum; - int keyend; - int64_t readversion; - int randstrlen; - int rangei; +int run_one_transaction(FDBTransaction* transaction, mako_args_t* args, mako_stats_t* stats, char* keystr, + char* keystr2, char* valstr) { + int i; + int count; + int rc; + struct timespec timer_start, timer_end; + struct timespec timer_per_xact_start, timer_per_xact_end; + int docommit = 0; + int keynum; + int keyend; + int64_t readversion; + int randstrlen; + int rangei; +#if 0 /* this call conflicts with debug transaction */ /* make sure that the transaction object is clean */ fdb_transaction_reset(transaction); +#endif - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_start); - retryTxn: - for (i = 0; i < MAX_OP; i++) { - - if ((args->txnspec.ops[i][OP_COUNT] > 0) && (i != OP_COMMIT)) { - for (count = 0; count < args->txnspec.ops[i][OP_COUNT]; count++) { - - /* note: for simplicity, always generate a new key(s) even when retrying */ +retryTxn: + for (i = 0; i < MAX_OP; i++) { - /* pick a random key(s) */ - if (args->zipf) { - keynum = zipfian_next(); - } else { - keynum = urand(0, args->rows - 1); - } - genkey(keystr, keynum, args->rows, args->key_length + 1); - - /* range */ - if (args->txnspec.ops[i][OP_RANGE] > 0) { - keyend = keynum + args->txnspec.ops[i][OP_RANGE] - 1; /* inclusive */ - if (keyend > args->rows - 1) { - keyend = args->rows - 1; - } - genkey(keystr2, keyend, args->rows, args->key_length + 1); - } - - if (stats->xacts % args->sampling == 0) { - /* per op latency */ - clock_gettime(CLOCK_MONOTONIC, &timer_start); - } - - switch (i) { - case OP_GETREADVERSION: - rc = run_op_getreadversion(transaction, &readversion); - break; - case OP_GET: - rc = run_op_get(transaction, keystr, valstr, 0); - break; - case OP_GETRANGE: - rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, - args->txnspec.ops[i][OP_REVERSE]); - break; - case OP_SGET: - rc = run_op_get(transaction, keystr, valstr, 1); - break; - case OP_SGETRANGE: - rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, - args->txnspec.ops[i][OP_REVERSE]); - break; - case OP_UPDATE: - randstr(valstr, args->value_length + 1); - rc = run_op_update(transaction, keystr, valstr); - docommit = 1; - break; - case OP_INSERT: - randstr(keystr + KEYPREFIXLEN, - args->key_length - KEYPREFIXLEN + 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - rc = run_op_insert(transaction, keystr, valstr); - docommit = 1; - break; - case OP_INSERTRANGE: - randstrlen = args->key_length - KEYPREFIXLEN - - digits(args->txnspec.ops[i][OP_RANGE]); - randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { - sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", - digits(args->txnspec.ops[i][OP_RANGE]), rangei); - rc = run_op_insert(transaction, keystr, valstr); - if (rc != FDB_SUCCESS) - break; - } - docommit = 1; - break; - case OP_CLEAR: - rc = run_op_clear(transaction, keystr); - docommit = 1; - break; - case OP_SETCLEAR: - randstr(keystr + KEYPREFIXLEN, - args->key_length - KEYPREFIXLEN + 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - rc = run_op_insert(transaction, keystr, valstr); - if (rc == FDB_SUCCESS) { - /* commit insert so mutation goes to storage */ - rc = commit_transaction(transaction); - if (rc == FDB_SUCCESS) { - stats->ops[OP_COMMIT]++; - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, - OP_COMMIT, stats); - } else { - /* error */ - if (rc == FDB_ERROR_CONFLICT) { - stats->conflicts++; - } else { - stats->errors[OP_COMMIT]++; - } - if (rc == FDB_ERROR_ABORT) { - return rc; /* abort */ - } - goto retryTxn; - } - fdb_transaction_reset(transaction); - rc = run_op_clear(transaction, keystr); - } - docommit = 1; - break; - case OP_CLEARRANGE: - rc = run_op_clearrange(transaction, keystr, keystr2); - docommit = 1; - break; - case OP_SETCLEARRANGE: - randstrlen = args->key_length - KEYPREFIXLEN - - digits(args->txnspec.ops[i][OP_RANGE]); - randstr(keystr + KEYPREFIXLEN, - randstrlen + 1); /* make it (almost) unique */ - randstr(valstr, args->value_length + 1); - for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { - sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", - digits(args->txnspec.ops[i][OP_RANGE]), rangei); - if (rangei == 0) { - strcpy(keystr2, keystr); - keystr2[strlen(keystr)] = '\0'; - } - rc = run_op_insert(transaction, keystr, valstr); - /* rollback not necessary, move on */ - if (rc == FDB_ERROR_RETRY) { - goto retryTxn; - } else if (rc == FDB_ERROR_ABORT) { - return rc; /* abort */ - } - } - /* commit insert so mutation goes to storage */ - rc = commit_transaction(transaction); - if (rc == FDB_SUCCESS) { - stats->ops[OP_COMMIT]++; - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, - OP_COMMIT, stats); - } else { - /* error */ - if (rc == FDB_ERROR_CONFLICT) { - stats->conflicts++; - } else { - stats->errors[OP_COMMIT]++; - } - if (rc == FDB_ERROR_ABORT) { - return rc; /* abort */ - } - goto retryTxn; - } - fdb_transaction_reset(transaction); - rc = run_op_clearrange(transaction, keystr2, keystr); - docommit = 1; - break; - default: - fprintf(stderr, "ERROR: Unknown Operation %d\n", i); - break; + if ((args->txnspec.ops[i][OP_COUNT] > 0) && (i != OP_COMMIT)) { + for (count = 0; count < args->txnspec.ops[i][OP_COUNT]; count++) { + + /* note: for simplicity, always generate a new key(s) even when retrying */ + + /* pick a random key(s) */ + if (args->zipf) { + keynum = zipfian_next(); + } else { + keynum = urand(0, args->rows - 1); + } + genkey(keystr, keynum, args->rows, args->key_length + 1); + + /* range */ + if (args->txnspec.ops[i][OP_RANGE] > 0) { + keyend = keynum + args->txnspec.ops[i][OP_RANGE] - 1; /* inclusive */ + if (keyend > args->rows - 1) { + keyend = args->rows - 1; + } + genkey(keystr2, keyend, args->rows, args->key_length + 1); + } + + if (stats->xacts % args->sampling == 0) { + /* per op latency */ + clock_gettime(CLOCK_MONOTONIC, &timer_start); + } + + switch (i) { + case OP_GETREADVERSION: + rc = run_op_getreadversion(transaction, &readversion); + break; + case OP_GET: + rc = run_op_get(transaction, keystr, valstr, 0); + break; + case OP_GETRANGE: + rc = run_op_getrange(transaction, keystr, keystr2, valstr, 0, args->txnspec.ops[i][OP_REVERSE]); + break; + case OP_SGET: + rc = run_op_get(transaction, keystr, valstr, 1); + break; + case OP_SGETRANGE: + rc = run_op_getrange(transaction, keystr, keystr2, valstr, 1, args->txnspec.ops[i][OP_REVERSE]); + break; + case OP_UPDATE: + randstr(valstr, args->value_length + 1); + rc = run_op_update(transaction, keystr, valstr); + docommit = 1; + break; + case OP_INSERT: + randstr(keystr + KEYPREFIXLEN, args->key_length - KEYPREFIXLEN + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + rc = run_op_insert(transaction, keystr, valstr); + docommit = 1; + break; + case OP_INSERTRANGE: + randstrlen = args->key_length - KEYPREFIXLEN - digits(args->txnspec.ops[i][OP_RANGE]); + randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { + sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), + rangei); + rc = run_op_insert(transaction, keystr, valstr); + if (rc != FDB_SUCCESS) break; + } + docommit = 1; + break; + case OP_CLEAR: + rc = run_op_clear(transaction, keystr); + docommit = 1; + break; + case OP_SETCLEAR: + randstr(keystr + KEYPREFIXLEN, args->key_length - KEYPREFIXLEN + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + rc = run_op_insert(transaction, keystr, valstr); + if (rc == FDB_SUCCESS) { + /* commit insert so mutation goes to storage */ + rc = commit_transaction(transaction); + if (rc == FDB_SUCCESS) { + stats->ops[OP_COMMIT]++; + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + } else { + /* error */ + if (rc == FDB_ERROR_CONFLICT) { + stats->conflicts++; + } else { + stats->errors[OP_COMMIT]++; + } + if (rc == FDB_ERROR_ABORT) { + /* make sure to reset transaction */ + fdb_transaction_reset(transaction); + return rc; /* abort */ + } + goto retryTxn; + } + fdb_transaction_reset(transaction); + rc = run_op_clear(transaction, keystr); + } + docommit = 1; + break; + case OP_CLEARRANGE: + rc = run_op_clearrange(transaction, keystr, keystr2); + docommit = 1; + break; + case OP_SETCLEARRANGE: + randstrlen = args->key_length - KEYPREFIXLEN - digits(args->txnspec.ops[i][OP_RANGE]); + randstr(keystr + KEYPREFIXLEN, randstrlen + 1); /* make it (almost) unique */ + randstr(valstr, args->value_length + 1); + for (rangei = 0; rangei < args->txnspec.ops[i][OP_RANGE]; rangei++) { + sprintf(keystr + KEYPREFIXLEN + randstrlen, "%0.*d", digits(args->txnspec.ops[i][OP_RANGE]), + rangei); + if (rangei == 0) { + strcpy(keystr2, keystr); + keystr2[strlen(keystr)] = '\0'; + } + rc = run_op_insert(transaction, keystr, valstr); + /* rollback not necessary, move on */ + if (rc == FDB_ERROR_RETRY) { + goto retryTxn; + } else if (rc == FDB_ERROR_ABORT) { + /* make sure to reset transaction */ + fdb_transaction_reset(transaction); + return rc; /* abort */ + } + } + /* commit insert so mutation goes to storage */ + rc = commit_transaction(transaction); + if (rc == FDB_SUCCESS) { + stats->ops[OP_COMMIT]++; + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + } else { + /* error */ + if (rc == FDB_ERROR_CONFLICT) { + stats->conflicts++; + } else { + stats->errors[OP_COMMIT]++; + } + if (rc == FDB_ERROR_ABORT) { + /* make sure to reset transaction */ + fdb_transaction_reset(transaction); + return rc; /* abort */ + } + goto retryTxn; + } + fdb_transaction_reset(transaction); + rc = run_op_clearrange(transaction, keystr2, keystr); + docommit = 1; + break; + default: + fprintf(stderr, "ERROR: Unknown Operation %d\n", i); + break; + } + + if (stats->xacts % args->sampling == 0) { + clock_gettime(CLOCK_MONOTONIC, &timer_end); + if (rc == FDB_SUCCESS) { + /* per op latency, record successful transactions */ + update_op_lat_stats(&timer_start, &timer_end, i, stats); + } + } + + /* check rc and update stats */ + if (rc == FDB_SUCCESS) { + stats->ops[i]++; + } else { + /* error */ + if (rc == FDB_ERROR_CONFLICT) { + stats->conflicts++; + } else { + stats->errors[OP_COMMIT]++; + } + if (rc == FDB_ERROR_ABORT) { + /* make sure to reset transaction */ + fdb_transaction_reset(transaction); + return rc; /* abort */ + } + goto retryTxn; + } + } + } } - if (stats->xacts % args->sampling == 0) { - clock_gettime(CLOCK_MONOTONIC, &timer_end); - if (rc == FDB_SUCCESS) { - /* per op latency, record successful transactions */ - update_op_lat_stats(&timer_start, &timer_end, i, stats); - } + /* commit only successful transaction */ + if (docommit | args->commit_get) { + rc = commit_transaction(transaction); + if (rc == FDB_SUCCESS) { + /* success */ + stats->ops[OP_COMMIT]++; + clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); + update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, OP_COMMIT, stats); + } else { + /* error */ + if (rc == FDB_ERROR_CONFLICT) { + stats->conflicts++; + } else { + stats->errors[OP_COMMIT]++; + } + if (rc == FDB_ERROR_ABORT) { + /* make sure to reset transaction */ + fdb_transaction_reset(transaction); + return rc; /* abort */ + } + goto retryTxn; + } } - /* check rc and update stats */ - if (rc == FDB_SUCCESS) { - stats->ops[i]++; - } else { - /* error */ - if (rc == FDB_ERROR_CONFLICT) { - stats->conflicts++; - } else { - stats->errors[OP_COMMIT]++; - } - if (rc == FDB_ERROR_ABORT) { - return rc; /* abort */ - } - goto retryTxn; - } - } - } - } + stats->xacts++; - /* commit only successful transaction */ - if (docommit | args->commit_get) { - rc = commit_transaction(transaction); - if (rc == FDB_SUCCESS) { - /* success */ - stats->ops[OP_COMMIT]++; - clock_gettime(CLOCK_MONOTONIC, &timer_per_xact_end); - update_op_lat_stats(&timer_per_xact_start, &timer_per_xact_end, - OP_COMMIT, stats); - } else { - /* error */ - if (rc == FDB_ERROR_CONFLICT) { - stats->conflicts++; - } else { - stats->errors[OP_COMMIT]++; - } - if (rc == FDB_ERROR_ABORT) { - return rc; /* abort */ - } - goto retryTxn; - } - } - - stats->xacts++; - - return 0; + /* make sure to reset transaction */ + fdb_transaction_reset(transaction); + return 0; } +int run_workload(FDBTransaction* transaction, mako_args_t* args, int thread_tps, volatile double* throttle_factor, + int thread_iters, volatile int* signal, mako_stats_t* stats, int dotrace) { + int xacts = 0; + int64_t total_xacts = 0; + int rc = 0; + struct timespec timer_prev, timer_now; + char* keystr; + char* keystr2; + char* valstr; + int current_tps; + char* traceid; + int tracetimer = 0; -int run_workload(FDBTransaction *transaction, mako_args_t *args, - int thread_tps, volatile double *throttle_factor, - int thread_iters, volatile int *signal, mako_stats_t *stats) { - int xacts = 0; - int rc = 0; - struct timespec timer_prev, timer_now; - char *keystr; - char *keystr2; - char *valstr; - int current_tps; + if (thread_tps < 0) return 0; - if (thread_tps < 0) - return 0; + if (dotrace) { + traceid = (char*)malloc(32); + } - current_tps = (int)((double)thread_tps * *throttle_factor); - - keystr = (char *)malloc(sizeof(char) * args->key_length + 1); - if (!keystr) - return -1; - keystr2 = (char *)malloc(sizeof(char) * args->key_length + 1); - if (!keystr2) { - free(keystr); - return -1; - } - valstr = (char *)malloc(sizeof(char) * args->value_length + 1); - if (!valstr) { - free(keystr); - free(keystr2); - return -1; - } - - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_prev); - - /* main transaction loop */ - while (1) { - - if ((thread_tps > 0) && (xacts >= current_tps)) { - /* throttling is on */ - - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || - ((timer_now.tv_sec == timer_prev.tv_sec + 1) && - (timer_now.tv_nsec > timer_prev.tv_nsec))) { - /* more than 1 second passed, no need to throttle */ - xacts = 0; - timer_prev.tv_sec = timer_now.tv_sec; - timer_prev.tv_nsec = timer_now.tv_nsec; - /* update throttle rate */ current_tps = (int)((double)thread_tps * *throttle_factor); - } else { - /* 1 second not passed, throttle */ - usleep(1000); - continue; - } - } - rc = run_one_transaction(transaction, args, stats, keystr, keystr2, valstr); - if (rc) { - /* FIXME: run_one_transaction should return something meaningful */ - fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc); - } + keystr = (char*)malloc(sizeof(char) * args->key_length + 1); + if (!keystr) return -1; + keystr2 = (char*)malloc(sizeof(char) * args->key_length + 1); + if (!keystr2) { + free(keystr); + return -1; + } + valstr = (char*)malloc(sizeof(char) * args->value_length + 1); + if (!valstr) { + free(keystr); + free(keystr2); + return -1; + } - if (thread_iters > 0) { - if (thread_iters == xacts) { - /* xact limit reached */ - break; - } - } else if (*signal == SIGNAL_RED) { - /* signal turned red, target duration reached */ - break; - } - xacts++; - } - free(keystr); - free(keystr2); - free(valstr); + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_prev); - return rc; + /* main transaction loop */ + while (1) { + + if (((thread_tps > 0) && (xacts >= current_tps)) /* throttle on */ || dotrace /* transaction tracing on */) { + + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + if ((timer_now.tv_sec > timer_prev.tv_sec + 1) || + ((timer_now.tv_sec == timer_prev.tv_sec + 1) && (timer_now.tv_nsec > timer_prev.tv_nsec))) { + /* more than 1 second passed, no need to throttle */ + xacts = 0; + timer_prev.tv_sec = timer_now.tv_sec; + timer_prev.tv_nsec = timer_now.tv_nsec; + + /* update throttle rate */ + if (thread_tps > 0) { + current_tps = (int)((double)thread_tps * *throttle_factor); + } + + /* enable transaction trace */ + if (dotrace) { + tracetimer++; + if (tracetimer == dotrace) { + fdb_error_t err; + tracetimer = 0; + snprintf(traceid, 32, "makotrace%019lld", total_xacts); + fprintf(debugme, "DEBUG: txn tracing %s\n", traceid); + err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER, + (uint8_t*)traceid, strlen(traceid)); + if (err) { + fprintf(stderr, "ERROR: FDB_TR_OPTION_DEBUG_TRANSACTION_IDENTIFIER: %s\n", + fdb_get_error(err)); + } + err = fdb_transaction_set_option(transaction, FDB_TR_OPTION_LOG_TRANSACTION, (uint8_t*)NULL, 0); + if (err) { + fprintf(stderr, "ERROR: FDB_TR_OPTION_LOG_TRANSACTION: %s\n", fdb_get_error(err)); + } + } + } + + } else { + if (thread_tps > 0) { + /* 1 second not passed, throttle */ + usleep(1000); + continue; + } + } + } /* throttle or txntrace */ + + rc = run_one_transaction(transaction, args, stats, keystr, keystr2, valstr); + if (rc) { + /* FIXME: run_one_transaction should return something meaningful */ + fprintf(annoyme, "ERROR: run_one_transaction failed (%d)\n", rc); + } + + if (thread_iters > 0) { + if (thread_iters == xacts) { + /* xact limit reached */ + break; + } + } else if (*signal == SIGNAL_RED) { + /* signal turned red, target duration reached */ + break; + } + xacts++; + total_xacts++; + } + free(keystr); + free(keystr2); + free(valstr); + if (dotrace) { + free(traceid); + } + + return rc; } - /* mako worker thread */ -void *worker_thread(void *thread_args) { - int worker_id = ((thread_args_t *)thread_args)->process->worker_id; - int thread_id = ((thread_args_t *)thread_args)->thread_id; - mako_args_t *args = ((thread_args_t *)thread_args)->process->args; - FDBDatabase *database = ((thread_args_t *)thread_args)->process->database; - fdb_error_t err; - int rc; - FDBTransaction *transaction; - int thread_tps = 0; - int thread_iters = 0; - int op; - volatile int *signal = &((thread_args_t *)thread_args)->process->shm->signal; - volatile double *throttle_factor = &((thread_args_t *)thread_args)->process->shm->throttle_factor; - volatile int *readycount = - &((thread_args_t *)thread_args)->process->shm->readycount; - mako_stats_t *stats = - (void *)((thread_args_t *)thread_args)->process->shm + - sizeof(mako_shmhdr_t) /* skip header */ - + (sizeof(mako_stats_t) * (worker_id * args->num_threads + thread_id)); +void* worker_thread(void* thread_args) { + int worker_id = ((thread_args_t*)thread_args)->process->worker_id; + int thread_id = ((thread_args_t*)thread_args)->thread_id; + mako_args_t* args = ((thread_args_t*)thread_args)->process->args; + FDBDatabase* database = ((thread_args_t*)thread_args)->process->database; + fdb_error_t err; + int rc; + FDBTransaction* transaction; + int thread_tps = 0; + int thread_iters = 0; + int op; + int dotrace = (worker_id == 0 && thread_id == 0 && args->txntrace) ? args->txntrace : 0; + volatile int* signal = &((thread_args_t*)thread_args)->process->shm->signal; + volatile double* throttle_factor = &((thread_args_t*)thread_args)->process->shm->throttle_factor; + volatile int* readycount = &((thread_args_t*)thread_args)->process->shm->readycount; + mako_stats_t* stats = (void*)((thread_args_t*)thread_args)->process->shm + sizeof(mako_shmhdr_t) /* skip header */ + + (sizeof(mako_stats_t) * (worker_id * args->num_threads + thread_id)); - /* init latency */ - for (op = 0; op < MAX_OP; op++) { - stats->latency_us_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ - stats->latency_us_max[op] = 0; - stats->latency_us_total[op] = 0; - } + /* init latency */ + for (op = 0; op < MAX_OP; op++) { + stats->latency_us_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ + stats->latency_us_max[op] = 0; + stats->latency_us_total[op] = 0; + } - fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, - args->num_processes, thread_id, args->num_threads, - (unsigned int)pthread_self()); + fprintf(debugme, "DEBUG: worker_id:%d (%d) thread_id:%d (%d) (tid:%d)\n", worker_id, args->num_processes, thread_id, + args->num_threads, (unsigned int)pthread_self()); - if (args->tpsmax) { - thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, - args->num_processes, args->num_threads); - } + if (args->tpsmax) { + thread_tps = compute_thread_tps(args->tpsmax, worker_id, thread_id, args->num_processes, args->num_threads); + } - if (args->iteration) { - thread_iters = compute_thread_iters(args->iteration, worker_id, thread_id, - args->num_processes, args->num_threads); - } + if (args->iteration) { + thread_iters = + compute_thread_iters(args->iteration, worker_id, thread_id, args->num_processes, args->num_threads); + } - /* create my own transaction object */ - err = fdb_database_create_transaction(database, &transaction); - check_fdb_error(err); + /* create my own transaction object */ + err = fdb_database_create_transaction(database, &transaction); + check_fdb_error(err); - /* i'm ready */ - __sync_fetch_and_add(readycount, 1); - while (*signal == SIGNAL_OFF) { - usleep(10000); /* 10ms */ - } + /* i'm ready */ + __sync_fetch_and_add(readycount, 1); + while (*signal == SIGNAL_OFF) { + usleep(10000); /* 10ms */ + } - /* clean */ - if (args->mode == MODE_CLEAN) { - rc = cleanup(transaction, args); - if (rc < 0) { - fprintf(stderr, "ERROR: cleanup failed\n"); - } - } + /* clean */ + if (args->mode == MODE_CLEAN) { + rc = cleanup(transaction, args); + if (rc < 0) { + fprintf(stderr, "ERROR: cleanup failed\n"); + } + } - /* build/popualte */ - else if (args->mode == MODE_BUILD) { - rc = populate(transaction, args, worker_id, thread_id, thread_tps, stats); - if (rc < 0) { - fprintf(stderr, "ERROR: populate failed\n"); - } - } + /* build/popualte */ + else if (args->mode == MODE_BUILD) { + rc = populate(transaction, args, worker_id, thread_id, thread_tps, stats); + if (rc < 0) { + fprintf(stderr, "ERROR: populate failed\n"); + } + } - /* run the workload */ - else if (args->mode == MODE_RUN) { - rc = run_workload(transaction, args, thread_tps, throttle_factor, - thread_iters, signal, stats); - if (rc < 0) { - fprintf(stderr, "ERROR: run_workload failed\n"); - } - } + /* run the workload */ + else if (args->mode == MODE_RUN) { + rc = run_workload(transaction, args, thread_tps, throttle_factor, thread_iters, signal, stats, dotrace); + if (rc < 0) { + fprintf(stderr, "ERROR: run_workload failed\n"); + } + } - /* fall through */ + /* fall through */ failExit: - fdb_transaction_destroy(transaction); - pthread_exit(0); + fdb_transaction_destroy(transaction); + pthread_exit(0); } - /* mako worker process */ -int worker_process_main(mako_args_t *args, int worker_id, mako_shmhdr_t *shm) { - int i; - pthread_t - network_thread; /* handle for thread which invoked fdb_run_network() */ - pthread_t *worker_threads; +int worker_process_main(mako_args_t* args, int worker_id, mako_shmhdr_t* shm) { + int i; + pthread_t network_thread; /* handle for thread which invoked fdb_run_network() */ + pthread_t* worker_threads; #if FDB_API_VERSION < 610 - FDBCluster *cluster; + FDBCluster* cluster; #endif - process_info_t process; - thread_args_t *thread_args; - int rc; - fdb_error_t err; + process_info_t process; + thread_args_t* thread_args; + int rc; + fdb_error_t err; - process.worker_id = worker_id; - process.args = args; - process.shm = (mako_shmhdr_t *)shm; + process.worker_id = worker_id; + process.args = args; + process.shm = (mako_shmhdr_t*)shm; - fprintf(debugme, "DEBUG: worker %d started\n", worker_id); + fprintf(debugme, "DEBUG: worker %d started\n", worker_id); - /* Everything starts from here */ - err = fdb_select_api_version(args->api_version); - check_fdb_error(err); + /* Everything starts from here */ + err = fdb_select_api_version(args->api_version); + check_fdb_error(err); - /* enable flatbuffers if specified */ - if (args->flatbuffers) { + /* enable flatbuffers if specified */ + if (args->flatbuffers) { #ifdef FDB_NET_OPTION_USE_FLATBUFFERS - fprintf(debugme, "DEBUG: Using flatbuffers\n"); - err = - fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS, - (uint8_t *)&args->flatbuffers, sizeof(uint8_t)); - if (err) { - fprintf( - stderr, - "ERROR: fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS): %s\n", - fdb_get_error(err)); - } + fprintf(debugme, "DEBUG: Using flatbuffers\n"); + err = fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS, (uint8_t*)&args->flatbuffers, sizeof(uint8_t)); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option(FDB_NET_OPTION_USE_FLATBUFFERS): %s\n", fdb_get_error(err)); + } #else - fprintf(printme, "INFO: flatbuffers is not supported in FDB API version %d\n", - FDB_API_VERSION); + fprintf(printme, "INFO: flatbuffers is not supported in FDB API version %d\n", FDB_API_VERSION); #endif - } + } - /* enable tracing if specified */ - if (args->trace) { - fprintf(debugme, "DEBUG: Enable Tracing (%s)\n", (args->tracepath[0] == '\0') - ? "current directory" - : args->tracepath); - err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, - (uint8_t *)args->tracepath, - strlen(args->tracepath)); - if (err) { - fprintf( - stderr, - "ERROR: fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE): %s\n", - fdb_get_error(err)); - } - } + /* enable tracing if specified */ + if (args->trace) { + fprintf(debugme, "DEBUG: Enable Tracing in %s (%s)\n", (args->traceformat == 0) ? "XML" : "JSON", + (args->tracepath[0] == '\0') ? "current directory" : args->tracepath); + err = fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE, (uint8_t*)args->tracepath, strlen(args->tracepath)); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option(FDB_NET_OPTION_TRACE_ENABLE): %s\n", fdb_get_error(err)); + } + if (args->traceformat == 1) { + err = fdb_network_set_option(FDB_NET_OPTION_TRACE_FORMAT, (uint8_t*)"json", 4); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option(FDB_NET_OPTION_TRACE_FORMAT): %s\n", fdb_get_error(err)); + } + } + } - /* enable knobs if specified */ - if (args->knobs[0] != '\0') { - char delim[] = ", "; - char *knob = strtok(args->knobs, delim); - while (knob != NULL) { - fprintf(debugme, "DEBUG: Setting client knobs: %s\n", knob); - err = fdb_network_set_option(FDB_NET_OPTION_KNOB, (uint8_t *)knob, - strlen(knob)); - if (err) { - fprintf(stderr, "ERROR: fdb_network_set_option: %s\n", - fdb_get_error(err)); - } - knob = strtok(NULL, delim); - } - } + /* enable knobs if specified */ + if (args->knobs[0] != '\0') { + char delim[] = ", "; + char* knob = strtok(args->knobs, delim); + while (knob != NULL) { + fprintf(debugme, "DEBUG: Setting client knobs: %s\n", knob); + err = fdb_network_set_option(FDB_NET_OPTION_KNOB, (uint8_t*)knob, strlen(knob)); + if (err) { + fprintf(stderr, "ERROR: fdb_network_set_option: %s\n", fdb_get_error(err)); + } + knob = strtok(NULL, delim); + } + } - /* Network thread must be setup before doing anything */ - fprintf(debugme, "DEBUG: fdb_setup_network\n"); - err = fdb_setup_network(); - check_fdb_error(err); + /* Network thread must be setup before doing anything */ + fprintf(debugme, "DEBUG: fdb_setup_network\n"); + err = fdb_setup_network(); + check_fdb_error(err); - /* Each worker process will have its own network thread */ - fprintf(debugme, "DEBUG: creating network thread\n"); - rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void *)args); - if (rc != 0) { - fprintf(stderr, "ERROR: Cannot create a network thread\n"); - return -1; - } + /* Each worker process will have its own network thread */ + fprintf(debugme, "DEBUG: creating network thread\n"); + rc = pthread_create(&network_thread, NULL, fdb_network_thread, (void*)args); + if (rc != 0) { + fprintf(stderr, "ERROR: Cannot create a network thread\n"); + return -1; + } - /*** let's party! ***/ + /*** let's party! ***/ - /* set up cluster and datbase for workder threads */ + /* set up cluster and datbase for workder threads */ #if FDB_API_VERSION < 610 - /* cluster */ - f = fdb_create_cluster(args->cluster_file); - fdb_block_wait(f); - err = fdb_future_get_cluster(f, &cluster); - check_fdb_error(err); - fdb_future_destroy(f); + /* cluster */ + f = fdb_create_cluster(args->cluster_file); + fdb_block_wait(f); + err = fdb_future_get_cluster(f, &cluster); + check_fdb_error(err); + fdb_future_destroy(f); - /* database */ - /* big mystery -- do we ever have a database named other than "DB"? */ - f = fdb_cluster_create_database(cluster, (uint8_t *)"DB", 2); - fdb_block_wait(f); - err = fdb_future_get_database(f, &process.database); - check_fdb_error(err); - fdb_future_destroy(f); + /* database */ + /* big mystery -- do we ever have a database named other than "DB"? */ + f = fdb_cluster_create_database(cluster, (uint8_t*)"DB", 2); + fdb_block_wait(f); + err = fdb_future_get_database(f, &process.database); + check_fdb_error(err); + fdb_future_destroy(f); #else /* >= 610 */ - fdb_create_database(args->cluster_file, &process.database); + fdb_create_database(args->cluster_file, &process.database); #endif - fprintf(debugme, "DEBUG: creating %d worker threads\n", args->num_threads); - worker_threads = (pthread_t *)calloc(sizeof(pthread_t), args->num_threads); - if (!worker_threads) { - fprintf(stderr, "ERROR: cannot allocate worker_threads\n"); - goto failExit; - } + fprintf(debugme, "DEBUG: creating %d worker threads\n", args->num_threads); + worker_threads = (pthread_t*)calloc(sizeof(pthread_t), args->num_threads); + if (!worker_threads) { + fprintf(stderr, "ERROR: cannot allocate worker_threads\n"); + goto failExit; + } - /* spawn worker threads */ - thread_args = - (thread_args_t *)calloc(sizeof(thread_args_t), args->num_threads); - if (!thread_args) { - fprintf(stderr, "ERROR: cannot allocate thread_args\n"); - goto failExit; - } + /* spawn worker threads */ + thread_args = (thread_args_t*)calloc(sizeof(thread_args_t), args->num_threads); + if (!thread_args) { + fprintf(stderr, "ERROR: cannot allocate thread_args\n"); + goto failExit; + } - for (i = 0; i < args->num_threads; i++) { - thread_args[i].thread_id = i; - thread_args[i].process = &process; - rc = pthread_create(&worker_threads[i], NULL, worker_thread, - (void *)&thread_args[i]); - if (rc != 0) { - fprintf(stderr, "ERROR: cannot create a new worker thread %d\n", i); - /* ignore this thread? */ - } - } + for (i = 0; i < args->num_threads; i++) { + thread_args[i].thread_id = i; + thread_args[i].process = &process; + rc = pthread_create(&worker_threads[i], NULL, worker_thread, (void*)&thread_args[i]); + if (rc != 0) { + fprintf(stderr, "ERROR: cannot create a new worker thread %d\n", i); + /* ignore this thread? */ + } + } - /*** party is over ***/ + /*** party is over ***/ - /* wait for everyone to finish */ - for (i = 0; i < args->num_threads; i++) { - fprintf(debugme, "DEBUG: worker_thread %d joining\n", i); - rc = pthread_join(worker_threads[i], NULL); - if (rc != 0) { - fprintf(stderr, "ERROR: threads %d failed to join\n", i); - } - } + /* wait for everyone to finish */ + for (i = 0; i < args->num_threads; i++) { + fprintf(debugme, "DEBUG: worker_thread %d joining\n", i); + rc = pthread_join(worker_threads[i], NULL); + if (rc != 0) { + fprintf(stderr, "ERROR: threads %d failed to join\n", i); + } + } failExit: - if (worker_threads) - free(worker_threads); - if (thread_args) - free(thread_args); + if (worker_threads) free(worker_threads); + if (thread_args) free(thread_args); - /* clean up database and cluster */ - fdb_database_destroy(process.database); + /* clean up database and cluster */ + fdb_database_destroy(process.database); #if FDB_API_VERSION < 610 - fdb_cluster_destroy(cluster); + fdb_cluster_destroy(cluster); #endif - /* stop the network thread */ - fprintf(debugme, "DEBUG: fdb_stop_network\n"); - err = fdb_stop_network(); - check_fdb_error(err); + /* stop the network thread */ + fprintf(debugme, "DEBUG: fdb_stop_network\n"); + err = fdb_stop_network(); + check_fdb_error(err); - /* wait for the network thread to join */ - fprintf(debugme, "DEBUG: network_thread joining\n"); - rc = pthread_join(network_thread, NULL); - if (rc != 0) { - fprintf(stderr, "ERROR: network thread failed to join\n"); - } + /* wait for the network thread to join */ + fprintf(debugme, "DEBUG: network_thread joining\n"); + rc = pthread_join(network_thread, NULL); + if (rc != 0) { + fprintf(stderr, "ERROR: network thread failed to join\n"); + } - return 0; + return 0; } - /* initialize the parameters with default values */ -int init_args(mako_args_t *args) { - int i; - if (!args) - return -1; - memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ - args->api_version = fdb_get_max_api_version(); - args->json = 0; - args->num_processes = 1; - args->num_threads = 1; - args->mode = MODE_INVALID; - args->rows = 100000; - args->seconds = 30; - args->iteration = 0; - args->tpsmax = 0; - args->tpsmin = -1; - args->tpsinterval = 10; - args->tpschange = TPS_SIN; - args->sampling = 1000; - args->key_length = 32; - args->value_length = 16; - args->zipf = 0; - args->commit_get = 0; - args->verbose = 1; - args->flatbuffers = 0; /* internal */ - args->knobs[0] = '\0'; - args->trace = 0; - args->tracepath[0] = '\0'; - for (i = 0; i < MAX_OP; i++) { - args->txnspec.ops[i][OP_COUNT] = 0; - } - return 0; +int init_args(mako_args_t* args) { + int i; + if (!args) return -1; + memset(args, 0, sizeof(mako_args_t)); /* zero-out everything */ + args->api_version = fdb_get_max_api_version(); + args->json = 0; + args->num_processes = 1; + args->num_threads = 1; + args->mode = MODE_INVALID; + args->rows = 100000; + args->seconds = 30; + args->iteration = 0; + args->tpsmax = 0; + args->tpsmin = -1; + args->tpsinterval = 10; + args->tpschange = TPS_SIN; + args->sampling = 1000; + args->key_length = 32; + args->value_length = 16; + args->zipf = 0; + args->commit_get = 0; + args->verbose = 1; + args->flatbuffers = 0; /* internal */ + args->knobs[0] = '\0'; + args->trace = 0; + args->tracepath[0] = '\0'; + args->traceformat = 0; /* default to client's default (XML) */ + args->txntrace = 0; + for (i = 0; i < MAX_OP; i++) { + args->txnspec.ops[i][OP_COUNT] = 0; + } + return 0; } - /* parse transaction specification */ -int parse_transaction(mako_args_t *args, char *optarg) { - char *ptr = optarg; - int op = 0; - int rangeop = 0; - int num; - int error = 0; +int parse_transaction(mako_args_t* args, char* optarg) { + char* ptr = optarg; + int op = 0; + int rangeop = 0; + int num; + int error = 0; - for (op = 0; op < MAX_OP; op++) { - args->txnspec.ops[op][OP_COUNT] = 0; - args->txnspec.ops[op][OP_RANGE] = 0; - } + for (op = 0; op < MAX_OP; op++) { + args->txnspec.ops[op][OP_COUNT] = 0; + args->txnspec.ops[op][OP_RANGE] = 0; + } - op = 0; - while (*ptr) { - if (strncmp(ptr, "grv", 3) == 0) { - op = OP_GETREADVERSION; - ptr += 3; - } else if (strncmp(ptr, "gr", 2) == 0) { - op = OP_GETRANGE; - rangeop = 1; - ptr += 2; - } else if (strncmp(ptr, "g", 1) == 0) { - op = OP_GET; - ptr++; - } else if (strncmp(ptr, "sgr", 3) == 0) { - op = OP_SGETRANGE; - rangeop = 1; - ptr += 3; - } else if (strncmp(ptr, "sg", 2) == 0) { - op = OP_SGET; - ptr += 2; - } else if (strncmp(ptr, "u", 1) == 0) { - op = OP_UPDATE; - ptr++; - } else if (strncmp(ptr, "ir", 2) == 0) { - op = OP_INSERTRANGE; - rangeop = 1; - ptr += 2; - } else if (strncmp(ptr, "i", 1) == 0) { - op = OP_INSERT; - ptr++; - } else if (strncmp(ptr, "cr", 2) == 0) { - op = OP_CLEARRANGE; - rangeop = 1; - ptr += 2; - } else if (strncmp(ptr, "c", 1) == 0) { - op = OP_CLEAR; - ptr++; - } else if (strncmp(ptr, "scr", 3) == 0) { - op = OP_SETCLEARRANGE; - rangeop = 1; - ptr += 3; - } else if (strncmp(ptr, "sc", 2) == 0) { - op = OP_SETCLEAR; - ptr += 2; - } else { - fprintf(debugme, "Error: Invalid transaction spec: %s\n", ptr); - error = 1; - break; - } + op = 0; + while (*ptr) { + if (strncmp(ptr, "grv", 3) == 0) { + op = OP_GETREADVERSION; + ptr += 3; + } else if (strncmp(ptr, "gr", 2) == 0) { + op = OP_GETRANGE; + rangeop = 1; + ptr += 2; + } else if (strncmp(ptr, "g", 1) == 0) { + op = OP_GET; + ptr++; + } else if (strncmp(ptr, "sgr", 3) == 0) { + op = OP_SGETRANGE; + rangeop = 1; + ptr += 3; + } else if (strncmp(ptr, "sg", 2) == 0) { + op = OP_SGET; + ptr += 2; + } else if (strncmp(ptr, "u", 1) == 0) { + op = OP_UPDATE; + ptr++; + } else if (strncmp(ptr, "ir", 2) == 0) { + op = OP_INSERTRANGE; + rangeop = 1; + ptr += 2; + } else if (strncmp(ptr, "i", 1) == 0) { + op = OP_INSERT; + ptr++; + } else if (strncmp(ptr, "cr", 2) == 0) { + op = OP_CLEARRANGE; + rangeop = 1; + ptr += 2; + } else if (strncmp(ptr, "c", 1) == 0) { + op = OP_CLEAR; + ptr++; + } else if (strncmp(ptr, "scr", 3) == 0) { + op = OP_SETCLEARRANGE; + rangeop = 1; + ptr += 3; + } else if (strncmp(ptr, "sc", 2) == 0) { + op = OP_SETCLEAR; + ptr += 2; + } else { + fprintf(debugme, "Error: Invalid transaction spec: %s\n", ptr); + error = 1; + break; + } - /* count */ - num = 0; - if ((*ptr < '0') || (*ptr > '9')) { - num = 1; /* if omitted, set it to 1 */ - } else { - while ((*ptr >= '0') && (*ptr <= '9')) { - num = num * 10 + *ptr - '0'; - ptr++; - } - } - /* set count */ - args->txnspec.ops[op][OP_COUNT] = num; + /* count */ + num = 0; + if ((*ptr < '0') || (*ptr > '9')) { + num = 1; /* if omitted, set it to 1 */ + } else { + while ((*ptr >= '0') && (*ptr <= '9')) { + num = num * 10 + *ptr - '0'; + ptr++; + } + } + /* set count */ + args->txnspec.ops[op][OP_COUNT] = num; - if (rangeop) { - if (*ptr != ':') { - error = 1; - break; - } else { - ptr++; /* skip ':' */ - /* check negative '-' sign */ - if (*ptr == '-') { - args->txnspec.ops[op][OP_REVERSE] = 1; - ptr++; - } else { - args->txnspec.ops[op][OP_REVERSE] = 0; - } - num = 0; - if ((*ptr < '0') || (*ptr > '9')) { - error = 1; - break; - } - while ((*ptr >= '0') && (*ptr <= '9')) { - num = num * 10 + *ptr - '0'; - ptr++; - } - /* set range */ - args->txnspec.ops[op][OP_RANGE] = num; - } - } - rangeop = 0; - } + if (rangeop) { + if (*ptr != ':') { + error = 1; + break; + } else { + ptr++; /* skip ':' */ + /* check negative '-' sign */ + if (*ptr == '-') { + args->txnspec.ops[op][OP_REVERSE] = 1; + ptr++; + } else { + args->txnspec.ops[op][OP_REVERSE] = 0; + } + num = 0; + if ((*ptr < '0') || (*ptr > '9')) { + error = 1; + break; + } + while ((*ptr >= '0') && (*ptr <= '9')) { + num = num * 10 + *ptr - '0'; + ptr++; + } + /* set range */ + args->txnspec.ops[op][OP_RANGE] = num; + } + } + rangeop = 0; + } - if (error) { - fprintf(stderr, "ERROR: invalid transaction specification %s\n", optarg); - return -1; - } + if (error) { + fprintf(stderr, "ERROR: invalid transaction specification %s\n", optarg); + return -1; + } - if (args->verbose == VERBOSE_DEBUG) { - for (op = 0; op < MAX_OP; op++) { - fprintf(debugme, "DEBUG: OP: %d: %d: %d\n", op, args->txnspec.ops[op][0], - args->txnspec.ops[op][1]); - } - } + if (args->verbose == VERBOSE_DEBUG) { + for (op = 0; op < MAX_OP; op++) { + fprintf(debugme, "DEBUG: OP: %d: %d: %d\n", op, args->txnspec.ops[op][0], args->txnspec.ops[op][1]); + } + } - return 0; + return 0; } - void usage() { - printf("Usage:\n"); - printf("%-24s%s\n", "-h, --help", "Print this message"); - printf("%-24s%s\n", " --version", "Print FDB version"); - printf("%-24s%s\n", "-v, --verbose", "Specify verbosity"); - printf("%-24s%s\n", "-a, --api_version=API_VERSION", "Specify API_VERSION to use"); - printf("%-24s%s\n", "-c, --cluster=FILE", "Specify FDB cluster file"); - printf("%-24s%s\n", "-p, --procs=PROCS", - "Specify number of worker processes"); - printf("%-24s%s\n", "-t, --threads=THREADS", - "Specify number of worker threads"); - printf("%-24s%s\n", "-r, --rows=ROWS", "Specify number of records"); - printf("%-24s%s\n", "-s, --seconds=SECONDS", - "Specify the test duration in seconds\n"); - printf("%-24s%s\n", "", "This option cannot be specified with --iteration."); - printf("%-24s%s\n", "-i, --iteration=ITERS", - "Specify the number of iterations.\n"); - printf("%-24s%s\n", "", "This option cannot be specified with --seconds."); - printf("%-24s%s\n", " --keylen=LENGTH", "Specify the key lengths"); - printf("%-24s%s\n", " --vallen=LENGTH", "Specify the value lengths"); - printf("%-24s%s\n", "-x, --transaction=SPEC", "Transaction specification"); - printf("%-24s%s\n", " --tps|--tpsmax=TPS", "Specify the target max TPS"); - printf("%-24s%s\n", " --tpsmin=TPS", "Specify the target min TPS"); - printf("%-24s%s\n", " --tpsinterval=SEC", "Specify the TPS change interval (Default: 10 seconds)"); - printf("%-24s%s\n", " --tpschange=", "Specify the TPS change type (Default: sin)"); - printf("%-24s%s\n", " --sampling=RATE", - "Specify the sampling rate for latency stats"); - printf("%-24s%s\n", "-m, --mode=MODE", - "Specify the mode (build, run, clean)"); - printf("%-24s%s\n", "-z, --zipf", - "Use zipfian distribution instead of uniform distribution"); - printf("%-24s%s\n", " --commitget", "Commit GETs"); - printf("%-24s%s\n", " --trace", "Enable tracing"); - printf("%-24s%s\n", " --tracepath=PATH", "Set trace file path"); - printf("%-24s%s\n", " --knobs=KNOBS", "Set client knobs"); - printf("%-24s%s\n", " --flatbuffers", "Use flatbuffers"); + printf("Usage:\n"); + printf("%-24s %s\n", "-h, --help", "Print this message"); + printf("%-24s %s\n", " --version", "Print FDB version"); + printf("%-24s %s\n", "-v, --verbose", "Specify verbosity"); + printf("%-24s %s\n", "-a, --api_version=API_VERSION", "Specify API_VERSION to use"); + printf("%-24s %s\n", "-c, --cluster=FILE", "Specify FDB cluster file"); + printf("%-24s %s\n", "-p, --procs=PROCS", "Specify number of worker processes"); + printf("%-24s %s\n", "-t, --threads=THREADS", "Specify number of worker threads"); + printf("%-24s %s\n", "-r, --rows=ROWS", "Specify number of records"); + printf("%-24s %s\n", "-s, --seconds=SECONDS", "Specify the test duration in seconds\n"); + printf("%-24s %s\n", "", "This option cannot be specified with --iteration."); + printf("%-24s %s\n", "-i, --iteration=ITERS", "Specify the number of iterations.\n"); + printf("%-24s %s\n", "", "This option cannot be specified with --seconds."); + printf("%-24s %s\n", " --keylen=LENGTH", "Specify the key lengths"); + printf("%-24s %s\n", " --vallen=LENGTH", "Specify the value lengths"); + printf("%-24s %s\n", "-x, --transaction=SPEC", "Transaction specification"); + printf("%-24s %s\n", " --tps|--tpsmax=TPS", "Specify the target max TPS"); + printf("%-24s %s\n", " --tpsmin=TPS", "Specify the target min TPS"); + printf("%-24s %s\n", " --tpsinterval=SEC", "Specify the TPS change interval (Default: 10 seconds)"); + printf("%-24s %s\n", " --tpschange=", "Specify the TPS change type (Default: sin)"); + printf("%-24s %s\n", " --sampling=RATE", "Specify the sampling rate for latency stats"); + printf("%-24s %s\n", "-m, --mode=MODE", "Specify the mode (build, run, clean)"); + printf("%-24s %s\n", "-z, --zipf", "Use zipfian distribution instead of uniform distribution"); + printf("%-24s %s\n", " --commitget", "Commit GETs"); + printf("%-24s %s\n", " --trace", "Enable tracing"); + printf("%-24s %s\n", " --tracepath=PATH", "Set trace file path"); + printf("%-24s %s\n", " --trace_format ", "Set trace format (Default: json)"); + printf("%-24s %s\n", " --txntrace=sec", "Specify transaction tracing interval (Default: 0)"); + printf("%-24s %s\n", " --knobs=KNOBS", "Set client knobs"); + printf("%-24s %s\n", " --flatbuffers", "Use flatbuffers"); } - /* parse benchmark paramters */ -int parse_args(int argc, char *argv[], mako_args_t *args) { - int rc; - int c; - int idx; - while (1) { - const char *short_options = "a:c:p:t:r:s:i:x:v:m:hjz"; - static struct option long_options[] = { - /* name, has_arg, flag, val */ - {"api_version", required_argument, NULL, 'a'}, - {"cluster", required_argument, NULL, 'c'}, - {"procs", required_argument, NULL, 'p'}, - {"threads", required_argument, NULL, 't'}, - {"rows", required_argument, NULL, 'r'}, - {"seconds", required_argument, NULL, 's'}, - {"iteration", required_argument, NULL, 'i'}, - {"keylen", required_argument, NULL, ARG_KEYLEN}, - {"vallen", required_argument, NULL, ARG_VALLEN}, - {"transaction", required_argument, NULL, 'x'}, - {"tps", required_argument, NULL, ARG_TPS}, - {"tpsmax", required_argument, NULL, ARG_TPSMAX}, - {"tpsmin", required_argument, NULL, ARG_TPSMIN}, - {"tpsinterval", required_argument, NULL, ARG_TPSINTERVAL}, - {"tpschange", required_argument, NULL, ARG_TPSCHANGE}, - {"sampling", required_argument, NULL, ARG_SAMPLING}, - {"verbose", required_argument, NULL, 'v'}, - {"mode", required_argument, NULL, 'm'}, - {"knobs", required_argument, NULL, ARG_KNOBS}, - {"tracepath", required_argument, NULL, ARG_TRACEPATH}, - /* no args */ - {"help", no_argument, NULL, 'h'}, - {"json", no_argument, NULL, 'j'}, - {"zipf", no_argument, NULL, 'z'}, - {"commitget", no_argument, NULL, ARG_COMMITGET}, - {"flatbuffers", no_argument, NULL, ARG_FLATBUFFERS}, - {"trace", no_argument, NULL, ARG_TRACE}, - {"version", no_argument, NULL, ARG_VERSION}, - {NULL, 0, NULL, 0}}; - idx = 0; - c = getopt_long(argc, argv, short_options, long_options, &idx); - if (c < 0) - break; - switch (c) { - case '?': - case 'h': - usage(); - return -1; - case 'a': - args->api_version = atoi(optarg); - break; - case 'c': - strcpy(args->cluster_file, optarg); - break; - case 'p': - args->num_processes = atoi(optarg); - break; - case 't': - args->num_threads = atoi(optarg); - break; - case 'r': - args->rows = atoi(optarg); - break; - case 's': - args->seconds = atoi(optarg); - break; - case 'i': - args->iteration = atoi(optarg); - break; - case 'x': - rc = parse_transaction(args, optarg); - if (rc < 0) - return -1; - break; - case 'v': - args->verbose = atoi(optarg); - break; - case 'z': - args->zipf = 1; - break; - case 'm': - if (strcmp(optarg, "clean") == 0) { - args->mode = MODE_CLEAN; - } else if (strcmp(optarg, "build") == 0) { - args->mode = MODE_BUILD; - } else if (strcmp(optarg, "run") == 0) { - args->mode = MODE_RUN; - } - break; - case ARG_KEYLEN: - args->key_length = atoi(optarg); - break; - case ARG_VALLEN: - args->value_length = atoi(optarg); - break; - case ARG_TPS: - case ARG_TPSMAX: - args->tpsmax = atoi(optarg); - break; - case ARG_TPSMIN: - args->tpsmin = atoi(optarg); - break; - case ARG_TPSINTERVAL: - args->tpsinterval = atoi(optarg); - break; - case ARG_TPSCHANGE: - if (strcmp(optarg, "sin") == 0) - args->tpschange = TPS_SIN; - else if (strcmp(optarg, "square") == 0) - args->tpschange = TPS_SQUARE; - else if (strcmp(optarg, "pulse") == 0) - args->tpschange = TPS_PULSE; - else { - fprintf(stderr, "--tpschange must be sin, square or pulse\n"); - return -1; - } - break; - case ARG_SAMPLING: - args->sampling = atoi(optarg); - break; - case ARG_VERSION: - fprintf(stderr, "Version: %d\n", FDB_API_VERSION); - exit(0); - break; - case ARG_COMMITGET: - args->commit_get = 1; - break; - case ARG_FLATBUFFERS: - args->flatbuffers = 1; - break; - case ARG_KNOBS: - memcpy(args->knobs, optarg, strlen(optarg) + 1); - break; - case ARG_TRACE: - args->trace = 1; - break; - case ARG_TRACEPATH: - args->trace = 1; - memcpy(args->tracepath, optarg, strlen(optarg) + 1); - break; - } - } - if ((args->tpsmin == -1) || (args->tpsmin > args->tpsmax)) { - args->tpsmin = args->tpsmax; - } +int parse_args(int argc, char* argv[], mako_args_t* args) { + int rc; + int c; + int idx; + while (1) { + const char* short_options = "a:c:p:t:r:s:i:x:v:m:hjz"; + static struct option long_options[] = { /* name, has_arg, flag, val */ + { "api_version", required_argument, NULL, 'a' }, + { "cluster", required_argument, NULL, 'c' }, + { "procs", required_argument, NULL, 'p' }, + { "threads", required_argument, NULL, 't' }, + { "rows", required_argument, NULL, 'r' }, + { "seconds", required_argument, NULL, 's' }, + { "iteration", required_argument, NULL, 'i' }, + { "keylen", required_argument, NULL, ARG_KEYLEN }, + { "vallen", required_argument, NULL, ARG_VALLEN }, + { "transaction", required_argument, NULL, 'x' }, + { "tps", required_argument, NULL, ARG_TPS }, + { "tpsmax", required_argument, NULL, ARG_TPSMAX }, + { "tpsmin", required_argument, NULL, ARG_TPSMIN }, + { "tpsinterval", required_argument, NULL, ARG_TPSINTERVAL }, + { "tpschange", required_argument, NULL, ARG_TPSCHANGE }, + { "sampling", required_argument, NULL, ARG_SAMPLING }, + { "verbose", required_argument, NULL, 'v' }, + { "mode", required_argument, NULL, 'm' }, + { "knobs", required_argument, NULL, ARG_KNOBS }, + { "tracepath", required_argument, NULL, ARG_TRACEPATH }, + { "trace_format", required_argument, NULL, ARG_TRACEFORMAT }, + { "txntrace", required_argument, NULL, ARG_TXNTRACE }, + /* no args */ + { "help", no_argument, NULL, 'h' }, + { "json", no_argument, NULL, 'j' }, + { "zipf", no_argument, NULL, 'z' }, + { "commitget", no_argument, NULL, ARG_COMMITGET }, + { "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS }, + { "trace", no_argument, NULL, ARG_TRACE }, + { "version", no_argument, NULL, ARG_VERSION }, + { NULL, 0, NULL, 0 } + }; + idx = 0; + c = getopt_long(argc, argv, short_options, long_options, &idx); + if (c < 0) break; + switch (c) { + case '?': + case 'h': + usage(); + return -1; + case 'a': + args->api_version = atoi(optarg); + break; + case 'c': + strcpy(args->cluster_file, optarg); + break; + case 'p': + args->num_processes = atoi(optarg); + break; + case 't': + args->num_threads = atoi(optarg); + break; + case 'r': + args->rows = atoi(optarg); + break; + case 's': + args->seconds = atoi(optarg); + break; + case 'i': + args->iteration = atoi(optarg); + break; + case 'x': + rc = parse_transaction(args, optarg); + if (rc < 0) return -1; + break; + case 'v': + args->verbose = atoi(optarg); + break; + case 'z': + args->zipf = 1; + break; + case 'm': + if (strcmp(optarg, "clean") == 0) { + args->mode = MODE_CLEAN; + } else if (strcmp(optarg, "build") == 0) { + args->mode = MODE_BUILD; + } else if (strcmp(optarg, "run") == 0) { + args->mode = MODE_RUN; + } + break; + case ARG_KEYLEN: + args->key_length = atoi(optarg); + break; + case ARG_VALLEN: + args->value_length = atoi(optarg); + break; + case ARG_TPS: + case ARG_TPSMAX: + args->tpsmax = atoi(optarg); + break; + case ARG_TPSMIN: + args->tpsmin = atoi(optarg); + break; + case ARG_TPSINTERVAL: + args->tpsinterval = atoi(optarg); + break; + case ARG_TPSCHANGE: + if (strcmp(optarg, "sin") == 0) + args->tpschange = TPS_SIN; + else if (strcmp(optarg, "square") == 0) + args->tpschange = TPS_SQUARE; + else if (strcmp(optarg, "pulse") == 0) + args->tpschange = TPS_PULSE; + else { + fprintf(stderr, "--tpschange must be sin, square or pulse\n"); + return -1; + } + break; + case ARG_SAMPLING: + args->sampling = atoi(optarg); + break; + case ARG_VERSION: + fprintf(stderr, "Version: %d\n", FDB_API_VERSION); + exit(0); + break; + case ARG_COMMITGET: + args->commit_get = 1; + break; + case ARG_FLATBUFFERS: + args->flatbuffers = 1; + break; + case ARG_KNOBS: + memcpy(args->knobs, optarg, strlen(optarg) + 1); + break; + case ARG_TRACE: + args->trace = 1; + break; + case ARG_TRACEPATH: + args->trace = 1; + memcpy(args->tracepath, optarg, strlen(optarg) + 1); + break; + case ARG_TRACEFORMAT: + if (strncmp(optarg, "json", 5) == 0) { + args->traceformat = 1; + } else if (strncmp(optarg, "xml", 4) == 0) { + args->traceformat = 0; + } else { + fprintf(stderr, "Error: Invalid trace_format %s\n", optarg); + exit(0); + } + break; + case ARG_TXNTRACE: + args->txntrace = atoi(optarg); + break; + } + } + if ((args->tpsmin == -1) || (args->tpsmin > args->tpsmax)) { + args->tpsmin = args->tpsmax; + } - if (args->verbose >= VERBOSE_DEFAULT) { - printme = stdout; - } else { - printme = fopen("/dev/null", "w"); - } - if (args->verbose >= VERBOSE_ANNOYING) { - annoyme = stdout; - } else { - annoyme = fopen("/dev/null", "w"); - } - if (args->verbose >= VERBOSE_DEBUG) { - debugme = stdout; - } else { - debugme = fopen("/dev/null", "w"); - } - - return 0; + if (args->verbose >= VERBOSE_DEFAULT) { + printme = stdout; + } else { + printme = fopen("/dev/null", "w"); + } + if (args->verbose >= VERBOSE_ANNOYING) { + annoyme = stdout; + } else { + annoyme = fopen("/dev/null", "w"); + } + if (args->verbose >= VERBOSE_DEBUG) { + debugme = stdout; + } else { + debugme = fopen("/dev/null", "w"); + } + + return 0; } - -int validate_args(mako_args_t *args) { - if (args->mode == MODE_INVALID) { - fprintf(stderr, "ERROR: --mode has to be set\n"); - return -1; - } - if (args->rows <= 0) { - fprintf(stderr, "ERROR: --rows must be a positive integer\n"); - return -1; - } - if (args->key_length < 0) { - fprintf(stderr, "ERROR: --keylen must be a positive integer\n"); - return -1; - } - if (args->value_length < 0) { - fprintf(stderr, "ERROR: --vallen must be a positive integer\n"); - return -1; - } - if (args->key_length < 4 /* "mako" */ + digits(args->rows)) { - fprintf(stderr, - "ERROR: --keylen must be larger than %d to store \"mako\" prefix " - "and maximum row number\n", - 4 + digits(args->rows)); - return -1; - } - if (args->mode == MODE_RUN) { - if ((args->seconds > 0) && (args->iteration > 0)) { - fprintf(stderr, "ERROR: Cannot specify seconds and iteration together\n"); - return -1; - } - if ((args->seconds == 0) && (args->iteration == 0)) { - fprintf(stderr, "ERROR: Must specify either seconds or iteration\n"); - return -1; - } - } - return 0; +int validate_args(mako_args_t* args) { + if (args->mode == MODE_INVALID) { + fprintf(stderr, "ERROR: --mode has to be set\n"); + return -1; + } + if (args->rows <= 0) { + fprintf(stderr, "ERROR: --rows must be a positive integer\n"); + return -1; + } + if (args->key_length < 0) { + fprintf(stderr, "ERROR: --keylen must be a positive integer\n"); + return -1; + } + if (args->value_length < 0) { + fprintf(stderr, "ERROR: --vallen must be a positive integer\n"); + return -1; + } + if (args->key_length < 4 /* "mako" */ + digits(args->rows)) { + fprintf(stderr, + "ERROR: --keylen must be larger than %d to store \"mako\" prefix " + "and maximum row number\n", + 4 + digits(args->rows)); + return -1; + } + if (args->mode == MODE_RUN) { + if ((args->seconds > 0) && (args->iteration > 0)) { + fprintf(stderr, "ERROR: Cannot specify seconds and iteration together\n"); + return -1; + } + if ((args->seconds == 0) && (args->iteration == 0)) { + fprintf(stderr, "ERROR: Must specify either seconds or iteration\n"); + return -1; + } + } + return 0; } - /* stats output formatting */ #define STR2(x) #x #define STR(x) STR2(x) #define STATS_TITLE_WIDTH 12 #define STATS_FIELD_WIDTH 12 -void print_stats(mako_args_t *args, mako_stats_t *stats, struct timespec *now, - struct timespec *prev) { - int i, j; - int op; - int print_err; - static uint64_t ops_total_prev[MAX_OP] = {0}; - uint64_t ops_total[MAX_OP] = {0}; - static uint64_t errors_total_prev[MAX_OP] = {0}; - uint64_t errors_total[MAX_OP] = {0}; - uint64_t errors_diff[MAX_OP] = {0}; - static uint64_t totalxacts_prev = 0; - uint64_t totalxacts = 0; - static uint64_t conflicts_prev = 0; - uint64_t conflicts = 0; - double durationns = (now->tv_sec - prev->tv_sec) * 1000000000.0 + - (now->tv_nsec - prev->tv_nsec); +void print_stats(mako_args_t* args, mako_stats_t* stats, struct timespec* now, struct timespec* prev) { + int i, j; + int op; + int print_err; + static uint64_t ops_total_prev[MAX_OP] = { 0 }; + uint64_t ops_total[MAX_OP] = { 0 }; + static uint64_t errors_total_prev[MAX_OP] = { 0 }; + uint64_t errors_total[MAX_OP] = { 0 }; + uint64_t errors_diff[MAX_OP] = { 0 }; + static uint64_t totalxacts_prev = 0; + uint64_t totalxacts = 0; + static uint64_t conflicts_prev = 0; + uint64_t conflicts = 0; + double durationns = (now->tv_sec - prev->tv_sec) * 1000000000.0 + (now->tv_nsec - prev->tv_nsec); - for (i = 0; i < args->num_processes; i++) { - for (j = 0; j < args->num_threads; j++) { - totalxacts += stats[(i * args->num_threads) + j].xacts; - conflicts += stats[(i * args->num_threads) + j].conflicts; - for (op = 0; op < MAX_OP; op++) { - ops_total[op] += stats[(i * args->num_threads) + j].ops[op]; - errors_total[op] += stats[(i * args->num_threads) + j].errors[op]; - } - } - } - printf("%" STR(STATS_TITLE_WIDTH) "s ", "OPS"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", - ops_total[op] - ops_total_prev[op]); - errors_diff[op] = errors_total[op] - errors_total_prev[op]; - print_err = (errors_diff[op] > 0); - ops_total_prev[op] = ops_total[op]; - errors_total_prev[op] = errors_total[op]; - } - } - /* TPS */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f ", - (totalxacts - totalxacts_prev) * 1000000000.0 / durationns); - totalxacts_prev = totalxacts; - - /* Conflicts */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", - (conflicts - conflicts_prev) * 1000000000.0 / durationns); - conflicts_prev = conflicts; - - if (print_err) { - printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_diff[op]); - } - } - printf("\n"); - } - return; -} - - -void print_stats_header(mako_args_t *args) { - int op; - int i; - - /* header */ - for (i = 0; i <= STATS_TITLE_WIDTH; i++) - printf(" "); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - switch (op) { - case OP_GETREADVERSION: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "GRV"); - break; - case OP_GET: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "GET"); - break; - case OP_GETRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "GETRANGE"); - break; - case OP_SGET: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGET"); - break; - case OP_SGETRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGETRANGE"); - break; - case OP_UPDATE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "UPDATE"); - break; - case OP_INSERT: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERT"); - break; - case OP_INSERTRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERTRANGE"); - break; - case OP_CLEAR: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEAR"); - break; - case OP_SETCLEAR: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLEAR"); - break; - case OP_CLEARRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEARRANGE"); - break; - case OP_SETCLEARRANGE: - printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLRRANGE"); - break; - } - } - } - printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS"); - printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); - - for (i = 0; i < STATS_TITLE_WIDTH; i++) - printf("="); - printf(" "); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0) { - for (i = 0; i < STATS_FIELD_WIDTH; i++) - printf("="); - printf(" "); - } - } - /* TPS */ - for (i = 0; i < STATS_FIELD_WIDTH; i++) - printf("="); - printf(" "); - /* Conflicts */ - for (i = 0; i < STATS_FIELD_WIDTH; i++) - printf("="); - printf("\n"); -} - - -void print_report(mako_args_t *args, mako_stats_t *stats, - struct timespec *timer_now, struct timespec *timer_start) { - int i, j, op; - uint64_t totalxacts = 0; - uint64_t conflicts = 0; - uint64_t totalerrors = 0; - uint64_t ops_total[MAX_OP] = {0}; - uint64_t errors_total[MAX_OP] = {0}; - uint64_t lat_min[MAX_OP] = {0}; - uint64_t lat_total[MAX_OP] = {0}; - uint64_t lat_samples[MAX_OP] = {0}; - uint64_t lat_max[MAX_OP] = {0}; - - uint64_t durationns = (timer_now->tv_sec - timer_start->tv_sec) * 1000000000 + - (timer_now->tv_nsec - timer_start->tv_nsec); - - for (op = 0; op < MAX_OP; op++) { - lat_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ - lat_max[op] = 0; - lat_total[op] = 0; - lat_samples[op] = 0; - } - - for (i = 0; i < args->num_processes; i++) { - for (j = 0; j < args->num_threads; j++) { - int idx = i * args->num_threads + j; - totalxacts += stats[idx].xacts; - conflicts += stats[idx].conflicts; - for (op = 0; op < MAX_OP; op++) { - if ((args->txnspec.ops[op][OP_COUNT] > 0) || (op == OP_COMMIT)) { - totalerrors += stats[idx].errors[op]; - ops_total[op] += stats[idx].ops[op]; - errors_total[op] += stats[idx].errors[op]; - lat_total[op] += stats[idx].latency_us_total[op]; - lat_samples[op] += stats[idx].latency_samples[op]; - if (stats[idx].latency_us_min[op] < lat_min[op]) { - lat_min[op] = stats[idx].latency_us_min[op]; - } - if (stats[idx].latency_us_max[op] > lat_max[op]) { - lat_max[op] = stats[idx].latency_us_max[op]; - } - } /* if count > 0 */ - } - } - } - - /* overall stats */ - printf("\n====== Total Duration %6.3f sec ======\n\n", - (double)durationns / 1000000000); - printf("Total Processes: %8d\n", args->num_processes); - printf("Total Threads: %8d\n", args->num_threads); - if (args->tpsmax == args->tpsmin) - printf("Target TPS: %8d\n", args->tpsmax); - else { - printf("Target TPS (MAX): %8d\n", args->tpsmax); - printf("Target TPS (MIN): %8d\n", args->tpsmin); - printf("TPS Interval: %8d\n", args->tpsinterval); - printf("TPS Change: "); - switch (args->tpschange) { - case TPS_SIN: printf("%8s\n", "SIN"); break; - case TPS_SQUARE: printf("%8s\n", "SQUARE"); break; - case TPS_PULSE: printf("%8s\n", "PULSE"); break; - } - } - printf("Total Xacts: %8lld\n", totalxacts); - printf("Total Conflicts: %8lld\n", conflicts); - printf("Total Errors: %8lld\n", totalerrors); - printf("Overall TPS: %8lld\n\n", totalxacts * 1000000000 / durationns); - - /* per-op stats */ - print_stats_header(args); - - /* OPS */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Total OPS"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op]); - } - } - /* TPS */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f ", - totalxacts * 1000000000.0 / durationns); - /* Conflicts */ - printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", - conflicts * 1000000000.0 / durationns); - - /* Errors */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Errors"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_total[op]); - } - } - printf("\n"); - - /* Min Latency */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Min (us)"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { - if (lat_min[op] == -1) { - printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); - } else { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_min[op]); - } - } - } - printf("\n"); - - /* Avg Latency */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Avg (us)"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { - if (lat_total[op]) { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", - lat_total[op] / lat_samples[op]); - } else { - printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); - } - } - } - printf("\n"); - - /* Max Latency */ - printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Max (us)"); - for (op = 0; op < MAX_OP; op++) { - if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { - if (lat_max[op] == 0) { - printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); - } else { - printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_max[op]); - } - } - } - printf("\n"); -} - - -int stats_process_main(mako_args_t *args, mako_stats_t *stats, - volatile double *throttle_factor, volatile int *signal) { - struct timespec timer_start, timer_prev, timer_now; - double sin_factor; - - /* wait until the signal turn on */ - while (*signal == SIGNAL_OFF) { - usleep(10000); /* 10ms */ - } - - if (args->verbose >= VERBOSE_DEFAULT) - print_stats_header(args); - - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); - timer_prev.tv_sec = timer_start.tv_sec; - timer_prev.tv_nsec = timer_start.tv_nsec; - while (*signal != SIGNAL_RED) { - usleep(100000); /* sleep for 100ms */ - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - - /* print stats every (roughly) 1 sec */ - if (timer_now.tv_sec > timer_prev.tv_sec) { - - /* adjust throttle rate if needed */ - if (args->tpsmax != args->tpsmin) { - /* set the throttle factor between 0.0 and 1.0 */ - switch (args->tpschange) { - case TPS_SIN: - sin_factor = sin((timer_now.tv_sec % args->tpsinterval) / (double)args->tpsinterval * M_PI * 2) / 2.0 + 0.5; - *throttle_factor = 1 - (sin_factor * (1.0 - ((double)args->tpsmin / args->tpsmax))); - break; - case TPS_SQUARE: - if (timer_now.tv_sec % args->tpsinterval < (args->tpsinterval / 2)) { - /* set to max */ - *throttle_factor = 1.0; - } else { - /* set to min */ - *throttle_factor = (double)args->tpsmin / (double)args->tpsmax; - } - break; - case TPS_PULSE: - if (timer_now.tv_sec % args->tpsinterval == 0) { - /* set to max */ - *throttle_factor = 1.0; - } else { - /* set to min */ - *throttle_factor = (double)args->tpsmin / (double)args->tpsmax; - } - break; + for (i = 0; i < args->num_processes; i++) { + for (j = 0; j < args->num_threads; j++) { + totalxacts += stats[(i * args->num_threads) + j].xacts; + conflicts += stats[(i * args->num_threads) + j].conflicts; + for (op = 0; op < MAX_OP; op++) { + ops_total[op] += stats[(i * args->num_threads) + j].ops[op]; + errors_total[op] += stats[(i * args->num_threads) + j].errors[op]; + } + } } - } - - if (args->verbose >= VERBOSE_DEFAULT) - print_stats(args, stats, &timer_now, &timer_prev); - timer_prev.tv_sec = timer_now.tv_sec; - timer_prev.tv_nsec = timer_now.tv_nsec; - } + printf("%" STR(STATS_TITLE_WIDTH) "s ", "OPS"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op] - ops_total_prev[op]); + errors_diff[op] = errors_total[op] - errors_total_prev[op]; + print_err = (errors_diff[op] > 0); + ops_total_prev[op] = ops_total[op]; + errors_total_prev[op] = errors_total[op]; + } + } + /* TPS */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f ", (totalxacts - totalxacts_prev) * 1000000000.0 / durationns); + totalxacts_prev = totalxacts; - } + /* Conflicts */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", (conflicts - conflicts_prev) * 1000000000.0 / durationns); + conflicts_prev = conflicts; - /* print report */ - if (args->verbose >= VERBOSE_DEFAULT) { - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - print_report(args, stats, &timer_now, &timer_start); - } - - return 0; + if (print_err) { + printf("%" STR(STATS_TITLE_WIDTH) "s ", "Errors"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_diff[op]); + } + } + printf("\n"); + } + return; } +void print_stats_header(mako_args_t* args) { + int op; + int i; -int main(int argc, char *argv[]) { - int rc; - mako_args_t args; - int p; - pid_t *worker_pids; - proc_type_t proc_type = proc_master; - int worker_id; - pid_t pid; - int status; - mako_shmhdr_t *shm; /* shmhdr + stats */ - int shmfd; - char shmpath[NAME_MAX]; - size_t shmsize; - mako_stats_t *stats; + /* header */ + for (i = 0; i <= STATS_TITLE_WIDTH; i++) printf(" "); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + switch (op) { + case OP_GETREADVERSION: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "GRV"); + break; + case OP_GET: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "GET"); + break; + case OP_GETRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "GETRANGE"); + break; + case OP_SGET: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGET"); + break; + case OP_SGETRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SGETRANGE"); + break; + case OP_UPDATE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "UPDATE"); + break; + case OP_INSERT: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERT"); + break; + case OP_INSERTRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "INSERTRANGE"); + break; + case OP_CLEAR: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEAR"); + break; + case OP_SETCLEAR: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLEAR"); + break; + case OP_CLEARRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "CLEARRANGE"); + break; + case OP_SETCLEARRANGE: + printf("%" STR(STATS_FIELD_WIDTH) "s ", "SETCLRRANGE"); + break; + } + } + } + printf("%" STR(STATS_FIELD_WIDTH) "s ", "TPS"); + printf("%" STR(STATS_FIELD_WIDTH) "s\n", "Conflicts/s"); - rc = init_args(&args); - if (rc < 0) { - fprintf(stderr, "ERROR: init_args failed\n"); - return -1; - } - rc = parse_args(argc, argv, &args); - if (rc < 0) { - /* usage printed */ - return 0; - } + for (i = 0; i < STATS_TITLE_WIDTH; i++) printf("="); + printf(" "); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0) { + for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); + printf(" "); + } + } + /* TPS */ + for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); + printf(" "); + /* Conflicts */ + for (i = 0; i < STATS_FIELD_WIDTH; i++) printf("="); + printf("\n"); +} - rc = validate_args(&args); - if (rc < 0) - return -1; +void print_report(mako_args_t* args, mako_stats_t* stats, struct timespec* timer_now, struct timespec* timer_start) { + int i, j, op; + uint64_t totalxacts = 0; + uint64_t conflicts = 0; + uint64_t totalerrors = 0; + uint64_t ops_total[MAX_OP] = { 0 }; + uint64_t errors_total[MAX_OP] = { 0 }; + uint64_t lat_min[MAX_OP] = { 0 }; + uint64_t lat_total[MAX_OP] = { 0 }; + uint64_t lat_samples[MAX_OP] = { 0 }; + uint64_t lat_max[MAX_OP] = { 0 }; - if (args.mode == MODE_CLEAN) { - /* cleanup will be done from a single thread */ - args.num_processes = 1; - args.num_threads = 1; - } + uint64_t durationns = + (timer_now->tv_sec - timer_start->tv_sec) * 1000000000 + (timer_now->tv_nsec - timer_start->tv_nsec); - if (args.mode == MODE_BUILD) { - if (args.txnspec.ops[OP_INSERT][OP_COUNT] == 0) { - parse_transaction(&args, "i100"); - } - } + for (op = 0; op < MAX_OP; op++) { + lat_min[op] = 0xFFFFFFFFFFFFFFFF; /* uint64_t */ + lat_max[op] = 0; + lat_total[op] = 0; + lat_samples[op] = 0; + } - /* create the shared memory for stats */ - sprintf(shmpath, "mako%d", getpid()); - shmfd = shm_open(shmpath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); - if (shmfd < 0) { - fprintf(stderr, "ERROR: shm_open failed\n"); - return -1; - } + for (i = 0; i < args->num_processes; i++) { + for (j = 0; j < args->num_threads; j++) { + int idx = i * args->num_threads + j; + totalxacts += stats[idx].xacts; + conflicts += stats[idx].conflicts; + for (op = 0; op < MAX_OP; op++) { + if ((args->txnspec.ops[op][OP_COUNT] > 0) || (op == OP_COMMIT)) { + totalerrors += stats[idx].errors[op]; + ops_total[op] += stats[idx].ops[op]; + errors_total[op] += stats[idx].errors[op]; + lat_total[op] += stats[idx].latency_us_total[op]; + lat_samples[op] += stats[idx].latency_samples[op]; + if (stats[idx].latency_us_min[op] < lat_min[op]) { + lat_min[op] = stats[idx].latency_us_min[op]; + } + if (stats[idx].latency_us_max[op] > lat_max[op]) { + lat_max[op] = stats[idx].latency_us_max[op]; + } + } /* if count > 0 */ + } + } + } - /* allocate */ - shmsize = sizeof(mako_shmhdr_t) + - (sizeof(mako_stats_t) * args.num_processes * args.num_threads); - if (ftruncate(shmfd, shmsize) < 0) { - fprintf(stderr, "ERROR: ftruncate (fd:%d size:%llu) failed\n", shmfd, - (unsigned long long)shmsize); - goto failExit; - } + /* overall stats */ + printf("\n====== Total Duration %6.3f sec ======\n\n", (double)durationns / 1000000000); + printf("Total Processes: %8d\n", args->num_processes); + printf("Total Threads: %8d\n", args->num_threads); + if (args->tpsmax == args->tpsmin) + printf("Target TPS: %8d\n", args->tpsmax); + else { + printf("Target TPS (MAX): %8d\n", args->tpsmax); + printf("Target TPS (MIN): %8d\n", args->tpsmin); + printf("TPS Interval: %8d\n", args->tpsinterval); + printf("TPS Change: "); + switch (args->tpschange) { + case TPS_SIN: + printf("%8s\n", "SIN"); + break; + case TPS_SQUARE: + printf("%8s\n", "SQUARE"); + break; + case TPS_PULSE: + printf("%8s\n", "PULSE"); + break; + } + } + printf("Total Xacts: %8lld\n", totalxacts); + printf("Total Conflicts: %8lld\n", conflicts); + printf("Total Errors: %8lld\n", totalerrors); + printf("Overall TPS: %8lld\n\n", totalxacts * 1000000000 / durationns); - /* map it */ - shm = (mako_shmhdr_t *)mmap(NULL, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, - shmfd, 0); - if (shm == MAP_FAILED) { - fprintf(stderr, "ERROR: mmap (fd:%d size:%llu) failed\n", shmfd, - (unsigned long long)shmsize); - goto failExit; - } + /* per-op stats */ + print_stats_header(args); - stats = (mako_stats_t *)((void *)shm + sizeof(mako_shmhdr_t)); + /* OPS */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Total OPS"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", ops_total[op]); + } + } + /* TPS */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f ", totalxacts * 1000000000.0 / durationns); + /* Conflicts */ + printf("%" STR(STATS_FIELD_WIDTH) ".2f\n", conflicts * 1000000000.0 / durationns); - /* initialize the shared memory */ - memset(shm, 0, shmsize); + /* Errors */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Errors"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 && op != OP_COMMIT) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", errors_total[op]); + } + } + printf("\n"); - /* get ready */ - shm->signal = SIGNAL_OFF; - shm->readycount = 0; - shm->throttle_factor = 1.0; + /* Min Latency */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Min (us)"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { + if (lat_min[op] == -1) { + printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); + } else { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_min[op]); + } + } + } + printf("\n"); - /* fork worker processes + 1 stats process */ - worker_pids = (pid_t *)calloc(sizeof(pid_t), args.num_processes + 1); - if (!worker_pids) { - fprintf(stderr, "ERROR: cannot allocate worker_pids (%d processes)\n", - args.num_processes); - goto failExit; - } + /* Avg Latency */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Avg (us)"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { + if (lat_total[op]) { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_total[op] / lat_samples[op]); + } else { + printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); + } + } + } + printf("\n"); - /* forking (num_process + 1) children */ - /* last process is the stats handler */ - for (p = 0; p < args.num_processes + 1; p++) { - pid = fork(); - if (pid != 0) { - /* master */ - worker_pids[p] = pid; - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: worker %d (PID:%d) forked\n", p, worker_pids[p]); - } - } else { - if (p < args.num_processes) { - /* worker process */ - proc_type = proc_worker; - worker_id = p; - } else { - /* stats */ - proc_type = proc_stats; - } - break; - } - } + /* Max Latency */ + printf("%-" STR(STATS_TITLE_WIDTH) "s ", "Lat Max (us)"); + for (op = 0; op < MAX_OP; op++) { + if (args->txnspec.ops[op][OP_COUNT] > 0 || op == OP_COMMIT) { + if (lat_max[op] == 0) { + printf("%" STR(STATS_FIELD_WIDTH) "s ", "N/A"); + } else { + printf("%" STR(STATS_FIELD_WIDTH) "lld ", lat_max[op]); + } + } + } + printf("\n"); +} - /* initialize the randomizer */ - srand(time(0) * getpid()); +int stats_process_main(mako_args_t* args, mako_stats_t* stats, volatile double* throttle_factor, volatile int* signal) { + struct timespec timer_start, timer_prev, timer_now; + double sin_factor; - /* initialize zipfian if necessary (per-process) */ - if (args.zipf) { - zipfian_generator(args.rows); - } + /* wait until the signal turn on */ + while (*signal == SIGNAL_OFF) { + usleep(10000); /* 10ms */ + } - if (proc_type == proc_worker) { - /* worker process */ - worker_process_main(&args, worker_id, shm); - /* worker can exit here */ - exit(0); - } else if (proc_type == proc_stats) { - /* stats */ - if (args.mode == MODE_CLEAN) { - /* no stats needed for clean mode */ - exit(0); - } - stats_process_main(&args, stats, &shm->throttle_factor, &shm->signal); - exit(0); - } + if (args->verbose >= VERBOSE_DEFAULT) print_stats_header(args); - /* master */ + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); + timer_prev.tv_sec = timer_start.tv_sec; + timer_prev.tv_nsec = timer_start.tv_nsec; + while (*signal != SIGNAL_RED) { + usleep(100000); /* sleep for 100ms */ + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - /* wait for everyone to be ready */ - while (shm->readycount < (args.num_processes * args.num_threads)) { - usleep(1000); - } - shm->signal = SIGNAL_GREEN; + /* print stats every (roughly) 1 sec */ + if (timer_now.tv_sec > timer_prev.tv_sec) { - if (args.mode == MODE_RUN) { - struct timespec timer_start, timer_now; + /* adjust throttle rate if needed */ + if (args->tpsmax != args->tpsmin) { + /* set the throttle factor between 0.0 and 1.0 */ + switch (args->tpschange) { + case TPS_SIN: + sin_factor = + sin((timer_now.tv_sec % args->tpsinterval) / (double)args->tpsinterval * M_PI * 2) / 2.0 + 0.5; + *throttle_factor = 1 - (sin_factor * (1.0 - ((double)args->tpsmin / args->tpsmax))); + break; + case TPS_SQUARE: + if (timer_now.tv_sec % args->tpsinterval < (args->tpsinterval / 2)) { + /* set to max */ + *throttle_factor = 1.0; + } else { + /* set to min */ + *throttle_factor = (double)args->tpsmin / (double)args->tpsmax; + } + break; + case TPS_PULSE: + if (timer_now.tv_sec % args->tpsinterval == 0) { + /* set to max */ + *throttle_factor = 1.0; + } else { + /* set to min */ + *throttle_factor = (double)args->tpsmin / (double)args->tpsmax; + } + break; + } + } - /* run the benchamrk */ + if (args->verbose >= VERBOSE_DEFAULT) print_stats(args, stats, &timer_now, &timer_prev); + timer_prev.tv_sec = timer_now.tv_sec; + timer_prev.tv_nsec = timer_now.tv_nsec; + } + } - /* if seconds is specified, stop child processes after the specified - * duration */ - if (args.seconds > 0) { - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: master sleeping for %d seconds\n", args.seconds); - } + /* print report */ + if (args->verbose >= VERBOSE_DEFAULT) { + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + print_report(args, stats, &timer_now, &timer_start); + } - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); - while (1) { - usleep(100000); /* sleep for 100ms */ - clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); - /* doesn't have to be precise */ - if (timer_now.tv_sec - timer_start.tv_sec > args.seconds) { - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: time's up (%d seconds)\n", args.seconds); - } - break; - } - } + return 0; +} - /* notify everyone the time's up */ - shm->signal = SIGNAL_RED; - } - } +int main(int argc, char* argv[]) { + int rc; + mako_args_t args; + int p; + pid_t* worker_pids; + proc_type_t proc_type = proc_master; + int worker_id; + pid_t pid; + int status; + mako_shmhdr_t* shm; /* shmhdr + stats */ + int shmfd; + char shmpath[NAME_MAX]; + size_t shmsize; + mako_stats_t* stats; - /* wait for worker processes to exit */ - for (p = 0; p < args.num_processes; p++) { - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: waiting worker %d (PID:%d) to exit\n", p, worker_pids[p]); - } - pid = waitpid(worker_pids[p], &status, 0 /* or what? */); - if (pid < 0) { - fprintf(stderr, "ERROR: waitpid failed for worker process PID %d\n", - worker_pids[p]); - } - if (args.verbose == VERBOSE_DEBUG) { - printf("DEBUG: worker %d (PID:%d) exited\n", p, worker_pids[p]); - } - } + rc = init_args(&args); + if (rc < 0) { + fprintf(stderr, "ERROR: init_args failed\n"); + return -1; + } + rc = parse_args(argc, argv, &args); + if (rc < 0) { + /* usage printed */ + return 0; + } - /* all worker threads finished, stop the stats */ - if (args.mode == MODE_BUILD || args.iteration > 0) { - shm->signal = SIGNAL_RED; - } + rc = validate_args(&args); + if (rc < 0) return -1; - /* wait for stats to stop */ - pid = waitpid(worker_pids[args.num_processes], &status, 0 /* or what? */); - if (pid < 0) { - fprintf(stderr, "ERROR: waitpid failed for stats process PID %d\n", - worker_pids[args.num_processes]); - } + if (args.mode == MODE_CLEAN) { + /* cleanup will be done from a single thread */ + args.num_processes = 1; + args.num_threads = 1; + } + + if (args.mode == MODE_BUILD) { + if (args.txnspec.ops[OP_INSERT][OP_COUNT] == 0) { + parse_transaction(&args, "i100"); + } + } + + /* create the shared memory for stats */ + sprintf(shmpath, "mako%d", getpid()); + shmfd = shm_open(shmpath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (shmfd < 0) { + fprintf(stderr, "ERROR: shm_open failed\n"); + return -1; + } + + /* allocate */ + shmsize = sizeof(mako_shmhdr_t) + (sizeof(mako_stats_t) * args.num_processes * args.num_threads); + if (ftruncate(shmfd, shmsize) < 0) { + fprintf(stderr, "ERROR: ftruncate (fd:%d size:%llu) failed\n", shmfd, (unsigned long long)shmsize); + goto failExit; + } + + /* map it */ + shm = (mako_shmhdr_t*)mmap(NULL, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); + if (shm == MAP_FAILED) { + fprintf(stderr, "ERROR: mmap (fd:%d size:%llu) failed\n", shmfd, (unsigned long long)shmsize); + goto failExit; + } + + stats = (mako_stats_t*)((void*)shm + sizeof(mako_shmhdr_t)); + + /* initialize the shared memory */ + memset(shm, 0, shmsize); + + /* get ready */ + shm->signal = SIGNAL_OFF; + shm->readycount = 0; + shm->throttle_factor = 1.0; + + /* fork worker processes + 1 stats process */ + worker_pids = (pid_t*)calloc(sizeof(pid_t), args.num_processes + 1); + if (!worker_pids) { + fprintf(stderr, "ERROR: cannot allocate worker_pids (%d processes)\n", args.num_processes); + goto failExit; + } + + /* forking (num_process + 1) children */ + /* last process is the stats handler */ + for (p = 0; p < args.num_processes + 1; p++) { + pid = fork(); + if (pid != 0) { + /* master */ + worker_pids[p] = pid; + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: worker %d (PID:%d) forked\n", p, worker_pids[p]); + } + } else { + if (p < args.num_processes) { + /* worker process */ + proc_type = proc_worker; + worker_id = p; + } else { + /* stats */ + proc_type = proc_stats; + } + break; + } + } + + /* initialize the randomizer */ + srand(time(0) * getpid()); + + /* initialize zipfian if necessary (per-process) */ + if (args.zipf) { + zipfian_generator(args.rows); + } + + if (proc_type == proc_worker) { + /* worker process */ + worker_process_main(&args, worker_id, shm); + /* worker can exit here */ + exit(0); + } else if (proc_type == proc_stats) { + /* stats */ + if (args.mode == MODE_CLEAN) { + /* no stats needed for clean mode */ + exit(0); + } + stats_process_main(&args, stats, &shm->throttle_factor, &shm->signal); + exit(0); + } + + /* master */ + + /* wait for everyone to be ready */ + while (shm->readycount < (args.num_processes * args.num_threads)) { + usleep(1000); + } + shm->signal = SIGNAL_GREEN; + + if (args.mode == MODE_RUN) { + struct timespec timer_start, timer_now; + + /* run the benchamrk */ + + /* if seconds is specified, stop child processes after the specified + * duration */ + if (args.seconds > 0) { + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: master sleeping for %d seconds\n", args.seconds); + } + + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_start); + while (1) { + usleep(100000); /* sleep for 100ms */ + clock_gettime(CLOCK_MONOTONIC_COARSE, &timer_now); + /* doesn't have to be precise */ + if (timer_now.tv_sec - timer_start.tv_sec > args.seconds) { + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: time's up (%d seconds)\n", args.seconds); + } + break; + } + } + + /* notify everyone the time's up */ + shm->signal = SIGNAL_RED; + } + } + + /* wait for worker processes to exit */ + for (p = 0; p < args.num_processes; p++) { + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: waiting worker %d (PID:%d) to exit\n", p, worker_pids[p]); + } + pid = waitpid(worker_pids[p], &status, 0 /* or what? */); + if (pid < 0) { + fprintf(stderr, "ERROR: waitpid failed for worker process PID %d\n", worker_pids[p]); + } + if (args.verbose == VERBOSE_DEBUG) { + printf("DEBUG: worker %d (PID:%d) exited\n", p, worker_pids[p]); + } + } + + /* all worker threads finished, stop the stats */ + if (args.mode == MODE_BUILD || args.iteration > 0) { + shm->signal = SIGNAL_RED; + } + + /* wait for stats to stop */ + pid = waitpid(worker_pids[args.num_processes], &status, 0 /* or what? */); + if (pid < 0) { + fprintf(stderr, "ERROR: waitpid failed for stats process PID %d\n", worker_pids[args.num_processes]); + } failExit: - if (worker_pids) - free(worker_pids); + if (worker_pids) free(worker_pids); - if (shm != MAP_FAILED) - munmap(shm, shmsize); + if (shm != MAP_FAILED) munmap(shm, shmsize); - if (shmfd) { - close(shmfd); - shm_unlink(shmpath); - } + if (shmfd) { + close(shmfd); + shm_unlink(shmpath); + } - return 0; + return 0; } diff --git a/bindings/c/test/mako/mako.h b/bindings/c/test/mako/mako.h old mode 100755 new mode 100644 index d924f8a648..4f703e7271 --- a/bindings/c/test/mako/mako.h +++ b/bindings/c/test/mako/mako.h @@ -3,7 +3,7 @@ #pragma once #ifndef FDB_API_VERSION -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #endif #include @@ -32,23 +32,22 @@ #define FDB_ERROR_ABORT -2 #define FDB_ERROR_CONFLICT -3 - /* transaction specification */ enum Operations { - OP_GETREADVERSION, - OP_GET, - OP_GETRANGE, - OP_SGET, - OP_SGETRANGE, - OP_UPDATE, - OP_INSERT, - OP_INSERTRANGE, - OP_CLEAR, - OP_SETCLEAR, - OP_CLEARRANGE, - OP_SETCLEARRANGE, - OP_COMMIT, - MAX_OP /* must be the last item */ + OP_GETREADVERSION, + OP_GET, + OP_GETRANGE, + OP_SGET, + OP_SGETRANGE, + OP_UPDATE, + OP_INSERT, + OP_INSERTRANGE, + OP_CLEAR, + OP_SETCLEAR, + OP_CLEARRANGE, + OP_SETCLEARRANGE, + OP_COMMIT, + MAX_OP /* must be the last item */ }; #define OP_COUNT 0 @@ -57,27 +56,25 @@ enum Operations { /* for long arguments */ enum Arguments { - ARG_KEYLEN, - ARG_VALLEN, - ARG_TPS, - ARG_COMMITGET, - ARG_SAMPLING, - ARG_VERSION, - ARG_KNOBS, - ARG_FLATBUFFERS, - ARG_TRACE, - ARG_TRACEPATH, - ARG_TPSMAX, - ARG_TPSMIN, - ARG_TPSINTERVAL, - ARG_TPSCHANGE + ARG_KEYLEN, + ARG_VALLEN, + ARG_TPS, + ARG_COMMITGET, + ARG_SAMPLING, + ARG_VERSION, + ARG_KNOBS, + ARG_FLATBUFFERS, + ARG_TRACE, + ARG_TRACEPATH, + ARG_TRACEFORMAT, + ARG_TPSMAX, + ARG_TPSMIN, + ARG_TPSINTERVAL, + ARG_TPSCHANGE, + ARG_TXNTRACE }; -enum TPSChangeTypes { - TPS_SIN, - TPS_SQUARE, - TPS_PULSE -}; +enum TPSChangeTypes { TPS_SIN, TPS_SQUARE, TPS_PULSE }; #define KEYPREFIX "mako" #define KEYPREFIXLEN 4 @@ -87,38 +84,40 @@ enum TPSChangeTypes { */ typedef struct { - /* for each operation, it stores "count", "range" and "reverse" */ - int ops[MAX_OP][3]; + /* for each operation, it stores "count", "range" and "reverse" */ + int ops[MAX_OP][3]; } mako_txnspec_t; #define KNOB_MAX 256 /* benchmark parameters */ typedef struct { - int api_version; - int json; - int num_processes; - int num_threads; - int mode; - int rows; /* is 2 billion enough? */ - int seconds; - int iteration; - int tpsmax; - int tpsmin; - int tpsinterval; - int tpschange; - int sampling; - int key_length; - int value_length; - int zipf; - int commit_get; - int verbose; - mako_txnspec_t txnspec; - char cluster_file[PATH_MAX]; - int trace; - char tracepath[PATH_MAX]; - char knobs[KNOB_MAX]; - uint8_t flatbuffers; + int api_version; + int json; + int num_processes; + int num_threads; + int mode; + int rows; /* is 2 billion enough? */ + int seconds; + int iteration; + int tpsmax; + int tpsmin; + int tpsinterval; + int tpschange; + int sampling; + int key_length; + int value_length; + int zipf; + int commit_get; + int verbose; + mako_txnspec_t txnspec; + char cluster_file[PATH_MAX]; + int trace; + char tracepath[PATH_MAX]; + int traceformat; /* 0 - XML, 1 - JSON */ + char knobs[KNOB_MAX]; + uint8_t flatbuffers; + int txntrace; } mako_args_t; /* shared memory */ @@ -127,34 +126,34 @@ typedef struct { #define SIGNAL_OFF 2 typedef struct { - int signal; - int readycount; - double throttle_factor; + int signal; + int readycount; + double throttle_factor; } mako_shmhdr_t; typedef struct { - uint64_t xacts; - uint64_t conflicts; - uint64_t ops[MAX_OP]; - uint64_t errors[MAX_OP]; - uint64_t latency_samples[MAX_OP]; - uint64_t latency_us_total[MAX_OP]; - uint64_t latency_us_min[MAX_OP]; - uint64_t latency_us_max[MAX_OP]; + uint64_t xacts; + uint64_t conflicts; + uint64_t ops[MAX_OP]; + uint64_t errors[MAX_OP]; + uint64_t latency_samples[MAX_OP]; + uint64_t latency_us_total[MAX_OP]; + uint64_t latency_us_min[MAX_OP]; + uint64_t latency_us_max[MAX_OP]; } mako_stats_t; /* per-process information */ typedef struct { - int worker_id; - FDBDatabase *database; - mako_args_t *args; - mako_shmhdr_t *shm; + int worker_id; + FDBDatabase* database; + mako_args_t* args; + mako_shmhdr_t* shm; } process_info_t; /* args for threads */ typedef struct { - int thread_id; - process_info_t *process; + int thread_id; + process_info_t* process; } thread_args_t; /* process type */ diff --git a/bindings/c/test/mako/mako.rst b/bindings/c/test/mako/mako.rst index 05dcb525fc..8a63944869 100644 --- a/bindings/c/test/mako/mako.rst +++ b/bindings/c/test/mako/mako.rst @@ -1,27 +1,27 @@ ############## -mako Benchmark +🦈 Mako Benchmark ############## -| mako (named after a small, but very fast shark) is a micro-benchmark for FoundationDB +| Mako (named after a very fast shark) is a micro-benchmark for FoundationDB | which is designed to be very light and flexible | so that you can stress a particular part of an FoundationDB cluster without introducing unnecessary overhead. How to Build ============ -| ``mako`` gets build automatically when you build FoundationDB. +| ``mako`` gets built automatically when you build FoundationDB. | To build ``mako`` manually, simply build ``mako`` target in the FoundationDB build directory. -| e.g. If you're using Unix Makefiles +| e.g. If you're using Unix Makefiles, type: | ``make mako`` Architecture ============ - mako is a stand-alone program written in C, - which communicates to FoundationDB using C binding API (``libfdb_c.so``) -- It creates one master process, and one or more worker processes (multi-process) -- Each worker process creates one or more multiple threads (multi-thread) -- All threads within the same process share the same network thread + which communicates to FoundationDB using C API (via ``libfdb_c.so``) +- It creates one master process, one stats emitter process, and one or more worker processes (multi-process) +- Each worker process creates one FDB network thread, and one or more worker threads (multi-thread) +- All worker threads within the same process share the same network thread Data Specification @@ -32,18 +32,18 @@ Data Specification Arguments ========= -- | ``--mode `` +- | ``-m | --mode `` | One of the following modes must be specified. (Required) | - ``clean``: Clean up existing data | - ``build``: Populate data | - ``run``: Run the benchmark -- | ``-a | --api_version `` - | FDB API version to use (Default: Latest) - - | ``-c | --cluster `` | FDB cluster file (Required) +- | ``-a | --api_version `` + | FDB API version to use (Default: Latest) + - | ``-p | --procs `` | Number of worker processes (Default: 1) @@ -51,7 +51,7 @@ Arguments | Number of threads per worker process (Default: 1) - | ``-r | --rows `` - | Number of rows populated (Default: 100000) + | Number of rows initially populated (Default: 100000) - | ``-s | --seconds `` | Test duration in seconds (Default: 30) @@ -113,10 +113,10 @@ Arguments Transaction Specification ========================= -| A transaction may contain multiple operations of multiple types. +| A transaction may contain multiple operations of various types. | You can specify multiple operations for one operation type by specifying "Count". -| For RANGE operations, "Range" needs to be specified in addition to "Count". -| Every transaction is committed unless it contains only GET / GET RANGE operations. +| For RANGE operations, the "Range" needs to be specified in addition to "Count". +| Every transaction is committed unless the transaction is read-only. Operation Types --------------- @@ -137,21 +137,22 @@ Format ------ | One operation type is defined as ```` or ``:``. | When Count is omitted, it's equivalent to setting it to 1. (e.g. ``g`` is equivalent to ``g1``) -| Multiple operation types can be concatenated. (e.g. ``g9u1`` = 9 GETs and 1 update) +| Multiple operation types within the same trancaction can be concatenated. (e.g. ``g9u1`` = 9 GETs and 1 update) Transaction Specification Examples ---------------------------------- -- | 100 GETs (No Commit) +- | 100 GETs (Non-commited) | ``g100`` -- | 10 GET RANGE with Range of 50 (No Commit) +- | 10 GET RANGE with Range of 50 (Non-commited) | ``gr10:50`` - | 90 GETs and 10 Updates (Committed) | ``g90u10`` -- | 80 GETs, 10 Updates and 10 Inserts (Committed) - | ``g90u10i10`` +- | 70 GETs, 10 Updates and 10 Inserts (Committed) + | ``g70u10i10`` + | This is 80-20. Execution Examples @@ -160,12 +161,14 @@ Execution Examples Preparation ----------- - Start the FoundationDB cluster and create a database -- Set LD_LIBRARY_PATH pointing to a proper ``libfdb_c.so`` +- Set ``LD_LIBRARY_PATH`` environment variable pointing to a proper ``libfdb_c.so`` shared library -Build ------ +Populate Initial Database +------------------------- ``mako --cluster /etc/foundationdb/fdb.cluster --mode build --rows 1000000 --procs 4`` +Note: You may be able to speed up the data population by increasing the number of processes or threads. Run --- +Run a mixed workload with a total of 8 threads for 60 seconds, keeping the throughput limited to 1000 TPS. ``mako --cluster /etc/foundationdb/fdb.cluster --mode run --rows 1000000 --procs 2 --threads 8 --transaction "g8ui" --seconds 60 --tps 1000`` diff --git a/bindings/c/test/mako/utils.c b/bindings/c/test/mako/utils.c old mode 100755 new mode 100644 index 9af57d58a4..20f5daee77 --- a/bindings/c/test/mako/utils.c +++ b/bindings/c/test/mako/utils.c @@ -1,81 +1,79 @@ -#include -#include -#include #include "utils.h" #include "mako.h" +#include +#include +#include /* uniform-distribution random */ int urand(int low, int high) { - double r = rand() / (1.0 + RAND_MAX); - int range = high - low + 1; - return (int)((r * range) + low); + double r = rand() / (1.0 + RAND_MAX); + int range = high - low + 1; + return (int)((r * range) + low); } /* random string */ /* len is the buffer size, must include null */ -void randstr(char *str, int len) { - int i; - for (i = 0; i < len-1; i++) { - str[i] = '!' + urand(0, 'z'-'!'); /* generage a char from '!' to 'z' */ - } - str[len-1] = '\0'; +void randstr(char* str, int len) { + int i; + for (i = 0; i < len - 1; i++) { + str[i] = '!' + urand(0, 'z' - '!'); /* generage a char from '!' to 'z' */ + } + str[len - 1] = '\0'; } /* random numeric string */ /* len is the buffer size, must include null */ -void randnumstr(char *str, int len) { - int i; - for (i = 0; i < len-1; i++) { - str[i] = '0' + urand(0, 9); /* generage a char from '!' to 'z' */ - } - str[len-1] = '\0'; +void randnumstr(char* str, int len) { + int i; + for (i = 0; i < len - 1; i++) { + str[i] = '0' + urand(0, 9); /* generage a char from '!' to 'z' */ + } + str[len - 1] = '\0'; } /* return the first key to be inserted */ int insert_begin(int rows, int p_idx, int t_idx, int total_p, int total_t) { - double interval = (double)rows / total_p / total_t; - return (int)(round(interval * ((p_idx * total_t) + t_idx))); + double interval = (double)rows / total_p / total_t; + return (int)(round(interval * ((p_idx * total_t) + t_idx))); } /* return the last key to be inserted */ int insert_end(int rows, int p_idx, int t_idx, int total_p, int total_t) { - double interval = (double)rows / total_p / total_t; - return (int)(round(interval * ((p_idx * total_t) + t_idx + 1) - 1)); + double interval = (double)rows / total_p / total_t; + return (int)(round(interval * ((p_idx * total_t) + t_idx + 1) - 1)); } /* devide val equally among threads */ int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, int total_t) { - int interval = val / total_p / total_t; - int remaining = val - (interval * total_p * total_t); - if ((p_idx * total_t + t_idx) < remaining) { - return interval+1; - } else if (interval == 0) { - return -1; - } - /* else */ - return interval; + int interval = val / total_p / total_t; + int remaining = val - (interval * total_p * total_t); + if ((p_idx * total_t + t_idx) < remaining) { + return interval + 1; + } else if (interval == 0) { + return -1; + } + /* else */ + return interval; } /* number of digits */ int digits(int num) { - int digits = 0; - while (num > 0) { - num /= 10; - digits++; - } - return digits; + int digits = 0; + while (num > 0) { + num /= 10; + digits++; + } + return digits; } - /* generate a key for a given key number */ /* len is the buffer size, key length + null */ -void genkey(char *str, int num, int rows, int len) { - int i; - int rowdigit = digits(rows); - sprintf(str, KEYPREFIX "%0.*d", rowdigit, num); - for (i = (KEYPREFIXLEN + rowdigit); i < len-1; i++) { - str[i] = 'x'; - } - str[len-1] = '\0'; +void genkey(char* str, int num, int rows, int len) { + int i; + int rowdigit = digits(rows); + sprintf(str, KEYPREFIX "%0.*d", rowdigit, num); + for (i = (KEYPREFIXLEN + rowdigit); i < len - 1; i++) { + str[i] = 'x'; + } + str[len - 1] = '\0'; } - diff --git a/bindings/c/test/mako/utils.h b/bindings/c/test/mako/utils.h old mode 100755 new mode 100644 index a95607ff6a..5fde4ef25a --- a/bindings/c/test/mako/utils.h +++ b/bindings/c/test/mako/utils.h @@ -9,12 +9,12 @@ int urand(int low, int high); /* write a random string of the length of (len-1) to memory pointed by str * with a null-termination character at str[len-1]. */ -void randstr(char *str, int len); +void randstr(char* str, int len); /* write a random numeric string of the length of (len-1) to memory pointed by str * with a null-termination character at str[len-1]. */ -void randnumstr(char *str, int len); +void randnumstr(char* str, int len); /* given the total number of rows to be inserted, * the worker process index p_idx and the thread index t_idx (both 0-based), @@ -27,26 +27,25 @@ int insert_begin(int rows, int p_idx, int t_idx, int total_p, int total_t); int insert_end(int rows, int p_idx, int t_idx, int total_p, int total_t); /* devide a value equally among threads */ -int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, - int total_t); +int compute_thread_portion(int val, int p_idx, int t_idx, int total_p, int total_t); /* similar to insert_begin/end, compute_thread_tps computes * the per-thread target TPS for given configuration. */ -#define compute_thread_tps(val, p_idx, t_idx, total_p, total_t) \ - compute_thread_portion(val, p_idx, t_idx, total_p, total_t) +#define compute_thread_tps(val, p_idx, t_idx, total_p, total_t) \ + compute_thread_portion(val, p_idx, t_idx, total_p, total_t) /* similar to compute_thread_tps, * compute_thread_iters computs the number of iterations. */ -#define compute_thread_iters(val, p_idx, t_idx, total_p, total_t) \ - compute_thread_portion(val, p_idx, t_idx, total_p, total_t) +#define compute_thread_iters(val, p_idx, t_idx, total_p, total_t) \ + compute_thread_portion(val, p_idx, t_idx, total_p, total_t) /* get the number of digits */ int digits(int num); /* generate a key for a given key number */ /* len is the buffer size, key length + null */ -void genkey(char *str, int num, int rows, int len); +void genkey(char* str, int num, int rows, int len); #endif /* UTILS_H */ diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index edfb9a96d1..7a265e7d0f 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -603,7 +603,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(620), "select API version", rs); + checkError(fdb_select_api_version(630), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index 1777150894..cbb7fcf304 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -244,7 +244,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(620), "select API version", rs); + checkError(fdb_select_api_version(630), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, keySize); diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index 895691c265..5fb4268b78 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -29,7 +29,7 @@ #include #ifndef FDB_API_VERSION -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #endif #include @@ -236,7 +236,7 @@ void* runNetwork() { FDBDatabase* openDatabase(struct ResultSet *rs, pthread_t *netThread) { checkError(fdb_setup_network(), "setup network", rs); - pthread_create(netThread, NULL, &runNetwork, NULL); + pthread_create(netThread, NULL, (void*)(&runNetwork), NULL); FDBDatabase *db; checkError(fdb_create_database(NULL, &db), "create database", rs); diff --git a/bindings/c/test/txn_size_test.c b/bindings/c/test/txn_size_test.c index 73ee9a82e6..4f2744d199 100644 --- a/bindings/c/test/txn_size_test.c +++ b/bindings/c/test/txn_size_test.c @@ -97,7 +97,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(620), "select API version", rs); + checkError(fdb_select_api_version(630), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, KEY_SIZE); diff --git a/bindings/c/test/workloads/SimpleWorkload.cpp b/bindings/c/test/workloads/SimpleWorkload.cpp index 3d61f85cb9..35b18f71a3 100644 --- a/bindings/c/test/workloads/SimpleWorkload.cpp +++ b/bindings/c/test/workloads/SimpleWorkload.cpp @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #include "foundationdb/fdb_c.h" #undef DLLEXPORT #include "workloads.h" @@ -258,7 +258,7 @@ struct SimpleWorkload : FDBWorkload { insertsPerTx = context->getOption("insertsPerTx", 100ul); opsPerTx = context->getOption("opsPerTx", 100ul); runFor = context->getOption("runFor", 10.0); - auto err = fdb_select_api_version(620); + auto err = fdb_select_api_version(630); if (err) { context->trace(FDBSeverity::Info, "SelectAPIVersionFailed", { { "Error", std::string(fdb_get_error(err)) } }); diff --git a/bindings/flow/CMakeLists.txt b/bindings/flow/CMakeLists.txt index d4b4a25b04..d57378744c 100644 --- a/bindings/flow/CMakeLists.txt +++ b/bindings/flow/CMakeLists.txt @@ -16,7 +16,7 @@ set(SRCS fdb_flow.actor.cpp fdb_flow.h) -add_flow_target(NAME fdb_flow SRCS ${SRCS} STATIC_LIBRARY) +add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS}) target_link_libraries(fdb_flow PUBLIC fdb_c) add_subdirectory(tester) diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 810f088e1c..3ed3d93700 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -25,6 +25,7 @@ #include "flow/DeterministicRandom.h" #include "flow/SystemMonitor.h" +#include "flow/TLSConfig.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. using namespace FDB; @@ -35,7 +36,7 @@ THREAD_FUNC networkThread(void* fdb) { } ACTOR Future _test() { - API *fdb = FDB::API::selectAPIVersion(620); + API *fdb = FDB::API::selectAPIVersion(630); auto db = fdb->createDatabase(); state Reference tr = db->createTransaction(); @@ -78,11 +79,11 @@ ACTOR Future _test() { } void fdb_flow_test() { - API *fdb = FDB::API::selectAPIVersion(620); + API *fdb = FDB::API::selectAPIVersion(630); fdb->setupNetwork(); startThread(networkThread, fdb); - g_network = newNet2( false ); + g_network = newNet2(TLSConfig()); openTraceFile(NetworkAddress(), 1000000, 1000000, "."); systemMonitor(); @@ -131,6 +132,8 @@ namespace FDB { GetRangeLimits limits = GetRangeLimits(), bool snapshot = false, bool reverse = false, FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; + + Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; void addReadConflictRange(KeyRangeRef const& keys) override; void addReadConflictKey(KeyRef const& key) override; @@ -345,6 +348,14 @@ namespace FDB { } ); } + Future TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) { + return backToFuture(fdb_transaction_get_estimated_range_size_bytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), [](Reference f) { + int64_t bytes; + throw_on_error(fdb_future_get_int64(f->f, &bytes)); + return bytes; + }); + } + void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { throw_on_error( fdb_transaction_add_conflict_range( tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ ) ); } diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index 90f77e67cc..e261052fae 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -23,7 +23,7 @@ #include -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #include #undef DLLEXPORT @@ -89,6 +89,8 @@ namespace FDB { streamingMode); } + virtual Future getEstimatedRangeSizeBytes(const KeyRange& keys) = 0; + virtual void addReadConflictRange(KeyRangeRef const& keys) = 0; virtual void addReadConflictKey(KeyRef const& key) = 0; diff --git a/bindings/flow/fdb_flow.vcxproj b/bindings/flow/fdb_flow.vcxproj deleted file mode 100755 index d6a71427c0..0000000000 --- a/bindings/flow/fdb_flow.vcxproj +++ /dev/null @@ -1,152 +0,0 @@ - - - - - - FDB_CLEAN_BUILD;%(PreprocessorDefinitions) - - - - Debug - X64 - - - Release - X64 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {2BA0A5E2-EB4C-4A32-948C-CBAABD77AF87} - v4.5.2 - Win32Proj - fdb_flow - - - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - $(IntDir)\$(MSBuildProjectName).log - - - - StaticLibrary - MultiByte - v141 - - - StaticLibrary - MultiByte - v141 - - - - - - - - - - true - ..\..\;C:\Program Files\boost_1_67_0;$(IncludePath) - - - false - ..\..\;C:\Program Files\boost_1_67_0;$(IncludePath) - - - - - - Level3 - false - ProgramDatabase - Disabled - EnableFastChecks - MultiThreadedDebug - TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - %(AdditionalIncludeDirectories);..\c - true - /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - Console - true - Advapi32.lib - - - $(TargetDir)flow.lib - - - - - Level3 - - - ProgramDatabase - Full - MultiThreaded - true - TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - %(AdditionalIncludeDirectories);..\c - NotSet - false - /bigobj @..\..\flow/no_intellisense.opt %(AdditionalOptions) - true - Speed - false - stdcpp17 - - - Console - true - false - false - Default - Advapi32.lib - /LTCG %(AdditionalOptions) - - - $(TargetDir)flow.lib - - - - - - - - - - diff --git a/bindings/flow/local.mk b/bindings/flow/local.mk deleted file mode 100644 index 7299e6a22a..0000000000 --- a/bindings/flow/local.mk +++ /dev/null @@ -1,44 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdb_flow_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS) -fdb_flow_LDFLAGS := -Llib -lfdb_c $(fdbrpc_LDFLAGS) -fdb_flow_LIBS := - -packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz: fdb_flow - @echo "Packaging fdb_flow" - @rm -rf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH) - @mkdir -p packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/flow packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/c/foundationdb - @cp lib/libfdb_flow.a packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/lib - @find bindings/flow -name '*.h' -not -path 'bindings/flow/tester/*' -exec cp {} packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/flow \; - @find bindings/c/foundationdb -name '*.h' -exec cp {} packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH)/include/bindings/c/foundationdb \; - @tar czf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz -C packages fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH) - @rm -rf packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH) - -FDB_FLOW: packages/fdb-flow-$(FLOWVER)-$(PLATFORM)-$(ARCH).tar.gz - -FDB_FLOW_clean: - @echo "Cleaning fdb_flow package" - @rm -rf packages/fdb-flow-*.tar.gz - -packages: FDB_FLOW -packages_clean: FDB_FLOW_clean diff --git a/bindings/flow/tester/CMakeLists.txt b/bindings/flow/tester/CMakeLists.txt index 4e017cddba..dd7f8988de 100644 --- a/bindings/flow/tester/CMakeLists.txt +++ b/bindings/flow/tester/CMakeLists.txt @@ -2,5 +2,5 @@ set(TEST_SRCS DirectoryTester.actor.cpp Tester.actor.cpp Tester.actor.h) -add_flow_target(NAME fdb_flow_tester EXECUTABLE SRCS ${TEST_SRCS}) +add_flow_target(EXECUTABLE NAME fdb_flow_tester SRCS ${TEST_SRCS}) target_link_libraries(fdb_flow_tester fdb_flow) diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index 52d193320e..a190299747 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -28,6 +28,7 @@ #include "bindings/flow/FDBLoanerTypes.h" #include "fdbrpc/fdbrpc.h" #include "flow/DeterministicRandom.h" +#include "flow/TLSConfig.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. // Otherwise we have to type setupNetwork(), FDB::open(), etc. @@ -216,19 +217,19 @@ ACTOR Future< Standalone > getRange(Reference tr, K } } -ACTOR static Future debugPrintRange(Reference tr, std::string subspace, std::string msg) { - if (!tr) - return Void(); - - Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff')))); - printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), - StringRef(subspace).printable().c_str(), results.size()); - for (auto & s : results) { - printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); - } - - return Void(); -} +//ACTOR static Future debugPrintRange(Reference tr, std::string subspace, std::string msg) { +// if (!tr) +// return Void(); +// +// Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + '\xff')))); +// printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), +// StringRef(subspace).printable().c_str(), results.size()); +// for (auto & s : results) { +// printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); +// } +// +// return Void(); +//} ACTOR Future stackSub(FlowTesterStack* stack) { if (stack->data.size() < 2) @@ -429,9 +430,8 @@ struct LogStackFunc : InstructionFunc { wait(logStack(data, entries, prefix)); entries.clear(); } - - wait(logStack(data, entries, prefix)); } + wait(logStack(data, entries, prefix)); return Void(); } @@ -638,6 +638,29 @@ struct GetFunc : InstructionFunc { const char* GetFunc::name = "GET"; REGISTER_INSTRUCTION_FUNC(GetFunc); +struct GetEstimatedRangeSize : InstructionFunc { + static const char* name; + + ACTOR static Future call(Reference data, Reference instruction) { + state std::vector items = data->stack.pop(2); + if (items.size() != 2) + return Void(); + + Standalone s1 = wait(items[0].value); + state Standalone beginKey = Tuple::unpack(s1).getString(0); + + Standalone s2 = wait(items[1].value); + state Standalone endKey = Tuple::unpack(s2).getString(0); + Future fsize = instruction->tr->getEstimatedRangeSizeBytes(KeyRangeRef(beginKey, endKey)); + int64_t size = wait(fsize); + data->stack.pushTuple(LiteralStringRef("GOT_ESTIMATED_RANGE_SIZE")); + + return Void(); + } +}; +const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE"; +REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize); + struct GetKeyFunc : InstructionFunc { static const char* name; @@ -1603,6 +1626,7 @@ struct UnitTestsFunc : InstructionFunc { tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS); Optional > _ = wait(tr->get(LiteralStringRef("\xff"))); tr->cancel(); @@ -1748,7 +1772,7 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi populateOpsThatCreateDirectories(); // FIXME // This is "our" network - g_network = newNet2(false); + g_network = newNet2(TLSConfig()); ASSERT(!API::isAPIVersionSelected()); try { @@ -1791,9 +1815,9 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi ACTOR void _test_versionstamp() { try { - g_network = newNet2(false); + g_network = newNet2(TLSConfig()); - API *fdb = FDB::API::selectAPIVersion(620); + API *fdb = FDB::API::selectAPIVersion(630); fdb->setupNetwork(); startThread(networkThread, fdb); diff --git a/bindings/flow/tester/fdb_flow_tester.vcxproj b/bindings/flow/tester/fdb_flow_tester.vcxproj deleted file mode 100644 index ef8405f60a..0000000000 --- a/bindings/flow/tester/fdb_flow_tester.vcxproj +++ /dev/null @@ -1,131 +0,0 @@ - - - - - -PRERELEASE - - - - - - - - Debug - x64 - - - Release - x64 - - - - - - - - - - - {086EB89C-CDBD-4ABE-8296-5CA224244C80} - Win32Proj - fdb_flow_tester - - - - Application - true - MultiByte - v141 - - - Application - false - false - MultiByte - v141 - - - - - - - - - - - - - true - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - $(IncludePath);../../../;C:\Program Files\boost_1_67_0 - - - false - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - $(IncludePath);../;C:\Program Files\boost_1_67_0 - - - - FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions) - stdcpp17 - - - - - - - Level3 - Disabled - TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;BOOST_ALL_NO_LIB;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - true - false - MultiThreadedDebug - @../../../flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - Console - true - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;$(SolutionDir)bin\$(Configuration)\fdb_c.lib;$(SolutionDir)bin\$(Configuration)\fdb_flow.lib;Advapi32.lib - - - - - - - - - Level3 - - - Full - true - TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - true - Speed - MultiThreaded - false - StreamingSIMDExtensions2 - @../flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - Console - true - false - false - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;$(SolutionDir)bin\$(Configuration)\fdb_c.lib;$(SolutionDir)bin\$(Configuration)\fdb_flow.lib;Advapi32.lib - Default - - - - - - - - - - - diff --git a/bindings/flow/tester/fdb_flow_tester.vcxproj.filters b/bindings/flow/tester/fdb_flow_tester.vcxproj.filters deleted file mode 100644 index f3a7efcce6..0000000000 --- a/bindings/flow/tester/fdb_flow_tester.vcxproj.filters +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/bindings/flow/tester/local.mk b/bindings/flow/tester/local.mk deleted file mode 100644 index 83444774bd..0000000000 --- a/bindings/flow/tester/local.mk +++ /dev/null @@ -1,41 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdb_flow_tester_CFLAGS := -Ibindings/c $(fdbrpc_CFLAGS) -fdb_flow_tester_LDFLAGS := -Llib $(fdbrpc_LDFLAGS) -lfdb_c -fdb_flow_tester_LIBS := lib/libfdb_flow.a lib/libflow.a lib/libfdb_c.$(DLEXT) - -fdb_flow_tester: lib/libfdb_c.$(DLEXT) - @mkdir -p bindings/flow/bin - @rm -f bindings/flow/bin/fdb_flow_tester - @cp bin/fdb_flow_tester bindings/flow/bin/fdb_flow_tester - -fdb_flow_tester_clean: _fdb_flow_tester_clean - -_fdb_flow_tester_clean: - @rm -rf bindings/flow/bin - -ifeq ($(PLATFORM),linux) - fdb_flow_tester_LDFLAGS += -static-libstdc++ -static-libgcc -ldl -lpthread -lrt -lm -else ifeq ($(PLATFORM),osx) - fdb_flow_tester_LDFLAGS += -lc++ -endif diff --git a/bindings/go/CMakeLists.txt b/bindings/go/CMakeLists.txt index 793089a3f7..701fa49ca8 100644 --- a/bindings/go/CMakeLists.txt +++ b/bindings/go/CMakeLists.txt @@ -99,6 +99,8 @@ function(build_go_package) endif() add_custom_command(OUTPUT ${outfile} COMMAND ${CMAKE_COMMAND} -E env ${go_env} + ${GO_EXECUTABLE} get -d ${GO_IMPORT_PATH}/${BGP_PATH} && + ${CMAKE_COMMAND} -E env ${go_env} ${GO_EXECUTABLE} install ${GO_IMPORT_PATH}/${BGP_PATH} DEPENDS ${fdb_options_file} COMMENT "Building ${BGP_NAME}") diff --git a/bindings/go/README.md b/bindings/go/README.md index 6e2a90a684..7a03ea1d6f 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -9,7 +9,7 @@ This package requires: - [Mono](http://www.mono-project.com/) (macOS or Linux) or [Visual Studio](https://www.visualstudio.com/) (Windows) (build-time only) - FoundationDB C API 2.0.x-6.1.x (part of the [FoundationDB client packages](https://apple.github.io/foundationdb/downloads.html#c)) -Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-620. +Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-630. To install this package, you can run the "fdb-go-install.sh" script (for versions 5.0.x and greater): diff --git a/bindings/go/fdb-go-install.sh b/bindings/go/fdb-go-install.sh index ff3c739cc8..148f4e50ea 100755 --- a/bindings/go/fdb-go-install.sh +++ b/bindings/go/fdb-go-install.sh @@ -25,6 +25,9 @@ platform=$(uname) if [[ "${platform}" == "Darwin" ]] ; then FDBLIBDIR="${FDBLIBDIR:-/usr/local/lib}" libfdbc="libfdb_c.dylib" +elif [[ "${platform}" == "FreeBSD" ]] ; then + FDBLIBDIR="${FDBLIBDIR:-/lib}" + libfdbc="libfdb_c.so" elif [[ "${platform}" == "Linux" ]] ; then libfdbc="libfdb_c.so" custom_libdir="${FDBLIBDIR:-}" @@ -248,8 +251,11 @@ else : elif [[ "${status}" -eq 0 ]] ; then echo "Building generated files." + if [[ "${platform}" == "FreeBSD" ]] ; then + cmd=( 'gmake' '-C' "${fdbdir}" 'bindings/c/foundationdb/fdb_c_options.g.h' ) + else cmd=( 'make' '-C' "${fdbdir}" 'bindings/c/foundationdb/fdb_c_options.g.h' ) - + fi echo "${cmd[*]}" if ! "${cmd[@]}" ; then let status="${status} + 1" diff --git a/bindings/go/include.mk b/bindings/go/include.mk deleted file mode 100644 index 358404309e..0000000000 --- a/bindings/go/include.mk +++ /dev/null @@ -1,103 +0,0 @@ -# -# include.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TARGETS += fdb_go fdb_go_tester -CLEAN_TARGETS += fdb_go_clean fdb_go_tester_clean - -GOPATH := $(CURDIR)/bindings/go/build -GO_IMPORT_PATH := github.com/apple/foundationdb/bindings/go/src -GO_DEST := $(GOPATH)/src/$(GO_IMPORT_PATH) - -.PHONY: fdb_go fdb_go_path fdb_go_fmt fdb_go_fmt_check fdb_go_tester fdb_go_tester_clean - -# We only override if the environment didn't set it (this is used by -# the fdbwebsite documentation build process) -GODOC_DIR ?= bindings/go - -CGO_CFLAGS := -I$(CURDIR)/bindings/c -CGO_LDFLAGS := -L$(CURDIR)/lib - -ifeq ($(PLATFORM),linux) - GOPLATFORM := linux_amd64 -else ifeq ($(PLATFORM),osx) - GOPLATFORM := darwin_amd64 -else - $(error Not prepared to compile on platform $(PLATFORM)) -endif - -GO_PACKAGE_OUTDIR := $(GOPATH)/pkg/$(GOPLATFORM)/$(GO_IMPORT_PATH) - -GO_PACKAGES := fdb fdb/tuple fdb/subspace fdb/directory -GO_PACKAGE_OBJECTS := $(addprefix $(GO_PACKAGE_OUTDIR)/,$(GO_PACKAGES:=.a)) - -GO_GEN := $(CURDIR)/bindings/go/src/fdb/generated.go -GO_SRC := $(shell find $(CURDIR)/bindings/go/src -name '*.go') $(GO_GEN) - -fdb_go: $(GO_PACKAGE_OBJECTS) $(GO_SRC) fdb_go_fmt_check - -fdb_go_fmt: $(GO_SRC) - @echo "Formatting fdb_go" - @gofmt -w $(GO_SRC) - -fdb_go_fmt_check: $(GO_SRC) - @echo "Checking fdb_go" - @bash -c 'fmtoutstr=$$(gofmt -l $(GO_SRC)) ; if [[ -n "$${fmtoutstr}" ]] ; then echo "Detected go formatting violations for the following files:" ; echo "$${fmtoutstr}" ; echo "Try running: make fdb_go_fmt"; exit 1 ; fi' - -$(GO_DEST)/.stamp: $(GO_SRC) - @echo "Creating fdb_go_path" - @mkdir -p $(GO_DEST) - @cp -r bindings/go/src/* $(GO_DEST) - @touch $(GO_DEST)/.stamp - -fdb_go_path: $(GO_DEST)/.stamp - -fdb_go_clean: - @echo "Cleaning fdb_go" - @rm -rf $(GOPATH) - -fdb_go_tester: $(GOPATH)/bin/_stacktester - -fdb_go_tester_clean: - @echo "Cleaning fdb_go_tester" - @rm -rf $(GOPATH)/bin - -$(GOPATH)/bin/_stacktester: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OBJECTS) - @echo "Compiling $(basename $(notdir $@))" - @go install $(GO_IMPORT_PATH)/_stacktester - -$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a - @echo "Compiling fdb/tuple" - @go install $(GO_IMPORT_PATH)/fdb/tuple - -$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a - @echo "Compiling fdb/subspace" - @go install $(GO_IMPORT_PATH)/fdb/subspace - -$(GO_PACKAGE_OUTDIR)/fdb/directory.a: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a - @echo "Compiling fdb/directory" - @go install $(GO_IMPORT_PATH)/fdb/directory - -$(GO_PACKAGE_OUTDIR)/fdb.a: $(GO_DEST)/.stamp lib/libfdb_c.$(DLEXT) $(GO_SRC) - @echo "Compiling fdb" - @go install $(GO_IMPORT_PATH)/fdb - -$(GO_GEN): bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options - @echo "Building $@" - @go run bindings/go/src/_util/translate_fdb_options.go < fdbclient/vexillographer/fdb.options > $@ diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 750327a57a..9737391569 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -569,6 +569,16 @@ func (sm *StackMachine) processInst(idx int, inst tuple.Tuple) { } sm.store(idx, res.(fdb.FutureByteSlice)) + case op == "GET_ESTIMATED_RANGE_SIZE": + r := sm.popKeyRange() + _, e := rt.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) { + _ = rtr.GetEstimatedRangeSizeBytes(r).MustGet() + sm.store(idx, []byte("GOT_ESTIMATED_RANGE_SIZE")) + return nil, nil + }) + if e != nil { + panic(e) + } case op == "COMMIT": sm.store(idx, sm.currentTransaction().Commit()) case op == "RESET": diff --git a/bindings/go/src/fdb/cluster.go b/bindings/go/src/fdb/cluster.go index 3ec92c2c5e..df895e9a51 100644 --- a/bindings/go/src/fdb/cluster.go +++ b/bindings/go/src/fdb/cluster.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include import "C" diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go index aca709e3d4..c9bf818fab 100644 --- a/bindings/go/src/fdb/database.go +++ b/bindings/go/src/fdb/database.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include import "C" diff --git a/bindings/go/src/fdb/directory/directoryPartition.go b/bindings/go/src/fdb/directory/directoryPartition.go index d6e0275f02..7702bd3e04 100644 --- a/bindings/go/src/fdb/directory/directoryPartition.go +++ b/bindings/go/src/fdb/directory/directoryPartition.go @@ -45,6 +45,10 @@ func (dp directoryPartition) Pack(t tuple.Tuple) fdb.Key { panic("cannot pack keys using the root of a directory partition") } +func (dp directoryPartition) PackWithVersionstamp(t tuple.Tuple) (fdb.Key, error) { + panic("cannot pack keys using the root of a directory partition") +} + func (dp directoryPartition) Unpack(k fdb.KeyConvertible) (tuple.Tuple, error) { panic("cannot unpack keys using the root of a directory partition") } diff --git a/bindings/go/src/fdb/directory/directorySubspace.go b/bindings/go/src/fdb/directory/directorySubspace.go index 2dd927b2da..f67c46bc33 100644 --- a/bindings/go/src/fdb/directory/directorySubspace.go +++ b/bindings/go/src/fdb/directory/directorySubspace.go @@ -23,6 +23,8 @@ package directory import ( + "fmt" + "strings" "github.com/apple/foundationdb/bindings/go/src/fdb" "github.com/apple/foundationdb/bindings/go/src/fdb/subspace" ) @@ -43,6 +45,18 @@ type directorySubspace struct { layer []byte } +// String implements the fmt.Stringer interface and returns human-readable +// string representation of this object. +func (ds directorySubspace) String() string { + var path string + if len(ds.path) > 0 { + path = "(" + strings.Join(ds.path, ",") + ")" + } else { + path = "nil" + } + return fmt.Sprintf("DirectorySubspace(%s, %s)", path, fdb.Printable(ds.Bytes())) +} + func (d directorySubspace) CreateOrOpen(t fdb.Transactor, path []string, layer []byte) (DirectorySubspace, error) { return d.dl.CreateOrOpen(t, d.dl.partitionSubpath(d.path, path), layer) } diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go index c09eadbb9f..5cfb157ad6 100644 --- a/bindings/go/src/fdb/doc.go +++ b/bindings/go/src/fdb/doc.go @@ -46,7 +46,7 @@ A basic interaction with the FoundationDB API is demonstrated below: func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(620) + fdb.MustAPIVersion(630) // Open the default database from the system cluster db := fdb.MustOpenDefault() @@ -139,6 +139,16 @@ error. The above example may be rewritten as: return []string{valueOne, valueTwo}, nil }) +MustGet returns nil (which is different from empty slice []byte{}), when the +key doesn't exist, and hence non-existence can be checked as follows: + + val := tr.Get(fdb.Key("foobar")).MustGet() + if val == nil { + fmt.Println("foobar does not exist.") + } else { + fmt.Println("foobar exists.") + } + Any panic that occurs during execution of the caller-provided function will be recovered by the (Database).Transact method. If the error is an FDB Error, it will either result in a retry of the function or be returned by Transact. If the diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go index 9380736b4e..94e699c89e 100644 --- a/bindings/go/src/fdb/errors.go +++ b/bindings/go/src/fdb/errors.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include import "C" diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go index 7d0f17fbe1..d0bfd5f699 100644 --- a/bindings/go/src/fdb/fdb.go +++ b/bindings/go/src/fdb/fdb.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include // #include import "C" @@ -108,7 +108,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // library, an error will be returned. APIVersion must be called prior to any // other functions in the fdb package. // -// Currently, this package supports API versions 200 through 620. +// Currently, this package supports API versions 200 through 630. // // Warning: When using the multi-version client API, setting an API version that // is not supported by a particular client library will prevent that client from @@ -116,7 +116,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { // the API version of your application after upgrading your client until the // cluster has also been upgraded. func APIVersion(version int) error { - headerVersion := 620 + headerVersion := 630 networkMutex.Lock() defer networkMutex.Unlock() @@ -128,7 +128,7 @@ func APIVersion(version int) error { return errAPIVersionAlreadySet } - if version < 200 || version > 620 { + if version < 200 || version > 630 { return errAPIVersionNotSupported } diff --git a/bindings/go/src/fdb/fdb_darwin.go b/bindings/go/src/fdb/fdb_darwin.go new file mode 100644 index 0000000000..c4157af70b --- /dev/null +++ b/bindings/go/src/fdb/fdb_darwin.go @@ -0,0 +1,5 @@ +package fdb + +//#cgo CFLAGS: -I/usr/local/include/ +//#cgo LDFLAGS: -L/usr/local/lib/ +import "C" diff --git a/bindings/go/src/fdb/fdb_windows.go b/bindings/go/src/fdb/fdb_windows.go new file mode 100644 index 0000000000..a27cd11eb7 --- /dev/null +++ b/bindings/go/src/fdb/fdb_windows.go @@ -0,0 +1,5 @@ +package fdb + +//#cgo CFLAGS: -I"C:/Program Files/foundationdb/include" +//#cgo LDFLAGS: -L"C:/Program Files/foundationdb/bin" -lfdb_c +import "C" diff --git a/bindings/go/src/fdb/futures.go b/bindings/go/src/fdb/futures.go index f94909d761..17ae1d70a4 100644 --- a/bindings/go/src/fdb/futures.go +++ b/bindings/go/src/fdb/futures.go @@ -23,7 +23,7 @@ package fdb // #cgo LDFLAGS: -lfdb_c -lm -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include // #include // @@ -268,6 +268,7 @@ type futureKeyValueArray struct { *future } +//go:nocheckptr func stringRefToSlice(ptr unsafe.Pointer) []byte { size := *((*C.int)(unsafe.Pointer(uintptr(ptr) + 8))) diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go index c5dfa08d4f..f8cf89f5fd 100644 --- a/bindings/go/src/fdb/generated.go +++ b/bindings/go/src/fdb/generated.go @@ -88,6 +88,13 @@ func (o NetworkOptions) SetTraceFormat(param string) error { return o.setOpt(34, []byte(param)) } +// Select clock source for trace files. now (default) or realtime are supported. +// +// Parameter: Trace clock source +func (o NetworkOptions) SetTraceClockSource(param string) error { + return o.setOpt(35, []byte(param)) +} + // Set internal tuning or debugging knobs // // Parameter: knob_name=knob_value @@ -297,7 +304,7 @@ func (o DatabaseOptions) SetTransactionTimeout(param int64) error { return o.setOpt(500, int64ToBytes(param)) } -// Set a timeout in milliseconds which, when elapsed, will cause a transaction automatically to be cancelled. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information. +// Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information. // // Parameter: number of times to retry func (o DatabaseOptions) SetTransactionRetryLimit(param int64) error { @@ -323,7 +330,7 @@ func (o DatabaseOptions) SetTransactionCausalReadRisky() error { return o.setOpt(504, nil) } -// Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated. +// Addresses returned by get_addresses_for_key include the port when enabled. As of api version 630, this option is enabled by default and setting this has no effect. func (o DatabaseOptions) SetTransactionIncludePortInAddress() error { return o.setOpt(505, nil) } @@ -343,7 +350,7 @@ func (o TransactionOptions) SetCausalReadDisable() error { return o.setOpt(21, nil) } -// Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated. +// Addresses returned by get_addresses_for_key include the port when enabled. As of api version 630, this option is enabled by default and setting this has no effect. func (o TransactionOptions) SetIncludePortInAddress() error { return o.setOpt(23, nil) } @@ -422,7 +429,7 @@ func (o TransactionOptions) SetDebugTransactionIdentifier(param string) error { return o.setOpt(403, []byte(param)) } -// Enables tracing for this transaction and logs results to the client trace logs. The DEBUG_TRANSACTION_IDENTIFIER option must be set before using this option, and client trace logging must be enabled and to get log output. +// Enables tracing for this transaction and logs results to the client trace logs. The DEBUG_TRANSACTION_IDENTIFIER option must be set before using this option, and client trace logging must be enabled to get log output. func (o TransactionOptions) SetLogTransaction() error { return o.setOpt(404, nil) } @@ -472,7 +479,7 @@ func (o TransactionOptions) SetSnapshotRywDisable() error { return o.setOpt(601, nil) } -// The transaction can read and write to locked databases, and is resposible for checking that it took the lock. +// The transaction can read and write to locked databases, and is responsible for checking that it took the lock. func (o TransactionOptions) SetLockAware() error { return o.setOpt(700, nil) } @@ -505,13 +512,14 @@ const ( // small portion of data is transferred to the client initially (in order to // minimize costs if the client doesn't read the entire range), and as the // caller iterates over more items in the range larger batches will be - // transferred in order to minimize latency. + // transferred in order to minimize latency. After enough iterations, the + // iterator mode will eventually reach the same byte limit as ``WANT_ALL`` StreamingModeIterator StreamingMode = 0 // Infrequently used. The client has passed a specific row limit and wants // that many rows delivered in a single batch. Because of iterator operation // in client drivers make request batches transparent to the user, consider - // “WANT_ALL“ StreamingMode instead. A row limit must be specified if this + // ``WANT_ALL`` StreamingMode instead. A row limit must be specified if this // mode is used. StreamingModeExact StreamingMode = 1 @@ -628,15 +636,15 @@ type ErrorPredicate int const ( - // Returns “true“ if the error indicates the operations in the transactions - // should be retried because of transient error. + // Returns ``true`` if the error indicates the operations in the + // transactions should be retried because of transient error. ErrorPredicateRetryable ErrorPredicate = 50000 - // Returns “true“ if the error indicates the transaction may have succeeded, - // though not in a way the system can verify. + // Returns ``true`` if the error indicates the transaction may have + // succeeded, though not in a way the system can verify. ErrorPredicateMaybeCommitted ErrorPredicate = 50001 - // Returns “true“ if the error indicates the transaction has not committed, - // though in a way that can be retried. + // Returns ``true`` if the error indicates the transaction has not + // committed, though in a way that can be retried. ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002 ) diff --git a/bindings/go/src/fdb/range.go b/bindings/go/src/fdb/range.go index 8273fe37fe..67a45c63b2 100644 --- a/bindings/go/src/fdb/range.go +++ b/bindings/go/src/fdb/range.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include import "C" @@ -54,7 +54,8 @@ type RangeOptions struct { // Reverse indicates that the read should be performed in lexicographic // (false) or reverse lexicographic (true) order. When Reverse is true and // Limit is non-zero, the last Limit key-value pairs in the range are - // returned. + // returned. Reading ranges in reverse is supported natively by the + // database and should have minimal extra cost. Reverse bool } diff --git a/bindings/go/src/fdb/snapshot.go b/bindings/go/src/fdb/snapshot.go index 18c77d79bb..ca21818729 100644 --- a/bindings/go/src/fdb/snapshot.go +++ b/bindings/go/src/fdb/snapshot.go @@ -86,3 +86,11 @@ func (s Snapshot) GetReadVersion() FutureInt64 { func (s Snapshot) GetDatabase() Database { return s.transaction.db } + +func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { + beginKey, endKey := r.FDBRangeKeys() + return s.getEstimatedRangeSizeBytes( + beginKey.FDBKey(), + endKey.FDBKey(), + ) +} diff --git a/bindings/go/src/fdb/subspace/subspace.go b/bindings/go/src/fdb/subspace/subspace.go index b779d5a9f7..65f97048c8 100644 --- a/bindings/go/src/fdb/subspace/subspace.go +++ b/bindings/go/src/fdb/subspace/subspace.go @@ -35,6 +35,8 @@ package subspace import ( "bytes" "errors" + "fmt" + "github.com/apple/foundationdb/bindings/go/src/fdb" "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" ) @@ -54,6 +56,15 @@ type Subspace interface { // Subspace prepended. Pack(t tuple.Tuple) fdb.Key + // PackWithVersionstamp returns the key encoding the specified tuple in + // the subspace so that it may be used as the key in fdb.Transaction's + // SetVersionstampedKey() method. The passed tuple must contain exactly + // one incomplete tuple.Versionstamp instance or the method will return + // with an error. The behavior here is the same as if one used the + // tuple.PackWithVersionstamp() method to appropriately pack together this + // subspace and the passed tuple. + PackWithVersionstamp(t tuple.Tuple) (fdb.Key, error) + // Unpack returns the Tuple encoded by the given key with the prefix of this // Subspace removed. Unpack will return an error if the key is not in this // Subspace or does not encode a well-formed Tuple. @@ -73,7 +84,7 @@ type Subspace interface { } type subspace struct { - b []byte + rawPrefix []byte } // AllKeys returns the Subspace corresponding to all keys in a FoundationDB @@ -96,36 +107,46 @@ func FromBytes(b []byte) Subspace { return subspace{s} } +// String implements the fmt.Stringer interface and return the subspace +// as a human readable byte string provided by fdb.Printable. +func (s subspace) String() string { + return fmt.Sprintf("Subspace(rawPrefix=%s)", fdb.Printable(s.rawPrefix)) +} + func (s subspace) Sub(el ...tuple.TupleElement) Subspace { return subspace{concat(s.Bytes(), tuple.Tuple(el).Pack()...)} } func (s subspace) Bytes() []byte { - return s.b + return s.rawPrefix } func (s subspace) Pack(t tuple.Tuple) fdb.Key { - return fdb.Key(concat(s.b, t.Pack()...)) + return fdb.Key(concat(s.rawPrefix, t.Pack()...)) +} + +func (s subspace) PackWithVersionstamp(t tuple.Tuple) (fdb.Key, error) { + return t.PackWithVersionstamp(s.rawPrefix) } func (s subspace) Unpack(k fdb.KeyConvertible) (tuple.Tuple, error) { key := k.FDBKey() - if !bytes.HasPrefix(key, s.b) { + if !bytes.HasPrefix(key, s.rawPrefix) { return nil, errors.New("key is not in subspace") } - return tuple.Unpack(key[len(s.b):]) + return tuple.Unpack(key[len(s.rawPrefix):]) } func (s subspace) Contains(k fdb.KeyConvertible) bool { - return bytes.HasPrefix(k.FDBKey(), s.b) + return bytes.HasPrefix(k.FDBKey(), s.rawPrefix) } func (s subspace) FDBKey() fdb.Key { - return fdb.Key(s.b) + return fdb.Key(s.rawPrefix) } func (s subspace) FDBRangeKeys() (fdb.KeyConvertible, fdb.KeyConvertible) { - return fdb.Key(concat(s.b, 0x00)), fdb.Key(concat(s.b, 0xFF)) + return fdb.Key(concat(s.rawPrefix, 0x00)), fdb.Key(concat(s.rawPrefix, 0xFF)) } func (s subspace) FDBRangeKeySelectors() (fdb.Selectable, fdb.Selectable) { diff --git a/bindings/go/src/fdb/subspace/subspace_test.go b/bindings/go/src/fdb/subspace/subspace_test.go new file mode 100644 index 0000000000..abc713a2fc --- /dev/null +++ b/bindings/go/src/fdb/subspace/subspace_test.go @@ -0,0 +1,15 @@ +package subspace + +import ( + "fmt" + "testing" +) + +func TestSubspaceString(t *testing.T) { + printed := fmt.Sprint(Sub([]byte("hello"), "world", 42, 0x99)) + expected := "Subspace(rawPrefix=\\x01hello\\x00\\x02world\\x00\\x15*\\x15\\x99)" + + if printed != expected { + t.Fatalf("printed subspace result differs, expected %v, got %v", expected, printed) + } +} diff --git a/bindings/go/src/fdb/transaction.go b/bindings/go/src/fdb/transaction.go index 02da338cf6..4102a0556b 100644 --- a/bindings/go/src/fdb/transaction.go +++ b/bindings/go/src/fdb/transaction.go @@ -22,7 +22,7 @@ package fdb -// #define FDB_API_VERSION 620 +// #define FDB_API_VERSION 630 // #include import "C" @@ -39,6 +39,7 @@ type ReadTransaction interface { GetReadVersion() FutureInt64 GetDatabase() Database Snapshot() Snapshot + GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 ReadTransactor } @@ -305,6 +306,28 @@ func (t Transaction) GetRange(r Range, options RangeOptions) RangeResult { return t.getRange(r, options, false) } +func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) FutureInt64 { + return &futureInt64{ + future: newFuture(C.fdb_transaction_get_estimated_range_size_bytes( + t.ptr, + byteSliceToPtr(beginKey), + C.int(len(beginKey)), + byteSliceToPtr(endKey), + C.int(len(endKey)), + )), + } +} + +// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes +// stored in the given range. +func (t Transaction) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 { + beginKey, endKey := r.FDBRangeKeys() + return t.getEstimatedRangeSizeBytes( + beginKey.FDBKey(), + endKey.FDBKey(), + ) +} + func (t *transaction) getReadVersion() FutureInt64 { return &futureInt64{ future: newFuture(C.fdb_transaction_get_read_version(t.ptr)), @@ -383,6 +406,9 @@ func (t *transaction) getApproximateSize() FutureInt64 { } } +// Returns a future that is the approximate transaction size so far in this +// transaction, which is the summation of the estimated size of mutations, +// read conflict ranges, and write conflict ranges. func (t Transaction) GetApproximateSize() FutureInt64 { return t.getApproximateSize() } diff --git a/bindings/go/src/fdb/tuple/tuple.go b/bindings/go/src/fdb/tuple/tuple.go index a37ce5f3e8..46b17061ba 100644 --- a/bindings/go/src/fdb/tuple/tuple.go +++ b/bindings/go/src/fdb/tuple/tuple.go @@ -43,6 +43,8 @@ import ( "fmt" "math" "math/big" + "strconv" + "strings" "github.com/apple/foundationdb/bindings/go/src/fdb" ) @@ -66,6 +68,48 @@ type TupleElement interface{} // packing T (modulo type normalization to []byte, uint64, and int64). type Tuple []TupleElement +// String implements the fmt.Stringer interface and returns human-readable +// string representation of this tuple. For most elements, we use the +// object's default string representation. +func (tuple Tuple) String() string { + sb := strings.Builder{} + printTuple(tuple, &sb) + return sb.String() +} + +func printTuple(tuple Tuple, sb *strings.Builder) { + sb.WriteString("(") + + for i, t := range tuple { + switch t := t.(type) { + case Tuple: + printTuple(t, sb) + case nil: + sb.WriteString("") + case string: + sb.WriteString(strconv.Quote(t)) + case UUID: + sb.WriteString("UUID(") + sb.WriteString(t.String()) + sb.WriteString(")") + case []byte: + sb.WriteString("b\"") + sb.WriteString(fdb.Printable(t)) + sb.WriteString("\"") + default: + // For user-defined and standard types, we use standard Go + // printer, which itself uses Stringer interface. + fmt.Fprintf(sb, "%v", t) + } + + if (i < len(tuple) - 1) { + sb.WriteString(", ") + } + } + + sb.WriteString(")") +} + // UUID wraps a basic byte array as a UUID. We do not provide any special // methods for accessing or generating the UUID, but as Go does not provide // a built-in UUID type, this simple wrapper allows for other libraries @@ -73,6 +117,10 @@ type Tuple []TupleElement // an instance of this type. type UUID [16]byte +func (uuid UUID) String() string { + return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]) +} + // Versionstamp is struct for a FoundationDB verionstamp. Versionstamps are // 12 bytes long composed of a 10 byte transaction version and a 2 byte user // version. The transaction version is filled in at commit time and the user @@ -82,6 +130,11 @@ type Versionstamp struct { UserVersion uint16 } +// Returns a human-readable string for this Versionstamp. +func (vs Versionstamp) String() string { + return fmt.Sprintf("Versionstamp(%s, %d)", fdb.Printable(vs.TransactionVersion[:]), vs.UserVersion) +} + var incompleteTransactionVersion = [10]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} const versionstampLength = 12 diff --git a/bindings/go/src/fdb/tuple/tuple_test.go b/bindings/go/src/fdb/tuple/tuple_test.go index 59c37bc7bc..602bdfe915 100644 --- a/bindings/go/src/fdb/tuple/tuple_test.go +++ b/bindings/go/src/fdb/tuple/tuple_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/gob" "flag" + "fmt" "math/rand" "os" "testing" @@ -118,3 +119,38 @@ func BenchmarkTuplePacking(b *testing.B) { }) } } + +func TestTupleString(t *testing.T) { + testCases :=[ ]struct { + input Tuple + expected string + }{ + { + Tuple{[]byte("hello"), "world", 42, 0x99}, + "(b\"hello\", \"world\", 42, 153)", + }, + { + Tuple{nil, Tuple{"Ok", Tuple{1, 2}, "Go"}, 42, 0x99}, + "(, (\"Ok\", (1, 2), \"Go\"), 42, 153)", + }, + { + Tuple{"Bool", true, false}, + "(\"Bool\", true, false)", + }, + { + Tuple{"UUID", testUUID}, + "(\"UUID\", UUID(1100aabb-ccdd-eeff-1100-aabbccddeeff))", + }, + { + Tuple{"Versionstamp", Versionstamp{[10]byte{0, 0, 0, 0xaa, 0, 0xbb, 0, 0xcc, 0, 0xdd}, 620}}, + "(\"Versionstamp\", Versionstamp(\\x00\\x00\\x00\\xaa\\x00\\xbb\\x00\\xcc\\x00\\xdd, 620))", + }, + } + + for _, testCase := range testCases { + printed := fmt.Sprint(testCase.input) + if printed != testCase.expected { + t.Fatalf("printed tuple result differs, expected %v, got %v", testCase.expected, printed) + } + } +} diff --git a/bindings/java/CMakeLists.txt b/bindings/java/CMakeLists.txt index 80ff1d1388..6d94d75b71 100644 --- a/bindings/java/CMakeLists.txt +++ b/bindings/java/CMakeLists.txt @@ -56,6 +56,7 @@ set(JAVA_BINDING_SRCS src/main/com/apple/foundationdb/testing/Promise.java src/main/com/apple/foundationdb/testing/PerfMetric.java src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java + src/main/com/apple/foundationdb/tuple/FastByteComparisons.java src/main/com/apple/foundationdb/tuple/IterableComparator.java src/main/com/apple/foundationdb/tuple/package-info.java src/main/com/apple/foundationdb/tuple/StringUtil.java @@ -169,8 +170,6 @@ file(WRITE ${MANIFEST_FILE} ${MANIFEST_TEXT}) add_jar(fdb-java ${JAVA_BINDING_SRCS} ${GENERATED_JAVA_FILES} ${CMAKE_SOURCE_DIR}/LICENSE OUTPUT_DIR ${PROJECT_BINARY_DIR}/lib VERSION ${CMAKE_PROJECT_VERSION} MANIFEST ${MANIFEST_FILE}) add_dependencies(fdb-java fdb_java_options fdb_java) -add_jar(foundationdb-tests SOURCES ${JAVA_TESTS_SRCS} INCLUDE_JARS fdb-java) -add_dependencies(foundationdb-tests fdb_java_options) # TODO[mpilman]: The java RPM will require some more effort (mostly on debian). However, # most people will use the fat-jar, so it is not clear how high this priority is. @@ -237,6 +236,16 @@ if(NOT OPEN_FOR_IDE) WORKING_DIRECTORY ${unpack_dir} DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib_copied COMMENT "Build ${target_jar}") + add_jar(foundationdb-tests SOURCES ${JAVA_TESTS_SRCS} INCLUDE_JARS fdb-java) + add_dependencies(foundationdb-tests fdb_java_options) + set(tests_jar ${jar_destination}/fdb-java-${CMAKE_PROJECT_VERSION}${prerelease_string}-tests.jar) + add_custom_command(OUTPUT ${tests_jar} + COMMAND ${CMAKE_COMMAND} -E copy foundationdb-tests.jar "${tests_jar}" + WORKING_DIRECTORY . + DEPENDS foundationdb-tests + COMMENT "Build ${tests_jar}") + add_custom_target(fdb-java-tests ALL DEPENDS ${tests_jar}) + add_dependencies(fdb-java-tests foundationdb-tests) add_custom_target(fat-jar ALL DEPENDS ${target_jar}) add_dependencies(fat-jar fdb-java) add_dependencies(fat-jar copy_lib) diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index 286197997f..808485486b 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -19,7 +19,7 @@ */ #include -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #include #include @@ -368,9 +368,11 @@ struct JVM { { { "send", "(JZ)V", reinterpret_cast(&promiseSend) } }); auto fdbClass = getClass("com/apple/foundationdb/FDB"); jmethodID selectMethod = - env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(IZ)Lcom/apple/foundationdb/FDB;"); + env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(I)Lcom/apple/foundationdb/FDB;"); checkException(); - env->CallStaticObjectMethod(fdbClass, selectMethod, jint(620), jboolean(false)); + auto fdbInstance = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(630)); + checkException(); + env->CallObjectMethod(fdbInstance, getMethod(fdbClass, "disableShutdownHook", "()V")); checkException(); } diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 5a49987a85..938ac498f3 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -21,7 +21,7 @@ #include #include -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #include @@ -36,6 +36,11 @@ static JavaVM* g_jvm = nullptr; static thread_local JNIEnv* g_thread_jenv = nullptr; // Defined for the network thread once it is running, and for any thread that has called registerCallback static thread_local jmethodID g_IFutureCallback_call_methodID = JNI_NULL; static thread_local bool is_external = false; +static jclass range_result_summary_class; +static jclass range_result_class; +static jclass string_class; +static jmethodID range_result_init; +static jmethodID range_result_summary_init; void detachIfExternalThread(void *ignore) { if(is_external && g_thread_jenv != nullptr) { @@ -275,10 +280,9 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureStrings_FutureString return JNI_NULL; } - jclass str_clazz = jenv->FindClass("java/lang/String"); if( jenv->ExceptionOccurred() ) return JNI_NULL; - jobjectArray arr = jenv->NewObjectArray(count, str_clazz, JNI_NULL); + jobjectArray arr = jenv->NewObjectArray(count, string_class, JNI_NULL); if( !arr ) { if( !jenv->ExceptionOccurred() ) throwOutOfMem(jenv); @@ -306,13 +310,6 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult throwParamNotNull(jenv); return JNI_NULL; } - - jclass resultCls = jenv->FindClass("com/apple/foundationdb/RangeResultSummary"); - if( jenv->ExceptionOccurred() ) - return JNI_NULL; - jmethodID resultCtorId = jenv->GetMethodID(resultCls, "", "([BIZ)V"); - if( jenv->ExceptionOccurred() ) - return JNI_NULL; FDBFuture *f = (FDBFuture *)future; @@ -337,7 +334,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult jenv->SetByteArrayRegion(lastKey, 0, kvs[count - 1].key_length, (jbyte *)kvs[count - 1].key); } - jobject result = jenv->NewObject(resultCls, resultCtorId, lastKey, count, (jboolean)more); + jobject result = jenv->NewObject(range_result_summary_class, range_result_summary_init, lastKey, count, (jboolean)more); if( jenv->ExceptionOccurred() ) return JNI_NULL; @@ -350,9 +347,6 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult throwParamNotNull(jenv); return JNI_NULL; } - - jclass resultCls = jenv->FindClass("com/apple/foundationdb/RangeResult"); - jmethodID resultCtorId = jenv->GetMethodID(resultCls, "", "([B[IZ)V"); FDBFuture *f = (FDBFuture *)future; @@ -414,7 +408,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureResults_FutureResult jenv->ReleaseByteArrayElements(keyValueArray, (jbyte *)keyvalues_barr, 0); jenv->ReleaseIntArrayElements(lengthArray, length_barr, 0); - jobject result = jenv->NewObject(resultCls, resultCtorId, keyValueArray, lengthArray, (jboolean)more); + jobject result = jenv->NewObject(range_result_class, range_result_init, keyValueArray, lengthArray, (jboolean)more); if( jenv->ExceptionOccurred() ) return JNI_NULL; @@ -646,6 +640,35 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return (jlong)f; } +JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeBytes(JNIEnv *jenv, jobject, jlong tPtr, + jbyteArray beginKeyBytes, jbyteArray endKeyBytes) { + if( !tPtr || !beginKeyBytes || !endKeyBytes) { + throwParamNotNull(jenv); + return 0; + } + FDBTransaction *tr = (FDBTransaction *)tPtr; + + uint8_t *startKey = (uint8_t *)jenv->GetByteArrayElements( beginKeyBytes, JNI_NULL ); + if(!startKey) { + if( !jenv->ExceptionOccurred() ) + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return 0; + } + + uint8_t *endKey = (uint8_t *)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + if (!endKey) { + jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); + if( !jenv->ExceptionOccurred() ) + throwRuntimeEx( jenv, "Error getting handle to native resources" ); + return 0; + } + + FDBFuture *f = fdb_transaction_get_estimated_range_size_bytes( tr, startKey, jenv->GetArrayLength( beginKeyBytes ), endKey, jenv->GetArrayLength( endKeyBytes ) ); + jenv->ReleaseByteArrayElements( beginKeyBytes, (jbyte *)startKey, JNI_ABORT ); + jenv->ReleaseByteArrayElements( endKeyBytes, (jbyte *)endKey, JNI_ABORT ); + return (jlong)f; +} + JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1set(JNIEnv *jenv, jobject, jlong tPtr, jbyteArray keyBytes, jbyteArray valueBytes) { if( !tPtr || !keyBytes || !valueBytes ) { throwParamNotNull(jenv); @@ -1013,8 +1036,43 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDB_Network_1stop(JNIEnv *jen } jint JNI_OnLoad(JavaVM *vm, void *reserved) { + JNIEnv *env; g_jvm = vm; - return JNI_VERSION_1_1; + if (vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK) { + return JNI_ERR; + } else { + jclass local_range_result_class = env->FindClass("com/apple/foundationdb/RangeResult"); + range_result_init = env->GetMethodID(local_range_result_class, "", "([B[IZ)V"); + range_result_class = (jclass) (env)->NewGlobalRef(local_range_result_class); + + jclass local_range_result_summary_class = env->FindClass("com/apple/foundationdb/RangeResultSummary"); + range_result_summary_init = env->GetMethodID(local_range_result_summary_class, "", "([BIZ)V"); + range_result_summary_class = (jclass) (env)->NewGlobalRef(local_range_result_summary_class); + + jclass local_string_class = env->FindClass("java/lang/String"); + string_class = (jclass) (env)->NewGlobalRef(local_string_class); + + return JNI_VERSION_1_6; + } +} + +// Is automatically called once the Classloader is destroyed +void JNI_OnUnload(JavaVM *vm, void *reserved) { + JNIEnv* env; + if (vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK) { + return; + } else { + // delete global references so the GC can collect them + if (range_result_summary_class != NULL) { + env->DeleteGlobalRef(range_result_summary_class); + } + if (range_result_class != NULL) { + env->DeleteGlobalRef(range_result_class); + } + if (string_class != NULL) { + env->DeleteGlobalRef(string_class); + } + } } #ifdef __cplusplus diff --git a/bindings/java/fdb_java.vcxproj b/bindings/java/fdb_java.vcxproj deleted file mode 100644 index d5cba49bc6..0000000000 --- a/bindings/java/fdb_java.vcxproj +++ /dev/null @@ -1,104 +0,0 @@ - - - - - -PRERELEASE - - - FDB_CLEAN_BUILD;%(PreprocessorDefinitions) - - - - - - Debug - x64 - - - Release - x64 - - - - {9617584C-22E8-4272-934F-733F378BF6AE} - java - - - - DynamicLibrary - true - MultiByte - v141 - - - DynamicLibrary - false - true - MultiByte - v141 - - - - - - - - - - ..\..\;C:\Program Files\Java\jdk6\include\win32;C:\Program Files\Java\jdk6\include;C:\Program Files\boost_1_67_0;$(IncludePath) - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - - - - true - $(SolutionDir)bin\$(Configuration)\fdb_c.lib;%(AdditionalDependencies) - - - - - Level3 - Disabled - %(AdditionalIncludeDirectories);$(SolutionDir)bindings\c - TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - MultiThreadedDebug - /bigobj "@$(SolutionDir)flow/no_intellisense.opt" %(AdditionalOptions) - stdcpp17 - - - Windows - - - - - Level3 - MaxSpeed - true - true - %(AdditionalIncludeDirectories);$(SolutionDir)bindings\c - TLS_DISABLED;WIN32;_WIN32_WINNT=_WIN32_WINNT_WS03;BOOST_ALL_NO_LIB;WINVER=_WIN32_WINNT_WS03;NTDDI_VERSION=NTDDI_WS03;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - MultiThreaded - /bigobj "@$(SolutionDir)flow/no_intellisense.opt" %(AdditionalOptions) - stdcpp17 - - - true - true - Windows - - - - - - - - - - - false - - - - - - diff --git a/bindings/java/local.mk b/bindings/java/local.mk deleted file mode 100644 index 30f9e25152..0000000000 --- a/bindings/java/local.mk +++ /dev/null @@ -1,222 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdb_java_LDFLAGS := -Llib -fdb_java_CFLAGS := $(fdbclient_CFLAGS) -Ibindings/c - -# We only override if the environment didn't set it (this is used by -# the fdbwebsite documentation build process) -JAVADOC_DIR ?= bindings/java - -fdb_java_LIBS := lib/libfdb_c.$(DLEXT) - -ifeq ($(RELEASE),true) - JARVER = $(VERSION) - APPLEJARVER = $(VERSION) -else - JARVER = $(VERSION)-PRERELEASE - APPLEJARVER = $(VERSION)-SNAPSHOT -endif - -ifeq ($(PLATFORM),linux) - JAVA_HOME ?= /usr/lib/jvm/java-8-openjdk-amd64 - fdb_java_CFLAGS += -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux - fdb_java_LDFLAGS += -static-libgcc - - java_ARCH := amd64 -else ifeq ($(PLATFORM),osx) - JAVA_HOME ?= $(shell /usr/libexec/java_home) - fdb_java_CFLAGS += -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/darwin - - java_ARCH := x86_64 -endif - -JAVA_GENERATED_SOURCES := bindings/java/src/main/com/apple/foundationdb/NetworkOptions.java bindings/java/src/main/com/apple/foundationdb/DatabaseOptions.java bindings/java/src/main/com/apple/foundationdb/TransactionOptions.java bindings/java/src/main/com/apple/foundationdb/StreamingMode.java bindings/java/src/main/com/apple/foundationdb/ConflictRangeType.java bindings/java/src/main/com/apple/foundationdb/MutationType.java bindings/java/src/main/com/apple/foundationdb/FDBException.java - -JAVA_SOURCES := $(JAVA_GENERATED_SOURCES) bindings/java/src/main/com/apple/foundationdb/*.java bindings/java/src/main/com/apple/foundationdb/async/*.java bindings/java/src/main/com/apple/foundationdb/tuple/*.java bindings/java/src/main/com/apple/foundationdb/directory/*.java bindings/java/src/main/com/apple/foundationdb/subspace/*.java bindings/java/src/test/com/apple/foundationdb/test/*.java - -fdb_java: bindings/java/foundationdb-client.jar bindings/java/foundationdb-tests.jar - -bindings/java/foundationdb-tests.jar: bindings/java/.classstamp - @echo "Building $@" - @jar cf $@ -C bindings/java/classes/test com/apple/foundationdb - -bindings/java/foundationdb-client.jar: bindings/java/.classstamp lib/libfdb_java.$(DLEXT) - @echo "Building $@" - @rm -rf bindings/java/classes/main/lib/$(PLATFORM)/$(java_ARCH) - @mkdir -p bindings/java/classes/main/lib/$(PLATFORM)/$(java_ARCH) - @cp lib/libfdb_java.$(DLEXT) bindings/java/classes/main/lib/$(PLATFORM)/$(java_ARCH)/libfdb_java.$(java_DLEXT) - @jar cf $@ -C bindings/java/classes/main com/apple/foundationdb -C bindings/java/classes/main lib - -fdb_java_jar_clean: - @rm -rf $(JAVA_GENERATED_SOURCES) - @rm -rf bindings/java/classes - @rm -f bindings/java/foundationdb-client.jar bindings/java/foundationdb-tests.jar bindings/java/.classstamp - -# Redefinition of a target already defined in generated.mk, but it's "okay" and the way things were done before. -fdb_java_clean: fdb_java_jar_clean - -bindings/java/src/main/com/apple/foundationdb/StreamingMode.java: bin/vexillographer.exe fdbclient/vexillographer/fdb.options - @echo "Building Java options" - @$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options java $(@D) - -bindings/java/src/main/com/apple/foundationdb/MutationType.java: bindings/java/src/main/com/apple/foundationdb/StreamingMode.java - @true - -bindings/java/src/main/com/apple/foundationdb/ConflictRangeType.java: bindings/java/src/main/com/apple/foundationdb/StreamingMode.java - @true - -bindings/java/src/main/com/apple/foundationdb/FDBException.java: bindings/java/src/main/com/apple/foundationdb/StreamingMode.java - @true - -bindings/java/src/main/com/apple/foundationdb/%Options.java: bindings/java/src/main/com/apple/foundationdb/StreamingMode.java - @true - -bindings/java/src/main/overview.html: bindings/java/src/main/overview.html.in $(ALL_MAKEFILES) versions.target - @m4 -DVERSION=$(VERSION) $< > $@ - -bindings/java/.classstamp: $(JAVA_SOURCES) - @echo "Compiling Java source" - @rm -rf bindings/java/classes - @mkdir -p bindings/java/classes/main - @mkdir -p bindings/java/classes/test - @$(JAVAC) $(JAVAFLAGS) -d bindings/java/classes/main bindings/java/src/main/com/apple/foundationdb/*.java bindings/java/src/main/com/apple/foundationdb/async/*.java bindings/java/src/main/com/apple/foundationdb/tuple/*.java bindings/java/src/main/com/apple/foundationdb/directory/*.java bindings/java/src/main/com/apple/foundationdb/subspace/*.java - @$(JAVAC) $(JAVAFLAGS) -cp bindings/java/classes/main -d bindings/java/classes/test bindings/java/src/test/com/apple/foundationdb/test/*.java - @echo timestamp > bindings/java/.classstamp - -javadoc: $(JAVA_SOURCES) bindings/java/src/main/overview.html - @echo "Generating Javadocs" - @mkdir -p $(JAVADOC_DIR)/javadoc/ - @javadoc -quiet -public -notimestamp -source 1.8 -sourcepath bindings/java/src/main \ - -overview bindings/java/src/main/overview.html -d $(JAVADOC_DIR)/javadoc/ \ - -windowtitle "FoundationDB Java Client API" \ - -doctitle "FoundationDB Java Client API" \ - -link "http://docs.oracle.com/javase/8/docs/api" \ - com.apple.foundationdb com.apple.foundationdb.async com.apple.foundationdb.tuple com.apple.foundationdb.directory com.apple.foundationdb.subspace - -javadoc_clean: - @rm -rf $(JAVADOC_DIR)/javadoc - @rm -f bindings/java/src/main/overview.html - -ifeq ($(PLATFORM),linux) - - # We only need javadoc from one source - TARGETS += javadoc - CLEAN_TARGETS += javadoc_clean - - # _release builds the lib on macOS and the jars (including the macOS lib) on Linux - TARGETS += fdb_java_release - CLEAN_TARGETS += fdb_java_release_clean - - ifneq ($(FATJAR),) - packages/fdb-java-$(JARVER).jar: $(MAC_OBJ_JAVA) $(WINDOWS_OBJ_JAVA) - endif - - bindings/java/pom.xml: bindings/java/pom.xml.in $(ALL_MAKEFILES) versions.target - @echo "Generating $@" - @m4 -DVERSION=$(JARVER) -DNAME=fdb-java $< > $@ - - bindings/java/fdb-java-$(APPLEJARVER).pom: bindings/java/pom.xml - @echo "Copying $@" - sed -e 's/-PRERELEASE/-SNAPSHOT/g' bindings/java/pom.xml > "$@" - - packages/fdb-java-$(JARVER).jar: fdb_java versions.target - @echo "Building $@" - @rm -f $@ - @rm -rf packages/jar_regular - @mkdir -p packages/jar_regular - @cd packages/jar_regular && unzip -qq $(TOPDIR)/bindings/java/foundationdb-client.jar - ifneq ($(FATJAR),) - @mkdir -p packages/jar_regular/lib/windows/amd64 - @mkdir -p packages/jar_regular/lib/osx/x86_64 - @cp $(MAC_OBJ_JAVA) packages/jar_regular/lib/osx/x86_64/libfdb_java.jnilib - @cp $(WINDOWS_OBJ_JAVA) packages/jar_regular/lib/windows/amd64/fdb_java.dll - endif - @cd packages/jar_regular && jar cf $(TOPDIR)/$@ * - @rm -r packages/jar_regular - @cd bindings && jar uf $(TOPDIR)/$@ ../LICENSE - - packages/fdb-java-$(JARVER)-tests.jar: fdb_java versions.target - @echo "Building $@" - @rm -f $@ - @cp $(TOPDIR)/bindings/java/foundationdb-tests.jar packages/fdb-java-$(JARVER)-tests.jar - - packages/fdb-java-$(JARVER)-sources.jar: $(JAVA_GENERATED_SOURCES) versions.target - @echo "Building $@" - @rm -f $@ - @jar cf $(TOPDIR)/$@ -C bindings/java/src/main com/apple/foundationdb - - packages/fdb-java-$(JARVER)-javadoc.jar: javadoc versions.target - @echo "Building $@" - @rm -f $@ - @cd $(JAVADOC_DIR)/javadoc/ && jar cf $(TOPDIR)/$@ * - @cd bindings && jar uf $(TOPDIR)/$@ ../LICENSE - - packages/fdb-java-$(JARVER)-bundle.jar: packages/fdb-java-$(JARVER).jar packages/fdb-java-$(JARVER)-javadoc.jar packages/fdb-java-$(JARVER)-sources.jar bindings/java/pom.xml bindings/java/fdb-java-$(APPLEJARVER).pom versions.target - @echo "Building $@" - @rm -f $@ - @rm -rf packages/bundle_regular - @mkdir -p packages/bundle_regular - @cp packages/fdb-java-$(JARVER).jar packages/fdb-java-$(JARVER)-javadoc.jar packages/fdb-java-$(JARVER)-sources.jar bindings/java/fdb-java-$(APPLEJARVER).pom packages/bundle_regular - @cp bindings/java/pom.xml packages/bundle_regular/pom.xml - @cd packages/bundle_regular && jar cf $(TOPDIR)/$@ * - @rm -rf packages/bundle_regular - - fdb_java_release: packages/fdb-java-$(JARVER)-bundle.jar packages/fdb-java-$(JARVER)-tests.jar - - fdb_java_release_clean: - @echo "Cleaning Java release" - @rm -f packages/fdb-java-*.jar packages/fdb-java-*-sources.jar bindings/java/pom.xml bindings/java/fdb-java-$(APPLEJARVER).pom - - # Linux is where we build all the java packages - packages: fdb_java_release - packages_clean: fdb_java_release_clean - - ifneq ($(FATJAR),) - MAC_OBJ_JAVA := lib/libfdb_java.jnilib-$(VERSION_ID) - WINDOWS_OBJ_JAVA := lib/fdb_java.dll-$(VERSION_ID) - endif - -else ifeq ($(PLATFORM),osx) - - TARGETS += fdb_java_release - CLEAN_TARGETS += fdb_java_release_clean - - fdb_java_release: lib/libfdb_java.$(DLEXT) - @mkdir -p lib - @rm -f lib/libfdb_java.$(java_DLEXT)-* - @cp lib/libfdb_java.$(DLEXT) lib/libfdb_java.$(java_DLEXT)-$(VERSION_ID) - @cp lib/libfdb_java.$(DLEXT)-debug lib/libfdb_java.$(java_DLEXT)-debug-$(VERSION_ID) - - fdb_java_release_clean: - @rm -f lib/libfdb_java.$(DLEXT)-* - @rm -f lib/libfdb_java.$(java_DLEXT)-* - - # macOS needs to put its java lib in packages - packages: fdb_java_lib_package - - fdb_java_lib_package: fdb_java_release - mkdir -p packages - cp lib/libfdb_java.$(java_DLEXT)-$(VERSION_ID) packages - cp lib/libfdb_java.$(java_DLEXT)-debug-$(VERSION_ID) packages - -endif diff --git a/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java b/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java index 576f971c11..3cd7125a97 100644 --- a/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java +++ b/bindings/java/src/junit/com/apple/foundationdb/tuple/ArrayUtilTests.java @@ -304,4 +304,58 @@ public class ArrayUtilTests { fail("Not yet implemented"); } + private static final int SAMPLE_COUNT = 1000000; + private static final int SAMPLE_MAX_SIZE = 2048; + private List unsafe; + private List java; + @Before + public void init() { + unsafe = new ArrayList(SAMPLE_COUNT); + java = new ArrayList(SAMPLE_COUNT); + Random random = new Random(); + for (int i = 0; i <= SAMPLE_COUNT; i++) { + byte[] addition = new byte[random.nextInt(SAMPLE_MAX_SIZE)]; + random.nextBytes(addition); + unsafe.add(addition); + java.add(addition); + } + } + + @Test + public void testComparatorSort() { + Collections.sort(unsafe, FastByteComparisons.lexicographicalComparerUnsafeImpl()); + Collections.sort(java, FastByteComparisons.lexicographicalComparerJavaImpl()); + Assert.assertTrue(unsafe.equals(java)); + } + + @Test + public void testUnsafeComparison() { + for (int i =0; i< SAMPLE_COUNT; i++) { + Assert.assertEquals(FastByteComparisons.lexicographicalComparerUnsafeImpl().compare(unsafe.get(i), java.get(i)), 0); + } + } + + @Test + public void testJavaComparison() { + for (int i =0; i< SAMPLE_COUNT; i++) { + Assert.assertEquals(FastByteComparisons.lexicographicalComparerJavaImpl().compare(unsafe.get(i), java.get(i)), 0); + } + } + + @Test + public void testUnsafeComparisonWithOffet() { + for (int i =0; i< SAMPLE_COUNT; i++) { + if (unsafe.get(i).length > 5) + Assert.assertEquals(FastByteComparisons.lexicographicalComparerUnsafeImpl().compareTo(unsafe.get(i), 4, unsafe.get(i).length - 4, java.get(i), 4, java.get(i).length - 4), 0); + } + } + + @Test + public void testJavaComparisonWithOffset() { + for (int i =0; i< SAMPLE_COUNT; i++) { + if (unsafe.get(i).length > 5) + Assert.assertEquals(FastByteComparisons.lexicographicalComparerJavaImpl().compareTo(unsafe.get(i), 4, unsafe.get(i).length - 4, java.get(i), 4, java.get(i).length - 4), 0); + } + } + } diff --git a/bindings/java/src/main/com/apple/foundationdb/FDB.java b/bindings/java/src/main/com/apple/foundationdb/FDB.java index f105f91983..ba96814cef 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDB.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDB.java @@ -35,7 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger; * This call is required before using any other part of the API. The call allows * an error to be thrown at this point to prevent client code from accessing a later library * with incorrect assumptions from the current version. The API version documented here is version - * {@code 620}.

+ * {@code 630}.

* FoundationDB encapsulates multiple versions of its interface by requiring * the client to explicitly specify the version of the API it uses. The purpose * of this design is to allow you to upgrade the server, client libraries, or @@ -85,6 +85,8 @@ public class FDB { private volatile boolean netStarted = false; private volatile boolean netStopped = false; volatile boolean warnOnUnclosed = true; + private boolean useShutdownHook = true; + private Thread shutdownHook; private final Semaphore netRunning = new Semaphore(1); private final NetworkOptions options; @@ -104,15 +106,8 @@ public class FDB { * Called only once to create the FDB singleton. */ private FDB(int apiVersion) { - this(apiVersion, true); - } - - private FDB(int apiVersion, boolean controlRuntime) { this.apiVersion = apiVersion; options = new NetworkOptions(this::Network_setOption); - if (controlRuntime) { - Runtime.getRuntime().addShutdownHook(new Thread(this::stopNetwork)); - } } /** @@ -167,9 +162,9 @@ public class FDB { * object.

* * Warning: When using the multi-version client API, setting an API version that - * is not supported by a particular client library will prevent that client from + * is not supported by a particular client library will prevent that client from * being used to connect to the cluster. In particular, you should not advance - * the API version of your application after upgrading your client until the + * the API version of your application after upgrading your client until the * cluster has also been upgraded. * * @param version the API version required @@ -177,13 +172,6 @@ public class FDB { * @return the FoundationDB API object */ public static FDB selectAPIVersion(final int version) throws FDBException { - return selectAPIVersion(version, true); - } - - /** - This function is called from C++ if the VM is controlled directly from FDB - */ - private static synchronized FDB selectAPIVersion(final int version, boolean controlRuntime) throws FDBException { if(singleton != null) { if(version != singleton.getAPIVersion()) { throw new IllegalArgumentException( @@ -193,13 +181,30 @@ public class FDB { } if(version < 510) throw new IllegalArgumentException("API version not supported (minimum 510)"); - if(version > 620) - throw new IllegalArgumentException("API version not supported (maximum 620)"); + if(version > 630) + throw new IllegalArgumentException("API version not supported (maximum 630)"); Select_API_version(version); - FDB fdb = new FDB(version, controlRuntime); + singleton = new FDB(version); - return singleton = fdb; + return singleton; + } + + /** + * Disables shutdown hook that stops network thread upon process shutdown. This is useful if you need to run + * your own shutdown hook that uses the FDB instance and you need to avoid race conditions + * with the default shutdown hook. Replacement shutdown hook should stop the network thread manually + * by calling {@link #stopNetwork}. + */ + public synchronized void disableShutdownHook() { + useShutdownHook = false; + if(shutdownHook != null) { + // If this method was called after network thread started and shutdown hook was installed, + // remove this hook + Runtime.getRuntime().removeShutdownHook(shutdownHook); + // Release thread reference for GC + shutdownHook = null; + } } /** @@ -405,6 +410,11 @@ public class FDB { if(netStarted) { return; } + if(useShutdownHook) { + // Register shutdown hook that stops network thread if user did not opt out + shutdownHook = new Thread(this::stopNetwork, "fdb-shutdown-hook"); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } Network_setup(); netStarted = true; @@ -497,4 +507,4 @@ public class FDB { private native boolean Error_predicate(int predicate, int code); private native long Database_create(String clusterFilePath) throws FDBException; -} +} \ No newline at end of file diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java index d6f1e4f935..09be8a353a 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java @@ -70,6 +70,16 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC return getKey_internal(selector, true); } + @Override + public CompletableFuture getEstimatedRangeSizeBytes(byte[] begin, byte[] end) { + return FDBTransaction.this.getEstimatedRangeSizeBytes(begin, end); + } + + @Override + public CompletableFuture getEstimatedRangeSizeBytes(Range range) { + return FDBTransaction.this.getEstimatedRangeSizeBytes(range); + } + /////////////////// // getRange -> KeySelectors /////////////////// @@ -257,6 +267,21 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC } } + @Override + public CompletableFuture getEstimatedRangeSizeBytes(byte[] begin, byte[] end) { + pointerReadLock.lock(); + try { + return new FutureInt64(Transaction_getEstimatedRangeSizeBytes(getPtr(), begin, end), executor); + } finally { + pointerReadLock.unlock(); + } + } + + @Override + public CompletableFuture getEstimatedRangeSizeBytes(Range range) { + return this.getEstimatedRangeSizeBytes(range.begin, range.end); + } + /////////////////// // getRange -> KeySelectors /////////////////// @@ -659,4 +684,5 @@ class FDBTransaction extends NativeObjectWrapper implements Transaction, OptionC private native long Transaction_watch(long ptr, byte[] key) throws FDBException; private native void Transaction_cancel(long cPtr); private native long Transaction_getKeyLocations(long cPtr, byte[] key); + private native long Transaction_getEstimatedRangeSizeBytes(long cPtr, byte[] keyBegin, byte[] keyEnd); } diff --git a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java index 63a5fa73c6..3dd11b77ff 100644 --- a/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java @@ -184,7 +184,9 @@ public interface ReadTransaction extends ReadTransactionContext { * first keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query * should not limit the number of results. If {@code reverse} is {@code true} rows * will be limited starting at the end of the range. - * @param reverse return results starting at the end of the range in reverse order + * @param reverse return results starting at the end of the range in reverse order. + * Reading ranges in reverse is supported natively by the database and should + * have minimal extra cost. * * @return a handle to access the results of the asynchronous call */ @@ -205,11 +207,22 @@ public interface ReadTransaction extends ReadTransactionContext { * first keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query * should not limit the number of results. If {@code reverse} is {@code true} rows * will be limited starting at the end of the range. - * @param reverse return results starting at the end of the range in reverse order + * @param reverse return results starting at the end of the range in reverse order. + * Reading ranges in reverse is supported natively by the database and should + * have minimal extra cost. * @param mode provide a hint about how the results are to be used. This * can provide speed improvements or efficiency gains based on the caller's * knowledge of the upcoming access pattern. * + *

+ * When converting the result of this query to a list using {@link AsyncIterable#asList()} with the {@code ITERATOR} streaming + * mode, the query is automatically modified to fetch results in larger batches. This is done because it is + * known in advance that the {@link AsyncIterable#asList()} function will fetch all results in the range. If a limit is specified, + * the {@code EXACT} streaming mode will be used, and otherwise it will use {@code WANT_ALL}. + * + * To achieve comparable performance when iterating over an entire range without using {@link AsyncIterable#asList()}, the same + * streaming mode would need to be used. + *

* @return a handle to access the results of the asynchronous call */ AsyncIterable getRange(KeySelector begin, KeySelector end, @@ -263,7 +276,9 @@ public interface ReadTransaction extends ReadTransactionContext { * first keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query * should not limit the number of results. If {@code reverse} is {@code true} rows * will be limited starting at the end of the range. - * @param reverse return results starting at the end of the range in reverse order + * @param reverse return results starting at the end of the range in reverse order. + * Reading ranges in reverse is supported natively by the database and should + * have minimal extra cost. * * @return a handle to access the results of the asynchronous call */ @@ -284,11 +299,22 @@ public interface ReadTransaction extends ReadTransactionContext { * first keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query * should not limit the number of results. If {@code reverse} is {@code true} rows * will be limited starting at the end of the range. - * @param reverse return results starting at the end of the range in reverse order + * @param reverse return results starting at the end of the range in reverse order. + * Reading ranges in reverse is supported natively by the database and should + * have minimal extra cost. * @param mode provide a hint about how the results are to be used. This * can provide speed improvements or efficiency gains based on the caller's * knowledge of the upcoming access pattern. * + *

+ * When converting the result of this query to a list using {@link AsyncIterable#asList()} with the {@code ITERATOR} streaming + * mode, the query is automatically modified to fetch results in larger batches. This is done because it is + * known in advance that the {@link AsyncIterable#asList()} function will fetch all results in the range. If a limit is specified, + * the {@code EXACT} streaming mode will be used, and otherwise it will use {@code WANT_ALL}. + * + * To achieve comparable performance when iterating over an entire range without using {@link AsyncIterable#asList()}, the same + * streaming mode would need to be used. + *

* @return a handle to access the results of the asynchronous call */ AsyncIterable getRange(byte[] begin, byte[] end, @@ -351,7 +377,9 @@ public interface ReadTransaction extends ReadTransactionContext { * first keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query * should not limit the number of results. If {@code reverse} is {@code true} rows * will be limited starting at the end of the range. - * @param reverse return results starting at the end of the range in reverse order + * @param reverse return results starting at the end of the range in reverse order. + * Reading ranges in reverse is supported natively by the database and should + * have minimal extra cost. * * @return a handle to access the results of the asynchronous call */ @@ -375,16 +403,47 @@ public interface ReadTransaction extends ReadTransactionContext { * first keys in the range. Pass {@link #ROW_LIMIT_UNLIMITED} if this query * should not limit the number of results. If {@code reverse} is {@code true} rows * will be limited starting at the end of the range. - * @param reverse return results starting at the end of the range in reverse order + * @param reverse return results starting at the end of the range in reverse order. + * Reading ranges in reverse is supported natively by the database and should + * have minimal extra cost. * @param mode provide a hint about how the results are to be used. This * can provide speed improvements or efficiency gains based on the caller's * knowledge of the upcoming access pattern. * + *

+ * When converting the result of this query to a list using {@link AsyncIterable#asList()} with the {@code ITERATOR} streaming + * mode, the query is automatically modified to fetch results in larger batches. This is done because it is + * known in advance that the {@link AsyncIterable#asList()} function will fetch all results in the range. If a limit is specified, + * the {@code EXACT} streaming mode will be used, and otherwise it will use {@code WANT_ALL}. + * + * To achieve comparable performance when iterating over an entire range without using {@link AsyncIterable#asList()}, the same + * streaming mode would need to be used. + *

* @return a handle to access the results of the asynchronous call */ AsyncIterable getRange(Range range, int limit, boolean reverse, StreamingMode mode); + + /** + * Gets an estimate for the number of bytes stored in the given range. + * + * @param begin the beginning of the range (inclusive) + * @param end the end of the range (exclusive) + * + * @return a handle to access the results of the asynchronous call + */ + CompletableFuture getEstimatedRangeSizeBytes(byte[] begin, byte[] end); + + /** + * Gets an estimate for the number of bytes stored in the given range. + * + * @param range the range of the keys + * + * @return a handle to access the results of the asynchronous call + */ + CompletableFuture getEstimatedRangeSizeBytes(Range range); + /** * Returns a set of options that can be set on a {@code Transaction} * diff --git a/bindings/java/src/main/com/apple/foundationdb/directory/DirectoryLayer.java b/bindings/java/src/main/com/apple/foundationdb/directory/DirectoryLayer.java index be802b0cb6..5ea5f3945c 100644 --- a/bindings/java/src/main/com/apple/foundationdb/directory/DirectoryLayer.java +++ b/bindings/java/src/main/com/apple/foundationdb/directory/DirectoryLayer.java @@ -817,9 +817,9 @@ public class DirectoryLayer implements Directory { private static long unpackLittleEndian(byte[] bytes) { assert bytes.length == 8; - int value = 0; + long value = 0; for(int i = 0; i < 8; ++i) { - value += (bytes[i] << (i * 8)); + value += (Byte.toUnsignedLong(bytes[i]) << (i * 8)); } return value; } diff --git a/bindings/java/src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java b/bindings/java/src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java index 0c4e5f6e68..16011e056e 100644 --- a/bindings/java/src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java +++ b/bindings/java/src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; * {@link #printable(byte[])} for debugging non-text keys and values. * */ -public class ByteArrayUtil { +public class ByteArrayUtil extends FastByteComparisons { /** * Joins a set of byte arrays into a larger array. The {@code interlude} is placed @@ -135,11 +135,7 @@ public class ByteArrayUtil { if(src.length < start + pattern.length) return false; - for(int i = 0; i < pattern.length; i++) - if(pattern[i] != src[start + i]) - return false; - - return true; + return compareTo(src, start, pattern.length, pattern, 0, pattern.length) == 0; } /** @@ -307,14 +303,7 @@ public class ByteArrayUtil { * {@code r}. */ public static int compareUnsigned(byte[] l, byte[] r) { - for(int idx = 0; idx < l.length && idx < r.length; ++idx) { - if(l[idx] != r[idx]) { - return (l[idx] & 0xFF) < (r[idx] & 0xFF) ? -1 : 1; - } - } - if(l.length == r.length) - return 0; - return l.length < r.length ? -1 : 1; + return compareTo(l, 0, l.length, r, 0, r.length); } /** @@ -328,15 +317,11 @@ public class ByteArrayUtil { * @return {@code true} if {@code array} starts with {@code prefix} */ public static boolean startsWith(byte[] array, byte[] prefix) { + // Short Circuit if(array.length < prefix.length) { return false; } - for(int i = 0; i < prefix.length; ++i) { - if(prefix[i] != array[i]) { - return false; - } - } - return true; + return compareTo(array, 0, prefix.length, prefix, 0, prefix.length) == 0; } /** diff --git a/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java b/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java new file mode 100644 index 0000000000..77add1db7f --- /dev/null +++ b/bindings/java/src/main/com/apple/foundationdb/tuple/FastByteComparisons.java @@ -0,0 +1,294 @@ +/* + * ByteArrayUtil.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.apple.foundationdb.tuple; + +import java.lang.reflect.Field; +import java.nio.ByteOrder; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Comparator; + +import sun.misc.Unsafe; + + +/** + * Utility code to do optimized byte-array comparison. + * This is borrowed and slightly modified from Guava's {@link UnsignedBytes} + * class to be able to compare arrays that start at non-zero offsets. + */ +abstract class FastByteComparisons { + + private static final int UNSIGNED_MASK = 0xFF; + /** + * Lexicographically compare two byte arrays. + * + * @param buffer1 left operand, expected to not be null + * @param buffer2 right operand, expected to not be null + * @param offset1 Where to start comparing in the left buffer, expected to be >= 0 + * @param offset2 Where to start comparing in the right buffer, expected to be >= 0 + * @param length1 How much to compare from the left buffer, expected to be >= 0 + * @param length2 How much to compare from the right buffer, expected to be >= 0 + * @return 0 if equal, < 0 if left is less than right, etc. + */ + public static int compareTo(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + return LexicographicalComparerHolder.BEST_COMPARER.compareTo( + buffer1, offset1, length1, buffer2, offset2, length2); + } + /** + * Interface for both the java and unsafe comparators + offset based comparisons. + * @param + */ + interface Comparer extends Comparator { + /** + * Lexicographically compare two byte arrays. + * + * @param buffer1 left operand + * @param buffer2 right operand + * @param offset1 Where to start comparing in the left buffer + * @param offset2 Where to start comparing in the right buffer + * @param length1 How much to compare from the left buffer + * @param length2 How much to compare from the right buffer + * @return 0 if equal, < 0 if left is less than right, etc. + */ + abstract public int compareTo(T buffer1, int offset1, int length1, + T buffer2, int offset2, int length2); + } + + /** + * Pure Java Comparer + * + * @return + */ + static Comparer lexicographicalComparerJavaImpl() { + return LexicographicalComparerHolder.PureJavaComparer.INSTANCE; + } + + /** + * Unsafe Comparer + * + * @return + */ + static Comparer lexicographicalComparerUnsafeImpl() { + return LexicographicalComparerHolder.UnsafeComparer.INSTANCE; + } + + + /** + * Provides a lexicographical comparer implementation; either a Java + * implementation or a faster implementation based on {@link Unsafe}. + * + *

Uses reflection to gracefully fall back to the Java implementation if + * {@code Unsafe} isn't available. + */ + private static class LexicographicalComparerHolder { + static final String UNSAFE_COMPARER_NAME = + LexicographicalComparerHolder.class.getName() + "$UnsafeComparer"; + + static final Comparer BEST_COMPARER = getBestComparer(); + /** + * Returns the Unsafe-using Comparer, or falls back to the pure-Java + * implementation if unable to do so. + */ + static Comparer getBestComparer() { + String arch = System.getProperty("os.arch"); + boolean unaligned = arch.equals("i386") || arch.equals("x86") + || arch.equals("amd64") || arch.equals("x86_64"); + if (!unaligned) + return lexicographicalComparerJavaImpl(); + try { + Class theClass = Class.forName(UNSAFE_COMPARER_NAME); + + // yes, UnsafeComparer does implement Comparer + @SuppressWarnings("unchecked") + Comparer comparer = + (Comparer) theClass.getEnumConstants()[0]; + return comparer; + } catch (Throwable t) { // ensure we really catch *everything* + return lexicographicalComparerJavaImpl(); + } + } + + /** + * Java Comparer doing byte by byte comparisons + * + */ + enum PureJavaComparer implements Comparer { + INSTANCE; + + /** + * + * CompareTo looking at two buffers. + * + * @param buffer1 left operand + * @param buffer2 right operand + * @param offset1 Where to start comparing in the left buffer + * @param offset2 Where to start comparing in the right buffer + * @param length1 How much to compare from the left buffer + * @param length2 How much to compare from the right buffer + * @return 0 if equal, < 0 if left is less than right, etc. + */ + @Override + public int compareTo(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + // Short circuit equal case + if (buffer1 == buffer2 && + offset1 == offset2 && + length1 == length2) { + return 0; + } + int end1 = offset1 + length1; + int end2 = offset2 + length2; + for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { + int a = (buffer1[i] & UNSIGNED_MASK); + int b = (buffer2[j] & UNSIGNED_MASK); + if (a != b) { + return a - b; + } + } + return length1 - length2; + } + + /** + * Supports Comparator + * + * @param o1 + * @param o2 + * @return comparison + */ + @Override + public int compare(byte[] o1, byte[] o2) { + return compareTo(o1, 0, o1.length, o2, 0, o2.length); + } + } + + /** + * + * Takes advantage of word based comparisons + * + */ + @SuppressWarnings("unused") // used via reflection + enum UnsafeComparer implements Comparer { + INSTANCE; + + static final Unsafe theUnsafe; + + /** + * The offset to the first element in a byte array. + */ + static final int BYTE_ARRAY_BASE_OFFSET; + + @Override + public int compare(byte[] o1, byte[] o2) { + return compareTo(o1, 0, o1.length, o2, 0, o2.length); + } + + static { + theUnsafe = (Unsafe) AccessController.doPrivileged( + (PrivilegedAction) () -> { + try { + Field f = Unsafe.class.getDeclaredField("theUnsafe"); + f.setAccessible(true); + return f.get(null); + } catch (NoSuchFieldException e) { + // It doesn't matter what we throw; + // it's swallowed in getBestComparer(). + throw new Error(); + } catch (IllegalAccessException e) { + throw new Error(); + } + }); + + BYTE_ARRAY_BASE_OFFSET = theUnsafe.arrayBaseOffset(byte[].class); + + // sanity check - this should never fail + if (theUnsafe.arrayIndexScale(byte[].class) != 1) { + throw new AssertionError(); + } + } + + static final boolean LITTLE_ENDIAN = + ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); + + /** + * Lexicographically compare two arrays. + * + * @param buffer1 left operand + * @param buffer2 right operand + * @param offset1 Where to start comparing in the left buffer + * @param offset2 Where to start comparing in the right buffer + * @param length1 How much to compare from the left buffer + * @param length2 How much to compare from the right buffer + * @return 0 if equal, < 0 if left is less than right, etc. + */ + @Override + public int compareTo(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + // Short circuit equal case + if (buffer1 == buffer2 && + offset1 == offset2 && + length1 == length2) { + return 0; + } + final int stride = 8; + final int minLength = Math.min(length1, length2); + int strideLimit = minLength & ~(stride - 1); + final long offset1Adj = offset1 + BYTE_ARRAY_BASE_OFFSET; + final long offset2Adj = offset2 + BYTE_ARRAY_BASE_OFFSET; + int i; + + /* + * Compare 8 bytes at a time. Benchmarking on x86 shows a stride of 8 bytes is no slower + * than 4 bytes even on 32-bit. On the other hand, it is substantially faster on 64-bit. + */ + for (i = 0; i < strideLimit; i += stride) { + long lw = theUnsafe.getLong(buffer1, offset1Adj + i); + long rw = theUnsafe.getLong(buffer2, offset2Adj + i); + if (lw != rw) { + if(!LITTLE_ENDIAN) { + return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) ? -1 : 1; + } + + /* + * We want to compare only the first index where left[index] != right[index]. This + * corresponds to the least significant nonzero byte in lw ^ rw, since lw and rw are + * little-endian. Long.numberOfTrailingZeros(diff) tells us the least significant + * nonzero bit, and zeroing out the first three bits of L.nTZ gives us the shift to get + * that least significant nonzero byte. This comparison logic is based on UnsignedBytes + * comparator from guava v21 + */ + int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7; + return ((int) ((lw >>> n) & UNSIGNED_MASK)) - ((int) ((rw >>> n) & UNSIGNED_MASK)); + } + } + + // The epilogue to cover the last (minLength % stride) elements. + for (; i < minLength; i++) { + int a = (buffer1[offset1 + i] & UNSIGNED_MASK); + int b = (buffer2[offset2 + i] & UNSIGNED_MASK); + if (a != b) { + return a - b; + } + } + return length1 - length2; + } + } + } +} \ No newline at end of file diff --git a/bindings/java/src/main/overview.html.in b/bindings/java/src/main/overview.html.in index 648a4e3478..fd7c6ac80d 100644 --- a/bindings/java/src/main/overview.html.in +++ b/bindings/java/src/main/overview.html.in @@ -13,7 +13,7 @@ and then added to your classpath.

Getting started

To start using FoundationDB from Java, create an instance of the {@link com.apple.foundationdb.FDB FoundationDB API interface} with the version of the -API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 620}). +API that you want to use (this release of the FoundationDB Java API supports versions between {@code 510} and {@code 630}). With this API object you can then open {@link com.apple.foundationdb.Cluster Cluster}s and {@link com.apple.foundationdb.Database Database}s and start using {@link com.apple.foundationdb.Transaction Transaction}s. @@ -29,7 +29,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java index 969b12b89a..3a153e3582 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java @@ -27,7 +27,7 @@ import com.apple.foundationdb.Database; import com.apple.foundationdb.FDB; public abstract class AbstractTester { - public static final int API_VERSION = 620; + public static final int API_VERSION = 630; protected static final int NUM_RUNS = 25; protected static final Charset ASCII = Charset.forName("ASCII"); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index 8756558676..a5da772494 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -223,6 +223,12 @@ public class AsyncStackTester { inst.push(inst.readTcx.readAsync(readTr -> readTr.get((byte[]) param))); }); } + else if (op == StackOperation.GET_ESTIMATED_RANGE_SIZE) { + List params = inst.popParams(2).join(); + return inst.readTr.getEstimatedRangeSizeBytes((byte[])params.get(0), (byte[])params.get(1)).thenAcceptAsync(size -> { + inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes()); + }, FDB.DEFAULT_EXECUTOR); + } else if(op == StackOperation.GET_RANGE) { return inst.popParams(5).thenComposeAsync(params -> { int limit = StackUtils.getInt(params.get(2)); @@ -667,10 +673,7 @@ public class AsyncStackTester { }; if(operations == null || ++currentOp == operations.size()) { - Transaction tr = db.createTransaction(); - - return tr.getRange(nextKey, endKey, 1000).asList() - .whenComplete((x, t) -> tr.close()) + return db.readAsync(readTr -> readTr.getRange(nextKey, endKey, 1000).asList()) .thenComposeAsync(next -> { if(next.size() < 1) { //System.out.println("No key found after: " + ByteArrayUtil.printable(nextKey.getKey())); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java index 86963c4496..f21aabeb6a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/BlockingBenchmark.java @@ -33,7 +33,7 @@ public class BlockingBenchmark { private static final int PARALLEL = 100; public static void main(String[] args) throws InterruptedException { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); // The cluster file DOES NOT need to be valid, although it must exist. // This is because the database is never really contacted in this test. diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java index fcc77ae854..53f13695c1 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java @@ -48,7 +48,7 @@ public class ConcurrentGetSetGet { } public static void main(String[] args) { - try(Database database = FDB.selectAPIVersion(620).open()) { + try(Database database = FDB.selectAPIVersion(630).open()) { new ConcurrentGetSetGet().apply(database); } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java index 40a781756a..c43dd71809 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/DirectoryTest.java @@ -33,7 +33,7 @@ import com.apple.foundationdb.directory.DirectorySubspace; public class DirectoryTest { public static void main(String[] args) throws Exception { try { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { runTests(db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/Example.java b/bindings/java/src/test/com/apple/foundationdb/test/Example.java index 5a31f7c566..74090eccc0 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/Example.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/Example.java @@ -26,7 +26,7 @@ import com.apple.foundationdb.tuple.Tuple; public class Example { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { // Run an operation on the database diff --git a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java index 78bc725450..aca9e918d2 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/IterableTest.java @@ -31,7 +31,7 @@ public class IterableTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java index 9018339175..70f688e46a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/LocalityTests.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.tuple.ByteArrayUtil; public class LocalityTests { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database database = fdb.open(args[0])) { try(Transaction tr = database.createTransaction()) { String[] keyAddresses = LocalityUtil.getAddressesForKey(tr, "a".getBytes()).join(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java index b6c5cfdfaf..014f1f038d 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ParallelRandomScan.java @@ -43,7 +43,7 @@ public class ParallelRandomScan { private static final int PARALLELISM_STEP = 5; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(620); + FDB api = FDB.selectAPIVersion(630); try(Database database = api.open(args[0])) { for(int i = PARALLELISM_MIN; i <= PARALLELISM_MAX; i += PARALLELISM_STEP) { runTest(database, i, ROWS, DURATION_MS); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java index 1ce5f657c3..3a99c68d56 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/RangeTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; import com.apple.foundationdb.async.AsyncIterable; public class RangeTest { - private static final int API_VERSION = 620; + private static final int API_VERSION = 630; public static void main(String[] args) { System.out.println("About to use version " + API_VERSION); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java index 44b1ee7b77..f873e954e1 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialInsertion.java @@ -34,7 +34,7 @@ public class SerialInsertion { private static final int NODES = 1000000; public static void main(String[] args) { - FDB api = FDB.selectAPIVersion(620); + FDB api = FDB.selectAPIVersion(630); try(Database database = api.open()) { long start = System.currentTimeMillis(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java index 49e51af299..cbcc2d713a 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java @@ -39,7 +39,7 @@ public class SerialIteration { private static final int THREAD_COUNT = 1; public static void main(String[] args) throws InterruptedException { - FDB api = FDB.selectAPIVersion(620); + FDB api = FDB.selectAPIVersion(630); try(Database database = api.open(args[0])) { for(int i = 1; i <= THREAD_COUNT; i++) { runThreadedTest(database, i); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java index c733f54e04..2aad1eb1bb 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialTest.java @@ -30,7 +30,7 @@ public class SerialTest { public static void main(String[] args) throws InterruptedException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { runTests(reps, db); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java index 16acc7c1a7..d324463408 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SnapshotTransactionTest.java @@ -39,7 +39,7 @@ public class SnapshotTransactionTest { private static final Subspace SUBSPACE = new Subspace(Tuple.from("test", "conflict_ranges")); public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { snapshotReadShouldNotConflict(db); snapshotShouldNotAddConflictRange(db); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java index 8d13aadde1..634a217c7f 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java @@ -56,6 +56,7 @@ enum StackOperation { GET_COMMITTED_VERSION, GET_APPROXIMATE_SIZE, GET_VERSIONSTAMP, + GET_ESTIMATED_RANGE_SIZE, SET_READ_VERSION, ON_ERROR, SUB, diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index 9586005a82..ee01a1c8c7 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -206,6 +206,11 @@ public class StackTester { CompletableFuture f = inst.readTcx.read(readTr -> readTr.get((byte[])params.get(0))); inst.push(f); } + else if (op == StackOperation.GET_ESTIMATED_RANGE_SIZE) { + List params = inst.popParams(2).join(); + Long size = inst.readTr.getEstimatedRangeSizeBytes((byte[])params.get(0), (byte[])params.get(1)).join(); + inst.push("GOT_ESTIMATED_RANGE_SIZE".getBytes()); + } else if(op == StackOperation.GET_RANGE) { List params = inst.popParams(5).join(); @@ -547,18 +552,15 @@ public class StackTester { @Override void executeOperations() { - KeySelector begin = nextKey; while(true) { - Transaction t = db.createTransaction(); - List keyValues = t.getRange(begin, endKey/*, 1000*/).asList().join(); - t.close(); + List keyValues = db.read(readTr -> readTr.getRange(nextKey, endKey/*, 1000*/).asList().join()); if(keyValues.size() == 0) { break; } //System.out.println(" * Got " + keyValues.size() + " instructions"); for(KeyValue next : keyValues) { - begin = KeySelector.firstGreaterThan(next.getKey()); + nextKey = KeySelector.firstGreaterThan(next.getKey()); processOp(next.getValue()); instructionIndex++; } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java index 599272f73c..c7aa190ce7 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/TupleTest.java @@ -50,7 +50,7 @@ public class TupleTest { public static void main(String[] args) throws NoSuchFieldException { final int reps = 1000; try { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); addMethods(); comparisons(); emptyTuple(); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java index 48a74f4d09..12bef587d2 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java @@ -32,7 +32,7 @@ import com.apple.foundationdb.tuple.Versionstamp; public class VersionstampSmokeTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database db = fdb.open()) { db.run(tr -> { tr.clear(Tuple.from("prefix").range()); diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java index 31076f1305..b204e842e5 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java @@ -34,7 +34,7 @@ import com.apple.foundationdb.Transaction; public class WatchTest { public static void main(String[] args) { - FDB fdb = FDB.selectAPIVersion(620); + FDB fdb = FDB.selectAPIVersion(630); try(Database database = fdb.open(args[0])) { database.options().setLocationCacheSize(42); try(Transaction tr = database.createTransaction()) { diff --git a/bindings/python/LICENSE b/bindings/python/LICENSE new file mode 100644 index 0000000000..19586598a8 --- /dev/null +++ b/bindings/python/LICENSE @@ -0,0 +1,207 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------- +SOFTWARE DISTRIBUTED WITH FOUNDATIONDB: + +The FoundationDB software includes a number of subcomponents with separate +copyright notices and license terms - please see the file ACKNOWLEDGEMENTS. +------------------------------------------------------------------------------- diff --git a/bindings/python/fdb/__init__.py b/bindings/python/fdb/__init__.py index 82ebb0d6e7..0d54c96b5f 100644 --- a/bindings/python/fdb/__init__.py +++ b/bindings/python/fdb/__init__.py @@ -52,7 +52,7 @@ def get_api_version(): def api_version(ver): - header_version = 620 + header_version = 630 if '_version' in globals(): if globals()['_version'] != ver: diff --git a/bindings/python/fdb/impl.py b/bindings/python/fdb/impl.py index c7d33f0fe9..5076136f81 100644 --- a/bindings/python/fdb/impl.py +++ b/bindings/python/fdb/impl.py @@ -449,6 +449,17 @@ class TransactionRead(_FDBBase): if isinstance(key, slice): return self.get_range(key.start, key.stop, reverse=(key.step == -1)) return self.get(key) + + def get_estimated_range_size_bytes(self, begin_key, end_key): + if begin_key is None: + begin_key = b'' + if end_key is None: + end_key = b'\xff' + return FutureInt64(self.capi.fdb_transaction_get_estimated_range_size_bytes( + self.tpointer, + begin_key, len(begin_key), + end_key, len(end_key) + )) class Transaction(TransactionRead): @@ -1220,6 +1231,8 @@ if platform.system() == 'Windows': capi_name = 'fdb_c.dll' elif platform.system() == 'Linux': capi_name = 'libfdb_c.so' +elif platform.system() == 'FreeBSD': + capi_name = 'libfdb_c.so' elif platform.system() == 'Darwin': capi_name = 'libfdb_c.dylib' elif sys.platform == 'win32': @@ -1424,6 +1437,9 @@ def init_c_api(): ctypes.c_int, ctypes.c_int] _capi.fdb_transaction_get_range.restype = ctypes.c_void_p + _capi.fdb_transaction_get_estimated_range_size_bytes.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int] + _capi.fdb_transaction_get_estimated_range_size_bytes.restype = ctypes.c_void_p + _capi.fdb_transaction_add_conflict_range.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_int, ctypes.c_int] _capi.fdb_transaction_add_conflict_range.restype = ctypes.c_int _capi.fdb_transaction_add_conflict_range.errcheck = check_error_code diff --git a/bindings/python/include.mk b/bindings/python/include.mk deleted file mode 100644 index f8fa0dd63a..0000000000 --- a/bindings/python/include.mk +++ /dev/null @@ -1,74 +0,0 @@ -# -# include.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile-gmake; -*- - -TARGETS += fdb_python -CLEAN_TARGETS += fdb_python_clean - -ifeq ($(RELEASE),true) - PYVER = $(VERSION) -else - PYVER = $(VERSION)a1 -endif - -fdb_python: bindings/python/fdb/fdboptions.py bindings/python/setup.py fdb_python_check - -bindings/python/fdb/fdboptions.py: bin/vexillographer.exe fdbclient/vexillographer/fdb.options - @echo "Building $@" - @$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options python $@ - -fdb_python_clean: - @echo "Cleaning fdb_python" - @rm -f bindings/python/fdb/fdboptions.py bindings/python/setup.py - -bindings/python/setup.py: bindings/python/setup.py.in $(ALL_MAKEFILES) versions.target - @echo "Generating $@" - @m4 -DVERSION=$(PYVER) $< > $@ - -fdb_python_check: bindings/python/setup.py bindings/python/fdb/*.py bindings/python/tests/*.py - @echo "Checking fdb_python" - @bash -c "if which pycodestyle &> /dev/null ; then pycodestyle bindings/python --config=bindings/python/setup.cfg ; else echo \"Skipped Python style check! Missing: pycodestyle\"; fi" - -fdb_python_sdist: fdb_python - @mkdir -p packages - @rm -rf bindings/python/dist - @cp LICENSE bindings/python/LICENSE - @cd bindings/python && python setup.py sdist - @rm bindings/python/LICENSE - @cp bindings/python/dist/*.tar.gz packages/ - -fdb_python_sdist_upload: fdb_python - @mkdir -p packages - @rm -rf bindings/python/dist - @cp LICENSE bindings/python/LICENSE - @cd bindings/python && python setup.py sdist upload -r apple-pypi - @rm bindings/python/LICENSE - @cp bindings/python/dist/*.tar.gz packages/ - -fdb_python_sdist_clean: - @echo "Cleaning fdb_python_sdist" - @rm -rf bindings/python/dist - @rm -f bindings/python/MANIFEST bindings/python/setup.py - @rm -f packages/foundationdb-*.tar.gz - -packages: fdb_python_sdist - -packages_clean: fdb_python_sdist_clean diff --git a/bindings/python/tests/size_limit_tests.py b/bindings/python/tests/size_limit_tests.py index 9c71942999..446f787bc1 100644 --- a/bindings/python/tests/size_limit_tests.py +++ b/bindings/python/tests/size_limit_tests.py @@ -22,7 +22,7 @@ import fdb import sys if __name__ == '__main__': - fdb.api_version(620) + fdb.api_version(630) @fdb.transactional def setValue(tr, key, value): diff --git a/bindings/python/tests/tester.py b/bindings/python/tests/tester.py index 8a7640a6c5..1a5cd1fa90 100644 --- a/bindings/python/tests/tester.py +++ b/bindings/python/tests/tester.py @@ -366,6 +366,10 @@ class Tester: inst.push(b'RESULT_NOT_PRESENT') else: inst.push(f) + elif inst.op == six.u("GET_ESTIMATED_RANGE_SIZE"): + begin, end = inst.pop(2) + estimatedSize = obj.get_estimated_range_size_bytes(begin, end).wait() + inst.push(b"GOT_ESTIMATED_RANGE_SIZE") elif inst.op == six.u("GET_KEY"): key, or_equal, offset, prefix = inst.pop(4) result = obj.get_key(fdb.KeySelector(key, or_equal, offset)) diff --git a/bindings/ruby/include.mk b/bindings/ruby/include.mk deleted file mode 100644 index 5537b02c95..0000000000 --- a/bindings/ruby/include.mk +++ /dev/null @@ -1,69 +0,0 @@ -# -# include.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile-gmake; -*- - -TARGETS += fdb_ruby fdb_ruby_gem -CLEAN_TARGETS += fdb_ruby_clean fdb_ruby_gem_clean - -ifeq ($(PLATFORM),linux) - packages: fdb_ruby_gem - - packages_clean: fdb_ruby_gem_clean -endif - -ifeq ($(PLATFORM),linux) - GEM := gem -else ifeq ($(PLATFORM),osx) - GEM := gem -else - $(error Not prepared to build a gem on platform $(PLATFORM)) -endif - -ifeq ($(RELEASE),true) - GEMVER = $(VERSION) -else - GEMVER = $(VERSION)PRERELEASE -endif - -fdb_ruby: bindings/ruby/lib/fdboptions.rb - -bindings/ruby/lib/fdboptions.rb: bin/vexillographer.exe fdbclient/vexillographer/fdb.options - @echo "Building $@" - @$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options ruby $@ - -fdb_ruby_clean: - @echo "Cleaning fdb_ruby" - @rm -f bindings/ruby/lib/fdboptions.rb - -fdb_ruby_gem_clean: - @echo "Cleaning RubyGem" - @rm -f packages/fdb-*.gem bindings/ruby/fdb.gemspec - -bindings/ruby/fdb.gemspec: bindings/ruby/fdb.gemspec.in $(ALL_MAKEFILES) versions.target - @m4 -DVERSION=$(GEMVER) $< > $@ - -fdb_ruby_gem: bindings/ruby/fdb.gemspec fdb_ruby - @echo "Packaging RubyGem" - @mkdir -p packages - @rm -f packages/fdb-*.gem - @cp LICENSE bindings/ruby/LICENSE - @(cd $( boost_1_67_0.tar.bz2 &&\ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha.txt &&\ - sha256sum -c boost-sha.txt &&\ + curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 &&\ + echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt &&\ + sha256sum -c boost-sha-67.txt &&\ tar -xjf boost_1_67_0.tar.bz2 &&\ - rm -rf boost_1_67_0.tar.bz2 boost-sha.txt boost_1_67_0/libs + rm -rf boost_1_67_0.tar.bz2 boost-sha-67.txt boost_1_67_0/libs &&\ + curl -L https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 &&\ + echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt &&\ + sha256sum -c boost-sha-72.txt &&\ + tar -xjf boost_1_72_0.tar.bz2 &&\ + rm -rf boost_1_72_0.tar.bz2 boost-sha-72.txt boost_1_72_0/libs # install cmake -RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz > /tmp/cmake.tar.gz &&\ +RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o /tmp/cmake.tar.gz &&\ echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\ sha256sum -c /tmp/cmake-sha.txt &&\ cd /tmp && tar xf cmake.tar.gz &&\ cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ &&\ rm -rf cmake.tar.gz cmake-3.13.4-Linux-x86_64 cmake-sha.txt -# install LibreSSL -RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip > ninja.zip &&\ +# install Ninja +RUN cd /tmp && curl -L https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip &&\ unzip ninja.zip && cd ninja-1.9.0 && scl enable devtoolset-8 -- ./configure.py --bootstrap && cp ninja /usr/bin &&\ - cd .. && rm -rf ninja-1.9.0 ninja.zip &&\ - curl -L https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz > /tmp/libressl.tar.gz &&\ - cd /tmp && echo "b8cb31e59f1294557bfc80f2a662969bc064e83006ceef0574e2553a1c254fd5 libressl.tar.gz" > libressl-sha.txt &&\ - sha256sum -c libressl-sha.txt && tar xf libressl.tar.gz &&\ - cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-8 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ - cd /tmp/libressl-2.8.2 && scl enable devtoolset-8 -- make -j`nproc` install &&\ - rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz + cd .. && rm -rf ninja-1.9.0 ninja.zip +# install openssl +RUN cd /tmp && curl -L https://www.openssl.org/source/openssl-1.1.1d.tar.gz -o openssl.tar.gz &&\ + echo "1e3a91bc1f9dfce01af26026f856e064eab4c8ee0a8f457b5ae30b40b8b711f2 openssl.tar.gz" > openssl-sha.txt &&\ + sha256sum -c openssl-sha.txt && tar -xzf openssl.tar.gz &&\ + cd openssl-1.1.1d && scl enable devtoolset-8 -- ./config CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ + scl enable devtoolset-8 -- make -j`nproc` && scl enable devtoolset-8 -- make -j1 install &&\ + ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ &&\ + cd /tmp/ && rm -rf /tmp/openssl-1.1.1d /tmp/openssl.tar.gz + +LABEL version=0.1.12 +ENV DOCKER_IMAGEVER=0.1.12 ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ -CMD scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash +CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash diff --git a/build/cmake/Dockerfile b/build/cmake/Dockerfile index 09f7b80cad..3f9d51a29a 100644 --- a/build/cmake/Dockerfile +++ b/build/cmake/Dockerfile @@ -13,10 +13,10 @@ RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.1 cd /tmp && tar xf cmake.tar.gz && cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ # install boost -RUN curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2 > /tmp/boost.tar.bz2 &&\ +RUN curl -L https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_72_0.tar.bz2 > /tmp/boost.tar.bz2 &&\ cd /tmp && echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost.tar.bz2" > boost-sha.txt &&\ - sha256sum -c boost-sha.txt && tar xf boost.tar.bz2 && cp -r boost_1_67_0/boost /usr/local/include/ &&\ - rm -rf boost.tar.bz2 boost_1_67_0 + sha256sum -c boost-sha.txt && tar xf boost.tar.bz2 && cp -r boost_1_72_0/boost /usr/local/include/ &&\ + rm -rf boost.tar.bz2 boost_1_72_0 # install mono (for actorcompiler) RUN yum install -y epel-release diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c index 80d05f591a..a15c1193e7 100644 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ b/build/cmake/package_tester/fdb_c_app/app.c @@ -1,7 +1,7 @@ -#define FDB_API_VERSION 620 +#define FDB_API_VERSION 630 #include int main(int argc, char* argv[]) { - fdb_select_api_version(620); + fdb_select_api_version(630); return 0; } diff --git a/build/cmake/package_tester/modules/tests.sh b/build/cmake/package_tester/modules/tests.sh index 8eef8c4e07..88709a7953 100644 --- a/build/cmake/package_tester/modules/tests.sh +++ b/build/cmake/package_tester/modules/tests.sh @@ -65,7 +65,7 @@ then python setup.py install successOr "Installing python bindings failed" popd - python -c 'import fdb; fdb.api_version(620)' + python -c 'import fdb; fdb.api_version(630)' successOr "Loading python bindings failed" # Test cmake and pkg-config integration: https://github.com/apple/foundationdb/issues/1483 diff --git a/build/concatinate_jsons.py b/build/concatinate_jsons.py deleted file mode 100755 index cf98c99987..0000000000 --- a/build/concatinate_jsons.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python - -import sys -import json - -lst = [] -for filename in sys.argv[1:]: - commands = json.load(open(filename)) - lst.extend(commands) - -json.dump(lst, open("compile_commands.json", "w")) diff --git a/build/csproj.mk b/build/csproj.mk deleted file mode 100644 index 7509aea584..0000000000 --- a/build/csproj.mk +++ /dev/null @@ -1,42 +0,0 @@ -# -# csproj.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -TARGETS += GENNAME -CLEAN_TARGETS += GENNAME()_clean - -GENNAME()_REFERENCES=-r:GENREFERENCES -GENNAME()_SOURCES=$(addprefix GENDIR/,GENSOURCES) - --include GENDIR/local.mk - -.PHONY: GENNAME()_clean GENNAME - -GENNAME: GENTARGET - -GENNAME()_clean: - @echo "Cleaning GENNAME" - @rm -f GENTARGET - -GENTARGET: $(GENNAME()_SOURCES) $(ALL_MAKEFILES) - @echo "Building $@" - @mkdir -p $(@D) - @$(MCS) $(GENNAME()_REFERENCES) $(GENNAME()_LOCAL_REFERENCES) $(GENNAME()_SOURCES) -target:GENOUTPUTTYPE -sdk:4 -out:$@ diff --git a/build/csprojtom4.py b/build/csprojtom4.py deleted file mode 100644 index c6d708ca5b..0000000000 --- a/build/csprojtom4.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python -# -# csprojtom4.py -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import print_function -import sys - -if len(sys.argv) != 2: - print( """Usage: - %s [input]""" % sys.argv[0] ) - sys.exit() - -csproj = sys.argv[1] - -from xml.dom.minidom import parse - -try: - dom = parse(csproj) -except: - print( "ERROR: Unable to open CSProj file %s" % csproj ) - sys.exit() - -outputType = dom.getElementsByTagName("OutputType")[0].childNodes[0].data -assemblyName = dom.getElementsByTagName("AssemblyName")[0].childNodes[0].data - -if outputType == "Exe": - print( "define(`GENTARGET', `bin/%s.exe')dnl" % assemblyName ) - print( "define(`GENOUTPUTTYPE', `exe')dnl" ) -elif outputType == "Library": - print( "define(`GENTARGET', `bin/%s.dll')dnl" % assemblyName ) - print( "define(`GENOUTPUTTYPE', `library')dnl" ) -else: - print( "ERROR: Unable to determine output type" ) - sys.exit() - -sources = [node.getAttribute("Include").replace('\\', '/') for node in - dom.getElementsByTagName("Compile")] -assemblies = [node.getAttribute("Include") for node in - dom.getElementsByTagName("Reference")] - -print( "define(`GENSOURCES', `%s')dnl" % ' '.join(sources) ) -print( "define(`GENREFERENCES', `%s')dnl" % ','.join(assemblies) ) diff --git a/build/ct.config.fdb b/build/ct.config.fdb deleted file mode 100644 index 1db2002394..0000000000 --- a/build/ct.config.fdb +++ /dev/null @@ -1,532 +0,0 @@ -# -# Automatically generated make config: don't edit -# crosstool-NG 1.20.0 Configuration -# Thu Apr 30 12:28:38 2015 -# -CT_CONFIGURE_has_xz=y -CT_MODULES=y - -# -# Paths and misc options -# - -# -# crosstool-NG behavior -# -# CT_OBSOLETE is not set -# CT_EXPERIMENTAL is not set -# CT_DEBUG_CT is not set - -# -# Paths -# -CT_LOCAL_TARBALLS_DIR="${CT_TOP_DIR}/.tarballs" -CT_SAVE_TARBALLS=y -CT_WORK_DIR="${CT_TOP_DIR}/.build" -CT_PREFIX_DIR="/opt/x-toolchain/" -CT_INSTALL_DIR="${CT_PREFIX_DIR}" -CT_RM_RF_PREFIX_DIR=y -CT_REMOVE_DOCS=y -CT_INSTALL_DIR_RO=y -CT_STRIP_ALL_TOOLCHAIN_EXECUTABLES=y - -# -# Downloading -# -# CT_FORBID_DOWNLOAD is not set -# CT_FORCE_DOWNLOAD is not set -CT_CONNECT_TIMEOUT=10 -# CT_ONLY_DOWNLOAD is not set -# CT_USE_MIRROR is not set - -# -# Extracting -# -# CT_FORCE_EXTRACT is not set -CT_OVERIDE_CONFIG_GUESS_SUB=y -# CT_ONLY_EXTRACT is not set -CT_PATCH_BUNDLED=y -# CT_PATCH_LOCAL is not set -# CT_PATCH_BUNDLED_LOCAL is not set -# CT_PATCH_LOCAL_BUNDLED is not set -# CT_PATCH_BUNDLED_FALLBACK_LOCAL is not set -# CT_PATCH_LOCAL_FALLBACK_BUNDLED is not set -# CT_PATCH_NONE is not set -CT_PATCH_ORDER="bundled" - -# -# Build behavior -# -CT_PARALLEL_JOBS=0 -CT_LOAD="" -CT_USE_PIPES=y -CT_EXTRA_CFLAGS_FOR_BUILD="" -CT_EXTRA_LDFLAGS_FOR_BUILD="" -CT_EXTRA_CFLAGS_FOR_HOST="" -CT_EXTRA_LDFLAGS_FOR_HOST="" -# CT_CONFIG_SHELL_SH is not set -# CT_CONFIG_SHELL_ASH is not set -CT_CONFIG_SHELL_BASH=y -# CT_CONFIG_SHELL_CUSTOM is not set -CT_CONFIG_SHELL="${bash}" - -# -# Logging -# -# CT_LOG_ERROR is not set -# CT_LOG_WARN is not set -# CT_LOG_INFO is not set -CT_LOG_EXTRA=y -# CT_LOG_ALL is not set -# CT_LOG_DEBUG is not set -CT_LOG_LEVEL_MAX="EXTRA" -# CT_LOG_SEE_TOOLS_WARN is not set -CT_LOG_PROGRESS_BAR=y -CT_LOG_TO_FILE=y -CT_LOG_FILE_COMPRESS=y - -# -# Target options -# -CT_ARCH="x86" -CT_ARCH_SUPPORTS_32=y -CT_ARCH_SUPPORTS_64=y -CT_ARCH_SUPPORTS_WITH_ARCH=y -CT_ARCH_SUPPORTS_WITH_CPU=y -CT_ARCH_SUPPORTS_WITH_TUNE=y -CT_ARCH_DEFAULT_32=y -CT_ARCH_ARCH="x86-64" -CT_ARCH_CPU="" -CT_ARCH_TUNE="k8" -# CT_ARCH_32 is not set -CT_ARCH_64=y -CT_ARCH_BITNESS=64 -CT_TARGET_CFLAGS="" -CT_TARGET_LDFLAGS="" -# CT_ARCH_alpha is not set -# CT_ARCH_arm is not set -# CT_ARCH_avr32 is not set -# CT_ARCH_blackfin is not set -# CT_ARCH_m68k is not set -# CT_ARCH_mips is not set -# CT_ARCH_powerpc is not set -# CT_ARCH_s390 is not set -# CT_ARCH_sh is not set -# CT_ARCH_sparc is not set -CT_ARCH_x86=y -CT_ARCH_alpha_AVAILABLE=y -CT_ARCH_arm_AVAILABLE=y -CT_ARCH_avr32_AVAILABLE=y -CT_ARCH_blackfin_AVAILABLE=y -CT_ARCH_m68k_AVAILABLE=y -CT_ARCH_microblaze_AVAILABLE=y -CT_ARCH_mips_AVAILABLE=y -CT_ARCH_powerpc_AVAILABLE=y -CT_ARCH_s390_AVAILABLE=y -CT_ARCH_sh_AVAILABLE=y -CT_ARCH_sparc_AVAILABLE=y -CT_ARCH_x86_AVAILABLE=y -CT_ARCH_SUFFIX="" - -# -# Generic target options -# -# CT_MULTILIB is not set -CT_ARCH_USE_MMU=y - -# -# Target optimisations -# -CT_ARCH_FLOAT="" - -# -# Toolchain options -# - -# -# General toolchain options -# -CT_FORCE_SYSROOT=y -CT_USE_SYSROOT=y -CT_SYSROOT_NAME="sysroot" -CT_SYSROOT_DIR_PREFIX="" -CT_WANTS_STATIC_LINK=y -# CT_STATIC_TOOLCHAIN is not set -CT_TOOLCHAIN_PKGVERSION="" -CT_TOOLCHAIN_BUGURL="" - -# -# Tuple completion and aliasing -# -CT_TARGET_VENDOR="nptl" -CT_TARGET_ALIAS_SED_EXPR="" -CT_TARGET_ALIAS="" - -# -# Toolchain type -# -CT_CROSS=y -# CT_CANADIAN is not set -CT_TOOLCHAIN_TYPE="cross" - -# -# Build system -# -CT_BUILD="" -CT_BUILD_PREFIX="" -CT_BUILD_SUFFIX="" - -# -# Misc options -# -# CT_TOOLCHAIN_ENABLE_NLS is not set - -# -# Operating System -# -CT_KERNEL_SUPPORTS_SHARED_LIBS=y -CT_KERNEL="linux" -CT_KERNEL_VERSION="3.8.13" -# CT_KERNEL_bare_metal is not set -CT_KERNEL_linux=y -# CT_KERNEL_windows is not set -CT_KERNEL_bare_metal_AVAILABLE=y -CT_KERNEL_linux_AVAILABLE=y -# CT_KERNEL_V_3_15 is not set -# CT_KERNEL_V_3_14 is not set -# CT_KERNEL_V_3_13 is not set -# CT_KERNEL_V_3_12 is not set -# CT_KERNEL_V_3_11 is not set -# CT_KERNEL_V_3_10 is not set -# CT_KERNEL_V_3_9 is not set -CT_KERNEL_V_3_8=y -# CT_KERNEL_V_3_7 is not set -# CT_KERNEL_V_3_6 is not set -# CT_KERNEL_V_3_5 is not set -# CT_KERNEL_V_3_4 is not set -# CT_KERNEL_V_3_3 is not set -# CT_KERNEL_V_3_2 is not set -# CT_KERNEL_V_3_1 is not set -# CT_KERNEL_V_3_0 is not set -# CT_KERNEL_V_2_6_39 is not set -# CT_KERNEL_V_2_6_38 is not set -# CT_KERNEL_V_2_6_37 is not set -# CT_KERNEL_V_2_6_36 is not set -# CT_KERNEL_V_2_6_33 is not set -# CT_KERNEL_V_2_6_32 is not set -# CT_KERNEL_V_2_6_31 is not set -# CT_KERNEL_V_2_6_27 is not set -# CT_KERNEL_LINUX_CUSTOM is not set -CT_KERNEL_windows_AVAILABLE=y - -# -# Common kernel options -# -CT_SHARED_LIBS=y - -# -# linux other options -# -CT_KERNEL_LINUX_VERBOSITY_0=y -# CT_KERNEL_LINUX_VERBOSITY_1 is not set -# CT_KERNEL_LINUX_VERBOSITY_2 is not set -CT_KERNEL_LINUX_VERBOSE_LEVEL=0 -CT_KERNEL_LINUX_INSTALL_CHECK=y - -# -# Binary utilities -# -CT_ARCH_BINFMT_ELF=y -CT_BINUTILS="binutils" -CT_BINUTILS_binutils=y - -# -# GNU binutils -# -# CT_BINUTILS_V_2_22 is not set -CT_BINUTILS_V_2_21_53=y -# CT_BINUTILS_V_2_21_1a is not set -# CT_BINUTILS_V_2_20_1a is not set -# CT_BINUTILS_V_2_19_1a is not set -# CT_BINUTILS_V_2_18a is not set -CT_BINUTILS_VERSION="2.21.53" -CT_BINUTILS_2_21_or_later=y -CT_BINUTILS_2_20_or_later=y -CT_BINUTILS_2_19_or_later=y -CT_BINUTILS_2_18_or_later=y -CT_BINUTILS_HAS_HASH_STYLE=y -CT_BINUTILS_HAS_GOLD=y -CT_BINUTILS_GOLD_SUPPORTS_ARCH=y -CT_BINUTILS_HAS_PLUGINS=y -CT_BINUTILS_HAS_PKGVERSION_BUGURL=y -CT_BINUTILS_FORCE_LD_BFD=y -CT_BINUTILS_LINKER_LD=y -# CT_BINUTILS_LINKER_LD_GOLD is not set -# CT_BINUTILS_LINKER_GOLD_LD is not set -CT_BINUTILS_LINKERS_LIST="ld" -CT_BINUTILS_LINKER_DEFAULT="bfd" -# CT_BINUTILS_PLUGINS is not set -CT_BINUTILS_EXTRA_CONFIG_ARRAY="" -# CT_BINUTILS_FOR_TARGET is not set - -# -# binutils other options -# - -# -# C-library -# -CT_LIBC="glibc" -CT_LIBC_VERSION="2.11" -# CT_LIBC_eglibc is not set -CT_LIBC_glibc=y -# CT_LIBC_musl is not set -# CT_LIBC_uClibc is not set -CT_LIBC_eglibc_AVAILABLE=y -CT_THREADS="nptl" -CT_LIBC_glibc_AVAILABLE=y -# CT_LIBC_GLIBC_V_2_19 is not set -# CT_LIBC_GLIBC_V_2_18 is not set -# CT_LIBC_GLIBC_V_2_17 is not set -# CT_LIBC_GLIBC_V_2_16_0 is not set -# CT_LIBC_GLIBC_V_2_15 is not set -# CT_LIBC_GLIBC_V_2_14_1 is not set -# CT_LIBC_GLIBC_V_2_14 is not set -# CT_LIBC_GLIBC_V_2_13 is not set -# CT_LIBC_GLIBC_V_2_12_2 is not set -# CT_LIBC_GLIBC_V_2_12_1 is not set -# CT_LIBC_GLIBC_V_2_11_1 is not set -CT_LIBC_GLIBC_V_2_11=y -# CT_LIBC_GLIBC_V_2_10_1 is not set -# CT_LIBC_GLIBC_V_2_9 is not set -# CT_LIBC_GLIBC_V_2_8 is not set -CT_LIBC_mingw_AVAILABLE=y -CT_LIBC_musl_AVAILABLE=y -CT_LIBC_newlib_AVAILABLE=y -CT_LIBC_none_AVAILABLE=y -CT_LIBC_uClibc_AVAILABLE=y -CT_LIBC_SUPPORT_THREADS_ANY=y -CT_LIBC_SUPPORT_THREADS_NATIVE=y - -# -# Common C library options -# -CT_THREADS_NATIVE=y -CT_LIBC_XLDD=y -CT_LIBC_GLIBC_PORTS_EXTERNAL=y -CT_LIBC_glibc_familly=y -CT_LIBC_GLIBC_EXTRA_CONFIG_ARRAY="" -CT_LIBC_GLIBC_CONFIGPARMS="" -CT_LIBC_GLIBC_EXTRA_CFLAGS="" -CT_LIBC_EXTRA_CC_ARGS="" -# CT_LIBC_DISABLE_VERSIONING is not set -CT_LIBC_OLDEST_ABI="" -CT_LIBC_GLIBC_FORCE_UNWIND=y -# CT_LIBC_GLIBC_USE_PORTS is not set -CT_LIBC_ADDONS_LIST="" -# CT_LIBC_LOCALES is not set -# CT_LIBC_GLIBC_KERNEL_VERSION_NONE is not set -CT_LIBC_GLIBC_KERNEL_VERSION_AS_HEADERS=y -# CT_LIBC_GLIBC_KERNEL_VERSION_CHOSEN is not set -CT_LIBC_GLIBC_MIN_KERNEL="3.8.13" - -# -# glibc other options -# - -# -# C compiler -# -CT_CC="gcc" -CT_CC_VERSION="4.9.1" -CT_CC_CORE_PASSES_NEEDED=y -CT_CC_CORE_PASS_1_NEEDED=y -CT_CC_CORE_PASS_2_NEEDED=y -CT_CC_gcc=y -# CT_CC_GCC_SHOW_LINARO is not set -CT_CC_V_4_9_1=y -# CT_CC_V_4_9_0 is not set -# CT_CC_V_4_8_3 is not set -# CT_CC_V_4_8_2 is not set -# CT_CC_V_4_8_1 is not set -# CT_CC_V_4_8_0 is not set -# CT_CC_V_4_7_4 is not set -# CT_CC_V_4_7_3 is not set -# CT_CC_V_4_7_2 is not set -# CT_CC_V_4_7_1 is not set -# CT_CC_V_4_7_0 is not set -# CT_CC_V_4_6_4 is not set -# CT_CC_V_4_6_3 is not set -# CT_CC_V_4_6_2 is not set -# CT_CC_V_4_6_1 is not set -# CT_CC_V_4_6_0 is not set -# CT_CC_V_4_5_3 is not set -# CT_CC_V_4_5_2 is not set -# CT_CC_V_4_5_1 is not set -# CT_CC_V_4_5_0 is not set -# CT_CC_V_4_4_7 is not set -# CT_CC_V_4_4_6 is not set -# CT_CC_V_4_4_5 is not set -# CT_CC_V_4_4_4 is not set -# CT_CC_V_4_4_3 is not set -# CT_CC_V_4_4_2 is not set -# CT_CC_V_4_4_1 is not set -# CT_CC_V_4_4_0 is not set -# CT_CC_V_4_3_6 is not set -# CT_CC_V_4_3_5 is not set -# CT_CC_V_4_3_4 is not set -# CT_CC_V_4_3_3 is not set -# CT_CC_V_4_3_2 is not set -# CT_CC_V_4_3_1 is not set -# CT_CC_V_4_2_4 is not set -# CT_CC_V_4_2_2 is not set -CT_CC_GCC_4_2_or_later=y -CT_CC_GCC_4_3_or_later=y -CT_CC_GCC_4_4_or_later=y -CT_CC_GCC_4_5_or_later=y -CT_CC_GCC_4_6_or_later=y -CT_CC_GCC_4_7_or_later=y -CT_CC_GCC_4_8_or_later=y -CT_CC_GCC_4_9=y -CT_CC_GCC_4_9_or_later=y -CT_CC_GCC_HAS_GRAPHITE=y -CT_CC_GCC_USE_GRAPHITE=y -CT_CC_GCC_HAS_LTO=y -CT_CC_GCC_USE_LTO=y -CT_CC_GCC_HAS_PKGVERSION_BUGURL=y -CT_CC_GCC_HAS_BUILD_ID=y -CT_CC_GCC_HAS_LNK_HASH_STYLE=y -CT_CC_GCC_USE_GMP_MPFR=y -CT_CC_GCC_USE_MPC=y -CT_CC_GCC_HAS_LIBQUADMATH=y -CT_CC_GCC_HAS_LIBSANITIZER=y -# CT_CC_LANG_FORTRAN is not set -CT_CC_SUPPORT_CXX=y -CT_CC_SUPPORT_FORTRAN=y -CT_CC_SUPPORT_JAVA=y -CT_CC_SUPPORT_ADA=y -CT_CC_SUPPORT_OBJC=y -CT_CC_SUPPORT_OBJCXX=y -CT_CC_SUPPORT_GOLANG=y - -# -# Additional supported languages: -# -CT_CC_LANG_CXX=y -# CT_CC_LANG_JAVA is not set - -# -# gcc other options -# -CT_CC_ENABLE_CXX_FLAGS="" -CT_CC_CORE_EXTRA_CONFIG_ARRAY="--with-pic" -CT_CC_EXTRA_CONFIG_ARRAY="--with-pic" -CT_CC_STATIC_LIBSTDCXX=y -# CT_CC_GCC_SYSTEM_ZLIB is not set - -# -# Optimisation features -# - -# -# Settings for libraries running on target -# -CT_CC_GCC_ENABLE_TARGET_OPTSPACE=y -# CT_CC_GCC_LIBMUDFLAP is not set -# CT_CC_GCC_LIBGOMP is not set -# CT_CC_GCC_LIBSSP is not set -# CT_CC_GCC_LIBQUADMATH is not set -# CT_CC_GCC_LIBSANITIZER is not set - -# -# Misc. obscure options. -# -CT_CC_CXA_ATEXIT=y -# CT_CC_GCC_DISABLE_PCH is not set -CT_CC_GCC_SJLJ_EXCEPTIONS=m -CT_CC_GCC_LDBL_128=m -# CT_CC_GCC_BUILD_ID is not set -CT_CC_GCC_LNK_HASH_STYLE_DEFAULT=y -# CT_CC_GCC_LNK_HASH_STYLE_SYSV is not set -# CT_CC_GCC_LNK_HASH_STYLE_GNU is not set -# CT_CC_GCC_LNK_HASH_STYLE_BOTH is not set -CT_CC_GCC_LNK_HASH_STYLE="" -CT_CC_GCC_DEC_FLOAT_AUTO=y -# CT_CC_GCC_DEC_FLOAT_BID is not set -# CT_CC_GCC_DEC_FLOAT_DPD is not set -# CT_CC_GCC_DEC_FLOATS_NO is not set - -# -# Debug facilities -# -# CT_DEBUG_dmalloc is not set -# CT_DEBUG_duma is not set -# CT_DEBUG_gdb is not set -# CT_DEBUG_ltrace is not set -# CT_DEBUG_strace is not set - -# -# Companion libraries -# -CT_COMPLIBS_NEEDED=y -CT_GMP_NEEDED=y -CT_MPFR_NEEDED=y -CT_ISL_NEEDED=y -CT_CLOOG_NEEDED=y -CT_MPC_NEEDED=y -CT_COMPLIBS=y -CT_GMP=y -CT_MPFR=y -CT_ISL=y -CT_CLOOG=y -CT_MPC=y -CT_GMP_V_5_1_3=y -# CT_GMP_V_5_1_1 is not set -# CT_GMP_V_5_0_2 is not set -# CT_GMP_V_5_0_1 is not set -# CT_GMP_V_4_3_2 is not set -# CT_GMP_V_4_3_1 is not set -# CT_GMP_V_4_3_0 is not set -CT_GMP_VERSION="5.1.3" -CT_MPFR_V_3_1_2=y -# CT_MPFR_V_3_1_0 is not set -# CT_MPFR_V_3_0_1 is not set -# CT_MPFR_V_3_0_0 is not set -# CT_MPFR_V_2_4_2 is not set -# CT_MPFR_V_2_4_1 is not set -# CT_MPFR_V_2_4_0 is not set -CT_MPFR_VERSION="3.1.2" -CT_ISL_V_0_12_2=y -# CT_ISL_V_0_11_1 is not set -CT_ISL_VERSION="0.12.2" -CT_CLOOG_V_0_18_1=y -# CT_CLOOG_V_0_18_0 is not set -CT_CLOOG_VERSION="0.18.1" -CT_CLOOG_0_18_or_later=y -CT_MPC_V_1_0_2=y -# CT_MPC_V_1_0_1 is not set -# CT_MPC_V_1_0 is not set -# CT_MPC_V_0_9 is not set -# CT_MPC_V_0_8_2 is not set -# CT_MPC_V_0_8_1 is not set -# CT_MPC_V_0_7 is not set -CT_MPC_VERSION="1.0.2" - -# -# Companion libraries common options -# -# CT_COMPLIBS_CHECK is not set - -# -# Companion tools -# - -# -# READ HELP before you say 'Y' below !!! -# -CT_COMP_TOOLS=y -CT_COMP_TOOLS_make=y -# CT_COMP_TOOLS_m4 is not set -# CT_COMP_TOOLS_autoconf is not set -# CT_COMP_TOOLS_automake is not set -# CT_COMP_TOOLS_libtool is not set diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml index 1853cc80fd..2f6fe49b42 100644 --- a/build/docker-compose.yaml +++ b/build/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: common: &common - image: foundationdb/foundationdb-build:0.1.9 + image: foundationdb/foundationdb-build:0.1.12 build-setup: &build-setup <<: *common @@ -36,11 +36,11 @@ services: release-packages: &release-packages <<: *release-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' snapshot-packages: &snapshot-packages <<: *build-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' prb-packages: <<: *snapshot-packages @@ -48,11 +48,11 @@ services: release-bindings: &release-bindings <<: *release-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' snapshot-bindings: &snapshot-bindings <<: *build-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' prb-bindings: <<: *snapshot-bindings @@ -60,7 +60,7 @@ services: snapshot-cmake: &snapshot-cmake <<: *build-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=0 -DVALGRIND=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' prb-cmake: <<: *snapshot-cmake @@ -68,7 +68,7 @@ services: snapshot-ctest: &snapshot-ctest <<: *build-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -L fast -j "$${MAKEJOBS}" --output-on-failure' prb-ctest: <<: *snapshot-ctest @@ -76,7 +76,7 @@ services: snapshot-correctness: &snapshot-correctness <<: *build-setup - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' + command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DCMAKE_COLOR_MAKEFILE=0 -DFDB_RELEASE=1 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' prb-correctness: <<: *snapshot-correctness diff --git a/build/gen_dev_docker.sh b/build/gen_dev_docker.sh new file mode 100755 index 0000000000..89129d5a86 --- /dev/null +++ b/build/gen_dev_docker.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -e + +# we first check whether the user is in the group docker +user=$(id -un) +DIR_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) +group=$(id -gn) +uid=$(id -u) +gid=$(id -g) +gids=( $(id -G) ) +groups=( $(id -Gn) ) +tmpdir="/tmp/fdb-docker-${DIR_UUID}" +image=fdb-dev + +pushd . +mkdir ${tmpdir} +cd ${tmpdir} + +echo + +cat <> Dockerfile +FROM foundationdb/foundationdb-dev:0.11.1 +RUN yum install -y sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +RUN groupadd -g 1100 sudo +EOF + +num_groups=${#gids[@]} +additional_groups="-G sudo" +for ((i=0;i> Dockerfile + if [ ${gids[i]} -ne ${gid} ] + then + additional_groups="${additional_groups},${gids[$i]}" + fi +done + +cat <> Dockerfile +RUN useradd -u ${uid} -g ${gid} ${additional_groups} -m ${user} + +USER ${user} +CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash + +EOF + +echo "Created ${tmpdir}" +echo "Buidling Docker container ${image}" +sudo docker build -t ${image} . + +popd + +echo "Writing startup script" +mkdir -p $HOME/bin +cat < $HOME/bin/fdb-dev +#!/usr/bin/bash + +if [ -d "\${CCACHE_DIR}" ] +then + args="-v \${CCACHE_DIR}:\${CCACHE_DIR}" + args="\${args} -e CCACHE_DIR=\${CCACHE_DIR}" + args="\${args} -e CCACHE_UMASK=\${CCACHE_UMASK}" + ccache_args=\$args +fi + +if [ -t 1 ] ; then + TERMINAL_ARGS=-it `# Run in interactive mode and simulate a TTY` +else + TERMINAL_ARGS=-i `# Run in interactive mode` +fi + +sudo docker run --rm `# delete (temporary) image after return` \\ + \${TERMINAL_ARGS} \\ + --privileged=true `# Run in privileged mode ` \\ + --cap-add=SYS_PTRACE \\ + --security-opt seccomp=unconfined \\ + -v "${HOME}:${HOME}" `# Mount home directory` \\ + -w="\$(pwd)" \\ + \${ccache_args} \\ + ${image} "\$@" +EOF + +cat < $HOME/bin/clangd +#!/usr/bin/bash + +fdb-dev scl enable devtoolset-8 rh-python36 rh-ruby24 -- clangd +EOF + +if [[ ":$PATH:" != *":$HOME/bin:"* ]] +then + echo "WARNING: $HOME/bin is not in your PATH!" + echo -e "\tThis can cause problems with some scripts (like fdb-clangd)" +fi +chmod +x $HOME/bin/fdb-dev +chmod +x $HOME/bin/clangd +echo "To start the dev docker image run $HOME/bin/fdb-dev" +echo "$HOME/bin/clangd can be used for IDE integration" +echo "You can edit these files but be aware that this script will overwrite your changes if you rerun it" diff --git a/build/link-validate.sh b/build/link-validate.sh deleted file mode 100755 index ac2c799893..0000000000 --- a/build/link-validate.sh +++ /dev/null @@ -1,48 +0,0 @@ -#/bin/sh -# -# This script is used to validate the shared libraries - -verlte() { - [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] -} - -ALLOWED_SHARED_LIBS=("libdl.so.2" "libpthread.so.0" "librt.so.1" "libm.so.6" "libc.so.6" "ld-linux-x86-64.so.2" "libfdb_c.so") - -if [ "$#" -ne 2 ]; then - echo "USAGE: link-validate.sh BINNAME GLIBC_VERSION" - exit 1 -fi - -# Step 1: glibc version - -FAILED=0 -for i in $(objdump -T "$1" | awk '{print $5}' | grep GLIBC | sed 's/ *$//g' | sed 's/GLIBC_//' | sort | uniq); do - if ! verlte "$i" "$2"; then - if [[ $FAILED == 0 ]]; then - echo "!!! WARNING: DEPENDENCY ON NEWER LIBC DETECTED !!!" - fi - - objdump -T "$1" | grep GLIBC_$i | awk '{print $5 " " $6}' | grep "^GLIBC" | sort | awk '$0="\t"$0' - FAILED=1 - fi -done - -if [[ $FAILED == 1 ]]; then - exit 1 -fi - -# Step 2: Other dynamic dependencies - -for j in $(objdump -p "$1" | grep NEEDED | awk '{print $2}'); do - PRESENT=0 - for k in ${ALLOWED_SHARED_LIBS[@]}; do - if [[ "$k" == "$j" ]]; then - PRESENT=1 - break - fi - done - if ! [[ $PRESENT == 1 ]]; then - echo "!!! WARNING: UNKNOWN SHARED OBJECT DEPENDENCY DETECTED: $j !!!" - exit 1 - fi -done diff --git a/build/link-wrapper.sh b/build/link-wrapper.sh deleted file mode 100755 index 184d0393b1..0000000000 --- a/build/link-wrapper.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash - -set -e -OPTIONS='' - -# Get compiler version and major version -COMPILER_VER=$("${CC}" -dumpversion) -COMPILER_MAJVER="${COMPILER_VER%%\.*}" - -# Add linker, if specified and valid -# The linker to use for building: -# can be LD (system default, default choice), GOLD, LLD, or BFD -if [ -n "${USE_LD}" ] && \ - (([[ "${CC}" == *"gcc"* ]] && [ "${COMPILER_MAJVER}" -ge 9 ]) || \ - ([[ "${CXX}" == *"clang++"* ]] && [ "${COMPILER_MAJVER}" -ge 4 ]) ) -then - if [ "${PLATFORM}" == "linux" ]; then - if [ "${USE_LD}" == "BFD" ]; then - OPTIONS+='-fuse-ld=bfd -Wl,--disable-new-dtags' - elif [ "${USE_LD}" == "GOLD" ]; then - OPTIONS+='-fuse-ld=gold -Wl,--disable-new-dtags' - elif [ "${USE_LD}" == "LLD" ]; then - OPTIONS+='-fuse-ld=lld -Wl,--disable-new-dtags' - elif [ "${USE_LD}" != "DEFAULT" ] && [ "${USE_LD}" != "LD" ]; then - echo 'USE_LD must be set to DEFAULT, LD, BFD, GOLD, or LLD!' - exit 1 - fi - fi -fi - -case $1 in - Application | DynamicLibrary) - echo "Linking $3" - - if [ "$1" = "DynamicLibrary" ]; then - OPTIONS+=" -shared" - if [ "$PLATFORM" = "linux" ]; then - OPTIONS+=" -Wl,-z,noexecstack -Wl,-soname,$( basename $3 )" - elif [ "$PLATFORM" = "osx" ]; then - OPTIONS+=" -Wl,-dylib_install_name -Wl,$( basename $3 )" - fi - fi - - OPTIONS=$( eval echo "$OPTIONS $LDFLAGS \$$2_OBJECTS \$$2_LIBS \$$2_STATIC_LIBS_REAL \$$2_LDFLAGS -o $3" ) - - if [[ "${OPTIONS}" == *"-static-libstdc++"* ]]; then - staticlibs=() - staticpaths='' - if [[ "${CC}" == *"gcc"* ]]; then - staticlibs+=('libstdc++.a') - elif [[ "${CXX}" == *"clang++"* ]]; then - staticlibs+=('libc++.a' 'libc++abi.a') - fi - for staticlib in "${staticlibs[@]}"; do - staticpaths+="$("${CC}" -print-file-name="${staticlib}") " - done - OPTIONS=$( echo $OPTIONS | sed -e s,-static-libstdc\+\+,, -e s,\$,\ "${staticpaths}"\ -lm, ) - fi - - case $PLATFORM in - osx) - if [[ "${OPTIONS}" == *"-static-libgcc"* ]]; then - $( $CC -### $OPTIONS 2>&1 | grep '^ ' | sed -e s,^\ ,, -e s,-lgcc[^\ ]*,,g -e s,\",,g -e s,\$,\ `$CC -print-file-name=libgcc_eh.a`, -e s,10.8.2,10.6, ) - else - $CC $OPTIONS - fi - ;; - *) - $CC $OPTIONS - ;; - esac - - if [ -z "$UNSTRIPPED" ]; then - if [ -z "${NOSTRIP}" ]; then echo "Stripping $3"; else echo "Not stripping $3"; fi - - case $1 in - Application) - case $PLATFORM in - linux) - objcopy --only-keep-debug $3 $3.debug - if [ -z "${NOSTRIP}" ]; then strip --strip-debug --strip-unneeded $3; fi - objcopy --add-gnu-debuglink=$3.debug $3 - ./build/link-validate.sh $3 $4 - ;; - osx) - cp $3 $3.debug - if [ -z "${NOSTRIP}" ]; then strip $3; fi - ;; - *) - echo "I don't know how to strip a binary on $PLATFORM" - exit 1 - ;; - esac - ;; - DynamicLibrary) - cp $3 $3-debug - case $PLATFORM in - linux) - if [ -z "${NOSTRIP}" ]; then strip --strip-all $3; fi - ;; - osx) - if [ -z "${NOSTRIP}" ]; then strip -S -x $3; fi - ;; - *) - echo "I don't know how to strip a library on $PLATFORM" - exit 1 - ;; - esac - ;; - esac - fi - ;; - StaticLibrary) - echo "Archiving $3" - rm -f $3 - eval ar rcs $3 \$$2_OBJECTS - ;; - *) - echo "I don't know how to build a $1" - exit 1 - ;; -esac diff --git a/build/packages.mk b/build/packages.mk deleted file mode 100644 index f5f27f81f2..0000000000 --- a/build/packages.mk +++ /dev/null @@ -1,157 +0,0 @@ -# -# packages.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -TARGETS += packages -CLEAN_TARGETS += packages_clean - -PACKAGE_BINARIES = fdbcli fdbserver fdbbackup fdbmonitor fdbrestore fdbdr dr_agent backup_agent -PROJECT_BINARIES = $(addprefix bin/, $(PACKAGE_BINARIES)) -PACKAGE_CONTENTS := $(PROJECT_BINARIES) $(addprefix bin/, $(addsuffix .debug, $(PACKAGE_BINARIES))) lib/libfdb_c.$(DLEXT) bindings/python/fdb/fdboptions.py bindings/c/foundationdb/fdb_c_options.g.h - -packages: TGZ BINS FDBSERVERAPI - -TGZ: $(PACKAGE_CONTENTS) versions.target lib/libfdb_java.$(DLEXT) - @echo "Archiving tgz" - @mkdir -p packages - @rm -f packages/FoundationDB-$(PLATFORM)-*.tar.gz - @bash -c "tar -czf packages/FoundationDB-$(PLATFORM)-$(VERSION)-$(PKGRELEASE).tar.gz bin/{backup_agent{,.debug},fdbmonitor{,.debug},fdbcli{,.debug},fdbserver{,.debug},fdbbackup{,.debug},fdbdr{,.debug},fdbrestore{,.debug},dr_agent{,.debug},coverage.{fdbclient,fdbserver,fdbrpc,flow}.xml} lib/libfdb_c.$(DLEXT){,-debug} lib/libfdb_java.$(DLEXT)* bindings/python/fdb/*.py bindings/c/*.h" - -BINS: packages/foundationdb-binaries-$(VERSION)-$(PLATFORM).tar.gz - -packages_clean: - @echo "Cleaning packages" - @rm -f packages/FoundationDB-$(PLATFORM)-*.tar.gz packages/foundationdb-binaries-$(VERSION)-$(PLATFORM).tar.gz packages/fdb-tests-$(VERSION).tar.gz packages/fdb-headers-$(VERSION).tar.gz packages/fdb-bindings-$(VERSION).tar.gz packages/fdb-server-$(VERSION)-$(PLATFORM).tar.gz - -packages/foundationdb-binaries-$(VERSION)-$(PLATFORM).tar.gz: $(PROJECT_BINARIES) versions.target - @echo "Packaging binaries" - @mkdir -p packages - @rm -f packages/foundationdb-binaries-$(VERSION)-$(PLATFORM).tar.gz - @bash -c "tar -czf packages/foundationdb-binaries-$(VERSION)-$(PLATFORM).tar.gz $(PROJECT_BINARIES)" - -packages/fdb-server-$(VERSION)-$(PLATFORM).tar.gz: bin/fdbserver bin/fdbcli lib/libfdb_c.$(DLEXT) - @echo "Packaging fdb server api" - @rm -rf packages/fdbserverapi - @mkdir -p packages/fdbserverapi/bin packages/fdbserverapi/lib - @cp bin/fdbserver bin/fdbcli packages/fdbserverapi/bin/ - @cp lib/libfdb_c.$(DLEXT) packages/fdbserverapi/lib/ - @tar czf packages/fdb-server-$(VERSION)-$(PLATFORM).tar.gz -C packages/fdbserverapi/ . - @rm -rf packages/fdbserverapi - -FDBSERVERAPI: packages/fdb-server-$(VERSION)-$(PLATFORM).tar.gz - -FDBSERVERAPI_clean: - @echo "Cleaning fdb server api" - @rm -rf packages/fdb-server-$(VERSION)-$(PLATFORM).tar.gz packages/fdbserverapi - -ifeq ($(PLATFORM),linux) - DEB: packages/foundationdb-clients_$(VERSION)-$(PKGRELEASE)_amd64.deb packages/foundationdb-server_$(VERSION)-$(PKGRELEASE)_amd64.deb - - DEB_clean: - @echo "Cleaning deb" - @rm -f packages/foundationdb-server_*.deb packages/foundationdb-clients_*.deb - - DEB_FILES := $(addprefix packaging/deb/,builddebs.sh foundationdb-clients.control.in foundationdb-init foundationdb-server.control.in DEBIAN-foundationdb-clients/postinst $(addprefix DEBIAN-foundationdb-server/,conffiles postinst postrm preinst prerm)) - - packages/foundationdb-server_%.deb packages/foundationdb-clients_%.deb: $(PACKAGE_CONTENTS) versions.target $(DEB_FILES) - @echo "Packaging deb" - @mkdir -p packages - @rm -f packages/foundationdb-server_*.deb packages/foundationdb-clients_*.deb - @mkdir -p packaging/deb/DEBIAN-foundationdb-server packaging/deb/DEBIAN-foundationdb-clients - @for i in server clients; do \ - m4 -DVERSION=$(VERSION) -DRELEASE=$(PKGRELEASE) packaging/deb/foundationdb-$$i.control.in > packaging/deb/DEBIAN-foundationdb-$$i/control; \ - done - @packaging/deb/builddebs.sh - @rm packaging/deb/DEBIAN-*/control - - RPM: packages/foundationdb-server-$(VERSION)-$(PKGRELEASE).el6.x86_64.rpm packages/foundationdb-clients-$(VERSION)-$(PKGRELEASE).el6.x86_64.rpm packages/foundationdb-server-$(VERSION)-$(PKGRELEASE).el7.x86_64.rpm packages/foundationdb-clients-$(VERSION)-$(PKGRELEASE).el7.x86_64.rpm - - RPM_clean: - @echo "Cleaning rpm" - @rm -f packages/foundationdb-server-*.rpm packages/foundationdb-clients-*.rpm - - RPM_FILES := $(addprefix packaging/rpm/,buildrpms.sh foundationdb-init foundationdb.service foundationdb.spec.in) - - JAVA_RELEASE: fdb_java_release - - JAVA_RELEASE_clean: fdb_java_release_clean - - FDBTESTS: - @echo "Archiving fdbtests" - @mkdir -p packages - @rm -f packages/fdb-tests-$(VERSION).tar.gz - @bash -c "tar -czf packages/fdb-tests-$(VERSION).tar.gz -C tests ." - - FDBTESTS_clean: - @echo "Cleaning fdbtests" - @rm -f packages/fdb-tests-$(VERSION).tgz - - FDBBINDINGS: bindings - @echo "Archiving fdbbindings" - @mkdir -p packages - @rm -f packages/fdb-bindings-$(VERSION).tar.gz - @bash -c "tar -czf packages/fdb-bindings-$(VERSION).tar.gz -C bindings ." - - FDBBINDINGS_clean: - @echo "Cleaning fdbbindings" - @rm -f packages/fdb-bindings-$(VERSION).tgz - - FDBHEADERS: bindings/python/fdb/fdboptions.py bindings/c/foundationdb/fdb_c_options.g.h fdbclient/vexillographer/fdb.options - @echo "Archiving fdbheaders" - @mkdir -p packages - @rm -f packages/fdb-headers-$(VERSION).tar.gz - @bash -c "tar -czf packages/fdb-headers-$(VERSION).tar.gz -C $(shell pwd)/bindings/c/foundationdb fdb_c.h -C $(shell pwd)/bindings/c/foundationdb fdb_c_options.g.h -C $(shell pwd)/fdbclient/vexillographer fdb.options" - - FDBHEADERS_clean: - @echo "Cleaning fdbheaders" - @rm -f packages/fdb-headers-$(VERSION).tgz - - packages/foundationdb-server-%.el6.x86_64.rpm packages/foundationdb-clients-%.el6.x86_64.rpm packages/foundationdb-server-%.el7.x86_64.rpm packages/foundationdb-clients-%.el7.x86_64.rpm: $(PACKAGE_CONTENTS) versions.target $(RPM_FILES) - - packages/foundationdb-server-%.el6.x86_64.rpm packages/foundationdb-clients-%.el6.x86_64.rpm packages/foundationdb-server-%.el7.x86_64.rpm packages/foundationdb-clients-%.el7.x86_64.rpm: $(PACKAGE_CONTENTS) versions.target $(RPM_FILES) - @echo "Packaging rpm" - @mkdir -p packages - @rm -f packages/foundationdb-server-*.rpm packages/foundationdb-clients-*.rpm - @packaging/rpm/buildrpms.sh $(VERSION) $(PKGRELEASE) - - packages: DEB RPM JAVA_RELEASE FDBTESTS FDBHEADERS - - packages_clean: DEB_clean RPM_clean JAVA_RELEASE_clean FDBHEADERS_clean - -endif - -ifeq ($(PLATFORM),osx) - ifeq ($(RELEASE),true) - PKGFILE := packages/FoundationDB-$(VERSION).pkg - else - PKGFILE := packages/FoundationDB-$(VERSION)-PRERELEASE.pkg - endif - - PKG: $(PACKAGE_CONTENTS) versions.target - @mkdir -p packages - @rm -f packages/*.pkg - @packaging/osx/buildpkg.sh $(PKGFILE) $(VERSION) $(PKGRELEASE) - - packages: PKG - - packages_clean: packages_osx_clean - - packages_osx_clean: - @rm -f packages/*.pkg -endif diff --git a/build/project_commands.py b/build/project_commands.py deleted file mode 100755 index 458c9ff86f..0000000000 --- a/build/project_commands.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python - -import argparse -import json -import os -import os.path -import sys - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--cflags', help="$(CFLAGS)") - parser.add_argument('--cxxflags', help="$(CXXFLAGS)") - parser.add_argument('--sources', help="All the source files") - parser.add_argument('--out', help="Output file name") - return parser.parse_args() - -def main(): - args = parse_args() - cwd = os.getcwd() - - args.cflags = args.cflags.replace('-DNO_INTELLISENSE', '').replace("/opt/boost", cwd+"/../boost") - - commands = [] - for fname in args.sources.split(' '): - d = {} - d["directory"] = cwd - compiler = "" - if fname.endswith("cpp") or fname.endswith(".h"): - compiler = "clang++ -x c++ " + args.cflags + args.cxxflags - if fname.endswith("c"): - compiler = "clang -x c " + args.cflags - d["command"] = compiler - d["file"] = fname - commands.append(d) - - json.dump(commands, open(args.out, "w")) - -if __name__ == '__main__': - main() diff --git a/build/scver.mk b/build/scver.mk deleted file mode 100644 index 8dfeea9504..0000000000 --- a/build/scver.mk +++ /dev/null @@ -1,167 +0,0 @@ -# -# scver.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -######################################################################### -# -# This makefile will define the make variables related to source control -# variables, values, and settings -# -# -# Author: Alvin Moore -# Created: 15-08-01 -######################################################################### - - -# Retrieves the major version number from a version string -# Param: -# 1. String to parse in form 'major[.minor][.build]'. -MAJORVERFUNC = $(firstword $(subst ., ,$1)) - -# Retrieves the major version number from a version string -# If there is no minor part in the string, returns the second argument -# (if specified). -# Param: -# 1. String to parse in form 'major[.minor][.build]'. -# 2. (optional) Fallback value. -MINORVERFUNC = $(or $(word 2,$(subst ., ,$1)),$(value 2)) - -# Ensures that the specified directory is created -# Displays a creation message, if the directory does not exists -# Param: -# 1. Path to the directory to create -# 2. (optional) Display name of the directory. -CREATEDIRFUNC = if [ ! -d "$1" ]; then echo "`date +%F_%H-%M-%S` Creating $2 directory: $1"; mkdir -p "$1"; fi - - -# Make Environment Settings -# -ARCH := $(shell uname -m) -MAKEDIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -FDBDIR := $(abspath $(MAKEDIR)/..) -FDBPARENTDIR := $(abspath $(FDBDIR)/..) -FDBDIRBASE := $(shell basename $(FDBDIR)) -USERID := $(shell id -u) -USER := $(shell whoami) -PROCESSID := $(shell echo "$$$$") - -ifeq ($(PLATFORM),osx) - MD5SUM=md5 -else - MD5SUM=md5sum -endif - - -# -# Define the Java Variables -# - -# Determine the Java compiler, if not defined -ifndef JAVAC - JAVAC := $(shell which javac) - ifeq ($(JAVAC),) -$(warning JAVA compiler is not installed on $(PLATFORM) $(ARCH)) - endif -endif - -# Define the Java Flags based on Java version -ifdef JAVAC - JAVAVER := $(shell bash -c 'javac -version 2>&1 | cut -d\ -f2-') - JAVAVERMAJOR := $(call MAJORVERFUNC,$(JAVAVER)) - JAVAVERMINOR := $(call MINORVERFUNC,$(JAVAVER)) - ifneq ($(JAVAVERMAJOR),1) -$(warning Unable to compile source using Java version: $(JAVAVER) with compiler: $(JAVAC) on $(PLATFORM) $(ARCH)) - else - JAVAFLAGS := -Xlint -source 1.8 -target 1.8 - endif -endif - - -# Determine active Version Control -# -GITPRESENT := $(wildcard $(FDBDIR)/.git) -HGPRESENT := $(wildcard $(FDBDIR)/.hg) - -# Do not override version IDs if already set -ifneq ($(VERSION_ID),) -# Noop - -# Use Git, if not missing -else ifneq ($(GITPRESENT),) - SCVER := $(shell cd "$(FDBDIR)" && git --version 2>/dev/null) - ifneq ($(SCVER),) - VERSION_ID := $(shell cd "$(FDBDIR)" && git rev-parse --verify HEAD) - SOURCE_CONTROL := GIT - SCBRANCH := $(shell cd "$(FDBDIR)" && git rev-parse --abbrev-ref HEAD) - else -$(error Missing git executable on $(PLATFORM) ) - endif - -# Otherwise, use Mercurial -else ifneq ($(HGPRESENT),) - SCVER := $(shell cd "$(FDBDIR)" && hg --version 2>/dev/null) - ifdef SCVER - VERSION_ID := $(shell cd "$(FDBDIR)" && hg id -n) - SOURCE_CONTROL := MERCURIAL - SCBRANCH := $(shell cd "$(FDBDIR)" && hg branch) - else -$(error Missing hg executable on $(PLATFORM)) - endif - -# No version control system -else - FDBFILES := $(shell ls -la $(FDBDIR)) -$(error Missing source control information for source on $(PLATFORM) in directory: $(FDBDIR) with files: $(FDBFILES)) -endif - -# Set the RELEASE variable based on the KVRELEASE variable. -ifeq ($(KVRELEASE),1) - RELEASE := true -endif - -# Define the Package Release and the File Version -ifeq ($(RELEASE),true) - PKGRELEASE := 1 -else ifeq ($(PRERELEASE),true) - PKGRELEASE := 0.$(VERSION_ID).PRERELEASE -else - PKGRELEASE := 0INTERNAL -endif - - -info: - @echo "Displaying Make Information" - @echo "Version: $(VERSION)" - @echo "Package: $(PACKAGE_NAME)" - @echo "Version ID: $(VERSION_ID)" - @echo "Package ID: $(PKGRELEASE)" - @echo "SC Branch: $(SCBRANCH)" - @echo "Git Dir: $(GITPRESENT)" - @echo "Make Dir: $(MAKEDIR)" - @echo "Foundation Dir: $(FDBDIR)" - @echo "Fdb Dir Base: $(FDBDIRBASE)" - @echo "User Id: $(USERID)" - @echo "Java Version: ($(JAVAVERMAJOR).$(JAVAVERMINOR)) $(JAVAVER)" - @echo "Platform: $(PLATFORM)" -ifdef TLS_DISABLED - @echo "TLS: Disabled" -else - @echo "TLS: Enabled" -endif - @echo "" diff --git a/build/valgrind.mk b/build/valgrind.mk deleted file mode 100644 index 13dc8af093..0000000000 --- a/build/valgrind.mk +++ /dev/null @@ -1,36 +0,0 @@ -# -# valgrind.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#VALGRIND := 1 - -ifeq ($(VALGRIND), 1) -ifeq ($(DEBUGLEVEL), 1) -$(info Enabling Valgrind instrumentation) -endif - CFLAGS += -DVALGRIND=1 -DUSE_VALGRIND=1 - - ifeq ($(PLATFORM), linux) - CFLAGS += -I/usr/include/valgrind - else ifeq ($(PLATFORM), osx) - CFLAGS += -I/usr/local/include/valgrind - else - $(error valgrind not supported on platform $(PLATFORM)) - endif -endif diff --git a/build/vcxproj.mk b/build/vcxproj.mk deleted file mode 100644 index d608673d77..0000000000 --- a/build/vcxproj.mk +++ /dev/null @@ -1,137 +0,0 @@ -# -# vcxproj.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -TARGETS += GENNAME -CLEAN_TARGETS += GENNAME()_clean - -GENNAME()_ALL_SOURCES := $(addprefix GENDIR/,GENSOURCES) - -GENNAME()_BUILD_SOURCES := $(patsubst %.actor.cpp,${OBJDIR}/%.actor.g.cpp,$(filter-out %.h %.hpp,$(GENNAME()_ALL_SOURCES))) -GENNAME()_GENERATED_SOURCES := $(patsubst %.actor.h,%.actor.g.h,$(patsubst %.actor.cpp,${OBJDIR}/%.actor.g.cpp,$(filter %.actor.h %.actor.cpp,$(GENNAME()_ALL_SOURCES)))) -GENERATED_SOURCES += $(GENNAME()_GENERATED_SOURCES) - --include GENDIR/local.mk - -# We need to include the current directory for .g.actor.cpp files emitted into -# .objs that use includes not based at the root of fdb. -GENNAME()_CFLAGS := -I GENDIR -I ${OBJDIR}/GENDIR ${GENNAME()_CFLAGS} - -# If we have any static libs, we have to wrap them in the appropriate -# compiler flag magic -ifeq ($(GENNAME()_STATIC_LIBS),) - GENNAME()_STATIC_LIBS_REAL := -else -# MacOS doesn't recognize -Wl,-Bstatic, but is happy with -Bstatic -# gcc will handle both, so we prefer the non -Wl version - GENNAME()_STATIC_LIBS_REAL := -Bstatic $(GENNAME()_STATIC_LIBS) -Bdynamic -endif - -# If we have any -L directives in our LDFLAGS, we need to add those -# paths to the VPATH -VPATH += $(addprefix :,$(patsubst -L%,%,$(filter -L%,$(GENNAME()_LDFLAGS)))) - -IGNORE := $(shell echo $(VPATH)) - -GENNAME()_OBJECTS := $(addprefix $(OBJDIR)/,$(filter-out $(OBJDIR)/%,$(GENNAME()_BUILD_SOURCES:=.o))) $(filter $(OBJDIR)/%,$(GENNAME()_BUILD_SOURCES:=.o)) -GENNAME()_DEPS := $(addprefix $(DEPSDIR)/,$(GENNAME()_BUILD_SOURCES:=.d)) - -.PHONY: GENNAME()_clean GENNAME - -GENNAME: GENTARGET - -$(CMDDIR)/GENDIR/compile_commands.json: build/project_commands.py ${GENNAME()_ALL_SOURCES} - @mkdir -p $(basename $@) - @build/project_commands.py --cflags="$(CFLAGS) $(GENNAME()_CFLAGS)" --cxxflags="$(CXXFLAGS) $(GENNAME()_CXXFLAGS)" --sources="$(GENNAME()_ALL_SOURCES)" --out="$@" - --include $(GENNAME()_DEPS) - -$(OBJDIR)/GENDIR/%.actor.g.cpp: GENDIR/%.actor.cpp $(ACTORCOMPILER) - @echo "Actorcompiling $<" - @mkdir -p $(OBJDIR)/$(/dev/null - -GENDIR/%.actor.g.h: GENDIR/%.actor.h $(ACTORCOMPILER) - @if [ -e $< ]; then echo "Actorcompiling $<" ; $(MONO) $(ACTORCOMPILER) $< $@ >/dev/null ; fi -.PRECIOUS: $(OBJDIR)/GENDIR/%.actor.g.cpp GENDIR/%.actor.g.h - -# The order-only dependency on the generated .h files is to force make -# to actor compile all headers before attempting compilation of any .c -# or .cpp files. We have no mechanism to detect dependencies on -# generated headers before compilation. - -$(OBJDIR)/GENDIR/%.cpp.o: GENDIR/%.cpp $(ALL_MAKEFILES) | $(filter %.h,$(GENERATED_SOURCES)) - @echo "Compiling $(<:${OBJDIR}/%=%)" -ifeq ($(VERBOSE),1) - @echo $(CCACHE_CXX) $(CFLAGS) $(CXXFLAGS) $(GENNAME()_CFLAGS) $(GENNAME()_CXXFLAGS) -MMD -MT $@ -MF $(DEPSDIR)/$<.d.tmp -c $< -o $@ -endif - @mkdir -p $(DEPSDIR)/$(> $(DEPSDIR)/$<.d && \ - rm $(DEPSDIR)/$<.d.tmp - -$(OBJDIR)/GENDIR/%.cpp.o: $(OBJDIR)/GENDIR/%.cpp $(ALL_MAKEFILES) | $(filter %.h,$(GENERATED_SOURCES)) - @echo "Compiling $(<:${OBJDIR}/%=%)" -ifeq ($(VERBOSE),1) - @echo $(CCACHE_CXX) $(CFLAGS) $(CXXFLAGS) $(GENNAME()_CFLAGS) $(GENNAME()_CXXFLAGS) -MMD -MT $@ -MF $(DEPSDIR)/$<.d.tmp -c $< -o $@ -endif - @mkdir -p $(DEPSDIR)/$(> $(DEPSDIR)/$<.d && \ - rm $(DEPSDIR)/$<.d.tmp - -$(OBJDIR)/GENDIR/%.c.o: GENDIR/%.c $(ALL_MAKEFILES) | $(filter %.h,$(GENERATED_SOURCES)) - @echo "Compiling $<" -ifeq ($(VERBOSE),1) - @echo "$(CCACHE_CC) $(CFLAGS) $(GENNAME()_CFLAGS) -MMD -MT $@ -MF $(DEPSDIR)/$<.d.tmp -c $< -o $@" -endif - @mkdir -p $(DEPSDIR)/$(> $(DEPSDIR)/$<.d && \ - rm $(DEPSDIR)/$<.d.tmp - -$(OBJDIR)/GENDIR/%.S.o: GENDIR/%.S $(ALL_MAKEFILES) | $(filter %.h,$(GENERATED_SOURCES)) - @echo "Assembling $<" -ifeq ($(VERBOSE),1) - @echo "$(CCACHE_CC) $(CFLAGS) $(GENNAME()_CFLAGS) -MMD -MT $@ -MF $(DEPSDIR)/$<.d.tmp -c $< -o $@" -endif - @mkdir -p $(DEPSDIR)/$(> $(DEPSDIR)/$<.d && \ - rm $(DEPSDIR)/$<.d.tmp - -GENNAME()_clean: - @echo "Cleaning GENNAME" - @rm -f GENTARGET $(GENNAME()_GENERATED_SOURCES) GENTARGET().debug GENTARGET()-debug - @rm -rf $(DEPSDIR)/GENDIR - @rm -rf $(OBJDIR)/GENDIR - -GENTARGET: $(GENNAME()_OBJECTS) $(GENNAME()_LIBS) $(ALL_MAKEFILES) build/link-wrapper.sh build/link-validate.sh - @mkdir -p GENOUTDIR - @./build/link-wrapper.sh GENCONFIGTYPE GENNAME $@ $(TARGET_LIBC_VERSION) diff --git a/build/vcxprojtom4.py b/build/vcxprojtom4.py deleted file mode 100644 index 32ffec7987..0000000000 --- a/build/vcxprojtom4.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/python -# -# vcxprojtom4.py -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import print_function -import sys - -if len(sys.argv) != 2: - print( """Usage: - %s [input]""" % sys.argv[0] ) - sys.exit() - -vcxproj = sys.argv[1] - -from xml.dom.minidom import parse - -try: - dom = parse(vcxproj) -except: - print( "ERROR: Unable to open VCXProj file %s" % vcxproj ) - sys.exit() - -# We need to find out what kind of project/configuration we're going -# to build. FIXME: Right now we're hardcoded to look for the -# Release|X64 configuration/platform. - -groups = dom.getElementsByTagName("PropertyGroup") -for group in groups: - if group.getAttribute("Label").lower() == "configuration" and \ - group.getAttribute("Condition").lower() == "'$(configuration)|$(platform)'=='release|x64'": - ctnodes = group.getElementsByTagName("ConfigurationType") - configType = ctnodes[0].childNodes[0].data - break - -print( "define(`GENCONFIGTYPE', `%s')dnl" % configType ) - -if configType == "StaticLibrary": - print( "define(`GENTARGET', `lib/lib`'GENNAME.a')dnl" ) - print( "define(`GENOUTDIR', `lib')dnl" ) -elif configType == "DynamicLibrary": - print( "define(`GENTARGET', `lib/lib`'GENNAME.$(DLEXT)')dnl" ) - print( "define(`GENOUTDIR', `lib')dnl" ) -elif configType == "Application": - print( "define(`GENTARGET', `bin/'`GENNAME')dnl" ) - print( "define(`GENOUTDIR', `bin')dnl" ) -else: - print( "ERROR: Unable to determine configuration type" ) - sys.exit() - -sources = [node.getAttribute("Include").replace('\\', '/') for node in - dom.getElementsByTagName("ActorCompiler") + - dom.getElementsByTagName("ClCompile") + - dom.getElementsByTagName("ClInclude") - if not node.getElementsByTagName("ExcludedFromBuild") and node.hasAttribute("Include")] - -print( "define(`GENSOURCES', `%s')dnl" % ' '.join(sorted(sources)) ) diff --git a/cmake/AddFdbTest.cmake b/cmake/AddFdbTest.cmake index c494e19229..46c3fc3e8b 100644 --- a/cmake/AddFdbTest.cmake +++ b/cmake/AddFdbTest.cmake @@ -87,6 +87,9 @@ function(add_fdb_test) if (NOT "${ADD_FDB_TEST_TEST_NAME}" STREQUAL "") set(test_name ${ADD_FDB_TEST_TEST_NAME}) endif() + if((NOT test_name MATCHES "${TEST_INCLUDE}") OR (test_name MATCHES "${TEST_EXCLUDE}")) + return() + endif() math(EXPR test_idx "${CURRENT_TEST_INDEX} + ${NUM_TEST_FILES}") set(CURRENT_TEST_INDEX "${test_idx}" PARENT_SCOPE) # set( PARENT_SCOPE) doesn't set the @@ -160,8 +163,6 @@ function(create_test_package) string(SUBSTRING ${file} ${base_length} -1 rel_out_file) set(out_file ${CMAKE_BINARY_DIR}/packages/tests/${rel_out_file}) list(APPEND out_files ${out_file}) - get_filename_component(test_dir ${out_file} DIRECTORY) - file(MAKE_DIRECTORY packages/tests/${test_dir}) add_custom_command( OUTPUT ${out_file} DEPENDS ${file} @@ -184,15 +185,151 @@ function(create_test_package) file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/packages/${dest_dir}) endforeach() endforeach() - set(tar_file ${CMAKE_BINARY_DIR}/packages/correctness.tar.gz) + if(NOT USE_VALGRIND) + set(tar_file ${CMAKE_BINARY_DIR}/packages/correctness-${CMAKE_PROJECT_VERSION}.tar.gz) + add_custom_command( + OUTPUT ${tar_file} + DEPENDS ${out_files} + ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe + ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll + ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/correctnessTest.sh + ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/correctnessTimeout.sh + ${external_files} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/correctnessTest.sh + ${CMAKE_BINARY_DIR}/packages/joshua_test + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/correctnessTimeout.sh + ${CMAKE_BINARY_DIR}/packages/joshua_timeout + COMMAND ${CMAKE_COMMAND} -E tar cfz ${tar_file} ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe + ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll + ${CMAKE_BINARY_DIR}/packages/joshua_test + ${CMAKE_BINARY_DIR}/packages/joshua_timeout + ${out_files} + ${external_files} + COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_BINARY_DIR}/packages/joshua_test ${CMAKE_BINARY_DIR}/packages/joshua_timeout + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/packages + COMMENT "Package correctness archive" + ) + add_custom_target(package_tests ALL DEPENDS ${tar_file}) + # seems make needs this dependency while this does nothing with ninja + add_dependencies(package_tests strip_only_fdbserver TestHarness) + endif() + + if(USE_VALGRIND) + set(tar_file ${CMAKE_BINARY_DIR}/packages/valgrind-${CMAKE_PROJECT_VERSION}.tar.gz) + add_custom_command( + OUTPUT ${tar_file} + DEPENDS ${out_files} + ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe + ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll + ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/valgrindTest.sh + ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/valgrindTimeout.sh + ${external_files} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/valgrindTest.sh + ${CMAKE_BINARY_DIR}/packages/joshua_test + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/valgrindTimeout.sh + ${CMAKE_BINARY_DIR}/packages/joshua_timeout + COMMAND ${CMAKE_COMMAND} -E tar cfz ${tar_file} + ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe + ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll + ${CMAKE_BINARY_DIR}/packages/joshua_test + ${CMAKE_BINARY_DIR}/packages/joshua_timeout + ${out_files} + ${external_files} + COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_BINARY_DIR}/packages/joshua_test ${CMAKE_BINARY_DIR}/packages/joshua_timeout + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/packages + COMMENT "Package correctness archive" + ) + add_custom_target(package_valgrind_tests ALL DEPENDS ${tar_file}) + add_dependencies(package_valgrind_tests strip_only_fdbserver TestHarness) + endif() +endfunction() + +function(package_bindingtester) + if(WIN32 OR OPEN_FOR_IDE) + return() + elseif(APPLE) + set(fdbcName "libfdb_c.dylib") + else() + set(fdbcName "libfdb_c.so") + endif() + set(bdir ${CMAKE_BINARY_DIR}/bindingtester) + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/bindingtester) + set(outfiles ${bdir}/fdbcli ${bdir}/fdbserver ${bdir}/${fdbcName} ${bdir}/joshua_test ${bdir}/joshua_timeout) + add_custom_command( + OUTPUT ${outfiles} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/packages/bin/fdbcli + ${CMAKE_BINARY_DIR}/packages/bin/fdbserver + ${CMAKE_BINARY_DIR}/packages/lib/${fdbcName} + ${bdir} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/bindingTest.sh ${bdir}/joshua_test + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/bindingTimeout.sh ${bdir}/joshua_timeout + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/localClusterStart.sh ${bdir}/localClusterStart.sh + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/contrib/Joshua/scripts/bindingTestScript.sh ${bdir}/bindingTestScript.sh + COMMENT "Copy executes to bindingtester dir") + file(GLOB_RECURSE test_files ${CMAKE_SOURCE_DIR}/bindings/*) + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/bindingtester.touch" + COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/bindingtester/tests + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/bindingtester/tests + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/bindings ${CMAKE_BINARY_DIR}/bindingtester/tests + COMMAND ${CMAKE_COMMAND} -E touch "${CMAKE_BINARY_DIR}/bindingtester.touch" + COMMENT "Copy test files for bindingtester") + + add_custom_target(copy_binding_output_files DEPENDS ${CMAKE_BINARY_DIR}/bindingtester.touch python_binding fdb_flow_tester) + add_custom_command( + TARGET copy_binding_output_files + COMMAND ${CMAKE_COMMAND} -E copy $ ${bdir}/tests/flow/bin/fdb_flow_tester + COMMENT "Copy Flow tester for bindingtester") + + set(generated_binding_files python/fdb/fdboptions.py) + if(WITH_JAVA) + if(NOT FDB_RELEASE) + set(prerelease_string "-PRERELEASE") + else() + set(prerelease_string "") + endif() + add_custom_command( + TARGET copy_binding_output_files + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_BINARY_DIR}/packages/fdb-java-${CMAKE_PROJECT_VERSION}${prerelease_string}.jar + ${bdir}/tests/java/foundationdb-client.jar + COMMENT "Copy Java bindings for bindingtester") + add_dependencies(copy_binding_output_files fat-jar) + add_dependencies(copy_binding_output_files foundationdb-tests) + set(generated_binding_files ${generated_binding_files} java/foundationdb-tests.jar) + endif() + + if(WITH_GO AND NOT OPEN_FOR_IDE) + add_dependencies(copy_binding_output_files fdb_go_tester fdb_go) + add_custom_command( + TARGET copy_binding_output_files + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bindings/go/bin/_stacktester ${bdir}/tests/go/build/bin/_stacktester + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_BINARY_DIR}/bindings/go/src/github.com/apple/foundationdb/bindings/go/src/fdb/generated.go # SRC + ${bdir}/tests/go/src/fdb/ # DEST + COMMENT "Copy generated.go for bindingtester") + endif() + + foreach(generated IN LISTS generated_binding_files) + add_custom_command( + TARGET copy_binding_output_files + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/bindings/${generated} ${bdir}/tests/${generated} + COMMENT "Copy ${generated} to bindingtester") + endforeach() + + add_custom_target(copy_bindingtester_binaries + DEPENDS ${outfiles} "${CMAKE_BINARY_DIR}/bindingtester.touch" copy_binding_output_files) + add_dependencies(copy_bindingtester_binaries strip_only_fdbserver strip_only_fdbcli strip_only_fdb_c) + set(tar_file ${CMAKE_BINARY_DIR}/packages/bindingtester-${CMAKE_PROJECT_VERSION}.tar.gz) add_custom_command( OUTPUT ${tar_file} - DEPENDS ${out_files} - COMMAND ${CMAKE_COMMAND} -E tar cfz ${tar_file} ${CMAKE_BINARY_DIR}/packages/bin/fdbserver - ${out_files} ${external_files} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/packages - COMMENT "Package correctness archive" - ) - add_custom_target(package_tests DEPENDS ${tar_file}) - add_dependencies(package_tests strip_fdbserver) + COMMAND ${CMAKE_COMMAND} -E tar czf ${tar_file} * + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bindingtester + COMMENT "Pack bindingtester") + add_custom_target(bindingtester ALL DEPENDS ${tar_file}) + add_dependencies(bindingtester copy_bindingtester_binaries) endfunction() diff --git a/cmake/CompileBoost.cmake b/cmake/CompileBoost.cmake index ede9afd946..00d69b082f 100644 --- a/cmake/CompileBoost.cmake +++ b/cmake/CompileBoost.cmake @@ -1,4 +1,4 @@ -find_package(Boost 1.67) +find_package(Boost 1.72) if(Boost_FOUND) add_library(boost_target INTERFACE) @@ -6,8 +6,8 @@ if(Boost_FOUND) else() include(ExternalProject) ExternalProject_add(boostProject - URL "https://dl.bintray.com/boostorg/release/1.67.0/source/boost_1_67_0.tar.bz2" - URL_HASH SHA256=2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba + URL "https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.bz2" + URL_HASH SHA256=59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 CONFIGURE_COMMAND "" BUILD_COMMAND "" BUILD_IN_SOURCE ON diff --git a/cmake/CompilerChecks.cmake b/cmake/CompilerChecks.cmake new file mode 100644 index 0000000000..027be35796 --- /dev/null +++ b/cmake/CompilerChecks.cmake @@ -0,0 +1,53 @@ +include(CheckCXXCompilerFlag) + +function(env_set var_name default_value type docstring) + set(val ${default_value}) + if(DEFINED ENV{${var_name}}) + set(val $ENV{${var_name}}) + endif() + set(${var_name} ${val} CACHE ${type} "${docstring}") +endfunction() + +function(default_linker var_name) + if(APPLE) + set("${var_name}" "DEFAULT" PARENT_SCOPE) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + find_program(lld_path ld.lld "Path to LLD - is only used to determine default linker") + if(lld_path) + set("${var_name}" "LLD" PARENT_SCOPE) + else() + set("${var_name}" "DEFAULT" PARENT_SCOPE) + endif() + else() + set("${var_name}" "DEFAULT" PARENT_SCOPE) + endif() +endfunction() + +function(use_libcxx out) + if(APPLE OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set("${out}" ON PARENT_SCOPE) + else() + set("${out}" OFF PARENT_SCOPE) + endif() +endfunction() + +function(static_link_libcxx out) + if(APPLE) + set("${out}" OFF PARENT_SCOPE) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + default_linker(linker) + if(NOT linker STREQUAL "LLD") + set("${out}" OFF PARENT_SCOPE) + return() + endif() + find_library(libcxx_a libc++.a) + find_library(libcxx_abi libc++abi.a) + if(libcxx_a AND libcxx_abi) + set("${out}" ON PARENT_SCOPE) + else() + set("${out}" OFF PARENT_SCOPE) + endif() + else() + set("${out}" ON PARENT_SCOPE) + endif() +endfunction() diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 71b80e7ef6..20960a61fa 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -1,16 +1,28 @@ -set(USE_GPERFTOOLS OFF CACHE BOOL "Use gperfools for profiling") -set(USE_VALGRIND OFF CACHE BOOL "Compile for valgrind usage") -set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} CACHE BOOL "Use valgrind for ctest") -set(ALLOC_INSTRUMENTATION OFF CACHE BOOL "Instrument alloc") -set(WITH_UNDODB OFF CACHE BOOL "Use rr or undodb") -set(USE_ASAN OFF CACHE BOOL "Compile with address sanitizer") -set(USE_UBSAN OFF CACHE BOOL "Compile with undefined behavior sanitizer") -set(FDB_RELEASE OFF CACHE BOOL "This is a building of a final release") -set(USE_LD "DEFAULT" CACHE STRING "The linker to use for building: can be LD (system default, default choice), BFD, GOLD, or LLD") -set(USE_LIBCXX OFF CACHE BOOL "Use libc++") -set(USE_CCACHE OFF CACHE BOOL "Use ccache for compilation if available") -set(RELATIVE_DEBUG_PATHS OFF CACHE BOOL "Use relative file paths in debug info") -set(STATIC_LINK_LIBCXX ON CACHE BOOL "Statically link libstdcpp/libc++") +include(CompilerChecks) + +env_set(USE_GPERFTOOLS OFF BOOL "Use gperfools for profiling") +env_set(USE_DTRACE ON BOOL "Enable dtrace probes on supported platforms") +env_set(USE_VALGRIND OFF BOOL "Compile for valgrind usage") +env_set(USE_VALGRIND_FOR_CTEST ${USE_VALGRIND} BOOL "Use valgrind for ctest") +env_set(ALLOC_INSTRUMENTATION OFF BOOL "Instrument alloc") +env_set(WITH_UNDODB OFF BOOL "Use rr or undodb") +env_set(USE_ASAN OFF BOOL "Compile with address sanitizer") +env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer") +env_set(FDB_RELEASE OFF BOOL "This is a building of a final release") +env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available") +env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info") +env_set(USE_WERROR OFF BOOL "Compile with -Werror. Recommended for local development and CI.") +default_linker(_use_ld) +env_set(USE_LD "${_use_ld}" STRING + "The linker to use for building: can be LD (system default and same as DEFAULT), BFD, GOLD, or LLD - will be LLD for Clang if available, DEFAULT otherwise") +use_libcxx(_use_libcxx) +env_set(USE_LIBCXX "${_use_libcxx}" BOOL "Use libc++") +static_link_libcxx(_static_link_libcxx) +env_set(STATIC_LINK_LIBCXX "${_static_link_libcxx}" BOOL "Statically link libstdcpp/libc++") + +if(USE_LIBCXX AND STATIC_LINK_LIBCXX AND NOT USE_LD STREQUAL "LLD") + message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX with libc+++ only works if USE_LD=LLD") +endif() set(rel_debug_paths OFF) if(RELATIVE_DEBUG_PATHS) @@ -55,11 +67,6 @@ else() add_definitions(-DUSE_UCONTEXT) endif() -if ((NOT USE_CCACHE) AND (NOT "$ENV{USE_CCACHE}" STREQUAL "")) - if (("$ENV{USE_CCACHE}" STREQUAL "ON") OR ("$ENV{USE_CCACHE}" STREQUAL "1") OR ("$ENV{USE_CCACHE}" STREQUAL "YES")) - set(USE_CCACHE ON) - endif() -endif() if (USE_CCACHE) FIND_PROGRAM(CCACHE_FOUND "ccache") if(CCACHE_FOUND) @@ -70,13 +77,6 @@ if (USE_CCACHE) endif() endif() -if ((NOT USE_LIBCXX) AND (NOT "$ENV{USE_LIBCXX}" STREQUAL "")) - string(TOUPPER "$ENV{USE_LIBCXX}" USE_LIBCXXENV) - if (("${USE_LIBCXXENV}" STREQUAL "ON") OR ("${USE_LIBCXXENV}" STREQUAL "1") OR ("${USE_LIBCXXENV}" STREQUAL "YES")) - set(USE_LIBCXX ON) - endif() -endif() - include(CheckFunctionExists) set(CMAKE_REQUIRED_INCLUDES stdlib.h malloc.h) set(CMAKE_REQUIRED_LIBRARIES c) @@ -86,31 +86,28 @@ if(WIN32) # see: https://docs.microsoft.com/en-us/windows/desktop/WinProg/using-the-windows-headers # this sets the windows target version to Windows Server 2003 set(WINDOWS_TARGET 0x0502) - add_compile_options(/W3 /EHsc /bigobj $<$:/Zi> /MP /FC) + if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") + # TODO: This doesn't seem to be good style, but I couldn't find a better way so far + string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + endif() + add_compile_options(/W0 /EHsc /bigobj $<$:/Zi> /MP /FC /Gm-) add_compile_definitions(_WIN32_WINNT=${WINDOWS_TARGET} WINVER=${WINDOWS_TARGET} NTDDI_VERSION=0x05020000 BOOST_ALL_NO_LIB) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") else() set(GCC NO) set(CLANG NO) + set(ICC NO) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") set(CLANG YES) + elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + set(ICC YES) else() # This is not a very good test. However, as we do not really support many architectures # this is good enough for now set(GCC YES) endif() - # Use the linker environmental variable, if specified and valid - if ((USE_LD STREQUAL "DEFAULT") AND (NOT "$ENV{USE_LD}" STREQUAL "")) - string(TOUPPER "$ENV{USE_LD}" USE_LDENV) - if (("${USE_LDENV}" STREQUAL "LD") OR ("${USE_LDENV}" STREQUAL "GOLD") OR ("${USE_LDENV}" STREQUAL "LLD") OR ("${USE_LDENV}" STREQUAL "BFD") OR ("${USE_LDENV}" STREQUAL "DEFAULT")) - set(USE_LD "${USE_LDENV}") - else() - message (FATAL_ERROR "USE_LD must be set to DEFAULT, LD, BFD, GOLD, or LLD!") - endif() - endif() - # check linker flags. if (USE_LD STREQUAL "DEFAULT") set(USE_LD "LD") @@ -188,22 +185,23 @@ else() # -mavx # -msse4.2) - if ((NOT USE_VALGRIND) AND (NOT "$ENV{USE_VALGRIND}" STREQUAL "")) - if (("$ENV{USE_VALGRIND}" STREQUAL "ON") OR ("$ENV{USE_VALGRIND}" STREQUAL "1") OR ("$ENV{USE_VALGRIND}" STREQUAL "YES")) - set(USE_VALGRIND ON) - endif() - endif() - if (USE_VALGRIND) add_compile_options(-DVALGRIND -DUSE_VALGRIND) endif() if (CLANG) add_compile_options() + # Clang has link errors unless `atomic` is specifically requested. + if(NOT APPLE) + #add_link_options(-latomic) + endif() if (APPLE OR USE_LIBCXX) add_compile_options($<$:-stdlib=libc++>) add_compile_definitions(WITH_LIBCXX) if (NOT APPLE) - add_link_options(-lc++ -lc++abi -Wl,-build-id=sha1) + if (STATIC_LINK_LIBCXX) + add_link_options(-static-libgcc -nostdlib++ -Wl,-Bstatic -lc++ -lc++abi -Wl,-Bdynamic) + endif() + add_link_options(-stdlib=libc++ -Wl,-build-id=sha1) endif() endif() if (OPEN_FOR_IDE) @@ -223,11 +221,10 @@ else() if (USE_CCACHE) add_compile_options( -Wno-register - -Wno-error=unused-command-line-argument) + -Wno-unused-command-line-argument) endif() endif() - if (CMAKE_GENERATOR STREQUAL Xcode) - else() + if (USE_WERROR) add_compile_options(-Werror) endif() if (GCC) @@ -236,6 +233,9 @@ else() # Otherwise `state [[maybe_unused]] int x;` will issue a warning. # https://stackoverflow.com/questions/50646334/maybe-unused-on-member-variable-gcc-warns-incorrectly-that-attribute-is add_compile_options(-Wno-attributes) + elseif(ICC) + add_compile_options(-wd1879 -wd1011) + add_link_options(-static-intel) endif() add_compile_options(-Wno-error=format -Wunused-variable @@ -256,7 +256,7 @@ else() check_symbol_exists(DTRACE_PROBE sys/sdt.h SUPPORT_DTRACE) check_symbol_exists(aligned_alloc stdlib.h HAS_ALIGNED_ALLOC) message(STATUS "Has aligned_alloc: ${HAS_ALIGNED_ALLOC}") - if(SUPPORT_DTRACE) + if((SUPPORT_DTRACE) AND (USE_DTRACE)) add_compile_definitions(DTRACE_PROBES) endif() if(HAS_ALIGNED_ALLOC) diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake index 69e93cec06..7e42871cce 100644 --- a/cmake/FDBComponents.cmake +++ b/cmake/FDBComponents.cmake @@ -9,21 +9,33 @@ if(USE_VALGRIND) endif() ################################################################################ -# LibreSSL +# SSL ################################################################################ - -set(DISABLE_TLS OFF CACHE BOOL "Don't try to find LibreSSL and always build without TLS support") +include(CheckSymbolExists) + +set(DISABLE_TLS OFF CACHE BOOL "Don't try to find OpenSSL and always build without TLS support") if(DISABLE_TLS) set(WITH_TLS OFF) else() - set(LIBRESSL_USE_STATIC_LIBS TRUE) - find_package(LibreSSL) - if(LibreSSL_FOUND) - set(WITH_TLS ON) - add_compile_options(-DHAVE_OPENSSL) + set(OPENSSL_USE_STATIC_LIBS TRUE) + find_package(OpenSSL) + if(OPENSSL_FOUND) + set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) + check_symbol_exists("OPENSSL_INIT_NO_ATEXIT" "openssl/crypto.h" OPENSSL_HAS_NO_ATEXIT) + if(OPENSSL_HAS_NO_ATEXIT) + set(WITH_TLS ON) + add_compile_options(-DHAVE_OPENSSL) + else() + message(WARNING "An OpenSSL version was found, but it doesn't support OPENSSL_INIT_NO_ATEXIT - Will compile without TLS Support") + set(WITH_TLS OFF) + endif() else() - message(STATUS "LibreSSL NOT Found - Will compile without TLS Support") - message(STATUS "You can set LibreSSL_ROOT to the LibreSSL install directory to help cmake find it") + message(STATUS "OpenSSL was not found - Will compile without TLS Support") + message(STATUS "You can set OPENSSL_ROOT_DIR to help cmake find it") + set(WITH_TLS OFF) + endif() + if(WIN32) + message(STATUS "TLS is temporarilty disabled on macOS while libressl -> openssl transition happens") set(WITH_TLS OFF) endif() endif() @@ -33,9 +45,10 @@ endif() ################################################################################ set(WITH_JAVA OFF) -find_package(JNI 1.8 REQUIRED) +find_package(JNI 1.8) find_package(Java 1.8 COMPONENTS Development) -if(JNI_FOUND AND Java_FOUND AND Java_Development_FOUND) +# leave FreeBSD JVM compat for later +if(JNI_FOUND AND Java_FOUND AND Java_Development_FOUND AND NOT (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")) set(WITH_JAVA ON) include(UseJava) enable_language(Java) @@ -51,7 +64,7 @@ find_package(Python COMPONENTS Interpreter) if(Python_Interpreter_FOUND) set(WITH_PYTHON ON) else() - message(FATAL_ERROR "Could not found a suitable python interpreter") + #message(FATAL_ERROR "Could not found a suitable python interpreter") set(WITH_PYTHON OFF) endif() @@ -59,8 +72,8 @@ endif() # Pip ################################################################################ -find_package(Virtualenv) -if (Virtualenv_FOUND) +find_package(Python3 COMPONENTS Interpreter) +if (Python3_Interpreter_FOUND) set(WITH_DOCUMENTATION ON) else() set(WITH_DOCUMENTATION OFF) @@ -102,6 +115,8 @@ function(print_components) message(STATUS "Build Ruby bindings: ${WITH_RUBY}") message(STATUS "Build Python sdist (make package): ${WITH_PYTHON}") message(STATUS "Build Documentation (make html): ${WITH_DOCUMENTATION}") + message(STATUS "Build Bindings (depends on Python): ${WITH_PYTHON}") + message(STATUS "Configure CTest (depends on Python): ${WITH_PYTHON}") message(STATUS "=========================================") endfunction() diff --git a/cmake/FindVirtualenv.cmake b/cmake/FindVirtualenv.cmake deleted file mode 100644 index ace748f672..0000000000 --- a/cmake/FindVirtualenv.cmake +++ /dev/null @@ -1,20 +0,0 @@ -find_program(_VIRTUALENV_EXE virtualenv) - -# get version and test that program actually works -if(_VIRTUALENV_EXE) - execute_process( - COMMAND ${_VIRTUALENV_EXE} --version - RESULT_VARIABLE ret_code - OUTPUT_VARIABLE version_string - ERROR_VARIABLE error_output - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(ret_code EQUAL 0 AND NOT ERROR_VARIABLE) - # we found a working virtualenv - set(VIRTUALENV_EXE ${_VIRTUALENV_EXE}) - set(VIRTUALENV_VERSION version_string) - endif() -endif() - -find_package_handle_standard_args(Virtualenv - REQUIRED_VARS VIRTUALENV_EXE - VERSION_VAR ${VIRTUALENV_VERSION}) diff --git a/cmake/FlowCommands.cmake b/cmake/FlowCommands.cmake index 19df995f25..53cdd7a33b 100644 --- a/cmake/FlowCommands.cmake +++ b/cmake/FlowCommands.cmake @@ -130,17 +130,20 @@ function(strip_debug_symbols target) list(APPEND strip_command -o "${out_file}") add_custom_command(OUTPUT "${out_file}" COMMAND ${strip_command} $ + DEPENDS ${target} COMMENT "Stripping symbols from ${target}") - set(out_files "${out_file}") + add_custom_target(strip_only_${target} DEPENDS ${out_file}) if(is_exec AND NOT APPLE) add_custom_command(OUTPUT "${out_file}.debug" - COMMAND objcopy --only-keep-debug $ "${out_file}.debug" && - objcopy --add-gnu-debuglink="${out_file}.debug" ${out_file} + DEPENDS strip_only_${target} + COMMAND objcopy --verbose --only-keep-debug $ "${out_file}.debug" + COMMAND objcopy --verbose --add-gnu-debuglink="${out_file}.debug" "${out_file}" COMMENT "Copy debug symbols to ${out_name}.debug") - list(APPEND out_files "${out_file}.debug") + add_custom_target(strip_${target} DEPENDS "${out_file}.debug") + else() + add_custom_target(strip_${target}) + add_dependencies(strip_${target} strip_only_${target}) endif() - add_custom_target(strip_${target} DEPENDS ${out_files}) - add_dependencies(strip_${target} ${target}) add_dependencies(strip_targets strip_${target}) endfunction() @@ -182,12 +185,12 @@ function(add_flow_target) if(WIN32) add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}" COMMAND $ "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe} + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe} COMMENT "Compile actor: ${src}") else() add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${generated}" COMMAND ${MONO_EXECUTABLE} ${actor_exe} "${CMAKE_CURRENT_SOURCE_DIR}/${src}" "${CMAKE_CURRENT_BINARY_DIR}/${generated}" ${actor_compiler_flags} > /dev/null - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" actorcompiler ${actor_exe} + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${src}" ${actor_exe} COMMENT "Compile actor: ${src}") endif() else() @@ -217,15 +220,18 @@ function(add_flow_target) get_filename_component(dname ${CMAKE_CURRENT_SOURCE_DIR} NAME) string(REGEX REPLACE "\\..*" "" fname ${src}) string(REPLACE / _ fname ${fname}) - set_source_files_properties(${src} PROPERTIES COMPILE_DEFINITIONS FNAME=${dname}_${fname}) + #set_source_files_properties(${src} PROPERTIES COMPILE_DEFINITIONS FNAME=${dname}_${fname}) endforeach() set_property(TARGET ${AFT_NAME} PROPERTY SOURCE_FILES ${AFT_SRCS}) set_property(TARGET ${AFT_NAME} PROPERTY COVERAGE_FILTERS ${AFT_SRCS}) add_custom_target(${AFT_NAME}_actors DEPENDS ${generated_files}) + add_dependencies(${AFT_NAME}_actors actorcompiler) add_dependencies(${AFT_NAME} ${AFT_NAME}_actors) - assert_no_version_h(${AFT_NAME}_actors) + if(NOT WIN32) + assert_no_version_h(${AFT_NAME}_actors) + endif() generate_coverage_xml(${AFT_NAME}) if(strip_target) strip_debug_symbols(${AFT_NAME}) diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake index 75da8d5c64..76a9889dc9 100644 --- a/cmake/InstallLayout.cmake +++ b/cmake/InstallLayout.cmake @@ -131,9 +131,9 @@ set(install_destination_for_log_el6 "var/log/foundationdb") set(install_destination_for_log_el7 "var/log/foundationdb") set(install_destination_for_log_pm "") set(install_destination_for_data_tgz "lib/foundationdb") -set(install_destination_for_data_deb "var/lib/foundationdb") -set(install_destination_for_data_el6 "var/lib/foundationdb") -set(install_destination_for_data_el7 "var/lib/foundationdb") +set(install_destination_for_data_deb "var/lib/foundationdb/data") +set(install_destination_for_data_el6 "var/lib/foundationdb/data") +set(install_destination_for_data_el7 "var/lib/foundationdb/data") set(install_destination_for_data_pm "") set(generated_dir "${CMAKE_CURRENT_BINARY_DIR}/generated") @@ -320,9 +320,14 @@ set(CPACK_RPM_SERVER-EL7_USER_FILELIST "%config(noreplace) /etc/foundationdb/foundationdb.conf" "%attr(0700,foundationdb,foundationdb) /var/log/foundationdb" "%attr(0700, foundationdb, foundationdb) /var/lib/foundationdb") +set(CPACK_RPM_CLIENTS-EL6_USER_FILELIST "%dir /etc/foundationdb") +set(CPACK_RPM_CLIENTS-EL7_USER_FILELIST "%dir /etc/foundationdb") set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/sbin" "/usr/share/java" + "/usr/lib64/cmake" + "/etc/foundationdb" + "/usr/lib64/pkgconfig" "/usr/lib64/python2.7" "/usr/lib64/python2.7/site-packages" "/var" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt new file mode 100644 index 0000000000..6bc891b854 --- /dev/null +++ b/contrib/CMakeLists.txt @@ -0,0 +1,5 @@ +if(NOT WIN32) + add_subdirectory(monitoring) + add_subdirectory(TraceLogHelper) + add_subdirectory(TestHarness) +endif() diff --git a/contrib/Joshua/README.md b/contrib/Joshua/README.md new file mode 100644 index 0000000000..f3d98892c6 --- /dev/null +++ b/contrib/Joshua/README.md @@ -0,0 +1,23 @@ +# Overview + +This directory provides the files needed to create a Joshua correctness bundle for testing FoundationDB. + +Rigorous testing is central to our engineering process. The features of our core are challenging, requiring us to meet exacting standards of correctness and performance. Data guarantees and transactional integrity must be maintained not only during normal operations but over a broad range of failure scenarios. At the same time, we aim to achieve performance goals such as low latencies and near-linear scalability. To meet these challenges, we use a combined regime of robust simulation, live performance testing, and hardware-based failure testing. + +# Joshua + +Joshua is a powerful tool for testing system correctness. Our simulation technology, called Joshua, is enabled by and tightly integrated with `flow`, our programming language for actor-based concurrency. In addition to generating efficient production code, Flow works with Joshua for simulated execution. + +The major goal of Joshua is to make sure that we find and diagnose issues in simulation rather than the real world. Joshua runs tens of thousands of simulations every night, each one simulating large numbers of component failures. Based on the volume of tests that we run and the increased intensity of the failures in our scenarios, we estimate that we have run the equivalent of roughly one trillion CPU-hours of simulation on FoundationDB. + +Joshua is able to conduct a *deterministic* simulation of an entire FoundationDB cluster within a single-threaded process. Determinism is crucial in that it allows perfect repeatability of a simulated run, facilitating controlled experiments to home in on issues. The simulation steps through time, synchronized across the system, representing a larger amount of real time in a smaller amount of simulated time. In practice, our simulations usually have about a 10-1 factor of real-to-simulated time, which is advantageous for the efficiency of testing. + +We run a broad range of simulations testing various aspects of the system. For example, we run a cycle test that uses key-values pairs arranged in a ring that executes transactions to change the values in a manner designed to maintain the ring's integrity, allowing a clear test of transactional isolation. + +Joshua simulates all physical components of a FoundationDB system, beginning with the number and type of machines in the cluster. For example, Joshua models drive performance on each machine, including drive space and the possibility of the drive filling up. Joshua also models the network, allowing a small amount of code to specify delivery of packets. + +We use Joshua to simulate failures modes at the network, machine, and datacenter levels, including connection failures, degradation of machine performance, machine shutdowns or reboots, machines coming back from the dead, etc. We stress-test all of these failure modes, failing machines at very short intervals, inducing unusually severe loads, and delaying communications channels. + +For a while, there was an informal competition within the engineering team to design failures that found the toughest bugs and issues the most easily. After a period of one-upsmanship, the reigning champion is called "swizzle-clogging". To swizzle-clog, you first pick a random subset of nodes in the cluster. Then, you "clog" (stop) each of their network connections one by one over a few seconds. Finally, you unclog them in a random order, again one by one, until they are all up. This pattern seems to be particularly good at finding deep issues that only happen in the rarest real-world cases. + +Joshua's success has surpassed our expectation and has been vital to our engineering team. It seems unlikely that we would have been able to build FoundationDB without this technology. diff --git a/contrib/Joshua/scripts/bindingTest.sh b/contrib/Joshua/scripts/bindingTest.sh new file mode 100755 index 0000000000..8e2fde1f7d --- /dev/null +++ b/contrib/Joshua/scripts/bindingTest.sh @@ -0,0 +1,11 @@ +#!/bin/bash +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +pkill fdbserver +ulimit -S -c unlimited + +unset FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY +WORKDIR="$(pwd)/tmp/$$" +if [ ! -d "${WORKDIR}" ] ; then + mkdir -p "${WORKDIR}" +fi +DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" FDBSERVERPORT="${PORT_FDBSERVER:-4500}" ${SCRIPTDIR}/bindingTestScript.sh 1 diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh new file mode 100755 index 0000000000..9ef19ab1a6 --- /dev/null +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -0,0 +1,80 @@ +#/bin/bash +SCRIPTDIR=$( cd "${BASH_SOURCE[0]%\/*}" && pwd ) +cwd="$(pwd)" +BINDIR="${BINDIR:-${SCRIPTDIR}}" +LIBDIR="${BINDIR}:${LD_LIBRARY_PATH}" +SCRIPTID="${$}" +SAVEONERROR="${SAVEONERROR:-1}" +PYTHONDIR="${BINDIR}/tests/python" +testScript="${BINDIR}/tests/bindingtester/run_binding_tester.sh" +VERSION="1.6" + +source ${SCRIPTDIR}/localClusterStart.sh + +# Display syntax +if [ "$#" -lt 1 ] +then + echo "bindingTestScript.sh " + echo " version: ${VERSION}" + exit 1 +fi + +cycles="${1}" + +if [ "${DEBUGLEVEL}" -gt 0 ] +then + echo "Work dir: ${WORKDIR}" + echo "Bin dir: ${BINDIR}" + echo "Log dir: ${LOGDIR}" + echo "Python path: ${PYTHONDIR}" + echo "Lib dir: ${LIBDIR}" + echo "Server port: ${FDBSERVERPORT}" + echo "Script Id: ${SCRIPTID}" + echo "Version: ${VERSION}" +fi + +# Begin the cluster using the logic in localClusterStart.sh. +startCluster + +# Display user message +if [ "${status}" -ne 0 ]; then + : +elif ! displayMessage "Running binding tester" +then + echo 'Failed to display user message' + let status="${status} + 1" + +elif ! PYTHONPATH="${PYTHONDIR}" LD_LIBRARY_PATH="${LIBDIR}" FDB_CLUSTER_FILE="${FDBCONF}" LOGSTDOUT=1 CONSOLELOG="${WORKDIR}/console.log" "${testScript}" "${cycles}" "${WORKDIR}/errors/run.log" +then + if [ "${DEBUGLEVEL}" -gt 0 ]; then + printf "\n%-16s %-40s \n" "$(date '+%F %H-%M-%S')" "Failed to complete binding tester in ${SECONDS} seconds." + fi + let status="${status} + 1" + +elif [ "${DEBUGLEVEL}" -gt 0 ]; then + printf "\n%-16s %-40s \n" "$(date '+%F %H-%M-%S')" "Completed binding tester in ${SECONDS} seconds" +fi + +# Display directory and log information, if an error occurred +if [ "${status}" -ne 0 ] +then + ls "${WORKDIR}" > "${LOGDIR}/dir.log" + ps -eafw > "${LOGDIR}/process-preclean.log" + if [ -f "${FDBCONF}" ]; then + cp -f "${FDBCONF}" "${LOGDIR}/" + fi + # Display the severity errors + if [ -d "${LOGDIR}" ]; then + grep -ir 'Severity="40"' "${LOGDIR}" + fi +fi + +# Save debug information files, environment, and log information, if an error occurred +if [ "${status}" -ne 0 ] && [ "${SAVEONERROR}" -gt 0 ]; then + ps -eafw > "${LOGDIR}/process-exit.log" + netstat -na > "${LOGDIR}/netstat.log" + df -h > "${LOGDIR}/disk.log" + env > "${LOGDIR}/env.log" +fi + +exit "${status}" diff --git a/contrib/Joshua/scripts/bindingTimeout.sh b/contrib/Joshua/scripts/bindingTimeout.sh new file mode 100755 index 0000000000..d28fd938ce --- /dev/null +++ b/contrib/Joshua/scripts/bindingTimeout.sh @@ -0,0 +1,28 @@ +#!/bin/bash -u + +# Look for the start cluster log file. +notstarted=0 +for file in `find . -name startcluster.log` ; do + if [ -n "$(grep 'Could not create database' "${file}")" ] ; then + echo "${file}:" + cat "${file}" + echo + notstarted=1 + fi +done + +# Print information on how the server didn't start. +if [ "${notstarted}" -gt 0 ] ; then + for file in `find . -name fdbclient.log` ; do + echo "${file}:" + cat "${file}" + echo + done +fi + +# Print the test output. +for file in `find . -name console.log` ; do + echo "${file}:" + cat "${file}" + echo +done diff --git a/contrib/Joshua/scripts/correctnessTest.sh b/contrib/Joshua/scripts/correctnessTest.sh new file mode 100755 index 0000000000..5f6abd6926 --- /dev/null +++ b/contrib/Joshua/scripts/correctnessTest.sh @@ -0,0 +1,3 @@ +#!/bin/sh +OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}" +mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" false diff --git a/contrib/Joshua/scripts/correctnessTimeout.sh b/contrib/Joshua/scripts/correctnessTimeout.sh new file mode 100755 index 0000000000..7917aae591 --- /dev/null +++ b/contrib/Joshua/scripts/correctnessTimeout.sh @@ -0,0 +1,4 @@ +#!/bin/bash -u +for file in `find . -name 'trace*.xml'` ; do + mono ./bin/TestHarness.exe summarize "${file}" summary.xml "" JoshuaTimeout true +done diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh new file mode 100644 index 0000000000..3ba4cb9dcb --- /dev/null +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -0,0 +1,315 @@ +#!/bin/bash +SCRIPTDIR="${SCRIPTDIR:-$( cd "${BASH_SOURCE[0]%\/*}" && pwd )}" +DEBUGLEVEL="${DEBUGLEVEL:-1}" +WORKDIR="${WORKDIR:-${SCRIPTDIR}/tmp/fdb.work}" +LOGDIR="${WORKDIR}/log" +ETCDIR="${WORKDIR}/etc" +BINDIR="${BINDIR:-${SCRIPTDIR}}" +FDBSERVERPORT="${FDBSERVERPORT:-4500}" +FDBCONF="${ETCDIR}/fdb.cluster" +LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" + +# Initialize the variables +status=0 +messagetime=0 +messagecount=0 + +function log +{ + local status=0 + if [ "$#" -lt 1 ] + then + echo "Usage: log [echo]" + echo + echo "Logs the message and timestamp to LOGFILE (${LOGFILE}) and, if the" + echo "second argument is either not present or is set to 1, stdout." + let status="${status} + 1" + else + # Log to stdout. + if [ "$#" -lt 2 ] || [ "${2}" -ge 1 ] + then + echo "${1}" + fi + + # Log to file. + datestr=$(date +"%Y-%m-%d %H:%M:%S (%s)") + dir=$(dirname "${LOGFILE}") + if ! [ -d "${dir}" ] && ! mkdir -p "${dir}" + then + echo "Could not create directory to log output." + let status="${status} + 1" + elif ! [ -f "${LOGFILE}" ] && ! touch "${LOGFILE}" + then + echo "Could not create file ${LOGFILE} to log output." + let status="${status} + 1" + elif ! echo "[ ${datestr} ] ${1}" >> "${LOGFILE}" + then + echo "Could not log output to ${LOGFILE}." + let status="${status} + 1" + fi + fi + + return "${status}" +} + +# Display a message for the user. +function displayMessage +{ + local status=0 + + if [ "$#" -lt 1 ] + then + echo "displayMessage " + let status="${status} + 1" + elif ! log "${1}" 0 + then + log "Could not write message to file." + else + # Increment the message counter + let messagecount="${messagecount} + 1" + + # Display successful message, if previous message + if [ "${messagecount}" -gt 1 ] + then + # Determine the amount of transpired time + let timespent="${SECONDS}-${messagetime}" + + if [ "${DEBUGLEVEL}" -gt 0 ]; then + printf "... done in %3d seconds\n" "${timespent}" + fi + fi + + # Display message + if [ "${DEBUGLEVEL}" -gt 0 ]; then + printf "%-16s %-35s " "$(date "+%F %H-%M-%S")" "$1" + fi + + # Update the variables + messagetime="${SECONDS}" + fi + + return "${status}" +} + +# Create the directories used by the server. +function createDirectories { + # Display user message + if ! displayMessage "Creating directories" + then + echo 'Failed to display user message' + let status="${status} + 1" + + elif ! mkdir -p "${LOGDIR}" "${ETCDIR}" + then + log "Failed to create directories" + let status="${status} + 1" + + # Display user message + elif ! displayMessage "Setting file permissions" + then + log 'Failed to display user message' + let status="${status} + 1" + + elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli" + then + log "Failed to set file permissions" + let status="${status} + 1" + + else + while read filepath + do + if [ -f "${filepath}" ] && [ ! -x "${filepath}" ] + then + # if [ "${DEBUGLEVEL}" -gt 1 ]; then + # log " Enable executable: ${filepath}" + # fi + log " Enable executable: ${filepath}" "${DEBUGLEVEL}" + if ! chmod 755 "${filepath}" + then + log "Failed to set executable for file: ${filepath}" + let status="${status} + 1" + fi + fi + done < <(find "${BINDIR}" -iname '*.py' -o -iname '*.rb' -o -iname 'fdb_flow_tester' -o -iname '_stacktester' -o -iname '*.js' -o -iname '*.sh' -o -iname '*.ksh') + fi + + return ${status} +} + +# Create a cluster file for the local cluster. +function createClusterFile { + if [ "${status}" -ne 0 ]; then + : + # Display user message + elif ! displayMessage "Creating Fdb Cluster file" + then + log 'Failed to display user message' + let status="${status} + 1" + else + description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) + random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) + echo "$description:$random_str@127.0.0.1:${FDBSERVERPORT}" > "${FDBCONF}" + fi + + if [ "${status}" -ne 0 ]; then + : + elif ! chmod 0664 "${FDBCONF}"; then + log "Failed to set permissions on fdbconf: ${FDBCONF}" + let status="${status} + 1" + fi + + return ${status} +} + +# Start the server running. +function startFdbServer { + if [ "${status}" -ne 0 ]; then + : + elif ! displayMessage "Starting Fdb Server" + then + log 'Failed to display user message' + let status="${status} + 1" + + elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "auto:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/$$" &> "${LOGDIR}/fdbserver.log" & + then + log "Failed to start FDB Server" + # Maybe the server is already running + FDBSERVERID="$(pidof fdbserver)" + let status="${status} + 1" + else + FDBSERVERID="${!}" + fi + + if ! kill -0 ${FDBSERVERID} ; then + log "FDB Server start failed." + let status="${status} + 1" + fi + + return ${status} +} + +function getStatus { + if [ "${status}" -ne 0 ]; then + : + elif ! date &>> "${LOGDIR}/fdbclient.log" + then + log 'Failed to get date' + let status="${status} + 1" + elif ! "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 120 &>> "${LOGDIR}/fdbclient.log" + then + log 'Failed to get status from fdbcli' + let status="${status} + 1" + elif ! date &>> "${LOGDIR}/fdbclient.log" + then + log 'Failed to get date' + let status="${status} + 1" + fi + + return ${status} +} + +# Verify that the cluster is available. +function verifyAvailable { + # Verify that the server is running. + if ! kill -0 "${FDBSERVERID}" + then + log "FDB server process (${FDBSERVERID}) is not running" + let status="${status} + 1" + return 1 + + # Display user message. + elif ! displayMessage "Checking cluster availability" + then + log 'Failed to display user message' + let status="${status} + 1" + return 1 + + # Determine if status json says the database is available. + else + avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 10 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` + log "Avail value: ${avail}" "${DEBUGLEVEL}" + if [[ -n "${avail}" ]] ; then + return 0 + else + return 1 + fi + fi +} + +# Configure the database on the server. +function createDatabase { + if [ "${status}" -ne 0 ]; then + : + # Ensure that the server is running + elif ! kill -0 "${FDBSERVERID}" + then + log "FDB server process: (${FDBSERVERID}) is not running" + let status="${status} + 1" + + # Display user message + elif ! displayMessage "Creating database" + then + log 'Failed to display user message' + let status="${status} + 1" + elif ! echo "Client log:" &> "${LOGDIR}/fdbclient.log" + then + log 'Failed to create fdbclient.log' + let status="${status} + 1" + elif ! getStatus + then + log 'Failed to get status' + let status="${status} + 1" + + # Configure the database. + else + "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout 240 --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" + + if ! displayMessage "Checking if config succeeded" + then + log 'Failed to display user message.' + fi + + iteration=0 + while [[ "${iteration}" -lt 10 ]] && ! verifyAvailable + do + log "Database not created (iteration ${iteration})." + let iteration="${iteration} + 1" + done + + if ! verifyAvailable + then + log "Failed to create database via cli" + getStatus + cat "${LOGDIR}/fdbclient.log" + log "Ignoring -- moving on" + #let status="${status} + 1" + fi + fi + + return ${status} +} + +# Begin the local cluster from scratch. +function startCluster { + if [ "${status}" -ne 0 ]; then + : + elif ! createDirectories + then + log "Could not create directories." + let status="${status} + 1" + elif ! createClusterFile + then + log "Could not create cluster file." + let status="${status} + 1" + elif ! startFdbServer + then + log "Could not start FDB server." + let status="${status} + 1" + elif ! createDatabase + then + log "Could not create database." + let status="${status} + 1" + fi + + return ${status} +} diff --git a/contrib/Joshua/scripts/valgrindTest.sh b/contrib/Joshua/scripts/valgrindTest.sh new file mode 100755 index 0000000000..5409429691 --- /dev/null +++ b/contrib/Joshua/scripts/valgrindTest.sh @@ -0,0 +1,3 @@ +#!/bin/sh +OLDBINDIR="${OLDBINDIR:-/app/deploy/global_data/oldBinaries}" +mono bin/TestHarness.exe joshua-run "${OLDBINDIR}" true diff --git a/contrib/Joshua/scripts/valgrindTimeout.sh b/contrib/Joshua/scripts/valgrindTimeout.sh new file mode 100755 index 0000000000..b9d9e7ebad --- /dev/null +++ b/contrib/Joshua/scripts/valgrindTimeout.sh @@ -0,0 +1,6 @@ +#!/bin/bash -u +for file in `find . -name 'trace*.xml'` ; do + for valgrindFile in `find . -name 'valgrind*.xml'` ; do + mono ./bin/TestHarness.exe summarize "${file}" summary.xml "${valgrindFile}" JoshuaTimeout true + done +done diff --git a/contrib/TestHarness/CMakeLists.txt b/contrib/TestHarness/CMakeLists.txt new file mode 100644 index 0000000000..265ba847d2 --- /dev/null +++ b/contrib/TestHarness/CMakeLists.txt @@ -0,0 +1,15 @@ +set(SRCS + Program.cs + Properties/AssemblyInfo.cs) + +set(TEST_HARNESS_REFERENCES + "-r:System,System.Core,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml,${TraceLogHelperDll}") + +set(out_file ${CMAKE_BINARY_DIR}/packages/bin/TestHarness.exe) + +add_custom_command(OUTPUT ${out_file} + COMMAND ${MCS_EXECUTABLE} ARGS ${TEST_HARNESS_REFERENCES} ${SRCS} "-target:exe" "-out:${out_file}" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${SRCS} TraceLogHelper + COMMENT "Compile TestHarness" VERBATIM) +add_custom_target(TestHarness DEPENDS ${out_file}) diff --git a/contrib/TestHarness/Program.cs b/contrib/TestHarness/Program.cs new file mode 100644 index 0000000000..93b14d176c --- /dev/null +++ b/contrib/TestHarness/Program.cs @@ -0,0 +1,1509 @@ +/* + * Program.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Xml.Linq; +using System.IO; +using System.Diagnostics; +using System.ComponentModel; +using System.Runtime.InteropServices; +using System.Xml; + +namespace SummarizeTest +{ + static class Program + { + static Random random; + const int killSeconds = 60 * 30; + const int maxWarnings = 10; + const int maxStderrBytes = 1000; + const double unseedRatio = 0.05; + const double buggifyOnRatio = 0.8; + static string BINARY = "fdbserver" + (IsRunningOnMono() ? "" : ".exe"); + static string PLUGIN = "FDBLibTLS." + (IsRunningOnMono() ? "so" : "dll"); + static string OS_NAME = IsRunningOnMono() ? "linux" : "win"; + + static int Main(string[] args) + { + bool traceToStdout = false; + try + { + byte[] seed = new byte[4]; + new System.Security.Cryptography.RNGCryptoServiceProvider().GetBytes(seed); + random = new Random(new BinaryReader(new MemoryStream(seed)).ReadInt32()); + + if (args.Length < 1) + return UsageMessage(); + + if (args[0] == "summarize") + { + if (args.Length < 3) + return UsageMessage(); + string valgrindFileName = args.Length >= 4 ? args[3] : null; + string externalError = args.Length >= 5 ? args[4] : ""; + traceToStdout = args.Length == 6 && args[5] == "true"; + + int unseed; + bool retryableError; + //SOMEDAY: This only works if a run generated just one trace file. We should change the summarize command to take multiple trace files + return Summarize(new string[]{ args[1] }, args[2], null, null, null, null, null, null, valgrindFileName, -1, out unseed, out retryableError, true, + traceToStdout: traceToStdout, externalError: externalError); + } + else if (args[0] == "remote") + { + if (args.Length < 6) + return UsageMessage(); + + return Remote(args[1], args[2], double.Parse(args[3]), int.Parse(args[4]), args[5], args.Length == 7 ? args[6] : "default"); + } + else if (args[0] == "run") + { + try + { + if (args.Length < 6) + return UsageMessage(); + + string runDir = null; + if(args[1] != "temp") runDir = args[1]; + + bool useValgind = args.Length > 6 && args[6].ToLower() == "true"; + int maxTries = (args.Length > 7) ? int.Parse(args[7]) : 3; + return Run(args[2], args[3], args[4], args[5], null, runDir, null, useValgind, maxTries); + } + catch(Exception e) + { + var xout = new XElement("TestHarnessError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message)); + + AppendXmlMessageToSummary(args[5], xout); + throw; + } + } + else if (args[0] == "replay") + { + if (args.Length != 6) + return UsageMessage(); + + string runDir = null; + if (args[1] != "temp") runDir = args[1]; + + return Replay(args[2], args[3], args[4], args[5], runDir); + } + else if (args[0] == "auto") + { + if (args.Length < 4) + return UsageMessage(); + + string runDir = null; + if (args[1] != "temp") runDir = args[1]; + + string cacheDir = Path.Combine(runDir, "test_harness_cache"); + bool useValgrind = args.Length > 4 && args[4].ToLower() == "true"; + int maxTries = (args.Length > 5) ? int.Parse(args[5]) : 3; + + return Auto(args[2], Path.Combine(runDir, "runs"), args[3], cacheDir, useValgrind, maxTries); + } + else if (args[0] == "extract-errors") + { + if (args.Length != 3) + return UsageMessage(); + + return ExtractErrors(args[1], args[2]); + } + else if (args[0] == "joshua-run") + { + traceToStdout = true; + + try + { + string oldBinaryFolder = (args.Length > 1) ? args[1] : Path.Combine("/opt", "joshua", "global_data", "oldBinaries"); + bool useValgrind = args.Length > 2 && args[2].ToLower() == "true"; + int maxTries = (args.Length > 3) ? int.Parse(args[3]) : 3; + return Run(Path.Combine("bin", BINARY), "", "tests", "summary.xml", "error.xml", "tmp", oldBinaryFolder, useValgrind, maxTries, true, Path.Combine("/app", "deploy", "runtime", ".tls_5_1", PLUGIN)); + } + catch(Exception e) + { + var xout = new XElement("TestHarnessError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message)); + + AppendXmlMessageToSummary("summary.xml", xout, true); + throw; + } + } + + return UsageMessage(); + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Error:"); + Console.WriteLine(e.ToString()); + } + + return 100; + } + } + + static T Choice(this Random random, IList items) + { + return items[ random.Next(items.Count) ]; + } + + static bool IsRunningOnMono() + { + return Type.GetType("Mono.Runtime") != null; + } + + static int Replay(string fdbserverName, string tlsPluginFile, string inputSummaryFileName, string outputSummaryFileName, string runDir) + { + using (var summaryFileIn = System.IO.File.Open(inputSummaryFileName, + System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) + { + foreach (var test in Magnesium.XmlParser.Parse(summaryFileIn, inputSummaryFileName).OfType()) + { + if (test is Magnesium.Test) continue; // Only process the test plans + int unseed; + bool retryableError; + RunTest(fdbserverName, tlsPluginFile, outputSummaryFileName, null, test.randomSeed, test.Buggify, test.TestFile, runDir, test.TestUID, -1, out unseed, out retryableError, true, false); + } + } + return 0; + } + + // Ver format - x.x.x (eg, 5.0.1, 4.2.11) + // Returns true is ver1 is greater than or equal to ver2 + static bool versionGreaterThanOrEqual(string ver1, string ver2) + { + string[] tokens1 = ver1.Split('.'); + string[] tokens2 = ver2.Split('.'); + if (tokens1.Length != tokens2.Length || tokens2.Length != 3) + { + throw new ArgumentException("Invalid Version Format Version1: " + ver1 + " Version2: " + ver2); + } + int[] version1 = Array.ConvertAll(tokens1, int.Parse); + int[] version2 = Array.ConvertAll(tokens2, int.Parse); + return ((System.Collections.IStructuralComparable)version1).CompareTo(version2, System.Collections.Generic.Comparer.Default) >= 0; + } + + static int Run(string fdbserverName, string tlsPluginFile, string testFolder, string summaryFileName, string errorFileName, string runDir, string oldBinaryFolder, bool useValgrind, int maxTries, bool traceToStdout = false, string tlsPluginFile_5_1 = "") + { + int seed = random.Next(1000000000); + bool buggify = random.NextDouble() < buggifyOnRatio; + string testFile = null; + string testDir = ""; + string oldServerName = ""; + + if (Directory.Exists(testFolder)) + { + int poolSize = 0; + if( Directory.Exists(Path.Combine(testFolder, "rare")) ) poolSize += 1; + if( Directory.Exists(Path.Combine(testFolder, "slow")) ) poolSize += 5; + if( Directory.Exists(Path.Combine(testFolder, "fast")) ) poolSize += 14; + if( Directory.Exists(Path.Combine(testFolder, "restarting")) ) poolSize += 1; + + if( poolSize == 0 ) { + Console.WriteLine("Passed folder ({0}) did not have a fast, slow, rare, or restarting sub-folder", testFolder); + return 1; + } + int selection = random.Next(poolSize); + int selectionWindow = 0; + + if( Directory.Exists(Path.Combine(testFolder, "rare")) ) selectionWindow += 1; + if (selection < selectionWindow) + testDir = Path.Combine(testFolder, "rare"); + else + { + if (Directory.Exists(Path.Combine(testFolder, "restarting"))) selectionWindow += 1; + if (selection < selectionWindow) + testDir = Path.Combine(testFolder, "restarting"); + else + { + if (Directory.Exists(Path.Combine(testFolder, "slow"))) selectionWindow += 5; + if (selection < selectionWindow) + testDir = Path.Combine(testFolder, "slow"); + else + testDir = Path.Combine(testFolder, "fast"); + } + } + string[] files = Directory.GetFiles(testDir, "*", SearchOption.AllDirectories); + string[] uniqueFiles; + if (testDir.EndsWith("restarting")) + { + ISet uniqueFileSet = new HashSet(); + foreach(string file in files) { + uniqueFileSet.Add(file.Substring(0, file.LastIndexOf("-"))); // all restarting tests end with -1.txt or -2.txt + } + uniqueFiles = uniqueFileSet.ToArray(); + testFile = random.Choice(uniqueFiles); + string oldBinaryVersionLowerBound = "0.0.0"; + string lastFolderName = Path.GetFileName(Path.GetDirectoryName(testFile)); + if (lastFolderName.Contains("from_")) // Only perform upgrade tests from certain versions + { + oldBinaryVersionLowerBound = lastFolderName.Split('_').Last(); + } + string[] currentBinary = { fdbserverName }; + IEnumerable oldBinaries = Array.FindAll( + Directory.GetFiles(oldBinaryFolder), + x => versionGreaterThanOrEqual(Path.GetFileName(x).Split('-').Last(), oldBinaryVersionLowerBound)); + oldBinaries = oldBinaries.Concat(currentBinary); + oldServerName = random.Choice(oldBinaries.ToList()); + } + else + { + uniqueFiles = Directory.GetFiles(testDir); + testFile = random.Choice(uniqueFiles); + } + } + else if (File.Exists(testFolder)) + testFile = testFolder; + else + { + Console.WriteLine("Passed path ({0}) was not a folder or file", testFolder); + return 1; + } + + int result = 0; + bool unseedCheck = random.NextDouble() < unseedRatio; + for (int i = 0; i < maxTries; ++i) + { + bool logOnRetryableError = i == maxTries - 1; + bool retryableError = false; + + if (testDir.EndsWith("restarting")) + { + int expectedUnseed = -1; + int unseed; + string uid = Guid.NewGuid().ToString(); + bool useNewPlugin = oldServerName == fdbserverName || versionGreaterThanOrEqual(oldServerName.Split('-').Last(), "5.2.0"); + result = RunTest(oldServerName, useNewPlugin ? tlsPluginFile : tlsPluginFile_5_1, summaryFileName, errorFileName, seed, buggify, testFile + "-1.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, true, oldServerName, traceToStdout); + if (result == 0) + { + result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed+1, buggify, testFile + "-2.txt", runDir, uid, expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, true, false, oldServerName, traceToStdout); + } + } + else + { + int expectedUnseed = -1; + if (!useValgrind && unseedCheck) + { + result = RunTest(fdbserverName, tlsPluginFile, null, null, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), -1, out expectedUnseed, out retryableError, logOnRetryableError, false, false, false, "", traceToStdout); + } + + if (!retryableError) + { + int unseed; + result = RunTest(fdbserverName, tlsPluginFile, summaryFileName, errorFileName, seed, buggify, testFile, runDir, Guid.NewGuid().ToString(), expectedUnseed, out unseed, out retryableError, logOnRetryableError, useValgrind, false, false, "", traceToStdout); + } + } + + if (!retryableError) + { + return result; + } + } + + return result; + } + + private static int RunTest(string fdbserverName, string tlsPluginFile, string summaryFileName, string errorFileName, int seed, + bool buggify, string testFile, string runDir, string uid, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, bool useValgrind, bool restarting = false, + bool willRestart = false, string oldBinaryName = "", bool traceToStdout = false) + { + unseed = -1; + + retryableError = false; + string tempPath = Path.Combine(runDir != null ? Path.GetFullPath(runDir) : Path.GetTempPath(), uid); + string oldDir = Directory.GetCurrentDirectory(); + + try + { + fdbserverName = Path.GetFullPath(fdbserverName); + tlsPluginFile = (tlsPluginFile.Length > 0) ? Path.GetFullPath(tlsPluginFile) : ""; + testFile = Path.GetFullPath(testFile); + var ok = 0; + + if (summaryFileName != null) + summaryFileName = Path.GetFullPath(summaryFileName); + + Directory.CreateDirectory(tempPath); + Directory.SetCurrentDirectory(tempPath); + + if (!restarting) LogTestPlan(summaryFileName, testFile, seed, buggify, expectedUnseed != -1, uid, oldBinaryName); + + string valgrindOutputFile = null; + using (var process = new System.Diagnostics.Process()) + { + ErrorOutputListener errorListener = new ErrorOutputListener(); + process.StartInfo.UseShellExecute = false; + if (tlsPluginFile.Length > 0) { + process.StartInfo.EnvironmentVariables["FDB_TLS_PLUGIN"] = tlsPluginFile; + } + process.StartInfo.RedirectStandardOutput = true; + var args = ""; + if (willRestart && oldBinaryName.EndsWith("alpha6")) + { + args = string.Format("-Rs 1000000000 -r simulation {0} -s {1} -f \"{2}\" -b {3} --tls_plugin={4} --crash", + IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginFile); + } + else + { + args = string.Format("-Rs 1GB -r simulation {0} -s {1} -f \"{2}\" -b {3} --tls_plugin={4} --crash", + IsRunningOnMono() ? "" : "-q", seed, testFile, buggify ? "on" : "off", tlsPluginFile); + } + if (restarting) args = args + " --restarting"; + if (useValgrind && !willRestart) + { + valgrindOutputFile = string.Format("valgrind-{0}.xml", seed); + process.StartInfo.FileName = "valgrind"; + // Add extra debug directory, if environment variables is defined + string valgrindDbgDir = System.Environment.GetEnvironmentVariable("FDB_VALGRIND_DBGPATH"); + process.StartInfo.Arguments = (string.IsNullOrEmpty(valgrindDbgDir)) ? "" : string.Format("--extra-debuginfo-path={0} ", valgrindDbgDir); + + process.StartInfo.Arguments += + string.Format("--xml=yes --xml-file={0} -q {1} {2}", valgrindOutputFile, fdbserverName, args); + } + else + { + process.StartInfo.FileName = fdbserverName; + process.StartInfo.Arguments = args; + } + process.StartInfo.RedirectStandardError = true; + process.ErrorDataReceived += new DataReceivedEventHandler(errorListener.handleData); + + if (!traceToStdout) + { + Console.WriteLine("Executing {0} {1} in {2} (buggify: {3}, valgrind: {4})", + process.StartInfo.FileName, process.StartInfo.Arguments, tempPath, + buggify ? "on" : "off", + useValgrind ? "on" : "off"); + } + + process.Start(); + + // SOMEDAY: Do we want to actually do anything with standard output or error? + // Using standarderror requires async read, see http://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx + //process.StandardOutput.ReadToEnd(); + OutputCopier copier = new OutputCopier(process.StandardOutput); + Thread consoleThread = new Thread(new ThreadStart(copier.copyOutput)); + consoleThread.Start(); + + MemoryChecker memChecker = new MemoryChecker(process); + Thread memCheckThread = new Thread(new ThreadStart(memChecker.monitorMemory)); + memCheckThread.Start(); + + process.BeginErrorReadLine(); + + Stopwatch stopwatch = new Stopwatch(); + stopwatch.Start(); + + var ms = 1000 * killSeconds; + if (useValgrind) + ms *= 20; + bool killed = !process.WaitForExit(ms); // Wait "killSeconds" seconds here + + stopwatch.Stop(); + + if (!killed) + process.WaitForExit(); // If the process finished successfully, we call the parameterless WaitForExit to ensure that output buffers get flushed + else + { + // It seems that WaitForExit sometimes returns false well before the timeout has elapsed + // We won't report that as an error, but just in case the process didn't actually finish we attempt to + // kill it here + if (killed && ms - stopwatch.ElapsedMilliseconds > 10000) + killed = false; + + if (!traceToStdout) + { + Console.WriteLine("Warning: Killing process after {0} seconds...", killSeconds); + } + + try + { + process.Kill(); + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Warning: Process.Kill returned {0}", e); + } + } + + int waitSeconds = 60 * 2; + process.WaitForExit(1000 * waitSeconds); + if (!process.HasExited) + { + if(!traceToStdout) + { + Console.WriteLine("Warning: Unable to kill process after {0} seconds...", waitSeconds); + } + + var xout = new XElement("UnableToKillProcess", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways)); + + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + return 104; + } + } + if (!traceToStdout) + { + Console.WriteLine("Exit code is {0}", process.ExitCode); + } + memCheckThread.Join(); + consoleThread.Join(); + + var traceFiles = Directory.GetFiles(tempPath, "trace*.xml"); + if (traceFiles.Length == 0) + { + if (!traceToStdout) + { + Console.WriteLine("Warning: No trace file was generated, summary will not be affected."); + } + + var xout = new XElement((useValgrind ? "NoTraceFileGeneratedLibPos" : "NoTraceFileGenerated"), + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), + new XAttribute("Plugin", tlsPluginFile), + new XAttribute("MachineName", System.Environment.MachineName)); + + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + ok = useValgrind ? 0 : 103; + } + else + { + var result = Summarize(traceFiles, summaryFileName, errorFileName, killed, errorListener.Errors, + process.ExitCode, memChecker.MaxMem, uid, + valgrindOutputFile == null ? null : Path.Combine(tempPath, valgrindOutputFile), + expectedUnseed, out unseed, out retryableError, logOnRetryableError, willRestart, restarting, oldBinaryName, traceToStdout); + + if (result != 0) + { + ok = result; + } + } + + if (willRestart) foreach (var f in traceFiles) File.Delete(f); + + process.Close(); + } + if (!traceToStdout) + { + Console.WriteLine("Done (unseed {0})", unseed); + } + + return ok; + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Error in Run:"); + Console.WriteLine(e); + } + + var xout = new XElement("TestHarnessRunError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message)); + + AppendXmlMessageToSummary(summaryFileName, xout, traceToStdout, testFile, seed, buggify, expectedUnseed != -1, oldBinaryName); + return 101; + } + finally + { + Directory.SetCurrentDirectory(oldDir); + if (!willRestart) Directory.Delete(tempPath, true); + } + } + + class OutputCopier + { + private StreamReader reader; + + public OutputCopier(StreamReader streamReader) + { + this.reader = streamReader; + } + + public void copyOutput() + { + //Console.WriteLine(" Beginning console copy..."); + while (!reader.EndOfStream) + { + reader.ReadLine(); + //Console.WriteLine(" : " + s); + } + } + } + + private class ErrorOutputListener + { + public List Errors { get; set; } + int maxErrorLength; + int maxErrors; + + public ErrorOutputListener(int maxErrorLength = 100, int maxErrors = 10) + { + Errors = new List(); + this.maxErrorLength = maxErrorLength; + this.maxErrors = maxErrors; + } + + public bool hasError = false; + public void handleData(object sendingProcess, DataReceivedEventArgs errLine) + { + if(!String.IsNullOrEmpty(errLine.Data)) + { + hasError = true; + if(Errors.Count < maxErrors) + Errors.Add(errLine.Data.Substring(0, Math.Min(maxErrorLength, errLine.Data.Length))); + } + } + } + + class MemoryChecker + { + //private System.Diagnostics.Process process; + private long maxMem; + private Process process; + + public long MaxMem + { + get { return maxMem; } + set { maxMem = value; } + } + + public MemoryChecker(System.Diagnostics.Process process) + { + this.process = process; + this.maxMem = 0; + } + + public void monitorMemory() + { + while (true) + { + try + { + process.Refresh(); + if (process.HasExited) + return; + long mem = process.PrivateMemorySize64; + MaxMem = Math.Max(MaxMem, mem); + //Console.WriteLine(string.Format("Process used {0} bytes", MaxMem)); + Thread.Sleep(1000); + } + catch + { + return; + } + } + } + } + + static void LogTestPlan(string summaryFileName, string testFileName, int randomSeed, bool buggify, bool testDeterminism, string uid, string oldBinary="") + { + var xout = new XElement("TestPlan", + new XAttribute("TestUID", uid), + new XAttribute("RandomSeed", randomSeed), + new XAttribute("TestFile", testFileName), + new XAttribute("BuggifyEnabled", buggify ? "1" : "0"), + new XAttribute("DeterminismCheck", testDeterminism ? "1" : "0"), + new XAttribute("OldBinary", Path.GetFileName(oldBinary))); + AppendToSummary(summaryFileName, xout); + } + + // Parses the valgrind XML file and returns a list of "what" tags for each error. + // All errors for which the "kind" tag starts with "Leak" are ignored + static string[] ParseValgrindOutput(string valgrindOutputFileName, bool traceToStdout) + { + if (!traceToStdout) + { + Console.WriteLine("Reading vXML file: " + valgrindOutputFileName); + } + + ISet whats = new HashSet(); + XElement xdoc = XDocument.Load(valgrindOutputFileName).Element("valgrindoutput"); + foreach(var elem in xdoc.Elements()) { + if (elem.Name != "error") + continue; + string kind = elem.Element("kind").Value; + if(kind.StartsWith("Leak")) + continue; + whats.Add(elem.Element("what").Value); + } + return whats.ToArray(); + } + + static int Summarize(string[] traceFiles, string summaryFileName, + string errorFileName, bool? killed, List outputErrors, int? exitCode, long? peakMemory, + string uid, string valgrindOutputFileName, int expectedUnseed, out int unseed, out bool retryableError, bool logOnRetryableError, + bool willRestart = false, bool restarted = false, string oldBinaryName = "", bool traceToStdout = false, string externalError = "") + { + unseed = -1; + retryableError = false; + List errorList = new List(); + var xout = new XElement("Test"); + if (uid != null) + xout.Add(new XAttribute("TestUID", uid)); + bool ok = false; + string testFile = "Unknown"; + int testsPassed = 0, testCount = -1, warnings = 0, errors = 0; + bool testBeginFound = false, testEndFound = false, error = false; + string firstRetryableError = ""; + int stderrSeverity = (int)Magnesium.Severity.SevError; + + Dictionary, Magnesium.Severity> severityMap = new Dictionary, Magnesium.Severity>(); + Dictionary, bool> codeCoverage = new Dictionary, bool>(); + + foreach (var traceFileName in traceFiles) + { + if(!traceToStdout) { + Console.WriteLine("Summarizing {0}", traceFileName); + } + using (var traceFile = System.IO.File.Open(traceFileName, + System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) + { + try + { + foreach (var ev in Magnesium.XmlParser.Parse(traceFile, traceFileName)) + { + Magnesium.Severity newSeverity; + if (severityMap.TryGetValue(new KeyValuePair(ev.Type, ev.Severity), out newSeverity)) + ev.Severity = newSeverity; + if (ev.Severity >= Magnesium.Severity.SevWarnAlways && ev.DDetails.ContainsKey("ErrorIsInjectedFault")) + ev.Severity = Magnesium.Severity.SevWarn; + + if (ev.Type == "ProgramStart" && !testBeginFound + // Just in case the first ProgramStart seen is a Simulated one, ignore it + && (!ev.DDetails.ContainsKey("Simulated") || ev.Details.Simulated != "1")) + { + xout.Add( + new XAttribute("RandomSeed", ev.Details.RandomSeed), + new XAttribute("SourceVersion", ev.Details.SourceVersion), + new XAttribute("Time", ev.Details.ActualTime), + new XAttribute("BuggifyEnabled", ev.Details.BuggifyEnabled), + new XAttribute("DeterminismCheck", expectedUnseed != -1 ? "1" : "0"), + new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); + testBeginFound = true; + } + if (ev.Type == "Simulation") + { + xout.Add( + new XAttribute("TestFile", ev.Details.TestFile)); + testFile = ev.Details.TestFile.Substring(ev.Details.TestFile.IndexOf("tests")); + } + if (ev.Type == "ActualRun") + { + xout.Add( + new XAttribute("TestFile", ev.Details.RunID)); + } + if (ev.Type == "ElapsedTime" && !testEndFound) + { + testEndFound = true; + unseed = int.Parse(ev.Details.RandomUnseed); + if (expectedUnseed != -1 && expectedUnseed != unseed) + { + Magnesium.Severity severity; + if (!severityMap.TryGetValue(new KeyValuePair("UnseedMismatch", Magnesium.Severity.SevError), out severity)) + severity = Magnesium.Severity.SevError; + + if (severity >= Magnesium.Severity.SevWarnAlways) + { + xout.Add(new XElement("UnseedMismatch", + new XAttribute("Unseed", unseed), + new XAttribute("ExpectedUnseed", expectedUnseed), + new XAttribute("Severity", (int)severity))); + if ( severity == Magnesium.Severity.SevError ) { + error = true; + errorList.Add("UnseedMismatch"); + } + } + } + xout.Add( + new XAttribute("SimElapsedTime", ev.Details.SimTime), + new XAttribute("RealElapsedTime", ev.Details.RealTime), + new XAttribute("RandomUnseed", ev.Details.RandomUnseed)); + } + if (ev.Severity == Magnesium.Severity.SevWarnAlways) + { + if (warnings < maxWarnings) + { + xout.Add(new XElement(ev.Type, + new XAttribute("Severity", (int)ev.Severity), + ev.DDetails + //.Where(kv => true) + .Select(kv => new XAttribute(kv.Key, kv.Value)))); + } + warnings++; + } + if (ev.Severity >= Magnesium.Severity.SevError) + { + string errorString = ev.FormatTestError(true); + if (errorString.Contains("platform_error")) + { + if (!retryableError) + { + firstRetryableError = errorString; + } + retryableError = true; + } + if (errors < maxWarnings) + { + xout.Add(new XElement(ev.Type, + new XAttribute("Severity", (int)ev.Severity), + ev.DDetails + //.Where(kv => true) + .Select(kv => new XAttribute(kv.Key, kv.Value)))); + errorList.Add(errorString); + } + errors++; + error = true; + } + if (ev.Type == "CodeCoverage" && !willRestart) + { + bool covered = true; + if(ev.DDetails.ContainsKey("Covered")) + { + covered = int.Parse(ev.Details.Covered) != 0; + } + + var key = new Tuple(ev.Details.File, ev.Details.Line); + if (covered || !codeCoverage.ContainsKey(key)) + { + codeCoverage[key] = covered; + } + } + if (ev.Type == "FaultInjected" || (ev.Type == "BuggifySection" && ev.Details.Activated == "1")) + { + xout.Add(new XElement(ev.Type, new XAttribute("File", ev.Details.File), new XAttribute("Line", ev.Details.Line))); + } + if (ev.Type == "TestsExpectedToPass") + testCount = int.Parse(ev.Details.Count); + if (ev.Type == "TestResults" && ev.Details.Passed == "1") + testsPassed++; + if (ev.Type == "RemapEventSeverity") + severityMap[new KeyValuePair(ev.Details.TargetEvent, (Magnesium.Severity)int.Parse(ev.Details.OriginalSeverity))] = (Magnesium.Severity)int.Parse(ev.Details.NewSeverity); + if (ev.Type == "StderrSeverity") + stderrSeverity = int.Parse(ev.Details.NewSeverity); + } + + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine("Error summarizing {0}: {1}", traceFileName, e); + } + + error = true; + xout.Add(new XElement("SummarizationError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message))); + errorList.Add("SummarizationError " + e.Message); + break; + } + } + } + + if (externalError.Length > 0) { + xout.Add(new XElement(externalError, new XAttribute("Severity", (int)Magnesium.Severity.SevError))); + } + + foreach(var kv in codeCoverage) + { + var element = new XElement("CodeCoverage", new XAttribute("File", kv.Key.Item1), new XAttribute("Line", kv.Key.Item2)); + if(!kv.Value) + { + element.Add(new XAttribute("Covered", "0")); + } + + xout.Add(element); + } + + if (warnings > maxWarnings) + { + //error = true; + xout.Add(new XElement("WarningLimitExceeded", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), + new XAttribute("WarningCount", warnings))); + } + if (errors > maxWarnings) + { + error = true; + xout.Add(new XElement("ErrorLimitExceeded", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorCount", errors))); + errorList.Add("ErrorLimitExceeded"); + } + if (killed == true) + { + if (!retryableError) + { + firstRetryableError = "ExternalTimeout"; + } + + retryableError = true; + error = true; + xout.Add(new XElement("ExternalTimeout", new XAttribute("Severity", (int)Magnesium.Severity.SevError))); + } + if (outputErrors != null) + { + int stderrBytes = 0; + foreach (string err in outputErrors) + { + if (stderrSeverity == (int)Magnesium.Severity.SevError) + { + error = true; + } + + int remainingBytes = maxStderrBytes - stderrBytes; + if (remainingBytes > 0) + { + string outErr = (err.Length > remainingBytes) ? err.Substring(remainingBytes) + "..." : err; + + xout.Add(new XElement("StdErrOutput", + new XAttribute("Severity", stderrSeverity), + new XAttribute("Output", outErr))); + } + + stderrBytes += err.Length; + } + + if (stderrBytes > maxStderrBytes) + { + xout.Add(new XElement("StdErrOutputTruncated", + new XAttribute("Severity", stderrSeverity), + new XAttribute("BytesRemaining", stderrBytes - maxStderrBytes))); + } + } + if (exitCode.HasValue && exitCode != 0) + { + error = true; + xout.Add(new XElement("ExitCode", new XAttribute("Code", exitCode.Value), new XAttribute("Severity", (int)Magnesium.Severity.SevError))); + errorList.Add(string.Format("ExitCode 0x{0:x}", exitCode.Value)); + } + if (!testEndFound && !willRestart) + { + // We didn't terminate the test, but it didn't reach the end? + error = true; + xout.Add(new XElement("TestUnexpectedlyNotFinished"), new XAttribute("Severity", (int)Magnesium.Severity.SevError)); + errorList.Add("TestUnexpectedlyNotFinished"); + } + ok = testsPassed == testCount && testsPassed > 0 && !error; + xout.Add( + new XAttribute("Passed", testsPassed), + new XAttribute("Failed", testCount - testsPassed)); + if (peakMemory.HasValue) + xout.Add(new XAttribute("PeakMemory", peakMemory.Value)); + + if (valgrindOutputFileName != null && valgrindOutputFileName.Length > 0) + { + try + { + // If there are any errors reported "ok" will be set to false + var whats = ParseValgrindOutput(valgrindOutputFileName, traceToStdout); + foreach (var what in whats) + { + xout.Add(new XElement("ValgrindError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("What", what))); + ok = false; + error = true; + } + } + catch (Exception e) + { + if (!traceToStdout) + { + Console.WriteLine(e); + } + + error = true; + xout.Add(new XElement("ValgrindParseError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message))); + errorList.Add("Failed to parse valgrind output: " + e.Message); + } + } + + if (retryableError && !logOnRetryableError) + { + xout = new XElement("Test", xout.Attributes()); + xout.Add(new XElement("RetryingError", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways), + new XAttribute("What", firstRetryableError))); + } + else + { + xout.Add(new XAttribute("OK", ok || willRestart)); + } + + AppendToSummary(summaryFileName, xout, traceToStdout); + + if ((!retryableError || logOnRetryableError) && errorFileName != null && (errorList.Count > 0 || !ok) && !willRestart) + { + var errorText = string.Join("\n\t", errorList + .Concat((!ok && errorList.Count == 0) ? new string[] { "Failed with no explanation" } : new string[] { }) + .Distinct() + .ToArray()); + AppendToFile(errorFileName, string.Format("Test {0} failed with:\n\t{1}\n", testFile, errorText)); + } + if (!error) { + return 0; + } + else { + return 102; + } + } + + static int ExtractErrors(string summaryFileName, string errorSummaryFileName) + { + Console.WriteLine("Extracting from {0}", summaryFileName); + List xout = new List(); + var coverage = new Dictionary,Tuple>(); + using (var traceFile = System.IO.File.Open(summaryFileName, + System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite | System.IO.FileShare.Delete)) + { + try + { + var events = Magnesium.XmlParser.Parse(traceFile, summaryFileName, true); + events = Magnesium.TraceLogUtil.IdentifyFailedTestPlans(events); + foreach (var ev in events) + { + Magnesium.Test t = ev as Magnesium.Test; + if (t != null) + { + foreach (var tev in t.events) + { + if (tev.Type == "CodeCoverage" || tev.Type == "FaultInjected") + { + var keyTuple = Tuple.Create(tev.Details.File, int.Parse(tev.Details.Line)); + if (coverage.ContainsKey(keyTuple)) + { + var old = coverage[keyTuple]; + coverage[keyTuple] = Tuple.Create(old.Item1 + 1, old.Item2 + (t.ok ? 0 : 1)); + } + else + { + coverage[keyTuple] = Tuple.Create(1, (t.ok ? 0 : 1)); + } + } + } + if (!t.ok) + { + if (t.original != null) + { + foreach (var c in t.original.Elements("CodeCoverage")) + c.Remove(); + foreach (var f in t.original.Elements("FaultInjected")) + f.Remove(); + + xout.Add(t.original); + } + else + { + xout.Add(new XElement("Test", + new XAttribute("Type", t.Type), + new XAttribute("Time", t.Time), + new XAttribute("Machine", t.Machine), + new XAttribute("TestUID", t.TestUID), + new XAttribute("TestFile", t.TestFile), + new XAttribute("randomSeed", t.randomSeed), + new XAttribute("Buggify", t.Buggify), + new XAttribute("DeterminismCheck", t.DeterminismCheck), + new XAttribute("OldBinary", t.OldBinary), + new XElement("TestNotSummarized", + new XAttribute("Severity", (int)Magnesium.Severity.SevWarnAlways) + ) + ) + ); + } + } + } + } + } + catch (Exception e) + { + Console.WriteLine("Error summarizing {0}: {1}", summaryFileName, e); + xout.Add(new XElement("SummarizationError", + new XAttribute("Severity", (int)Magnesium.Severity.SevError), + new XAttribute("ErrorMessage", e.Message))); + //failedTests.Add("SummarizationError " + e.Message); + } + } + + foreach (var e in coverage) + { + xout.Add(new XElement("Event", + new XAttribute("Type", "CoverageSummary"), + new XAttribute("Time", 0), + new XAttribute("Machine", ""), + new XAttribute("File", e.Key.Item1), + new XAttribute("Line", e.Key.Item2), + new XAttribute("Covered", e.Value.Item1), + new XAttribute("Failed", e.Value.Item2))); + } + + AppendToErrorSummary(errorSummaryFileName, xout); + return 0; + } + + private static void AppendToErrorSummary(string summaryFileName, List elements) + { + if (summaryFileName == null) + return; + takeLock(summaryFileName); + try { + foreach (XElement e in elements) + AppendToSummary(summaryFileName, e, false, false); + } + finally + { + releaseLock(summaryFileName); + } + } + + private static void AppendToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, bool shouldLock = true) + { + if (traceToStdout) + { + using (var wr = System.Xml.XmlWriter.Create(Console.OpenStandardOutput(), new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true, Encoding = new System.Text.UTF8Encoding(false) })) + xout.WriteTo(wr); + Console.WriteLine(); + return; + } + + if (summaryFileName == null) + return; + if (shouldLock) + takeLock(summaryFileName); + try + { + + using (var f = System.IO.File.Open(summaryFileName, System.IO.FileMode.Append, System.IO.FileAccess.Write)) + { + if (f.Length == 0) + { + byte[] bytes = Encoding.UTF8.GetBytes(""); + f.Write(bytes, 0, bytes.Length); + } + using (var wr = System.Xml.XmlWriter.Create(f, new System.Xml.XmlWriterSettings() { OmitXmlDeclaration = true })) + xout.Save(wr); + var endl = Encoding.UTF8.GetBytes(Environment.NewLine); + f.Write(endl, 0, endl.Length); + } + } + finally + { + if (shouldLock) + releaseLock(summaryFileName); + } + } + private static void AppendXmlMessageToSummary(string summaryFileName, XElement xout, bool traceToStdout = false, string testFile = null, + int? seed = null, bool? buggify = null, bool? determinismCheck = null, string oldBinaryName = null) + { + var test = new XElement("Test", xout); + if(testFile != null) + test.Add(new XAttribute("TestFile", testFile)); + if(seed != null) + test.Add(new XAttribute("RandomSeed", seed)); + if(buggify != null) + test.Add(new XAttribute("BuggifyEnabled", buggify.Value ? "1" : "0")); + if(determinismCheck != null) + test.Add(new XAttribute("DeterminismCheck", determinismCheck.Value ? "1" : "0")); + if(oldBinaryName != null) + test.Add(new XAttribute("OldBinary", Path.GetFileName(oldBinaryName))); + + test.Add(xout); + AppendToSummary(summaryFileName, test, traceToStdout); + } + + private static void AppendToFile(string fileName, string content) + { + if (fileName == null) + return; + takeLock(fileName); + try + { + using (var f = System.IO.File.Open(fileName, System.IO.FileMode.Append, System.IO.FileAccess.Write)) + { + var endl = Encoding.UTF8.GetBytes(content); + f.Write(endl, 0, endl.Length); + } + } + finally + { + releaseLock(fileName); + } + } + + static int Remote(string queue, string fdbRoot, double addHours, int testCount, string testTypes, string userScope) + { + queue = Path.GetFullPath(queue); + fdbRoot = Path.GetFullPath(fdbRoot); + var output = Path.Combine(queue, "archive"); + var now = DateTime.Now; + string date = string.Format("{0}-{1:00}-{2:00}-{3:00}-{4:00}", + now.Year, now.Month, now.Day, now.Hour, now.Minute); + + if (!Directory.Exists(queue)) + Directory.CreateDirectory(queue); + + int maxCount = 0; + foreach (var f in Directory.GetFiles(queue, String.Format("{0}-*.xml", date))) + { + var count = Int32.Parse(f.Split('-')[5]); + maxCount = Math.Max(maxCount, count); + } + maxCount++; + + var suffix = String.Format("{0}-{1}-{2}", Environment.UserName, userScope, OS_NAME); + foreach (var f in Directory.GetFiles(queue, "*" + suffix + ".xml")) + File.Delete(f); + + string label = String.Format("{0}-{1}-{2}", date, maxCount, suffix); + + var testStaging = Path.Combine(output, label); + Directory.CreateDirectory(testStaging); + File.Create(Path.Combine(testStaging, "errors.txt")); + + var release = Path.Combine(fdbRoot, "bin"); + if(!IsRunningOnMono()) + release = Path.Combine(release, "Release"); + + File.Copy(Path.Combine(release, BINARY), Path.Combine(testStaging, BINARY)); + File.Copy(Path.Combine(fdbRoot, "tls-plugins", PLUGIN), Path.Combine(testStaging, PLUGIN)); + + if (IsRunningOnMono()) + File.Copy(Path.Combine(release, BINARY + ".debug"), Path.Combine(testStaging, BINARY + ".debug")); + + //using (var f = System.IO.File.Open( + // Path.Combine(testStaging, "summary.xml"), + // System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.Delete)) + //{ + // byte[] bytes = Encoding.UTF8.GetBytes(""); + // f.Write(bytes, 0, bytes.Length); + //} + + var coverageFiles = Directory.GetFiles(release, "coverage*.xml"); + foreach (var coverage in coverageFiles) + File.Copy(coverage, Path.Combine(testStaging, Path.GetFileName(coverage))); + + Directory.CreateDirectory(Path.Combine(testStaging, "tests")); + + if (testTypes == "fast" || testTypes == "all") + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "fast")), new DirectoryInfo(Path.Combine(testStaging, "tests", "fast"))); + if (testTypes == "restarting" || testTypes == "all") + { + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "restarting")), new DirectoryInfo(Path.Combine(testStaging, "tests", "restarting"))); + //CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "oldBinaries")), new DirectoryInfo(Path.Combine(testStaging, "tests", "oldBinaries"))); + } + if (testTypes == "all") + { + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "slow")), new DirectoryInfo(Path.Combine(testStaging, "tests", "slow"))); + CopyAll(new DirectoryInfo(Path.Combine(fdbRoot, "tests", "rare")), new DirectoryInfo(Path.Combine(testStaging, "tests", "rare"))); + } + + + if (testTypes != "fast" && testTypes != "all" && testTypes != "restarting") + { + FileInfo file = new FileInfo(testTypes); + Directory.CreateDirectory(Path.Combine(testStaging, "tests", file.Directory.Name)); + file.CopyTo(Path.Combine(testStaging, "tests", file.Directory.Name, file.Name), true); + } + + var e = + new XElement("TestDefinition", + new XElement("Duration", + new XAttribute("Hours", addHours)), + new XElement("TestCount", testCount)); + new XDocument(e).Save(Path.Combine(queue, label + ".xml")); + File.Copy(Path.Combine(queue, label + ".xml"), Path.Combine(testStaging, label + ".xml")); + + var summaryFile = Path.Combine(testStaging, "summary.xml"); + Console.WriteLine(label); + + if (!IsRunningOnMono()) + { + using (var mProcess = new System.Diagnostics.Process()) + { + mProcess.StartInfo.UseShellExecute = false; + mProcess.StartInfo.RedirectStandardOutput = true; + mProcess.StartInfo.FileName = "Magnesium.exe"; + mProcess.StartInfo.Arguments = "Summary " + summaryFile; + mProcess.Start(); + } + } + + return 0; + } + + public static void CopyAll(DirectoryInfo source, DirectoryInfo target, Func predicate = null) + { + // Check if the target directory exists, if not, create it. + if (!Directory.Exists(target.FullName)) + Directory.CreateDirectory(target.FullName); + + // Copy each file into it's new directory. + foreach (FileInfo fi in source.GetFiles()) + if (predicate == null || predicate(fi)) + fi.CopyTo(Path.Combine(target.ToString(), fi.Name), true); + + // Copy each subdirectory using recursion. + foreach (DirectoryInfo diSourceSubDir in source.GetDirectories()) + CopyAll(diSourceSubDir, target.CreateSubdirectory(diSourceSubDir.Name), predicate); + } + + static int Auto(string queueDirectory, string runDir, string shareDir, string cacheDir, bool useValgrind, int maxTries) + { + try + { + queueDirectory = Path.GetFullPath(queueDirectory); + while (true) + { + Test test = getTest(queueDirectory, runDir, shareDir, cacheDir); + Console.WriteLine("Running test {0}", test.label); + + // run test + test.run(useValgrind, maxTries); + } + } + catch (Exception e) + { + Console.WriteLine("Error: {0}", e); + return 100; + } + } + + class Test + { + public DateTime? testEnd; + public string queueDirectory; + public string label; + public string runDir; + public string inputDir; + public string outputDir; + public string oldBinaryDir; + public int testCount; + + public Test(string queueDirectory, string label, string runDir, string shareDir, string cacheDir) + { + this.queueDirectory = queueDirectory; + this.label = label; + this.runDir = runDir; + var specFile = Path.Combine(queueDirectory, label + ".xml"); + var testDef = XDocument.Load(specFile).Element("TestDefinition"); + var testDuration = double.Parse(testDef.Element("Duration").Attribute("Hours").Value); + var testBegin = File.GetCreationTime(specFile); + testEnd = testDuration < 0 ? (DateTime?)null : testBegin.AddHours(testDuration); + + testCount = int.Parse(testDef.Element("TestCount").Value); + outputDir = Path.Combine(queueDirectory, "archive", label); + Directory.CreateDirectory(outputDir); + + string oldBinarySourceDir = Path.Combine(shareDir, "oldBinaries"); + + if (cacheDir != null) + { + inputDir = Path.Combine(cacheDir, "archive", label); + Directory.CreateDirectory(Path.Combine(cacheDir, "archive")); + + if (!Directory.Exists(inputDir)) + { + takeLock(inputDir); + if (!Directory.Exists(inputDir)) + { + string tmpDir = Path.Combine(cacheDir, "archive.part", label + "." + Path.GetRandomFileName() + ".part"); + Directory.CreateDirectory(tmpDir); + CopyAll(new DirectoryInfo(outputDir), new DirectoryInfo(tmpDir), (FileInfo file) => + file.Name != "fdbserver.debug" && + !file.Name.StartsWith("summary-") && + file.Name != "errors.txt" + ); + Directory.Move(tmpDir, inputDir); + } + releaseLock(inputDir); + } + + oldBinaryDir = Path.Combine(cacheDir, "oldBinaries"); + + Directory.CreateDirectory(oldBinaryDir); + foreach (FileInfo fi in new DirectoryInfo(oldBinarySourceDir).GetFiles()) + { + var targetName = Path.Combine(oldBinaryDir, fi.Name); + if (!File.Exists(targetName) || fi.LastWriteTimeUtc != File.GetLastWriteTimeUtc(targetName)) + { + fi.CopyTo(targetName, true); + File.SetLastWriteTimeUtc(targetName, fi.LastWriteTimeUtc); + } + } + + foreach (FileInfo fi in new DirectoryInfo(oldBinaryDir).GetFiles()) + { + var targetName = Path.Combine(oldBinarySourceDir, fi.Name); + if (!File.Exists(targetName)) + fi.Delete(); + } + } + else + { + inputDir = outputDir; + oldBinaryDir = oldBinarySourceDir; + } + //Console.WriteLine("TestEnd {0}, now {1}, duration {2}, done {3}", testEnd, DateTime.Now, testDuration, done()); + } + public bool done() + { + return testEnd.HasValue && System.DateTime.Now > testEnd.Value; + } + public void run(bool useValgrind, int maxTries) + { + Run(Path.Combine(inputDir, BINARY), + Path.Combine(inputDir, PLUGIN), + Path.Combine(inputDir, "tests"), + Path.Combine(outputDir, "summary-" + Environment.MachineName + ".xml"), + Path.Combine(outputDir, "errors.txt"), + runDir, + oldBinaryDir, + useValgrind, + maxTries); + } + + public void finalize() + { + try + { + Console.WriteLine("Deleting: {0}", Path.Combine(queueDirectory, label + ".xml")); + File.Delete(Path.Combine(queueDirectory, label + ".xml")); + } + catch (Exception e) + { + Console.WriteLine("Error deleting queue folder: {0}", e.Message); + } + } + } + + static Test getTest(string parent, string runDir, string shareDir, string cacheDir) + { + while (true) + { + var testFiles = Directory.GetFiles(parent, String.Format("*{0}.xml", OS_NAME)); + // (if no tests, wait, try again) + if (testFiles.Length != 0) + { + /*try + {*/ + var testFile = testFiles[random.Next(testFiles.Length)]; + var test = new Test(parent, Path.GetFileNameWithoutExtension(testFile), runDir, shareDir, cacheDir); + if ((test.testCount < 0 || UpdateTestTotals(Path.Combine(test.outputDir, "testCount"), test.testCount)) && !test.done()) + return test; + else + test.finalize(); + /*} + catch (Exception) + { + // retry opening a test + }*/ + } + System.Threading.Thread.Sleep(1000); + } + } + + static void takeLock(string targetFile) + { + // Console.WriteLine("Attempting to take lock on {0}", targetFile); + string lockFile = targetFile + ".lock"; + while (true) + { + try + { + using (var f = System.IO.File.Open(lockFile, System.IO.FileMode.CreateNew)) + { + return; + } + } + catch (System.IO.IOException e) + { + Console.WriteLine("Waiting for file lock: {0}", e.Message); + System.Threading.Thread.Sleep(250); + } + } + } + + static void releaseLock(string targetFile) + { + File.Delete(targetFile + ".lock"); + } + + private static bool UpdateTestTotals(string countFileName, int desiredTestCount) + { + takeLock(countFileName); + try + { + using (var f = System.IO.File.Open(countFileName, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite)) + { + int currentCount = 0; + byte[] b; + + if (f.Length != 0) + { + b = new byte[f.Length]; + f.Read(b, 0, b.Length); + currentCount = int.Parse(Encoding.UTF8.GetString(b)); + } + + if (currentCount >= desiredTestCount) + return false; + + f.SetLength(0); + b = Encoding.UTF8.GetBytes(String.Format("{0}", currentCount + 1)); + f.Write(b, 0, b.Length); + + return true; + } + } + finally + { + releaseLock(countFileName); + } + } + + private static int UsageMessage() + { + Console.WriteLine("Usage:"); + Console.WriteLine(" TestHarness run [temp/runDir] [fdbserver[.exe]] [TLSplugin] [testfolder] [summary.xml] "); + Console.WriteLine(" TestHarness summarize [trace.xml] [summary.xml] "); + Console.WriteLine(" TestHarness replay [temp/runDir] [fdbserver[.exe]] [TLSplugin] [summary-in.xml] [summary-out.xml]"); + Console.WriteLine(" TestHarness auto [temp/runDir] [directory] [shareDir] "); + Console.WriteLine(" TestHarness remote [queue folder] [root foundation folder] [duration in hours] [amount of tests] [all/fast/] [scope]"); + Console.WriteLine(" TestHarness extract-errors [summary-file] [error-summary-file]"); + Console.WriteLine(" TestHarness joshua-run "); + Console.WriteLine("Version: 1.01"); + return 1; + } + } +} diff --git a/contrib/TestHarness/Properties/AssemblyInfo.cs b/contrib/TestHarness/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..aa153aca11 --- /dev/null +++ b/contrib/TestHarness/Properties/AssemblyInfo.cs @@ -0,0 +1,56 @@ +/* + * AssemblyInfo.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("TestHarness")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Apple Inc.")] +[assembly: AssemblyProduct("TestHarness")] +[assembly: AssemblyCopyright("Copyright © Apple Inc. 2013")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("92d9df90-6d48-4558-bb25-d878ea69b4bd")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/contrib/TestHarness/TestHarness.csproj b/contrib/TestHarness/TestHarness.csproj new file mode 100644 index 0000000000..73937e67bb --- /dev/null +++ b/contrib/TestHarness/TestHarness.csproj @@ -0,0 +1,62 @@ + + + + Debug + x86 + 8.0.30703 + 2.0 + {6A6B7D20-EB7E-4768-BDE2-21FE0C32F17A} + Exe + Properties + TestHarness + TestHarness + v4.0 + Client + 512 + $(SolutionDir)bin\$(Configuration)\ + $(SystemDrive)\temp\msvcfdb\$(Configuration)\TestHarness\ + + + true + DEBUG;TRACE + full + AnyCPU + prompt + true + + + TRACE + true + pdbonly + AnyCPU + prompt + true + + + + + + + + + + + + + + + + + {1FA45F13-1015-403C-9115-CEFFDD522B20} + TraceLogHelper + + + + + diff --git a/contrib/TraceLogHelper/CMakeLists.txt b/contrib/TraceLogHelper/CMakeLists.txt new file mode 100644 index 0000000000..f60cda56a4 --- /dev/null +++ b/contrib/TraceLogHelper/CMakeLists.txt @@ -0,0 +1,20 @@ +set(SRCS + Event.cs + JsonParser.cs + Properties/AssemblyInfo.cs + TraceLogUtil.cs + XmlParser.cs) + +set(TRACE_LOG_HELPER_REFERENCES + "-r:System,System.Core,System.Runtime.Serialization,System.Xml.Linq,System.Data.DataSetExtensions,Microsoft.CSharp,System.Data,System.Xml") + + +set(out_file ${CMAKE_BINARY_DIR}/packages/bin/TraceLogHelper.dll) + +add_custom_command(OUTPUT ${out_file} + COMMAND ${MCS_EXECUTABLE} ARGS ${TRACE_LOG_HELPER_REFERENCES} ${SRCS} "-target:library" "-out:${out_file}" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS ${SRCS} + COMMENT "Compile TraceLogHelper" VERBATIM) +add_custom_target(TraceLogHelper DEPENDS ${out_file}) +set(TraceLogHelperDll "${out_file}" PARENT_SCOPE) diff --git a/contrib/TraceLogHelper/Event.cs b/contrib/TraceLogHelper/Event.cs new file mode 100644 index 0000000000..66c9d1ad67 --- /dev/null +++ b/contrib/TraceLogHelper/Event.cs @@ -0,0 +1,263 @@ +/* + * Event.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Dynamic; + +namespace Magnesium +{ + public enum Severity + { + SevDebug=5, + SevInfo=10, + SevWarn=20, + SevWarnAlways=30, + SevError=40, + }; + + public class Event + { + public double Time { get; set; } + public Severity Severity { get; set; } + public string Type { get; set; } + public string Machine { get; set; } + public string ID { get; set; } + public string WorkerDesc { get { return Machine + " " + ID; } } + public string TraceFile { get; set; } + public System.Xml.Linq.XElement original { get; set; } + static MyExpando emptyDetails = new MyExpando(new Dictionary()); + MyExpando _Details = emptyDetails; + public dynamic Details { get{ return _Details; } } + public IDictionary DDetails { get { return _Details._members; } + set { + _Details = new MyExpando(value); + //foreach(var v in value) + // _Details.SetMember(v.Key, v.Value); + } + } + + public Event ShallowCopy() + { + return (Event)MemberwiseClone(); + } + + class MyExpando : DynamicObject + { + public MyExpando(IDictionary init) + { + _members = init; + } + + public IDictionary _members = + new Dictionary(); + + /// + /// When a new property is set, + /// add the property name and value to the dictionary + /// + public override bool TrySetMember + (SetMemberBinder binder, Object value) + { + if (!_members.ContainsKey(binder.Name)) + _members.Add(binder.Name, value); + else + _members[binder.Name] = value; + + return true; + } + + public bool SetMember(string name, Object value) + { + if (!_members.ContainsKey(name)) + _members.Add(name, value); + else + _members[name] = value; + + return true; + } + + /// + /// When user accesses something, return the value if we have it + /// + public override bool TryGetMember + (GetMemberBinder binder, out Object result) + { + if (_members.ContainsKey(binder.Name)) + { + result = _members[binder.Name]; + return true; + } + else + { + return base.TryGetMember(binder, out result); + } + } + + /// + /// If a property value is a delegate, invoke it + /// + public override bool TryInvokeMember + (InvokeMemberBinder binder, Object[] args, out Object result) + { + if (_members.ContainsKey(binder.Name) + && _members[binder.Name] is Delegate) + { + result = (_members[binder.Name] as Delegate).DynamicInvoke(args); + return true; + } + else + { + return base.TryInvokeMember(binder, args, out result); + } + } + + + /// + /// Return all dynamic member names + /// + /// + public override IEnumerable GetDynamicMemberNames() + { + return _members.Keys; + } + } + + public string FormatTestError(bool includeDetails) + { + string s = Type; + if (Type == "InternalError") + s = string.Format("{0} {1} {2}", Type, Details.File, Details.Line); + else if (Type == "TestFailure") + s = string.Format("{0} {1}", Type, Details.Reason); + else if (Type == "ValgrindError") + s = string.Format("{0} {1}", Type, Details.What); + else if (Type == "ExitCode") + s = string.Format("{0} 0x{1:x}", Type, int.Parse(Details.Code)); + else if (Type == "StdErrOutput") + s = string.Format("{0}: {1}", Type, Details.Output); + else if (Type == "BTreeIntegrityCheck") + s = string.Format("{0}: {1}", Type, Details.ErrorDetail); + if (DDetails.ContainsKey("Error")) + s += " " + Details.Error; + if (DDetails.ContainsKey("WinErrorCode")) + s += " " + Details.WinErrorCode; + if (DDetails.ContainsKey("LinuxErrorCode")) + s += " " + Details.LinuxErrorCode; + if (DDetails.ContainsKey("Status")) + s += " Status=" + Details.Status; + if (DDetails.ContainsKey("In")) + s += " In " + Details.In; + if (DDetails.ContainsKey("SQLiteError")) + s += string.Format(" SQLiteError={0}({1})", Details.SQLiteError, Details.SQLiteErrorCode); + if (DDetails.ContainsKey("Details") && includeDetails) + s += ": " + Details.Details; + + return s; + } + + }; + + public class TestPlan : Event + { + public string TestUID; + public string TestFile; + public int randomSeed; + public bool Buggify; + public bool DeterminismCheck; + public string OldBinary; + + }; + + public class Test : TestPlan + { + public string SourceVersion; + public double SimElapsedTime; + public double RealElapsedTime; + public bool ok; + public int passed, failed; + public int randomUnseed; + public long peakMemUsage; + + public Event[] events; // Summarized events during the test + }; + + public struct AreaGraphPoint + { + public double X { get; set; } + public double Y { get; set; } + }; + + public struct LineGraphPoint + { + public double X { get; set; } + public double Y { get; set; } + public object Category { get; set; } + }; + + public class Interval + { + public double Begin { get; set; } + public double End { get; set; } + public string Category { get; set; } + public string Color { get; set; } + public string Detail { get; set; } + public object Object { get; set; } + }; + + public class MachineRole : Interval + { + public string Machine { get; set; } + public string Role { get; set; } + }; + + public class Location + { + public string Name; + public int Y; + }; + + //Specifies the type of LocationTime being used + public enum LocationTimeOp + { + //Designates a location in the code at a particular time + Normal = 0, + + //Designates that this LocationTime maps one event ID to another + MapId + }; + + //Struct which houses a location in the code and a time which that location was hit + //The ID signifies the particular pass through the code + //Some of these objects act as special markers used to connect one ID to another + public struct LocationTime + { + public int id; + public double Time; + public Location Loc; + + public LocationTimeOp locationTimeOp; + + //If locationTimeOp == MapId, then this will hold the id of the event that should come after the id specified in the id field + public int childId; + }; +} \ No newline at end of file diff --git a/contrib/TraceLogHelper/JsonParser.cs b/contrib/TraceLogHelper/JsonParser.cs new file mode 100644 index 0000000000..996a1e0e3c --- /dev/null +++ b/contrib/TraceLogHelper/JsonParser.cs @@ -0,0 +1,93 @@ +/* + * JsonParser.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.Serialization.Json; +using System.Text; +using System.Xml; +using System.Xml.XPath; +using System.Xml.Linq; + +namespace Magnesium +{ + public static class JsonParser + { + static Random r = new Random(); + + public static IEnumerable Parse(System.IO.Stream stream, string file, + bool keepOriginalElement = false, double startTime = -1, double endTime = Double.MaxValue, + double samplingFactor = 1.0) + { + using (var reader = new System.IO.StreamReader(stream)) + { + string line; + while((line = reader.ReadLine()) != null) + { + XElement root = XElement.Load(JsonReaderWriterFactory.CreateJsonReader(new MemoryStream(Encoding.UTF8.GetBytes(line)), new XmlDictionaryReaderQuotas())); + Event ev = null; + try + { + ev = ParseEvent(root, file, keepOriginalElement, startTime, endTime, samplingFactor); + } + catch (Exception e) + { + throw new Exception(string.Format("Failed to parse {0}", root), e); + } + if (ev != null) yield return ev; + } + } + } + + private static Event ParseEvent(XElement xEvent, string file, bool keepOriginalElement, double startTime, double endTime, double samplingFactor) + { + if (samplingFactor != 1.0 && r.NextDouble() > samplingFactor) + return null; + + XElement trackLatestElement = xEvent.XPathSelectElement("//TrackLatestType"); + bool rolledEvent = trackLatestElement != null && trackLatestElement.Value.Equals("Rolled"); + String timeElement = (rolledEvent) ? "OriginalTime" : "Time"; + double eventTime = double.Parse(xEvent.XPathSelectElement("//" + timeElement).Value); + + if (eventTime < startTime || eventTime > endTime) + return null; + + return new Event { + Severity = (Severity)int.Parse(xEvent.XPathSelectElement("//Severity").ValueOrDefault("40")), + Type = string.Intern(xEvent.XPathSelectElement("//Type").Value), + Time = eventTime, + Machine = string.Intern(xEvent.XPathSelectElement("//Machine").Value), + ID = string.Intern(xEvent.XPathSelectElement("//ID").ValueOrDefault("0")), + TraceFile = file, + DDetails = xEvent.Elements() + .Where(a=>a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "ID" && a.Name != "Severity" && (!rolledEvent || a.Name != "OriginalTime")) + .ToDictionary(a=>string.Intern(a.Name.LocalName), a=>(object)a.Value), + original = keepOriginalElement ? xEvent : null, + }; + } + + private static string ValueOrDefault( this XElement attr, string def ) { + if (attr == null) return def; + else return attr.Value; + } + } +} diff --git a/contrib/TraceLogHelper/Properties/AssemblyInfo.cs b/contrib/TraceLogHelper/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..e467ef9b7a --- /dev/null +++ b/contrib/TraceLogHelper/Properties/AssemblyInfo.cs @@ -0,0 +1,56 @@ +/* + * AssemblyInfo.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("TraceLogHelper")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("Apple Inc.")] +[assembly: AssemblyProduct("TraceLogHelper")] +[assembly: AssemblyCopyright("Copyright © Apple Inc. 2013")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("ec8f2fdc-25fe-4958-b807-c30bd1da341b")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/contrib/TraceLogHelper/TraceLogHelper.csproj b/contrib/TraceLogHelper/TraceLogHelper.csproj new file mode 100644 index 0000000000..084a44733d --- /dev/null +++ b/contrib/TraceLogHelper/TraceLogHelper.csproj @@ -0,0 +1,58 @@ + + + + Debug + AnyCPU + 8.0.30703 + 2.0 + {1FA45F13-1015-403C-9115-CEFFDD522B20} + Library + Properties + TraceLogHelper + TraceLogHelper + v4.0 + 512 + + + true + full + false + ..\bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + ..\bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrib/TraceLogHelper/TraceLogUtil.cs b/contrib/TraceLogHelper/TraceLogUtil.cs new file mode 100644 index 0000000000..4e8dc69f2b --- /dev/null +++ b/contrib/TraceLogHelper/TraceLogUtil.cs @@ -0,0 +1,79 @@ +/* + * TraceLogUtil.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Magnesium +{ + public static class TraceLogUtil + { + public static IEnumerable IdentifyFailedTestPlans(IEnumerable events) + { + var failedPlans = new Dictionary(); + foreach (var ev in events) + { + var tp = ev as TestPlan; + if (tp == null || tp.TestUID == "") + { + yield return ev; + continue; + } + var t = tp as Test; + if (t == null) + { + if (!failedPlans.ContainsKey(tp.TestUID + tp.TraceFile)) + { + failedPlans.Add(tp.TestUID + tp.TraceFile, tp); + } + } + else + { + failedPlans.Remove(tp.TestUID + tp.TraceFile); + if ((tp.TraceFile != null) && tp.TraceFile.EndsWith("-2.txt")) failedPlans.Remove(tp.TestUID + tp.TraceFile.Split('-')[0] + "-1.txt"); + yield return ev; + } + } + foreach (var p in failedPlans.Values) + yield return new Test + { + Type = "FailedTestPlan", + Time = p.Time, + Machine = p.Machine, + TestUID = p.TestUID, + TestFile = p.TestFile, + randomSeed = p.randomSeed, + Buggify = p.Buggify, + DeterminismCheck = p.DeterminismCheck, + OldBinary = p.OldBinary, + events = new Event[] { + new Event { + Severity = Severity.SevWarnAlways, + Type = "TestNotSummarized", + Time = p.Time, + Machine = p.Machine + } + } + }; + } + } +} diff --git a/contrib/TraceLogHelper/XmlParser.cs b/contrib/TraceLogHelper/XmlParser.cs new file mode 100644 index 0000000000..17b2405060 --- /dev/null +++ b/contrib/TraceLogHelper/XmlParser.cs @@ -0,0 +1,194 @@ +/* + * XmlParser.cs + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml; +using System.Xml.Linq; + +namespace Magnesium +{ + public static class XmlParser + { + static Random r = new Random(); + + public static IEnumerable Parse(System.IO.Stream stream, string file, + bool keepOriginalElement = false, double startTime = -1, double endTime = Double.MaxValue, + double samplingFactor = 1.0) + { + using (var reader = XmlReader.Create(stream)) + { + reader.ReadToDescendant("Trace"); + reader.Read(); + foreach (var xev in StreamElements(reader)) + { + Event ev = null; + try + { + if (xev.Name == "Event") + ev = ParseEvent(xev, file, keepOriginalElement, startTime, endTime, samplingFactor); + else if (xev.Name == "Test") + ev = ParseTest(xev, file, keepOriginalElement); + else if (xev.Name == "TestPlan") + ev = ParseTestPlan(xev, file, keepOriginalElement); + } + catch (Exception e) + { + throw new Exception(string.Format("Failed to parse {0}", xev), e); + } + if (ev != null) yield return ev; + } + } + } + + private static Event ParseEvent(XElement xEvent, string file, bool keepOriginalElement, double startTime, double endTime, double samplingFactor) + { + if (samplingFactor != 1.0 && r.NextDouble() > samplingFactor) + return null; + + XAttribute trackLatestAttribute = xEvent.Attribute("TrackLatestType"); + bool rolledEvent = trackLatestAttribute != null && trackLatestAttribute.Value.Equals("Rolled"); + String timeAttribute = (rolledEvent) ? "OriginalTime" : "Time"; + double eventTime = double.Parse(xEvent.Attribute(timeAttribute).Value); + + if (eventTime < startTime || eventTime > endTime) + return null; + + return new Event { + Severity = (Severity)int.Parse(xEvent.Attribute("Severity").ValueOrDefault("40")), + Type = string.Intern(xEvent.Attribute("Type").Value), + Time = eventTime, + Machine = string.Intern(xEvent.Attribute("Machine").Value), + ID = string.Intern(xEvent.Attribute("ID").ValueOrDefault("0")), + TraceFile = file, + DDetails = xEvent.Attributes() + .Where(a=>a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "ID" && a.Name != "Severity" && (!rolledEvent || a.Name != "OriginalTime")) + .ToDictionary(a=>string.Intern(a.Name.LocalName), a=>(object)a.Value), + original = keepOriginalElement ? xEvent : null, + }; + } + + private static string ValueOrDefault( this XAttribute attr, string def ) { + if (attr == null) return def; + else return attr.Value; + } + + private static TestPlan ParseTestPlan(XElement xTP, string file, bool keepOriginalElement) + { + var time = double.Parse(xTP.Attribute("Time").ValueOrDefault("0")); + var machine = xTP.Attribute("Machine").ValueOrDefault(""); + return new TestPlan + { + TraceFile = file, + Type = "TestPlan", + Time = time, + Machine = machine, + TestUID = xTP.Attribute("TestUID").ValueOrDefault(""), + TestFile = xTP.Attribute("TestFile").ValueOrDefault(""), + randomSeed = int.Parse(xTP.Attribute("RandomSeed").ValueOrDefault("0")), + Buggify = xTP.Attribute("BuggifyEnabled").ValueOrDefault("1") != "0", + DeterminismCheck = xTP.Attribute("DeterminismCheck").ValueOrDefault("1") != "0", + OldBinary = xTP.Attribute("OldBinary").ValueOrDefault(""), + original = keepOriginalElement ? xTP : null, + }; + } + + private static Test ParseTest(XElement xTest, string file, bool keepOriginalElement) + { + var time = double.Parse(xTest.Attribute("Time").ValueOrDefault("0")); + var machine = xTest.Attribute("Machine").ValueOrDefault(""); + return new Test + { + TraceFile = file, + Type = "Test", + Time = time, + Machine = machine, + TestUID = xTest.Attribute("TestUID").ValueOrDefault(""), + TestFile = xTest.Attribute("TestFile").ValueOrDefault(""), + SourceVersion = xTest.Attribute("SourceVersion").ValueOrDefault(""), + ok = bool.Parse(xTest.Attribute("OK").ValueOrDefault("false")), + randomSeed = int.Parse(xTest.Attribute("RandomSeed").ValueOrDefault("0")), + randomUnseed = int.Parse(xTest.Attribute("RandomUnseed").ValueOrDefault("0")), + SimElapsedTime = double.Parse(xTest.Attribute("SimElapsedTime").ValueOrDefault("0")), + RealElapsedTime = double.Parse(xTest.Attribute("RealElapsedTime").ValueOrDefault("0")), + passed = int.Parse(xTest.Attribute("Passed").ValueOrDefault("0")), + failed = int.Parse(xTest.Attribute("Failed").ValueOrDefault("0")), + peakMemUsage = long.Parse(xTest.Attribute("PeakMemory").ValueOrDefault("0")), + Buggify = xTest.Attribute("BuggifyEnabled").ValueOrDefault("1") != "0", + DeterminismCheck = xTest.Attribute("DeterminismCheck").ValueOrDefault("1") != "0", + OldBinary = xTest.Attribute("OldBinary").ValueOrDefault(""), + original = keepOriginalElement ? xTest : null, + events = xTest.Elements().Select(e => + new Event { + Severity = (Severity)int.Parse(e.Attribute("Severity").ValueOrDefault("0")), + Type = e.Name.LocalName, + Time = time, + Machine = machine, + DDetails = e.Attributes() + .Where(a => a.Name != "Type" && a.Name != "Time" && a.Name != "Machine" && a.Name != "Severity") + .ToDictionary(a => a.Name.LocalName, a => (object)a.Value) + }).ToArray() + }; + } + + private static T Try(Func action, Func onError, Func isEOF) + { + try + { + return action(); + } + catch (Exception e) + { + if (isEOF()) + return onError(e); + else + throw e; + } + } + + private static IEnumerable StreamElements(this XmlReader reader) + { + while (!reader.EOF) + { + if (reader.NodeType == XmlNodeType.Element) + { + XElement node = null; + try + { + node = XElement.ReadFrom(reader) as XElement; + } + catch (Exception) { break; } + if (node != null) + yield return node; + } + else + { + try + { + reader.Read(); + } + catch (Exception) { break; } + } + } + } + } +} diff --git a/contrib/commit_debug.py b/contrib/commit_debug.py new file mode 100755 index 0000000000..7f6de3ff91 --- /dev/null +++ b/contrib/commit_debug.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +import argparse +import glob +import gzip +import os.path +import sys +import xml.sax +import heapq +import json + +# Usage: ./commit_debug.py trace.xml tracing.json +# ./commit_debug.py trace.xml.gz tracing.json +# ./commit_debug.py folder-of-traces/ tracing.json +# +# And then open Chrome, navigate to chrome://tracing , and load the tracing.json + +def parse_args(): + args = argparse.ArgumentParser() + args.add_argument('path') + args.add_argument('output') + return args.parse_args() + +# When encountering an event with this location, use this as the (b)egin or +# (e)nd of a span with a better given name +locationToPhase = { + "NativeAPI.commit.Before": [], + "MasterProxyServer.batcher": [("b", "Commit")], + "MasterProxyServer.commitBatch.Before": [], + "MasterProxyServer.commitBatch.GettingCommitVersion": [("b", "CommitVersion")], + "MasterProxyServer.commitBatch.GotCommitVersion": [("e", "CommitVersion")], + "Resolver.resolveBatch.Before": [("b", "Resolver.PipelineWait")], + "Resolver.resolveBatch.AfterQueueSizeCheck": [], + "Resolver.resolveBatch.AfterOrderer": [("e", "Resolver.PipelineWait"), ("b", "Resolver.Conflicts")], + "Resolver.resolveBatch.After": [("e", "Resolver.Conflicts")], + "MasterProxyServer.commitBatch.AfterResolution": [("b", "Proxy.Processing")], + "MasterProxyServer.commitBatch.ProcessingMutations": [], + "MasterProxyServer.commitBatch.AfterStoreCommits": [("e", "Proxy.Processing")], + "TLog.tLogCommit.BeforeWaitForVersion": [("b", "TLog.PipelineWait")], + "TLog.tLogCommit.Before": [("e", "TLog.PipelineWait")], + "TLog.tLogCommit.AfterTLogCommit": [("b", "TLog.FSync")], + "TLog.tLogCommit.After": [("e", "TLog.FSync")], + "MasterProxyServer.commitBatch.AfterLogPush": [("e", "Commit")], + "NativeAPI.commit.After": [], +} + +class CommitDebugHandler(xml.sax.ContentHandler, object): + def __init__(self, f): + self._f = f + self._f.write('[ ') # Trace viewer adds the missing ] for us + self._starttime = None + self._data = dict() + + def _emit(self, d): + self._f.write(json.dumps(d) + ', ') + + def startElement(self, name, attrs): + # I've flipped from using Async spans to Duration spans, because + # I kept on running into issues with trace viewer believeing there + # is no start or end of an emitted span even when there actually is. + + if name == "Event" and attrs.get('Type') == "CommitDebug": + if self._starttime is None: + self._starttime = float(attrs['Time']) + + attr_id = attrs['ID'] + # Trace viewer doesn't seem to care about types, so use host as pid and port as tid + (pid, tid) = attrs['Machine'].split(':') + traces = locationToPhase[attrs["Location"]] + for (phase, name) in traces: + if phase == "b": + self._data[(attrs['Machine'], name)] = float(attrs['Time']) + else: + starttime = self._data.get((attrs['Machine'], name)) + if starttime is None: + return + trace = { + # ts and dur are in microseconds + "ts": (starttime - self._starttime) * 1000 * 1000 + 0.001, + "dur": (float(attrs['Time']) - starttime) * 1000 * 1000, + "cat": "commit", + "name": name, + "ph": "X", + "pid": pid, + "tid": tid } + self._emit(trace) + + +def do_file(args, handler, filename): + openfn = gzip.open if filename.endswith('.gz') else open + try: + with openfn(filename) as f: + xml.sax.parse(f, handler) + except xml.sax._exceptions.SAXParseException as e: + print(e) + +def main(): + args = parse_args() + + handler = CommitDebugHandler(open(args.output, 'w')) + + def xmliter(filename): + for line in gzip.open(filename): + if line.startswith("') + for line in merged: + f.write(line[1]) + f.write('') + do_file(args, handler, combined_xml) + else: + do_file(args, handler, args.path) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/contrib/monitoring/CMakeLists.txt b/contrib/monitoring/CMakeLists.txt index 37aab4b0ef..4f5f2008c3 100644 --- a/contrib/monitoring/CMakeLists.txt +++ b/contrib/monitoring/CMakeLists.txt @@ -1 +1,2 @@ add_executable(actor_flamegraph actor_flamegraph.cpp) +target_link_libraries(actor_flamegraph PRIVATE Threads::Threads) diff --git a/contrib/transaction_profiling_analyzer.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py old mode 100644 new mode 100755 similarity index 97% rename from contrib/transaction_profiling_analyzer.py rename to contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py index c7d6e0c602..15fa19d166 --- a/contrib/transaction_profiling_analyzer.py +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer.py @@ -1,3 +1,24 @@ +#!/usr/bin/env python3 +# +# transaction_profiling_analyzer.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2020 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """ Requirements: python3 diff --git a/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer_tests.py b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer_tests.py new file mode 100755 index 0000000000..9b90ef1c70 --- /dev/null +++ b/contrib/transaction_profiling_analyzer/transaction_profiling_analyzer_tests.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# +# transaction_profiling_analyzer_tests.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2020 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from transaction_profiling_analyzer import RangeCounter +from sortedcontainers import SortedDict + +import random +import string +import unittest + + +class RangeCounterTest(unittest.TestCase): + def test_one_range(self): + rc = RangeCounter(1) + rc._insert_range("a", "b") + assert rc.ranges == SortedDict({"a": ("b", 1)}), rc.ranges + + def test_two_non_overlapping_desc(self): + rc = RangeCounter(1) + rc._insert_range("c", "d") + rc._insert_range("a", "b") + assert rc.ranges == SortedDict({"a": ("b", 1), "c": ("d", 1)}), rc.ranges + + def test_two_non_overlapping_asc(self): + rc = RangeCounter(1) + rc._insert_range("a", "b") + rc._insert_range("c", "d") + assert rc.ranges == SortedDict({"a": ("b", 1), "c": ("d", 1)}), rc.ranges + + def test_two_touching(self): + rc = RangeCounter(1) + rc._insert_range("a", "b") + rc._insert_range("b", "c") + assert rc.ranges == SortedDict({"a": ("b", 1), "b": ("c", 1)}), rc.ranges + assert rc.get_count_for_key('a') == 1 + assert rc.get_count_for_key('b') == 1 + assert rc.get_count_for_key('c') == 0 + + def test_two_duplicates(self): + rc = RangeCounter(1) + rc._insert_range("a", "b") + rc._insert_range("a", "b") + assert rc.ranges == SortedDict({"a": ("b", 2)}), rc.ranges + + def test_wholly_outside(self): + rc = RangeCounter(1) + rc._insert_range("b", "c") + rc._insert_range("a", "d") + assert rc.ranges == SortedDict({"a": ("b", 1), "b": ("c", 2), "c": ("d", 1)}), rc.ranges + + def test_wholly_inside(self): + rc = RangeCounter(1) + rc._insert_range("a", "d") + rc._insert_range("b", "c") + assert rc.ranges == SortedDict({"a": ("b", 1), "b": ("c", 2), "c": ("d", 1)}), rc.ranges + + def test_intersect_before(self): + rc = RangeCounter(1) + rc._insert_range("b", "d") + rc._insert_range("a", "c") + assert rc.ranges == SortedDict({"a": ("b", 1), "b": ("c", 2), "c": ("d", 1)}), rc.ranges + + def test_intersect_after(self): + rc = RangeCounter(1) + rc._insert_range("a", "c") + rc._insert_range("b", "d") + assert rc.ranges == SortedDict({"a": ("b", 1), "b": ("c", 2), "c": ("d", 1)}), rc.ranges + + def test_wide(self): + rc = RangeCounter(1) + rc._insert_range("a", "c") + rc._insert_range("e", "g") + rc._insert_range("i", "k") + rc._insert_range("b", "j") + assert rc.ranges == SortedDict({"a": ("b", 1), "b": ("c", 2), "c": ("e", 1), "e": ("g", 2), "g": ("i", 1), "i": ("j", 2), "j": ("k", 1)}), rc.ranges + + def test_random(self): + letters = string.ascii_lowercase + + for _ in range(0, 100): + rc = RangeCounter(1) + count_dict = {} + + def test_correct(): + for (k, v) in count_dict.items(): + rc_count = rc.get_count_for_key(k) + assert rc_count == v, "Counts for %s mismatch. Expected %d got %d" % (k, v, rc_count) + + for _ in range(0, 100): + i = random.randint(0, len(letters)-1) + j = random.randint(0, len(letters)-2) + if i == j: + j += 1 + start_index = min(i, j) + end_index = max(i, j) + start_key = letters[start_index] + end_key = letters[end_index] + rc._insert_range(start_key, end_key) + for letter in letters[start_index:end_index]: + if letter not in count_dict: + count_dict[letter] = 0 + count_dict[letter] = count_dict[letter] + 1 + + test_correct() + + +if __name__ == "__main__": + unittest.main() # run all tests diff --git a/design/backup-dataFormat.md b/design/backup-dataFormat.md index c3e13def0c..73942e41ef 100644 --- a/design/backup-dataFormat.md +++ b/design/backup-dataFormat.md @@ -1,10 +1,10 @@ ## FDB Backup Data Format ### Introduction -This document describes the data format of the files generated by FoundationDB (FDB) backup procedure. -The target readers who may benefit from reading this document are: -* who make changes on the current backup or restore procedure; -* who writes tools to digest the backup data for analytical purpose; +This document describes the data format of the files generated by FoundationDB (FDB) backup procedure. +The target readers who may benefit from reading this document are: +* who make changes on the current backup or restore procedure; +* who writes tools to digest the backup data for analytical purpose; * who wants to understand the internals of how backup and restore works. The description of the backup data format is based on FDB 5.2 to FDB 6.1. The backup data format may (although unlikely) change after FDB 6.1. @@ -12,27 +12,27 @@ The description of the backup data format is based on FDB 5.2 to FDB 6.1. The ba ### Files generated by backup The backup procedure generates two types of files: range files and log files. -* A range file describes key-value pairs in a range at the version when the backup process takes a snapshot of the range. Different range files have data for different ranges at different versions. -* A log file describes the mutations taken from a version v1 to v2 during the backup procedure. +* A range file describes key-value pairs in a range at the version when the backup process takes a snapshot of the range. Different range files have data for different ranges at different versions. +* A log file describes the mutations taken from a version v1 to v2 during the backup procedure. With the key-value pairs in range file and the mutations in log file, the restore procedure can restore the database into a consistent state at a user-provided version vk if the backup data is claimed by the restore as restorable at vk. (The details of determining if a set of backup data is restorable at a version is out of scope of this document and can be found at [backup.md](https://github.com/xumengpanda/foundationdb/blob/cd873831ecd18653c5bf459d6f72d14a99b619c4/design/backup.md). ### Filename conventions -The backup files will be saved in a directory (i.e., url) specified by users. Under the directory, the range files are in the `snapshots` folder. The log files are in the `logs` folder. +The backup files will be saved in a directory (i.e., url) specified by users. Under the directory, the range files are in the `snapshots` folder. The log files are in the `logs` folder. The convention of the range filename is ` snapshots/snapshot,beginVersion,beginVersion,blockSize`, where `beginVersion` is the version when the key-values in the range file are recorded, and blockSize is the size of data blocks in the range file. The convention of the log filename is `logs/,versionPrefix/log,beginVersion,endVersion,randomUID, blockSize`, where the versionPrefix is a 2-level path (`x/y`) where beginVersion should go such that `x/y/*` contains (10^smallestBucket) possible versions; the randomUID is a random UID, the `beginVersion` and `endVersion` are the version range (left inclusive, right exclusive) when the mutations are recorded; and the `blockSize` is the data block size in the log file. We will use an example to explain what each field in the range and log filename means. -Suppose under the backup directory, we have a range file `snapshots/snapshot,78994177,78994177,97` and a log file `logs/0000/0000/log,78655645,98655645,149a0bdfedecafa2f648219d5eba816e,1048576`. +Suppose under the backup directory, we have a range file `snapshots/snapshot,78994177,78994177,97` and a log file `logs/0000/0000/log,78655645,98655645,149a0bdfedecafa2f648219d5eba816e,1048576`. The range file’s filename tells us that all key-value pairs decoded from the file are the KV value in DB at the version `78994177`. The data block size is `97` bytes. -The log file’s filename tells us that the mutations in the log file were the mutations in the DB during the version range `[78655645,98655645)`, and the data block size is `1048576` bytes. +The log file’s filename tells us that the mutations in the log file were the mutations in the DB during the version range `[78655645,98655645)`, and the data block size is `1048576` bytes. -### Data format in a range file -A range file can have one to many data blocks. Each data block has a set of key-value pairs. +### Data format in a range file +A range file can have one to many data blocks. Each data block has a set of key-value pairs. A data block is encoded as follows: `Header startKey k1v1 k2v2 Padding`. @@ -44,7 +44,7 @@ A data block is encoded as follows: `Header startKey k1v1 k2v2 Padding`. H = header P = padding a...z = keys v = value | = block boundary - Encoded file: H a cv dv ev P | H e ev fv gv hv P | H h hv iv jv z + Encoded file: H a cv dv P | H e ev fv gv hv P | H h hv iv jv z Decoded in blocks yields: Block 1: range [a, e) with kv pairs cv, dv Block 2: range [e, h) with kv pairs ev, fv, gv @@ -58,19 +58,19 @@ The code that decodes a range block is in `ACTOR Future>> decodeLogFileBlock(Reference file, int64_t offset, int len)`. ### Endianness -When the restore decodes a serialized integer from the backup file, it needs to convert the serialized value from big endian to little endian. +When the restore decodes a serialized integer from the backup file, it needs to convert the serialized value from big endian to little endian. -The reason is as follows: When the backup procedure transfers the data to remote blob store, the backup data is encoded in big endian. However, FoundationDB currently only run on little endian machines. The endianness affects the interpretation of an integer, so we must perform the endianness convertion. \ No newline at end of file +The reason is as follows: When the backup procedure transfers the data to remote blob store, the backup data is encoded in big endian. However, FoundationDB currently only run on little endian machines. The endianness affects the interpretation of an integer, so we must perform the endianness convertion. \ No newline at end of file diff --git a/design/backup.md b/design/backup.md index 80247462eb..072e8fb729 100644 --- a/design/backup.md +++ b/design/backup.md @@ -17,7 +17,7 @@ KV ranges {(a-b, v0), (c-d, v1), (e-f, v2) ... (y-z, v10)}. With mutation log recorded all along, we can still use the simple backup-restore scheme described above on sub keyspaces seperately. Assuming we did record mutation log from v0 to vn, that allows us to restore - + * Keyspace a-b to any version between v0 and vn * Keyspace c-d to any version between v1 and vn * Keyspace y-z to any version between v10 and vn diff --git a/design/backup_v2_partitioned_logs.md b/design/backup_v2_partitioned_logs.md new file mode 100644 index 0000000000..2fb6528baf --- /dev/null +++ b/design/backup_v2_partitioned_logs.md @@ -0,0 +1,336 @@ +# The New FDB Backup System: Requirements & Design + +Github tracking issue: https://github.com/apple/foundationdb/issues/1003 + +## Purpose and Audience + +The purpose of this document is to capture functional requirements as well as propose a high level design for implementation of the new backup system in FoundationDB. The intended audience for this document includes: + +* **FDB users** - Users can understand what are the changes in the new backup system, especially how to start a backup using the new backup system. The restore for new backup is handled by the [Performant Restore System](https://github.com/apple/foundationdb/issues/1049). +* **SRE's and Support** - can understand the high level architecture and know the requirements, including the metrics, tooling, and documentation to ensure that the new FDB backup can be supported. +* **Developers** - can know why this feature is needed, what it does, and how it is to be implemented. The hope is that this document becomes the starting point for any developer wishing to understand or be involved in the related aspects of FDB. + +## Functional Requirements + +As an essential component of a database system, backup and restore is commonly used technique for disaster recovery, reliability, audit and compliance purposes. The current FDB backup system consumes about half of the cluster’s write bandwidth, causes write skew among storage servers, increases storage space usage, and results in data balancing. The new backup system aims to double cluster’s write bandwidth for *HA clusters* (old DR clusters still need old style backup system). + +## Background + +FDB backup system continuously scan the database’s key-value space, save key-value pairs and mutations at versions into range files and log files in blob storage. Specifically, mutation logs are generated at Proxy, and are written to transaction logs along with regular mutations. In production clusters like CK clusters, backup system is always on, which means each mutation is written twice to transaction logs, consuming about half of write bandwidth and about 40% of Proxy CPU time. + +The design of old backup system is [here](https://github.com/apple/foundationdb/blob/master/design/backup.md), and the data format of range files and mutations files is [here](https://github.com/apple/foundationdb/blob/master/design/backup-dataFormat.md). The technical overview of FDB is [here](https://github.com/apple/foundationdb/wiki/Technical-Overview-of-the-Database). The FDB recovery is described in this [doc](https://github.com/apple/foundationdb/blob/master/design/recovery-internals.md). + + +## Terminology + +* **Blob storage**: blob storage is an object storage for unstructed data. Backup files are encoded in binary format and saved in blob storage, e.g., Amazon S3. +* **Version**: FDB continuously generate increasing number as version and use version to decide mutation ordering. Version number typically advance one million per second. To restore a FDB cluster to a specified date and time, the restore system first convert the date and time to the corresponding version number and restore the cluster to the version number. +* **Epoch**: A generation of FDB’s transaction system. After a component of the transaction system failed, FDB automatically initiates a recovery and restores the system in a new healthy generation, which is called an epoch. +* **Backup worker**: is a new role added to the FDB cluster that is responsible for pulling mutations from transaction logs and saving them to blob storage. +* **Tag**: A tag is a short address for a mutation’s destination, which includes a locality (`int8_t`, representing the data center ID and a negative number denotes special system locality) and an ID (`int16_t`). The idea is that the tag is a small data structure that consumes less bytes than using IP addresses or storage server’s UIDs (16 bytes each), since tags are associated with each mutation and are stored both in memory and on disk. +* **Tag partitioned log system**: FDB’s write-ahead log is a tag partitioned log system, where each mutation is assigned a number of tags. +* **Log router tag**: is a special system tag, e.g., `-2:0` where locality `-2` means log router tag and `0` means ID. If attached to a mutation, originally this tag means the mutation should be sent to a remote log router. In the new backup system, we reuse this tag for backup workers to receive all mutations in a number of partitioned streams. +* **Restorable version:** The version that a backup can be restored to. A version `v` is a restorable version if the entire key-space and mutations in version `[v1, v)` are recorded in backup files. +* **Node**: A node is a machine or a process in a cluster. + +## Detailed Feature Requirements + +Feature priorities: Feature 1, 2, 3, 4, 5 are must-have; Feature 6 is better to have. + +1. **Write bandwidth reduction by half**: removes the requirement to generate backup mutations at the Proxy, thus reduce TLog write bandwidth usage by half and significantly improve Proxy CPU usage; +2. **Correctness**: The restored database must be consistent: each *restored* state (i.e., key-value pair) at a version `v` must match the original state at version `v`. +3. **Performance**: The backup system should be performant, mostly measured as a small CPU overhead on transaction logs and backup workers. The version lag on backup workers is an indicator of performance. +4. **Fault-tolerant**: The backup system should be fault-tolerant to node failures in the FDB cluster. +5. **Restore ready**: The new backup system should be restored by the Performant Restore System. As a fallback for new performant restore system, we can convert new backup logs into the format of old backup logs, thus enabling restore of the new backup with existing old restore system. +6. **Backward compatibility**: The new backup system should allow both old style backup and DR (FDB 6.2 and below) to be performed, as well as support new backup in FDB 6.3 and above. + +## Security and Privacy Requirements + +**Security**: The backup system’s components are assumed to be trusted components, because they are running on the nodes in a FDB cluster. The transmission from cluster to blob store is through SSL connections. Blob credentials are passed in from “fdbserver” command line. + +**Privacy**: Backup data are stored in blob store with appropriate access control. Data retention policy can be set with “fdbbackup” tool to delete older backup data. + +## Operational and Maintainability Requirements + +This section discusses changes that may need to be identified or accounted for on the back-end in order to support the feature from a monitoring or management perspective. + +### Tooling / Front-End + +Workflow is needed for DBA to start, pause, resume, abort the new type of backups. The difference from the old type of backups should be only a flag change for starting the backup. The FDB cluster then generates backups as specified by the flag. + +A command line tool `fdbconvert` has been written to convert new backup logs into the format of old backup logs. Thus, if the new restore system has issues, we can still restore the new backup with existing old restore system. + +**Deployment instructions for tooling development** + +* A new stateless role “`Backup Worker`” (or “`BW`” for abbreviation) is introduced in a FDB cluster. The number of BW processes is based on the number of log routers (usually they are the same). If there is no log routers, the number of transaction logs is used. Note that occasionally the cluster may recruit more backup workers for version ranges in the old epoch. Since these version ranges are small, the resource requirements for these short-lived backup workers are very small. +* As in the old backup system, backup agents need to be started for saving snapshot files to blob storage. In contrast, backup workers in the new backup system running in the primary DC are responsible for saving mutation logs to blob storage. +* Backup worker’s memory should be large enough to hold 10s of seconds worth of mutation data from TLogs. The memory requirement can be calculated as: `WriteThroughput * BufferPeriod / partitions + SafetyMargin`, where `WriteThroughput` is the aggregated TLog write bandwidth, `partitions` is the number of log router tags. +* A new process class “backup” is defined for backup workers. +* How to start a new type backup: e.g., + + ``` + fdbbackup start -C fdb.cluster -p -d blob_url + ``` + +### KPI's and Health + +The solution must provide at least the following KPIs: + +* How fast (MB/s) does the transaction logs commit writes (already existed); +* How much backup data has been processed; +* An estimation of backup delay; + +### Customer Care + +The feature does not require any specific customer care awareness or interaction. + +### Roll-out + +The feature must follow the usual roll-out process. It needs to coexist with the existing backup system and periodically restore clusters to test its correctness. Only after we gain enough confidence will we deprecate the existing backup system. + +Note the new backup system is designed for HA clusters. Existing DR clusters still uses the old backup system. Thus, rolling out of the new backup system is only for HA clusters. + +### Quota + +This feature requires a blob storage for saving all log files. The blob storage must have enough: + +* disk capacity for all backup data; +* write bandwidth for uploading backup data; +* file count for backup data: the new backup system stored partitioned mutation logs, thus expecting several time increases of the file count. + +## Success Criteria + +* Write bandwidth reduction meets the expectation: TLog write bandwidth is reduced by half; +* New backup workflow is available to SREs; +* Continuous backup and restore should be performed to validate the restore. + +# Design + +**One sentence summary**: the new backup system introduces a new role, backup worker, to pull mutations from transaction logs and save them, thus removing the burden of saving mutation logs into the database. + +The old backup system writes the mutation log to the database itself, thus doubling the write bandwidth usage. Backup agents later fetch mutation logs from the database, upload them to blob storage, and then remove the mutation logs from the database. + +This project saves the mutation log to blob storage directly from the FDB cluster, which should almost double the database's write bandwidth when backup is enabled. In FDB, every mutation already has exactly one log router tag, so the idea of the new system is to backup data for each log router tag individually (i.e., saving mutation logs into multiple partitioned logs). During restore time, these partitioned mutation logs are combined together to form a continuous mutation log stream. + +## Design choices + +**Design question 1**: Should backup workers be recruited as part of log system or not? +There are two design alternatives: + +1. Backup worker is external to the log system. In other words, backup workers survive master recovery. Thus, backup workers are recruited and monitored by the cluster controller. + 1. The advantage is that the failure of backup workers does not cause master recovery. + 2. The disadvantage is that backup workers need to monitor master recovery, especially configuration changes. Because the number of log routers can change after a recovery, we might need to recruit more backup workers for an increase and need to pause/shutdown backup workers for a decrease, which complicates the recruitment logic; or we might need to changing the mapping of tags to backup workers, which is also complex. A further complication is that backup workers need to constantly monitor master recovery and be very careful about the version boundary between two consecutive epochs, because the number of tags may change. +2. Backup worker is recruited during master recovery as part of log system. The Master recruits a fixed number of backup workers, i.e., the same number as LogRouters. + 1. The advantage is that recruiting and mapping from backup worker to LogRouter tags are simple, i.e., one tag per worker. + 2. The disadvantages is that backup workers are tied with master recovery -- a failure of a backup worker results in a master recovery, and a master recovery stops old backup workers and starts new ones. + +**Decision**: We choose the second approach for the simplicity of the recruiting process and handling of mapping of LogRouter tags to backup workers. + +**Design question 2**: Place of backup workers on the primary or remote Data Center (DC)? +Placing backup workers on the primary side has the advantage of supporting any deployment configurations (single DC, multi DC). + +Placing on the remote is desirable to reduce the workload on the primary DC’s transaction logs. Since log routers on the remote side is already pulling mutations from primary DC, backup workers can simply pull from these log routers. + +**Decision**: We choose to recruit backup workers on the primary DC, because not all clusters are configured with multiple DCs and the backup system needs to support all types of deployment. + +## Design Assumptions + +The design proposed below is based upon the following assumptions: + +* Blob system has enough write bandwidth and storage space for backup workers to save log files. +* FDB cluster has enough stateless processes to run as backup workers and these processes have memory capacity to buffer 10s of seconds of commit data. + +## Design Challenges + +The requirement of the new backup system raises several design challenges: + +1. Correctness of the new backup files. Backup files must be complete and accurate to capture all data, otherwise we end up with corrupted data in the backup. The challenge here is to make sure no mutation is missing, even when the FDB cluster experiences failures and has to perform recovery. +2. Testing of the new backup system. How can we test the new backup system when there is no restore system available? We need to verify backup files are correct without performing a full restore. + +## System components + +**Backup Worker**: This is a new role introduced in the new backup system. A backup worker is a `fdbserver` process running inside a FDB cluster, responsible for pulling mutations from transaction logs and saving the mutations to blob storage. + +**Master**: The master is responsible for coordinating the transition of the FDB transaction sub-system from one generation to the next. In particular, the master recruits backup workers during the recovery. + +**Transaction Logs (TLogs)**: The transaction logs make mutations durable to disk for fast commit latencies. The logs receive commits from the proxy in version order, and only respond to the proxy once the data has been written and fsync'ed to an append only mutation log on disk. Storage servers retrieve mutations from TLogs. Once the storage servers have persisted mutations, storage servers then pop the mutations from the TLogs. + +**Proxy**: The proxies are responsible for providing read versions, committing transactions, and tracking the storage servers responsible for each range of keys. In the old backup system, Proxies are responsible to group mutations into backup mutations and write them to the database. + +## System overview + +From an end-to-end perspective, the new backup system works in the following steps: + +1. Operators issue a new backup request via `fdbbackup` command line tool; +2. FDB cluster receives the request and registers the request in the database (internal `TaskBucket` and system keys); +3. Backup workers monitor changes to system keys, register the request in its own internal queue, and starts logging mutations for the request key range; at the same time, backup agents (scheduled by `TaskBucket`) starts taking snapshots of key ranges in the database; +4. Periodically, backup workers upload mutations to the requested blob storage, and save the progress into the database; +5. The backup is restorable when backup workers have saved versions that are larger than the complete snapshot’s end version, and the backup is stopped if a stop on restorable flag is set in the request. + +The new backup has four major components: 1) backup workers; 2) recruitment of backup workers; 3) extension of tag partitioned log system to support pseudo tags; 4) integration with existing `TaskBucket` based backup command interface; and 5) integration with the Performant Restore System. + +### Backup workers + +Backup worker is a new role introduced in the new backup system. A backup worker is responsible for pulling mutations from transaction logs and saving the mutations to blob storage. Internally, a backup worker maintains a message buffer, which keeps mutations pulled from transaction logs, but have not been saved to blob storage yet. Periodically, the backup worker parses mutations in the message buffer, extracts those mutations that are within user specified key ranges, and then uploads mutation data to blob storage. After data is saved, the backup worker removes these messages from its internal buffer and saves its progress in the database, so that after a failure, a new backup worker starts from the previously saved version. + +Backup worker has two modes of operation: *no-op* mode, and *working* mode. When there is no active backup in the cluster, backup worker operates in the no-op mode, which simply obtains the recently committed version from Proxies and then pops mutations from transaction logs. After operators submit a new backup request to the cluster, backup workers transition into the working mode that starts pulling mutations from transaction logs and saving the mutation data to blob storage. + +In the working mode, the popping of backup workers need to follow a strictly increasing version order. For the same tag, there could be multiple backup workers, each is responsible for a different epoch. These backup workers must coordinating their popping order, otherwise the backup can miss some mutation data. This coordination among backup workers is achieved by deferring popping of a later epoch and only allowing the oldest epoch to pop first. After the oldest epoch has finished, these corresponding backup workers notifies the master, which will then advances the oldest backup epoch so that the next epoch can proceed the popping. + +A subtle issue for a displaced backup worker (i.e., being displaced because a new epoch begins), is that the last pop of the backup worker can cause missing version ranges in mutation logs. This is because the transaction for saving the progress may be delayed during recovery. As a result, the master could already recruited a new backup worker for the old epoch starting at the previously saved progress version. Then the saving transaction succeeds, and the worker pops mutations that the new backup worker is supposed to save, resulting in missing data for new backup worker’s log. The solution to this problem can be: 1) the old backup worker aborts immediately after knowing itself is displaced, thus not trying to save its progress; or 2) the old backup worker skip its last pop, since the next epoch will pop versions larger than its progress. Because the second approach avoids doing duplicated work in the new epoch, we choose to the second approach. + +Finally, multiple concurrent backups are supported. Each backup worker keeps track of current backup jobs and saves mutations to corresponding backup containers for the same batch of mutations. + +### Recruitment of Backup workers + +Backup workers are recruited during master recovery as part of log system. The Master recruits a fixed number of backup workers, one for each log router tag. During the recruiting process, the master sends backup worker initialization request as: + +``` +struct InitializeBackupRequest { + UID reqId; + LogEpoch epoch; // epoch this worker is recruited + LogEpoch backupEpoch; // epoch that this worker actually works on + Tag routerTag; + Version startVersion; + Optional endVersion; // Only present for unfinished old epoch + ReplyPromise reply; + … // additional methods elided +}; + +``` + +Note we need two epochs here: one for the recruited epoch and one for backing up epoch. The recruited epoch is the epoch of the log system, which is used by a backup worker to find out if it works for the current epoch. If so, the worker should save its progress and immediately exit. The `backupEpoch` is used for saving progress. The `backupEpoch` is usually the same as the epoch that the worker is recruited. However, it can be some earlier epoch than the recruiting epoch, signifying that the worker is responsible for data in that earlier epoch. In this case, when the worker is done and exits, the master should not flag its departure as a trigger of recovery. This is solved by the following protocol: + +1. The backup worker finishes its work, including saving progress to the key value store and uploading to cloud storage, and then sends a `BackupWorkerDoneRequest` to the master; +2. The master receives the request, removes the worker from its log system, and updates the oldest backing up epoch `oldestBackupEpoch`; +3. The master sends backup a reply message to the backup worker and registers the new log system with cluster controller; +4. The backup worker exits after receiving the reply. Other backup workers in the system get the new log system from the cluster controller. If a backup worker’s `backupEpoch` is equal to `oldestBackupEpoch`, then the worker may start popping from TLogs. + +Note `oldestBackupEpoch` is introduced to prevent a backup worker for a newer epoch from popping when there are backup workers for older epochs. Otherwise, these older backup workers may lose data. + +### Extension of tag partitioned log system to support pseudo tags + +The tag partitioned log system is modeled like a FIFO queue, where Proxies push mutations to the queue and Storage Servers or Log Routers pop mutations from the queue. Specifically, consumers of the tag partitioned log system use two operations, `peek` and `pop`, to read mutations for a given tag and to pop mutations from the queue. Because Proxies assign each mutation a unique log router tag, the backup system reuses this tag to obtain the whole mutation stream. As a result, each log router tag now has two consumers, a log router and a backup worker. + +To support multiple consumers of the log router tag, the peek and pop has been extended to support pseudo tags. In other words, each log router tag can be mapped to multiple pseudo tags. Log routers and Backup workers still `peek` mutations with the log router tag, but `pop` with different pseudo tags. Only after both pseudo tags are popped, TLogs can pop the mutations from its internal queue. + +Note the introduction of pseudo tags opens the possibility for more usage scenarios. For instance, a change stream can be implemented with a pseudo tag, where the new consumer can look at each mutation and emit mutations on specified key ranges. + +### Integration with existing taskbucket based backup command interface + +We strive to keep the operational interface the same as the old backup system. That is, the new backup is initiated by the client as before with an additional flag. FDB cluster receives the backup request, sees the flag being set, and uses the new system for generating mutation logs. + +By default, backup workers are not enabled in the system. When operators submit a new backup request for the first time, the database performs a configuration change (`backup_worker_enabled:=1`) that enables backup workers. + +The operator’s backup request can indicate if an old backup or a new backup is used. This is a command line option (i.e., `-p` or `--partitioned_log`) in the `fdbbackup` command. A backup request of the new type is started in the following steps: + +1. Operators use `fdbbackup` tool to write the backup range to a system key, i.e., `\xff\x02/backupStarted`. +2. All backup workers monitor the key `\xff\x02/backupStarted`, see the change, and start logging mutations. +3. After all backup workers have started, the `fdbbackup` tool initiates the backup of all or specified key ranges by issuing a transaction `Ts`. + +Compared to the old backup system, the above step 1 and 2 are new and is only triggered if client requests for a new type of backup. The purpose is to allow backup workers to function as no-op if there are no ongoing backups. However, the backup workers should still continuously pop their corresponding tags, otherwise mutations will be kept in the TLog. In order to know the version to pop, backup workers can obtain the read version from any proxy. Because the read version must be a committed version, so popping to this version is safe. + +**Backup Submission Protocol** +Protocol for `submitBackup()` to ensure that all backup workers of the current epoch have started logging mutations: + +1. After the `submitBackup()` call, the task bucket (i.e., `StartFullBackupTaskFunc`) starts by creating a `BackupConfig` object in the system key space. +2. Each backup worker monitors the `\xff\x02/backupStarted` key and notices the new backup job. Then the backup worker inserts the new job into its internal queue, and writes to `startedBackupWorkers` key in the `BackupConfig` object if the worker’s `backupEpoch` is the current epoch. Among these workers, the worker with Log Router Tag `-2:0` monitors the `startedBackupWorkers` key, and sets `allWorkerStarted` key after all workers have updated the `startedBackupWorkers` key. +3. The task bucket watches change to the `startedBackupWorkers` key and declares the job submission successful. + +This protocol was implemented after another abandoned protocol: the `startedBackupWorkers` key is set after all backup workers have saved logs with versions larger than the version of `submitBackup()` call. This protocol fails if there is already a backup job and there is a backup worker that doesn’t notice the change to the `\xff\x02/backupStarted` key. As a result, the worker is saving versions larger than the new job’s start version, but in the old backup container. Thus the new container misses some mutations. + +**Protocol for Determining A Backup is Restorable** + +1. Each backup worker independently logs mutations to a backup container and updates its progress in the system key space. +2. The worker with Log Router Tag `-2:0` of current epoch monitors all workers’ progress. If the oldest backup epoch is the current epoch (i.e, there are no backup workers for any old epochs, thus no version ranges missing before this epoch), this worker updates `latestBackupWorkerSavedVersion` key in the `BackupConfig` object with the minimum saved version among workers. +3. The client calls `describeBackup()`, which eventually calls `getLatestRestorableVersion` to read the value from the `latestBackupWorkerSavedVersion` key. If this version is larger than the first snapshot’s end version, then the backup is restorable. + +**Pause and Resume Backups** +The command line for pause or resume backups remains the same, but the implementation for the new backup system is different from the old one. This is because in the old backup system, both mutation logs and range logs are handled by `TaskBucket`, an asynchronous task scheduling framework that stores states in the FDB database. Thus, the old backup system simply pauses or resumes the `TaskBucket`. In the new backup system, mutation logs are generated by backup workers, thus the pause or resume command needs to tell all backup workers to pause or resume pulling mutations from TLogs. Specifically, + +1. The operator issues a pause or resume request that upates both the `TaskBucket` and `\xff\x02/backupPaused` key. +2. Each backup worker monitors the `\xff\x02/backupPaused` key and notices the change. Then the backup worker pauses or resumes pulling from TLogs. + +**Backup Container Changes** + +* Partitioned mutation logs are stored in `plogs/XXXX/XXXX` directory and their names are in the format of `log,[startVersion],[endVersion],[UID],[N-of-M],[blockSize]`, where `M` is total partition number, `N` can be any number from `0` to `M - 1`. In contrast, old mutation logs are stored in `logs/XXXX/XXXX` directory and are named differently. +* To restore a version range, all partitioned logs for the range needs to be available. The restore process should read all partitioned logs, and combine mutations from different logs into one mutation stream, ordered by `(commit_version, subsequence)` pair. It is guaranteed that all mutations form a total order. Note in the old backup files, there is no subsequence number, as each version’s mutations are serialized in order in one file. + +### Integration with the [Performant Restore System](https://github.com/apple/foundationdb/issues/1049) + +As discussed above, the new backup system split mutation logs into multiple partitions. Thus, the restore process must verify the backup files are continuous for all partitions with the restore’s version range. This is possible because each log file name has the information about its partition number and the total number of partitions. + +Once the restore system verifies the version range is continuous, the restore system needs to filter out duplicated version range among different log files (both log continuity analysis and dedup logic are implemented in `BackupContainer` abstraction). A given version range may be stored in **multiple** mutation log files. This can happen because a recruited backup worker can upload mutation files successfully, but doesn’t save the progress before another recovery happens. As a result, the new epoch tries to backup this version range again, producing the same version ranges (though the file names are different). + +Finally, the restore system loads the same version’s mutations from all partitions, and then merges these mutations in the order of their subsequence number before they are applied on the restore cluster. Note the mutations in the old backup system lack subsequence numbers. As a result, restoring old backups needs to assign subsequence number to mutations. + +## Ordered and Complete Guarantee of Mutation Logs + +The backup system must generate log files that the restore system can apply all the mutations on the backup cluster in the same order exactly once. + +**Ordering guarantee**. To maintain the ordering of mutations, each mutation is stored with its commit version and a subsequence number, both are assigned by Proxies during commit. The restore system can load all mutations and derive a total order among all the mutations. + +**Completeness guarantee**. All mutations should be saved in log files. We cannot allow any mutations missing from the backup. This is guaranteed by the fault tolerance discussed below. Essentially all backup workers checkpoint their progress in the database. After the recovery, the new master reads previous checkpoints and recruit new backup workers for any missing version ranges. + +## Backup File Format + +The old backup file format is documented [here](https://github.com/apple/foundationdb/blob/release-6.2/design/backup-dataFormat.md). We can’t use this file format, because our backup files are created for log router tags. When there are more than one log routers (almost always the case), the mutations in one transaction can be given different log router tags. As a result, for the same version, mutations are distributed in many files. Another subtle issue is that, there can be two mutations, (e.g., `a = 1` and `a = 2` in a transaction), which are given two different tags. We have to preserve the order of these two mutations in the restore process. Even though the order is saved in the sub-sequence number of a version, we still need to merge mutations from multiple files and apply them in the correct order. + +In the new backup system, mutation log file is named as `log,[startVersion],[endVersion],[UID],[N-of-M],[blockSize]`, where `startVersion` is inclusive and `endVersion` is *not* inclusive, e.g., `log,332850851,332938927,7be23c0a3e80df8ab1530fa76fa66980,1-of-4,1048576`. With the information from all file names, the restore process can find all files for a version range, i.e., versions intersect with the range and all log router tags. “`M`” is the total number of tags, and “`N`” is from `0` to `m - 1`.Note `tagId` is not required in the old backup filename, since all mutations for a version are included in one file. + +Each file content is a list of fixed size blocks. Each block contains a sequence of mutations, where each mutation consists of a serialized `Version`, `int32_t`, `int32_t`, (all these three numbers are in big endian) and `Mutation`, where `Mutation` is of format `type|kLen|vLen|Key|Value`, where `type` is the mutation type (e.g., `Set` or `Clear`), `kLen` and `vLen` respectively are the lengths of the key and value in the mutation. `Key` and `Value` are the serialized value of the Key and Value in the mutation. The paddings at the end of the block are bytes of `0xFF`. + +``` +`` +`` +`` +`…` +` +` +``` + +Note the big Endianness for version is required, as `0xFF` is used as the padding to indicate block end. A little endian number can easily be mistaken as the end. In contrast, big endian for version almost guarantee the first byte is not `0xFF` (should always be `0x00`). + +## Performance optimization + +### Future Optimizations + +Add a metadata file describe the backup file: + +* The number of mutations; +* The number of atomic operations; +* key range and version range of mutations in each backup file; + +The information can be used to optimize the restore process. For instance, the number of mutations can be used to make better load balancing decisions; if there is no atomic operations, the restore can apply mutation in a backward fashion -- skipping mutations with earlier versions. + +## Fault Tolerance + +Failures of a backup worker will trigger a master recovery. After the recovery, the new master recruits a new set of backup workers. Among them, a new backup worker shall continue the work of the failed backup worker from the previous epoch. + +The interesting part is the handling of old epochs, since the backup workers for the old epoch are in the “displaced” state and should exit. So the basic idea is that we need a set of backup workers for the data left in the old epochs. To figure out the set of data not backed up yet, the master first loads saved backup progress data ` `from the database, and then computes for each epoch, what version ranges have not been backed up. For each of the version range and tag, master recruit a worker to resume the backup for that version range and tag. Note that this worker has a different worker UID from the worker in the original epoch. As a result, for a given epoch and a tag, there might be multiple progress status, as these workers are recruited at different epochs. + +## KPI's and Metrics + +The backup system emits the following metrics: + +* How much backup data has been processed: the backup command line tool `fdbbackup` can show the status of backup, including the size of mutation logs (`LogBytes written`) and snapshots (`RangeBytes written`). By taking two consecutive backup status, the backup speed can be estimated as (`2nd_LogBytes - 1st_LogBytes) / interval`. +* An estimation of backup delay: Each backup worker emits `BackupWorkerMetrics` trace events every 5 seconds, which includes `SavedVersion`, `MinKnownCommittedVersion`, and `MsgQ`. The backup delay can be estimated as (`MinKnownCommittedVersion - SavedVersion) / 1,000,000` seconds, which is the difference between a worker’s saved version and current committed version, divided by 1M version per second. `MsgQ` is the queue size of memory buffer of the backup worker. + +## Controlling Properties + +System operator can control the following backup properties: + +* **Backup key ranges**: The non-overlapped key ranges that will be backed up to the blob storage. +* **Blob url**: The root path in blob that host all backup files. +* **Performance knobs**: The knobs that control the performance + * The backup interval (knob `BACKUP_UPLOAD_DELAY`) for saving mutation logs to blob storage; + +## Testing + +The feature will be tested both in simulation and in real clusters: + +* New test cases are added into the test folder in FDB. The nightly correctness (i.e., simulation) tests will test the correctness of both backup and restore. +* Tests will be added to constantly backup a cluster with the new backup system and restore the backup to ensure the restore works on real clusters. During the time period of active backup, the cluster should have better write performance than using old backup system. +* Tests should also be conducted with production data. This ensures backup data is restorable and catches potential bugs in backup and restore. This test is preferably conducted regularly, e.g., weekly per cluster. + +Before the restore system is available, the testing strategy for backup files is to keep old backup system running. Thus, both new backup files and old backup files are generated. Then both types of log files are decoded and compared against. The new backup file is considered correct if its content matches the content of old log files. diff --git a/design/recovery-internals.md b/design/recovery-internals.md index 3ede735e13..c9d8631ddc 100644 --- a/design/recovery-internals.md +++ b/design/recovery-internals.md @@ -67,7 +67,7 @@ The transaction system state before the recovery is the starting point for the c ## Phase 2: LOCKING_CSTATE -This phase locks the coordinated state (cstate) to make sure there is only one master who can change the cstate. Otherwise, we may end up with more than one master accepting commits after the recovery. To achieve that, the master needs to get currently alive tLogs’ interfaces and sends commands to tLogs to lock their states, preventing them from accepting any further writes. +This phase locks the coordinated state (cstate) to make sure there is only one master who can change the cstate. Otherwise, we may end up with more than one master accepting commits after the recovery. To achieve that, the master needs to get currently alive tLogs’ interfaces and sends commands to tLogs to lock their states, preventing them from accepting any further writes. Recall that `ServerDBInfo` has master's interface and is propogated by CC to every process in a cluster. The current running tLogs can use the master interface in its `ServerDBInfo` to send itself's interface to master. Master simply waits on receiving the `TLogRejoinRequest` streams: for each tLog’s interface received, the master compares the interface ID with the tLog ID read from cstate. Once the master collects enough old tLog interfaces, it will use the interfaces to lock those tLogs. diff --git a/design/special-key-space.md b/design/special-key-space.md new file mode 100644 index 0000000000..15386de508 --- /dev/null +++ b/design/special-key-space.md @@ -0,0 +1,81 @@ +# Special-Key-Space +This document discusses why we need the proposed special-key-space framwork. And for what problems the framework aims to solve and in what scenarios a developer should use it. + +## Motivation +Currently, there are several client functions implemented as FDB calls by passing through special keys(`prefixed with \xff\xff`). Below are all existing features: +- **status/json**: `get("\xff\xff/status/json")` +- **cluster_file_path**: `get("\xff\xff/cluster_file_path)` +- **connection_string**: `get("\xff\xff/connection_string)` +- **worker_interfaces**: `getRange("\xff\xff/worker_interfaces", )` +- **conflicting-keys**: `getRange("\xff\xff/transaction/conflicting_keys/", "\xff\xff/transaction/conflicting_keys/\xff")` + +At present, implementions are hard-coded and the pain points are obvious: +- **Maintainability**: As more features added, the hard-coded snippets are hard to maintain +- **Granularity**: It is impossible to scale up and down. For example, you want a cheap call like `get("\xff\xff/status/json/")` instead of calling `status/json` and parsing the results. On the contrary, sometime you want to aggregate results from several similiar features like `getRange("\xff\xff/transaction/, \xff\xff/transaction/\xff")` to get all transaction related info. Both of them are not achievable at present. +- **Consistency**: While using FDB calls like `get` or `getRange`, the behavior that the result of `get("\xff\xff/B")` is not included in `getRange("\xff\xff/A", "\xff\xff/C")` is inconsistent with general FDB calls. + +Consequently, the special-key-space framework wants to integrate all client functions using special keys(`prefixed with \xff`) and solve the pain points listed above. + +## When +If your feature is exposing information to clients and the results are easily formatted as key-value pairs, then you can use special-key-space to implement your client function. + +## How +If you choose to use, you need to implement a function class that inherits from `SpecialKeyRangeBaseImpl`, which has an abstract method `Future> getRange(Reference ryw, KeyRangeRef kr)`. +This method can be treated as a callback, whose implementation details are determined by the developer. +Once you fill out the method, register the function class to the corresponding key range. +Below is a detailed example. +```c++ +// Implement the function class, +// the corresponding key range is [\xff\xff/example/, \xff\xff/example/\xff) +class SKRExampleImpl : public SpecialKeyRangeBaseImpl { +public: + explicit SKRExampleImpl(KeyRangeRef kr): SpecialKeyRangeBaseImpl(kr) { + // Our implementation is quite simple here, the key-value pairs are formatted as: + // \xff\xff/example/ : + CountryToCapitalCity[LiteralStringRef("USA")] = LiteralStringRef("Washington, D.C."); + CountryToCapitalCity[LiteralStringRef("UK")] = LiteralStringRef("London"); + CountryToCapitalCity[LiteralStringRef("Japan")] = LiteralStringRef("Tokyo"); + CountryToCapitalCity[LiteralStringRef("China")] = LiteralStringRef("Beijing"); + } + // Implement the getRange interface + Future> getRange(Reference ryw, + KeyRangeRef kr) const override { + + Standalone result; + for (auto const& country : CountryToCapitalCity) { + // the registered range here: [\xff\xff/example/, \xff\xff/example/\xff] + Key keyWithPrefix = country.first.withPrefix(range.begin); + // check if any valid keys are given in the range + if (kr.contains(keyWithPrefix)) { + result.push_back(result.arena(), KeyValueRef(keyWithPrefix, country.second)); + result.arena().dependsOn(keyWithPrefix.arena()); + } + } + return result; + } +private: + std::map CountryToCapitalCity; +}; +// Instantiate the function object +// In development, you should have a function object pointer in DatabaseContext(DatabaseContext.h) and initialize in DatabaseContext's constructor(NativeAPI.actor.cpp) +const KeyRangeRef exampleRange(LiteralStringRef("\xff\xff/example/"), LiteralStringRef("\xff\xff/example/\xff")); +SKRExampleImpl exampleImpl(exampleRange); +// Assuming the database handler is `cx`, register to special-key-space +// In development, you should register all function objects in the constructor of DatabaseContext(NativeAPI.actor.cpp) +cx->specialKeySpace->registerKeyRange(exampleRange, &exampleImpl); +// Now any ReadYourWritesTransaction associated with `cx` is able to query the info +state ReadYourWritesTransaction tr(cx); +// get +Optional res1 = wait(tr.get("\xff\xff/example/Japan")); +ASSERT(res1.present() && res.getValue() == LiteralStringRef("Tokyo")); +// getRange +// Note: for getRange(key1, key2), both key1 and key2 should prefixed with \xff\xff +// something like getRange("normal_key", "\xff\xff/...") is not supported yet +Standalone res2 = wait(tr.getRange(LiteralStringRef("\xff\xff/example/U"), LiteralStringRef("\xff\xff/example/U\xff"))); +// res2 should contain USA and UK +ASSERT( + res2.size() == 2 && + res2[0].value == LiteralStringRef("London") && + res2[1].value == LiteralStringRef("Washington, D.C.") +); +``` \ No newline at end of file diff --git a/design/tlog-forward-compatibility.md.html b/design/tlog-forward-compatibility.md.html new file mode 100644 index 0000000000..9ba8f4ba6d --- /dev/null +++ b/design/tlog-forward-compatibility.md.html @@ -0,0 +1,217 @@ + + +# Forward Compatibility for Transaction Logs + +## Background + +A repeated concern with adopting FoundationDB has been that upgrades are one +way, with no supported rollback. If one were to upgrade a cluster running 6.0 +to a 6.1, then there's no way to roll back to 6.0 if the new version results in +worse client application performance or unavailability. In the interest of +increasing adoption, work has begun on supporting on-disk forward +compatibility, which allows for upgrades to be rolled back. + +The traditional way of allowing roll backs is to have one version, `N`, that +introduces a feature, but is left as disabled. `N+1` enables the feature, and +then `N+2` removes whatever was deprecated in `N`. However, FDB currently has +a 6 month release cadence, and waiting 6 months to be able to use a new feature +in production is unacceptably long. Thus, the goal is to have a way to be able +to have a sane and user-friendly, rollback-supporting upgrade path, but still +allow features to be used immediately if desired. + +This document also carries two specific restrictions to the scope of what it covers: + +1. This document specifically is **not** a discussion of network protocol + compatibility nor supporting rolling upgrades. Rolling upgrades of FDB are + still discouraged, and minor versions are still protocol incompatible with + each other. +2. This only covers the proposed design of how forward compatibility for + transaction logs will be handled, and not forward compatibility for + FoundationDB as a whole. There are other parts of the system that durably + store data, the coordinators and storage servers, that will not be discussed. + +## Overview + +A new configuration option, `log_version`, will be introduced to allow a user +to control which on-disk format the transaction logs are allowed to use. Not +every release will affect the on-disk format of the transaction logs, so +`log_version` is an opaque integer that is incremented by one whenever the +on-disk format of the transaction log is changed. + +`log_version` is set by from `fdbcli`, with an invocation looking like +`$ fdbcli -C cluster.file --exec "configure log_version:=2"`. Note that `:=` +is used instead of `=`, to keep the convention in `fdbcli` that configuration +options that users aren't expected to need (or wish) to modify are set with +`:=`. + +Right now, FDB releases and `log_version` values are as follows: + +| Release | Log Version | +| ------- | ----------- | +| pre-5.2 | 1 | +| 5.2-6.0 | 2 | +| 6.1+ | 3 | +| 6.2 | 4 | +| 6.3 | 5 | + +If a user does not specify any configuration for `log_version`, then +`log_version` will be set so that rolling back to the previous minor version of +FDB will be possible. FDB will always support loading files generated by +default from the next minor version. It will be possible to configure +`log_version` to a higher value on the release that introduces it, it the user +is willing to sacrifice the ability to roll back. + +This means FDB's releases will work like the following: + +| | 6.0 | 6.1 | 6.2 | 6.3 | +|--------------|-----|-----|-------|---------| +| Configurable | 2 | 2,3 | 3,4 | 4,5 | +| Default | 2 | 2 | 3 | 4 | +| Recoverable | 2 | 2,3 | 2,3,4 | 2,3,4,5 | + +Where... + +* "configurable" means values considered an acceptable configuration setting for `fdbcli> configure log_version:=N`. +* "default" means what `log_version` will be if you don't configure it. +* "recoverable" means that FDB can load files that were generated from the specified `log_version`. + +Configuring to a `log_version` will cause FDB to use the maximum of that +`log_version` and default `log_version`. The default `log_version` will always +be the minimum configurable log version. This is done so that manually setting +`log_version` once, and then upgrading FDB multiple times, will eventually +cause a low `log_version` left in the database configuration to act as a +request for the default. Configuring `log_version` to a very high number (e.g. 9999) +will cause FDB to always use the highest available log version. + +As a concrete example, 6.1 will introduce a new transaction log feature with +on-disk format implications. If you wish to use it, you'll first have to +`configure log_version:=3`. Otherwise, after upgrading to FDB6.2, it will +become the default. If problems are discovered when upgrading to FDB6.2, then +roll back to FDB6.1. (Theoretically. See scope restrictions above.) + +## Detailed Implementation + +`fdbcli> configure log_version:=3` sets `\xff/conf/log_version` to `3`. This +version is also persisted as part of the `LogSystemConfig` and thus +`DBCoreState`, so that any code handling the log system will have access to the +`log_version` that was used to create it. + +Changing `log_version` will result in a recovery, and FoundationDB will recover +into the requested transaction log implementation. This involves locking the +previous generation of transaction logs, and then recruiting a new generation +of transaction logs. FDB will load `\xff/conf/log_version` as the requested +`log_version`, and when sending a `InitializeTLogRequest` to recruit a new +transaction log, it uses the maximum of the requested log version and the +default `log_version`. + +A worker, when receiving an `InitializeTLogRequest`, will initialize a +transaction log corresponding to the requested `log_version`. Transaction logs +can pack multiple generations of transaction logs into the same shared entity, +a `SharedTLog`. `SharedTLog` instances correspond to one set of files, and +will only contain transaction log generations of the same `log_version`. + +This allows us to have multiple generations of transaction logs running within +one worker that have different `log_version`s, and if the worker crashes and +restarts, we need to be able to recreate those transaction log instances. + +Transaction logs maintain two types of files, one is a pair files prefixed with +`logqueue-` that are the DiskQueue, and the other is the metadata store, which +is normally a mini `ssd-2` storage engine running within the transaction log. + +When a worker first starts, it scans its data directory for any files that were +instances of a transaction log. It then needs to construct a transaction log +instance that can read the format of the file to be able to reconnect the data +in the files back to the FDB cluster, so that it can be used in a recovery if +needed. + +This presents a problem that the worker needs to know all the configuration +options that were used to decide the file format of the transaction log +*before* it can rejoin a cluster and get far enough through a recovery to find +out what that configuration was. To get around this, the relevant +configuration options have been added to the file name so that they're +available when scanning the list of files. + +Currently, FDB identifies a transaction log instance via seeing a file that starts +with `log-`, which represents the metadata store. This filename has the format +of `log-.` where UUID is the `logId`, and SUFFIX tells us if the +metadata store is a memory or ssd storage engine file. + +This format is being changed to `log2-<KV PAIRS>-.`, where KV +PAIRS is a small amount of information encoded into the file name to give us +the metadata *about* the file that is required. According to POSIX, the +characters allowed for "fully portable filenames" are `A–Z a–z 0–9 . _ -` and +the filename length should stay under 255 characters. This leaves only `_` as +the only character not already used. Therefore, the KV pair encoding +`K1_V1_K2_V2_...`, so keys and values separated by an `_`, and kv pairs are +also separated by an `_`. + +The currently supported keys are: + +V +: A copy of `log_version` + +LS +: `log_spill`, a new configuration option in 6.1 + +and any unrecognized keys are ignored, which will likely help forward compatibility. + +An example file name is `log2-V_3_LS_2-46a5f353ac18d787852d44c3a2e51527-0.fdq`. + +### Testing + +`SimulationConfig` has been changed to randomly set `log_version` according to +what is supported. This means that with restarting upgrade tests that simulate +upgrading from `N` to `N+1`, the `N+1` version will see files that came from an +FDB running with any `log_version` value that was previously supported. If +`N+1` can't handle the files correctly, then the simulation test will fail. + +`ConfigureTest` tries randomly toggling `log_version` up and down in a live +database, along with all the other log related options. Some are valid, some +are invalid and should be rejected, or will cause ASSERTs in later parts of the +code. + +I've added a new test, `ConfigureTestRestart` that tests changing +configurations and then upgrading FDB, to cover testing that upgrades still +happen correctly when `log_version` has been changed. This also verifies that +on-disk formats for those `log_version`s are still loadable by future FDB +versions. + +There are no tests that mix the `ConfigureDatabase` and `Attrition` workloads. +It would be good to do so, to cover the case of `log_version` changes in the +presence of failures, but one cannot be added easily. The simulator calculates +what processes/machines are safe to kill by looking at the current +configuration. For `ConfigureTest`, this isn't good enough, because `triple` +could mean that there are three replicas, or that the FDB cluster just changed +from `single` to `triple` and only have one replica of data until data +distribution finishes. It would be good to add a `ConfigureKillTest` sometime +in the future. + +For FDB to actually announce that rolling back from `N+1` to `N` is supported, +there will need to be downgrade tests from `N+1` to `N` also. The default in +`N+1` should always be recoverable within `N`. As FDB isn't promising forward +compatibility yet, these tests haven't been implemented. + +# Transaction Log Forward Compatibility Operational Guide + +## Notable Behavior Changes + +When release notes mention a new `log_version` is available, after deploying +that release, it's worth considering upgrading `log_version`. Doing so will +allow a controlled upgrade, and reduce the number of new changes that will +take effect when upgrading to the next release. + +## Observability + +* When running with a non-default `log_version`, the setting will appear in `fdbcli> status`. + +## Monitoring and Alerting + +If anyone is doing anything that relies on the file names the transaction log uses, they'll be changing. + + + + + + + + diff --git a/design/tlog-spilling.md.html b/design/tlog-spilling.md.html new file mode 100644 index 0000000000..aee572b597 --- /dev/null +++ b/design/tlog-spilling.md.html @@ -0,0 +1,680 @@ + + +# TLog Spill-By-Reference Design + +## Background + +(This assumes a basic familiarity with [FoundationDB's architecture](https://www.youtu.be/EMwhsGsxfPU).) + +Transaction logs are a distributed Write-Ahead-Log for FoundationDB. They +receive commits from proxies, and are responsible for durably storing those +commits, and making them available to storage servers for reading. + +Clients send *mutations*, the list of their set, clears, atomic operations, +etc., to proxies. Proxies collect mutations into a *batch*, which is the list +of all changes that need to be applied to the database to bring it from version +`N-1` to `N`. Proxies then walk through their in-memory mapping of shard +boundaries to associate one or more *tags*, a small integer uniquely +identifying a destination storage server, with each mutation. They then send a +*commit*, the full list of `(tags, mutation)` for each mutation in a batch, to +the transaction logs. + +The transaction log has two responsibilities: it must persist the commits to +disk and notify the proxy when a commit is durably stored, and it must make the +commit available for consumption by the storage server. Each storage server +*peeks* its own tag, which requests all mutations from the transaction log with +the given tag at a given version or above. After a storage server durably +applies the mutations to disk, it *pops* the transaction logs with the same tag +and its new durable version, notifying the transaction logs that they may +discard mutations with the given tag and a lesser version. + +To persist commits, a transaction log appends commits to a growable on-disk +ring buffer, called a *disk queue*, in version order. Commit data is *pushed* +onto the disk queue, and when all mutations in the oldest commit persisted are +no longer needed, the disk queue is *popped* to trim its tail. + +To make commits available to storage servers efficiently, a transaction log +maintains a copy of the commit in-memory, and maintains one queue per tag that +indexes the location of each mutation in each commit with the specific tag, +sequentially. This way, responding to a peek from a storage server only +requires sequentailly walking through the queue, and copying each mutation +referenced into the response buffer. + +Transaction logs internally handle commits via performing two operations +concurrently. First, they walk through each mutation in the commit, and push +the mutation onto an in-memory queue of mutations destined for that tag. +Second, they include the data in the next batch of pages to durably persist to +disk. These in-memory queues are popped from when the corresponding storage +server has persisted the data to its own disk. The disk queue only exists to +allow the in-memory queues to be rebuilt if the transaction log crashes, is +never read from except during a transaction log recovering post-crash, and is +popped when the oldest version it contains is no longer needed in memory. + +TLogs will need to hold the last 5-7 seconds of mutations. In normal +operation, the default 1.5GB of memory is enough such that the last 5-7 seconds +of commits should almost always fit in memory. However, in the presence of +failures, the transaction log can be required to buffer significantly more +data. Most notably, when a storage server fails, its tag isn't popped until +data distribution is able to re-replicate all of the shards that storage server +was responsible for to other storage servers. Before that happens, mutations +will accumulate on the TLog destined for the failed storage server, in case it +comes back and is able to rejoin the cluster. + +When this accumulation causes the memory required to hold all the unpopped data +to exceed `TLOG_SPILL_THREASHOLD` bytes, the transaction log offloads the +oldest data to disk. This writing of data to disk to reduce TLog memory +pressure is referred to as *spilling*. + +************************************************************** +* Transaction Log * +* * +* * +* +------------------+ pushes +------------+ * +* | Incoming Commits |----------->| Disk Queue | +------+ * +* +------------------+ +------------+ |SQLite| * +* | ^ +------+ * +* | | ^ * +* | pops | * +* +------+-------+------+ | writes * +* | | | | | * +* v v v +----------+ * +* in-memory +---+ +---+ +---+ |Spill Loop| * +* queues | 1 | | 2 | | 3 | +----------+ * +* per-tag | | | | | | ^ * +* |...| |...| |...| | * +* | | | | * +* v v v | * +* +-------+------+--------------+ * +* queues spilled on overflow * +* * +************************************************************** + +## Overview + +Previously, spilling would work by writing the data to a SQLite B-tree. The +key would be `(tag, version)`, and the value would be all the mutations +destined for the given tag at the given version. Peek requests have a start +version, that is the latest version for which the storage server knows about, +and the TLog responds by range-reading the B-tree from the start version. Pop +requests allow the TLog to forget all mutations for a tag until a specific +version, and the TLog thus issues a range clear from `(tag, 0)` to +`(tag, pop_version)`. After spilling, the durably written data in the disk +queue would be trimmed to only include from the spilled version on, as any +required data is now entirely, durably held in the B-tree. As the entire value +is copied into the B-tree, this method of spilling will be referred to as +*spill-by-value* in the rest of this document. + +Unfortunately, it turned out that spilling in this fashion greatly impacts TLog +performance. A write bandwidth saturation test was run against a cluster, with +a modification to the transaction logs to have them act as if there was one +storage server that was permanently failed; it never sent pop requests to allow +the TLog to remove data from memory. After 15min, the write bandwidth had +reduced to 30% of its baseline. After 30min, that became 10%. After 60min, +that became 5%. Writing entire values gives an immediate 3x additional write +amplification, and the actual write amplification increases as the B-tree gets +deeper. (This is an intentional illustration of the worst case, due to the +workload being a saturating write load.) + +With the recent multi-DC/multi-region work, a failure of a remote data center +would cause transaction logs to need to buffer all commits, as every commit is +tagged as destined for the remote datacenter. This would rapidly push +transaction logs into a spilling regime, and thus write bandwidth would begin +to rapidly degrade. It is unacceptable for a remote datacenter failure to so +drastically affect the primary datacenter's performance in the case of a +failure, so a more performant way of spilling data is required. + +Whereas spill-by-value copied the entire mutation into the B-tree and removes +it from the disk queue, spill-by-reference leaves the mutations in the disk +queue and writes a pointer to it into the B-tree. Performance experiments +revealed that the TLog's performance while spilling was dictated more by the +number of writes done to the SQLite B-tree, than by the size of those writes. +Thus, "spill-by-reference" being able to do a significantly better batching +with its writes to the B-tree is more important than that it writes less data +in aggregate. Spill-by-reference significantly reduces the volume of data +written to the B-tree, and the less data that we write, the more we can batch +versions to be written together. + +************************************************************************ +* DiskQueue * +* * +* ------- Index in B-tree ------- ---- Index in memory ---- * +* / \ / \ * +* +-----------------------------------+-----------------------------+ * +* | Spilled Data | Most Recent Data | * +* +-----------------------------------+-----------------------------+ * +* lowest version highest version * +* * +************************************************************************ + +Spill-by-reference works by taking a larger range of versions, and building a +single key-value pair per tag that describes where in the disk queue is every +relevant commit for that tag. Concretely, this takes the form +`(tag, last_version) -> [(version, start, end, mutation_bytes), ...]`, where: + + * `tag` is the small integer representing the storage server this mutation batch is destined for. + * `last_version` is the last/maximum version contained in the value's batch. + * `version` is the version of the commit that this index entry points to. + * `start` is an index into the disk queue of where to find the beginning of the commit. + * `end` is an index into the disk queue of where the end of the commit is. + * `mutation_bytes` is the number of bytes in the commit that are relevant for this tag. + +And then writing only once per tag spilled into the B-tree for each iteration +through spilling. This turns the number of writes into the B-Tree from +`O(tags * versions)` to `O(tags)`. + +Note that each tuple in the list represents a commit, and not a mutation. This +means that peeking spilled commits will involve reading all mutations that were +a part of the commit, and then filtering them to only the ones that have the +tag of interest. Alternatively, one could have each tuple represent a mutation +within a commit, to prevent over-reading when peeking. There exist +pathological workloads for each strategy. The purpose of this work is most +importantly to support spilling of log router tags. These exist on every +mutation, so that it will get copied to other datacenters. This is the exact +pathological workload for recording each mutation individually, because it only +increases the number of IO operations used to read the same amount of data. +For a wider set of workloads, there's room to establish a heuristic as to when +to record mutation(s) versus the entire commit, but performance testing hasn't +surfaced this as important enough to include in the initial version of this +work. + +Peeking spilled data now works by issuing a range read to the B-tree from +`(tag, peek_begin)` to `(tag, infinity)`. This is why the key contains the +last version of the batch, rather than the beginning, so that a range read from +the peek request's version will always return all relevant batches. For each +batched tuple, if the version is greater than our peek request's version, then +we read the commit containing that mutation from disk, extract the relevant +mutations, and append them to our response. There is a target size of the +response, 150KB by default. As we iterate through the tuples, we sum +`mutation_bytes`, which already informs us how many bytes of relevant mutations +we'll get from a given commit. This allows us to make sure we won't waste disk +IOs on reads that will end up being discarded as unnecessary. + +Popping spilled data works similarly to before, but now requires recovering +information from disk. Previously, we would maintain a map from version to +location in the disk queue for every version we hadn't yet spilled. Once +spilling has copied the value into the B-tree, knowing where the commit was in +the disk queue is useless to us, and is removed. In spill-by-reference, that +information is still needed to know how to map "pop until version 7" to "pop +until byte 87" in the disk queue. Unfortunately, keeping this information in +memory would result in TLogs slowly consuming more and more +memory[^versionmap-memory] as more data is spilled. Instead, we issue a range +read of the B-tree from `(tag, pop_version)` to `(tag, infinity)` and look at +the first commit we find with a version greater than our own. We then use its +starting disk queue location as the limit of what we could pop the disk queue +until for this tag. + +[^versionmap-memory]: Pessimistic assumptions would suggest that a TLog spilling 1TB of data would require ~50GB of memory to hold this map, which isn't acceptable. + +## Detailed Implementation + +The rough outline of concrete changes proposed looks like: + +1. Allow a new TLog and old TLog to co-exist and be configurable, upgradeable, and recoverable +1. Modify spilling in new TLogServer +1. Modify peeking in new TLogServer +1. Modify popping in new TLogServer +1. Spill txsTag specially + +### Configuring and Upgrading + +Modifying how transaction logs spill data is a change to the on-disk files of +transaction logs. The work for enabling safe upgrades and rollbacks of +persistent state changes to transaction logs was split off into a seperate +design document: "Forward Compatibility for Transaction Logs". + +That document describes a `log_version` configuration setting that controls the +availability of new transaction log features. A similar configuration setting +was created, `log_spill`, that at `log_version>=3`, one may `fdbcli> +configure log_spill:=2` to enable spill-by-reference. Only FDB 6.1 or newer +will be unable to recover transaction log files that were using +spill-by-reference. FDB 6.2 will use spill-by-reference by default. + +| FDB Version | Default | Configurable | +|-------------|---------|--------------| +| 6.0 | No | No | +| 6.1 | No | Yes | +| 6.2 | Yes | Yes | + +If running FDB 6.1, the full command to enable spill-by-reference is +`fdbcli> configure log_version:=3 log_spill:=2`. + +The TLog implementing spill-by-value was moved to `OldTLogServer_6_0.actor.cpp` +and namespaced similarly. `tLogFnForOptions` takes a `TLogOptions`, which is +the version and spillType, and returns the correct TLog implementation +according to those settings. We maintain a map of +`(TLogVersion, StoreType, TLogSpillType)` to TLog instance, so that only +one SharedTLog exists per configuration variant. + +### Generations + +As a background, each time FoundationDB goes through a recovery, it will +recruit a new generation of transaction logs. This new generation of +transaction logs will often be recruited on the same worker that hosted the +previous generation's transaction log. The old generation of transaction logs +will only shut down once all the data that they have has been fully popped. +This means that there can be multiple instances of a transaction log in the +same process. + +Naively, this would create resource issues. Each instance would think that it +is allowed its own 1.5GB buffer of in-memory mutations. Instead, internally to +the TLog implmentation, the transaction log is split into two parts. A +`SharedTLog` is all the data that should be shared across multiple generations. +A TLog is all the data that is private to one generation. Most notably, the +1.5GB mutation buffer and the on-disk files are owned by the `SharedTLog`. The +index for the data added to that buffer is maintained within each TLog. In the +code, a SharedTLog is `struct TLogData`, and a TLog is `struct LogData`. +(I didn't choose these names.) + +This background is required, because one needs to keep in mind that we might be +committing in one TLog instance, a different one might be spilling, and yet +another might be the one popping data. + +********************************************************* +* SharedTLog * +* * +* +--------+--------+--------+--------+--------+ * +* | TLog 1 | TLog 2 | TLog 3 | TLog 4 | TLog 5 | * +* +--------+--------+--------+--------+--------+ * +* ^ popping ^spilling ^committing * +********************************************************* + +Conceptually, this is because each TLog owns a separate part of the same Disk +Queue file. The earliest TLog instance needs to be the one that controls when +the earliest part of the file can be discarded. We spill in version order, and +thus whatever TLog is responsible for the earliest unspilled version needs to +be the one doing the spilling. We always commit the newest version, so the +newest TLog must be the one writing to the disk queue and inserting new data +into the buffer of mutations. + + +### Spilling + +`updatePersistentData()` is the core of the spilling loop, that takes a new +persistent data version, writes the in-memory index for all commits less than +that version to disk, and then removes them from memory. By contact, once +spilling commits an updated persistentDataVersion to the B-tree, then those +bytes will not need to be recovered into memory after a crash, nor will the +in-memory bytes be needed to serve a peek response. + +Our new method of spilling iterates through each tag, and builds up a +`vector` for each tag, where `SpilledData` is: + +``` CPP +struct SpilledData { + Version version; + IDiskQueue::location start; + uint32_t length; + uint32_t mutationBytes; +}; +``` + +And then this vector is serialized, and written to the B-tree as +`(logId, tag, max(SpilledData.version))` = `serialized(vector)` + +As we iterate through each commit, we record the number of mutation bytes in +this commit that have our tag of interest. This is so that later, peeking can +read exactly the number of commits that it needs from disk. + +Although the focus of this project is on the topic of spilling, the code +implementing itself saw the least amount of total change. + +### Peeking + +A `TLogPeekRequest` contains a `Tag` and a `Version`, and is a request for all +commits with the specified tag with a commit version greater than or equal to +the given version. The goal is to return a 150KB block of mutations. + +When servicing a peek request, we will read up to 150KB of mutations from the +in-memory index. If the peek version is lower than the version that we've +spilled to disk, then we consult the on-disk index for up to 150KB of +mutations. (If we tried to read from disk first, and then read from memory, we +would then be racing with the spilling loop moving data from memory to disk.) + +************************************************************************** +* * +* +---------+ Tag +---------+ Tag +--------+ * +* | Peek |-------->| Spilled | ...------------->| Memory | * +* | Request | Version | Index | Version | Index | * +* +---------+ +---------+ +--------+ * +* | | * +* +-----------------+-----------------+ | * +* / \ Start=100 _/ \_ Start=500 + Start=900 + Ptr=0xF00 * +* / \ Length=50 / \ Length=70 / \ Length=30 / \ Length=30 * +* +------------------------------------------------+------------------+ * +* | Disk Queue | Also In Memory | * +* +------------------------------------------------+------------------+ * +* * +************************************************************************** + +Spill-by-value and memory storage engine only ever read from the DiskQueue when +recovering, and read the entire file linearly. Therefore, `IDiskQueue` had no +API for random reads to the DiskQueue. That ability is now required for +peeking, and thus, `IDiskQueue`'s API has been enhanced correspondingly: + +``` CPP +enum class CheckHashes { NO, YES }; + +class IDiskQueue { + // ... + Future> read(location start, location end, CheckHashes ch); + // ... +}; +``` + +Internally, the DiskQueue adds page headers every 4K, which are stripped out +from the returned data. Therefore, the length of the result will not be the +same as `end-start`, intentionally. For this reason, the API is `(start, end)` +and not `(start, length)`. + +Spilled data, when using spill-by-value, was resistent to bitrot via data being +checksummed interally within SQLite's B-tree. Now that reads can be done +directly, the responsibility for verifing data integrity falls upon the +DiskQueue. `CheckHashes::YES` will cause the DiskQueue to use the checksum in +each DiskQueue page to verify data integrity. If an externally maintained +checksums exists to verify the returned data, then `CheckHashes::NO` can be +used to elide the checksumming. A page failing its checksum will cause the +transaction log to die with an `io_error()`. + +What is read from disk is a `TLogQueueEntry`: + +``` CPP +struct TLogQueueEntryRef { + UID id; + Version version; + Version knownCommittedVersion; + StringRef messages; +} +``` + +Which provides the commit version and the logId of the TLog generation that +produced this commit, in addition to all of the mutations for that version. +(`knownCommittedVersion` is only used during FDB's recovery process.) + +### Popping + +As storage servers persist data, they send `pop(tag, version)` requests to the +transaction log to notify it that it is allowed to discard data for `tag` up +through `version`. Once all the tags have been popped from the oldest commit +in the DiskQueue, the tail of the DiskQueue can be discarded to reclaim space. + +If our popped version is in the range of what has been spilled, then we need to +consult our on-disk index to see what is the next location in the disk queue +that has data which is useful to us. This act would race with the spilling +loop changing what data is spilled, and thus disk queue popping +(`popDiskQueue()`) was made to run serially after spilling completes. + +Also due to spilling and popping largely overlapping in state, the disk queue +popping loop does not immediately react to a pop request from a storage server +changing the popped version for a tag. Spilling saves the popped version for +each tag when the spill loop runs, and if that version changed, then +`popDiskQueue()` refreshes its knowledge of what the minimum location in the +disk queue is required for that tag. We can pop the disk queue to the minimum +of all minimum tag locations, or to the minimum location needed for an +in-memory mutation if there is no spilled data. + +As a post implementation note, this ended up being a "here be dragons" +experience, with a surprising number of edge cases in races between +spilling/popping, various situations of having/not having/having inaccurate +data for tags, or that tags can stop being pushed to when storage servers are +removed but their corresponding `TagData` is never removed. + +### Transaction State Store + +For FDB to perform a recovery, there is information that it needs to know about +the database, such as the configuration, worker exclusions, backup status, etc. +These values are stored into the database in the `\xff` system keyspace. +However, during a recovery, FDB can't read this data from the storage servers, +because recovery hasn't completed, so it doesn't know who the storage servers +are yet. Thus, a copy of this data is held in-memory on every proxy in the +*transaction state store*, and durably persisted as a part of commits on the +transaction logs. Being durably stored on the transaction logs means the list +of transaction logs can be fetched from the coordinators, and then used to load +the rest of the information about the database. + +The in-memory storage engine writes an equal amount of mutations and snapshot +data to a queue, an when a full snapshot of the data has been written, deletes +the preceeding snapshot and begins writing a new one. When backing an +in-memory storage engine with the transaction logs, the +`LogSystemDiskQueueAdapter` implements writing to a queue as committing +mutations to the transaction logs with a special tag of `txsTag`, and deleting +the preceeding snapshot as popping the transaction logs for the tag of `txsTag` +until the version where the last full snapshot began. + +This means that unlike every other commit that is tagged and stored on the +transaction logs, `txsTag` signifies data that is: + +1. Committed to infrequently +2. Only peeked on recovery +3. Popped infrequently, and a large portion of the data is popped at once +4. A small total volume of data + +The most problematic of these is the infrequent popping. Unpopped data will be +spilled after some time, and if `txsTag` data is spilled and not popped, it +will prevent the DiskQueue from being popped as well. This will cause the +DiskQueue to grow continuously. The infrequent commits and small data volume +means that there benefits of spill-by-reference over spill-by-value don't apply +for this tag. + +Thus, even when configured to spill-by-reference, `txsTag` is spilled by value. + +### Disk Queue Recovery + +If a transaction log dies and restarts, all commits that were in memory at the +time of the crash must be loaded back into memory. Recovery is blocked on this +process, as there might have been a commit to the transaction state store +immediately before crashing, and that data needs to be fully readable during a +recovery. + +In spill-by-value, the DiskQueue only ever contained commits that were also +held in memory, and thus recovery would need to read up to 1.5GB of data. With +spill-by-reference, the DiskQueue could theoretically contain terrabytes of +data. To keep recovery times boundedly low, FDB must still only read the +commits that need to be loaded back into memory. + +This is done by persisting the location in the DiskQueue of the last spilled +commit to the SQLite B-Tree. This is done in the same transaction as the +spilling of that commit. This provides an always accurate pointer to where +data that needs to be loaded into memory begins. The pointer is to the +beginning of the last commit rather than the end, to make sure that the pointer +is always contained within the DiskQueue. This provides extra sanity checking +on the validity of the DiskQueue's contents at recovery, at the cost of +potentially reading 10MB more than what would be required. + +## Testing + +Correctness bugs in spilling would manifest as data corruption, which is well covered by simulation. +The only special testing code added was to enable changing `log_spill` in `ConfigureTest`. +This covers switching between spilling methods in the presence of faults. + +An `ASSERT` was added to simulation that verifies that commits read from the +DiskQueue on recovery are only the commits which have not been spilled. + +The rest of the testing is to take a physical cluster and try the extremes that +can only happen at scale: + +* Verify that recovery times are not impacted when a large amount of data is spilled +* Verify that long running tests hit a steady state of memory usage (and thus there are likely no leaks). +* Plot how quickly (MB/s) a remote datacenter can catch up in old vs new spilling strategy +* See what happens when there's 1 tlog and more than 100 storage servers. + * Verify that peek requests get limited + * See if tlog commits can get starved by excessive peeking + +# TLog Spill-By-Reference Operational Guide + +## Notable Behavior Changes + +TL;DR: Spilling involves less IOPS and is faster. Peeking involves more IOPS and is slower. Popping involves >0 IOPS. + +### Spilling + +The most notable effect of the spilling changes is that the Disk Queue files +will now grow to potentially terrabytes in size. + + 1. Spilling will occur in larger batches, which will result in a more +sawtooth-like `BytesInput - BytesDurable` value. I'm not aware that this will have any meaningful impact. + + * Disk queue files will grow when spilling is happening + * Alerting based on DQ file size is no longer appropriate + +As a curious aside, throughput decreases as spilled volume increases, which +quite possibly worked as accidental backpressure. As a feature, this no longer +exists, but means write-heavy workloads can drown storage servers faster than +before. + +### Peeking + +Peeking has seen tremendous changes. Its involves more IO operations and memory usage. + +The expected implication of this are: + +1. A peek of spilled data will involve a burst of IO operations. + + Theoretically, this burst can drown out queued write operations to disk, + thus and slowing down TLog commits. This hasn't been observed in testing. + + Low IOPS devices, such as HDD or network attached storage, would struggle + more here than locally attached SSD. + +2. Generating a peek response of 150KB could require reading 100MB of data, and allocating buffers to hold that 100MB. + + OOMs were observed in early testing. Code has been added to specifically + limit how much memory can be allocated for serving a signle peek request + and all concurrent peek requests, with knobs to allow tuning this per + deployment configuration. + +### Popping + +Popping will transition from being an only in-memory operation to one that +can involve reads from disk if the popped tag has spilled data. + +Due to a strange quirk, TLogs will allocate up to 2GB of memory as a read cache +for SQLite's B-tree. The expected maximum size of the B-tree has drastically +reduced, so these reads should almost never actually hit disk. The number of +writes to disk will stay the same, so performance should stay unchanged. + +### Disk Queues + +This work should have a minimal impact on recovery times, which is why recovery +hasn't been significantly mentioned in this document. However, there are two +minor impacts on recovery times: + +1. Larger disk queue file means more file to zero out in the case of recovery. + + This should be negligable when fallocate `ZERO_RANGE` is available, because then it's only a metadata operation. + +2. A larger file means more bisection iterations to find the first page. + + If we say Disk Queue files are typically ~4GB now, and people are unlikely + to have more than 4TB drives, then this means in the worst case, another 8 + sequential IOs will need to be done when first recovering a disk queue file + to find the most recent page with a binary search. + + If this turns out to be an issue, it's trivial to address. There's no + reason to do only a binary search when drives support parallel requests. A + 32-way search could reasonably be done, and would would make a 4TB Disk + Queue file faster to recover than a 4GB one currently. + +3. Disk queue files can now shrink. + + The particular logic currently used is that: + + If one file is significantly larger than the other file, then it will be + truncated to the size of the other file. This resolves situations where a + particular storage server or remote DC being down causes one DiskQueue file + to be grown to a massive size, and then the data is rapidly popped. + + Otherwise, If the files are of reasonably similar size, then we'll take + `pushLocation - popLocation` as the number of "active" bytes, and then + shrink the file by `TLOG_DISK_QUEUE_SHRINK_BYTES` bytes if the file is + larger than `active + TLOG_DISK_QUEUE_EXTENSION_BYTES + TLOG_DISK_QUEUE_SHRINK_BYTES`. + +## Knobs + +`REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT` +: How many bytes of mutations should be spilled at once in a spill-by-reference TLog.
+ Increasing it could increase throughput in spilling regimes.
+ Decreasing it will decrease how sawtooth-like TLog memory usage is.
+ +`UPDATE_STORAGE_BYTE_LIMIT` +: How many bytes of mutations should be spilled at once in a spill-by-value TLog.
+ This knob is pre-existing, and has only been "changed" to only apply to spill-by-value.
+ +`TLOG_SPILL_REFERENCE_MAX_BATCHES_PER_PEEK` +: How many batches of spilled data index batches should be read from disk to serve one peek request.
+ Increasing it will potentially increase the throughput of peek requests.
+ Decreasing it will decrease the number of read IOs done per peek request.
+ +`TLOG_SPILL_REFERENCE_MAX_BYTES_PER_BATCH` +: How many bytes a batch of spilled data indexes can be.
+ Increasing it will increase TLog throughput while spilling.
+ Decreasing it will decrease the latency and increase the throughput of peek requests.
+ +`TLOG_SPILL_REFERENCE_MAX_PEEK_MEMORY_BYTES` +: How many bytes of memory can be allocated to hold the results of reads from disk to respond to peek requests.
+ Increasing it will increase the number of parallel peek requests a TLog can handle at once.
+ Decreasing it will reduce TLog memory usage.
+ If increased, `--max_memory` should be increased by the same amount.
+ +`TLOG_DISK_QUEUE_EXTENSION_BYTES` +: When a DiskQueue needs to extend a file, by how many bytes should it extend the file.
+ Increasing it will reduce metadata operations done to the drive, and likely tail commit latency.
+ Decreasing it will reduce allocated but unused space in the DiskQueue files.
+ Note that this was previously hardcoded to 20MB, and is only being promoted to a knob.
+ +`TLOG_DISK_QUEUE_SHRINK_BYTES` +: If a DiskQueue file has extra space left when switching to the other file, by how many bytes should it be shrunk.
+ Increasing this will cause disk space to be returned to the OS faster.
+ Decreasing this will decrease TLog tail latency due to filesystem metadata updates.
+ +## Observability + +With the new changes, we must ensure that sufficent information has been exposed such that: + +1. If something goes wrong in production, we can understand what and why from trace logs. +2. We can understand if the TLog is performing suboptimally, and if so, which knob we should change and by how much. + +The following metrics were added to `TLogMetrics`: + +### Spilling + +### Peeking + +`PeekMemoryRequestsStalled` +: The number of peek requests that are blocked on acquiring memory for reads. + +`PeekMemoryReserved` +: The amount of memory currently reserved for serving peek requests. + +### Popping + +`QueuePoppedVersion` +: The oldest version that's still useful. + +`MinPoppedTagLocality` +: The locality of the tag that's preventing the DiskQueue from being further popped. + +`MinPoppedTagId` +: The id of the tag that's preventing the DiskQueue from being further popped. + +## Monitoring and Alerting + +To answer questions like: + +1. What new graphs should exist? +2. What old graphs might exist that would no longer be meaningful? +3. What alerts might exist that need to be changed? +4. What alerts should be created? + +Of which I'm aware of: + +* Any current alerts on "Disk Queue files more than [constant size] GB" will need to be removed. +* Any alerting or monitoring of `log*.sqlite` as an indication of spilling will no longer be effective. + +* A graph of `BytesInput - BytesPopped` will give an idea of the number of "active" bytes in the DiskQueue file. + + + + + + + diff --git a/documentation/CMakeLists.txt b/documentation/CMakeLists.txt index 83fabf20ba..ccd60a2bbd 100644 --- a/documentation/CMakeLists.txt +++ b/documentation/CMakeLists.txt @@ -10,7 +10,7 @@ set(pip_command ${venv_dir}/bin/pip${EXE_SUFFIX}) set(python_command ${venv_dir}/bin/python${EXE_SUFFIX}) add_custom_command(OUTPUT ${venv_dir}/venv_setup - COMMAND ${VIRTUALENV_EXE} venv && + COMMAND ${Python3_EXECUTABLE} -m venv venv && ${CMAKE_COMMAND} -E copy ${sphinx_dir}/.pip.conf ${venv_dir}/pip.conf && . ${venv_dir}/bin/activate && ${pip_command} install --upgrade pip && @@ -59,6 +59,38 @@ endfunction() message(STATUS "Add html target") add_documentation_target(GENERATOR html) +set(DOCSERVER_PORT "-1" CACHE STRING "Port to which the documentation server should bind (negative means cmake will choose one)") + +if(DOCSERVER_PORT GREATER_EQUAL 0) + set(port ${DOCSERVER_PORT}) +else() + if(WIN32) + if(DEFINED $ENV{USERNAME}) + set(username $ENV{USERNAME}) + else() + set(username "dummy_user") + endif() + else() + if(DEFINED $ENV{USER}) + set(username $ENV{USER}) + else() + set(username "dummy_user") + endif() + endif() + string(MD5 username_hash ${username}) + # cmake math function can only use 64 bit signed integers - so we just truncate the string + string(SUBSTRING "${username_hash}" 0 15 username_hash_small) + message(STATUS math(EXPR port "(0x${username_hash_small} % 8000) + 8000" OUTPUT_FORMAT DECIMAL)) + math(EXPR port "(0x${username_hash_small} % 8000) + 8000" OUTPUT_FORMAT DECIMAL) + message(STATUS "Port is ${port}") +endif() + +add_custom_target(docpreview + COMMAND ${python_command} -m http.server ${port} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html + USES_TERMINAL) +add_dependencies(docpreview html) + set(tar_file ${CMAKE_BINARY_DIR}/packages/${CMAKE_PROJECT_NAME}-docs-${FDB_VERSION}.tar.gz) add_custom_command( OUTPUT ${tar_file} diff --git a/documentation/Makefile b/documentation/Makefile deleted file mode 100644 index 256b2b7bd5..0000000000 --- a/documentation/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -include ../build/scver.mk - -docprereqs: - $(MAKE) -C sphinx html - cp -r ../bindings/java/javadoc sphinx/.out/html - rm -f sphinx/.out/html/documentation - ln -s . sphinx/.out/html/documentation - -docpreview: docprereqs - @( cd sphinx/.out/html && ! grep FIXME * && python -m SimpleHTTPServer $$((0x$$(echo ${USER} | $(MD5SUM) | awk '{print $$1}' | cut -c1-8)%8000+8000)) ) - -docpreview_clean: - $(MAKE) -C sphinx clean - -docpackage: docprereqs - $(MAKE) -C sphinx package diff --git a/documentation/sphinx/Makefile b/documentation/sphinx/Makefile deleted file mode 100644 index ea672b8aa4..0000000000 --- a/documentation/sphinx/Makefile +++ /dev/null @@ -1,107 +0,0 @@ -# Makefile for Sphinx documentation -# -# local vars: -PROJECT_NAME := foundationdb-docs - -GIT_HEAD_REF := $(shell git rev-parse --short HEAD) -GIT_BRANCH := $(shell git symbolic-ref --short HEAD) -GIT_REPO_URL := $(shell git config --get remote.origin.url) - -# You can set these variables from the command line. -SPHINXOPTS := -c . -PAPER = -ROOTDIR := $(CURDIR) -BUILDDIR := $(ROOTDIR)/.out -DISTDIR := $(ROOTDIR)/.dist -VENVDIR := $(ROOTDIR)/.out/venv -SPHINXBUILD = $(VENVDIR)/bin/sphinx-build -SPHINXAUTOBUILD = $(VENVDIR)/bin/sphinx-autobuild -TEMPLATEDIR = $(ROOTDIR)/_templates - -# virtualenv for sphinx-build -VENV_VERSION ?= virtualenv-13.0.1 -VENV_URL_BASE ?= https://pypi.python.org -VENV_URL ?= $(VENV_URL_BASE)/packages/source/v/virtualenv/$(VENV_VERSION).tar.gz - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source - -BUILDINFO = "
Ref:%h
Updated:%cd
Committer:%cn

View on GitHub
" - -.PHONY: default help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext buildsphinx publish uptodate - -default: html - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " livehtml to launch a local webserver that auto-updates as changes are made" - @echo " publish to build the html and push it to GitHub pages" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " buildsphinx to install sphinx binary in virtualenv" - -buildsphinx: - if [ ! -e $(SPHINXBUILD) ]; then \ - mkdir $(BUILDDIR); \ - cd $(BUILDDIR); \ - curl -OL $(VENV_URL); \ - tar zxvf $(VENV_VERSION).tar.gz; \ - python2 ./$(VENV_VERSION)/virtualenv.py venv; \ - fi - . $(VENVDIR)/bin/activate && \ - cp .pip.conf $(VENVDIR)/pip.conf && \ - pip install --upgrade pip && \ - pip install --upgrade -r $(ROOTDIR)/requirements.txt; - -clean: - rm -rf $(BUILDDIR) - -cleanhtml: - rm -rf $(BUILDDIR)/html - -cleanvirtualenv: - rm -rf $(VENVDIR) - -html: buildsphinx cleanhtml - $(SPHINXBUILD) -W -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -check: checkwarnings linkcheck - -checkwarnings: buildsphinx - $(SPHINXBUILD) -n -W -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo "Check finished." - -livehtml: html - $(SPHINXAUTOBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - -# removed html prerequisite because it is previously explictly invoked -package: - mkdir -p $(DISTDIR) - rm -f $(DISTDIR)/$(PROJECT_NAME)-$(VERSION).tar.gz - cd $(BUILDDIR)/html && tar czf $(DISTDIR)/$(PROJECT_NAME)-$(VERSION).tar.gz . diff --git a/documentation/sphinx/extensions/rubydomain.py b/documentation/sphinx/extensions/rubydomain.py index 540f8487d3..1e5fb0bce4 100755 --- a/documentation/sphinx/extensions/rubydomain.py +++ b/documentation/sphinx/extensions/rubydomain.py @@ -502,7 +502,7 @@ class RubyModuleIndex(Index): ignores = self.domain.env.config['modindex_common_prefix'] ignores = sorted(ignores, key=len, reverse=True) # list of all modules, sorted by module name - modules = sorted(self.domain.data['modules'].iteritems(), + modules = sorted(iter(self.domain.data['modules'].items()), key=lambda x: x[0].lower()) # sort out collapsable modules prev_modname = '' @@ -551,7 +551,7 @@ class RubyModuleIndex(Index): collapse = len(modules) - num_toplevels < num_toplevels # sort by first letter - content = sorted(content.iteritems()) + content = sorted(content.items()) return content, collapse @@ -609,10 +609,10 @@ class RubyDomain(Domain): ] def clear_doc(self, docname): - for fullname, (fn, _) in self.data['objects'].items(): + for fullname, (fn, _) in list(self.data['objects'].items()): if fn == docname: del self.data['objects'][fullname] - for modname, (fn, _, _, _) in self.data['modules'].items(): + for modname, (fn, _, _, _) in list(self.data['modules'].items()): if fn == docname: del self.data['modules'][modname] @@ -704,9 +704,9 @@ class RubyDomain(Domain): contnode, name) def get_objects(self): - for modname, info in self.data['modules'].iteritems(): + for modname, info in self.data['modules'].items(): yield (modname, modname, 'module', info[0], 'module-' + modname, 0) - for refname, (docname, type) in self.data['objects'].iteritems(): + for refname, (docname, type) in self.data['objects'].items(): yield (refname, refname, type, docname, refname, 1) diff --git a/documentation/sphinx/requirements.txt b/documentation/sphinx/requirements.txt index 0cfa0c9f71..46b6da8b01 100644 --- a/documentation/sphinx/requirements.txt +++ b/documentation/sphinx/requirements.txt @@ -1,4 +1,5 @@ --index-url https://pypi.python.org/simple +setuptools>=20.10.0 sphinx==1.5.6 sphinx-bootstrap-theme==0.4.8 -pygments-style-solarized \ No newline at end of file +pygments-style-solarized diff --git a/documentation/sphinx/source/administration.rst b/documentation/sphinx/source/administration.rst index 14999a4e74..4041ce5eda 100644 --- a/documentation/sphinx/source/administration.rst +++ b/documentation/sphinx/source/administration.rst @@ -177,7 +177,7 @@ You can add new machines to a cluster at any time: 5) If you have previously :ref:`excluded ` a machine from the cluster, you will need to take it off the exclusion list using the ``include `` command of fdbcli before it can be a full participant in the cluster. - .. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled. +.. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled. .. _removing-machines-from-a-cluster: @@ -192,26 +192,26 @@ To temporarily or permanently remove one or more machines from a FoundationDB cl 3) Use the ``exclude`` command in ``fdbcli`` on the machines you plan to remove: - :: +:: - user@host1$ fdbcli - Using cluster file `/etc/foundationdb/fdb.cluster'. + user@host1$ fdbcli + Using cluster file `/etc/foundationdb/fdb.cluster'. - The database is available. + The database is available. - Welcome to the fdbcli. For help, type `help'. - fdb> exclude 1.2.3.4 1.2.3.5 1.2.3.6 - Waiting for state to be removed from all excluded servers. This may take a while. - It is now safe to remove these machines or processes from the cluster. + Welcome to the fdbcli. For help, type `help'. + fdb> exclude 1.2.3.4 1.2.3.5 1.2.3.6 + Waiting for state to be removed from all excluded servers. This may take a while. + It is now safe to remove these machines or processes from the cluster. - - ``exclude`` can be used to exclude either machines (by specifying an IP address) or individual processes (by specifying an ``IP``:``PORT`` pair). - .. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled. - - Excluding a server doesn't shut it down immediately; data on the machine is first moved away. When the ``exclude`` command completes successfully (by returning control to the command prompt), the machines that you specified are no longer required to maintain the configured redundancy mode. A large amount of data might need to be transferred first, so be patient. When the process is complete, the excluded machine or process can be shut down without fault tolerance or availability consequences. - - If you interrupt the exclude command with Ctrl-C after seeing the "waiting for state to be removed" message, the exclusion work will continue in the background. Repeating the command will continue waiting for the exclusion to complete. To reverse the effect of the ``exclude`` command, use the ``include`` command. +``exclude`` can be used to exclude either machines (by specifying an IP address) or individual processes (by specifying an ``IP``:``PORT`` pair). + +.. note:: Addresses have the form ``IP``:``PORT``. This form is used even if TLS is enabled. + +Excluding a server doesn't shut it down immediately; data on the machine is first moved away. When the ``exclude`` command completes successfully (by returning control to the command prompt), the machines that you specified are no longer required to maintain the configured redundancy mode. A large amount of data might need to be transferred first, so be patient. When the process is complete, the excluded machine or process can be shut down without fault tolerance or availability consequences. + +If you interrupt the exclude command with Ctrl-C after seeing the "waiting for state to be removed" message, the exclusion work will continue in the background. Repeating the command will continue waiting for the exclusion to complete. To reverse the effect of the ``exclude`` command, use the ``include`` command. Excluding a server with the ``failed`` flag will shut it down immediately; it will assume that it has already become unrecoverable or unreachable, and will not attempt to move the data on the machine away. This may break the guarantee required to maintain the configured redundancy mode, which will be checked internally, and the command may be denied if the guarantee is violated. This safety check can be ignored by using the command ``exclude FORCE failed``. @@ -320,9 +320,9 @@ Running backups Number of backups currently running. Different backups c Running DRs Number of DRs currently running. Different DRs could be streaming different prefixes and/or to different DR clusters. ====================== ========================================================================================================== -The "Memory availability" is a conservative estimate of the minimal RAM available to any ``fdbserver`` process across all machines in the cluster. This value is calculated in two steps. Memory available per process is first calculated *for each machine* by taking: +The "Memory availability" is a conservative estimate of the minimal RAM available to any ``fdbserver`` process across all machines in the cluster. This value is calculated in two steps. Memory available per process is first calculated *for each machine* by taking:: - availability = ((total - committed) + sum(processSize)) / processes + availability = ((total - committed) + sum(processSize)) / processes where: @@ -492,6 +492,19 @@ If a process has had more than 10 TCP segments retransmitted in the last 5 secon 10.0.4.1:4500 ( 3% cpu; 2% machine; 0.004 Gbps; 0% disk; REXMIT! 2.5 GB / 4.1 GB RAM ) +Machine-readable status +-------------------------------- + +The status command can provide a complete summary of statistics about the cluster and the database with the ``json`` argument. Full documentation for ``status json`` output can be found :doc:`here `. +From the output of ``status json``, operators can find useful health metrics to determine whether or not their cluster is hitting performance limits. + +====================== ============================================================================================================== +Ratekeeper limit ``cluster.qos.transactions_per_second_limit`` contains the number of read versions per second that the cluster can give out. A low ratekeeper limit indicates that the cluster performance is limited in some way. The reason for a low ratekeeper limit can be found at ``cluster.qos.performance_limited_by``. ``cluster.qos.released_transactions_per_second`` describes the number of read versions given out per second, and can be used to tell how close the ratekeeper is to throttling. +Storage queue size ``cluster.qos.worst_queue_bytes_storage_server`` contains the maximum size in bytes of a storage queue. Each storage server has mutations that have not yet been made durable, stored in its storage queue. If this value gets too large, it indicates a storage server is falling behind. A large storage queue will cause the ratekeeper to increase throttling. However, depending on the configuration, the ratekeeper can ignore the worst storage queue from one fault domain. Thus, ratekeeper uses ``cluster.qos.limiting_queue_bytes_storage_server`` to determine the throttling level. +Durable version lag ``cluster.qos.worst_durability_lag_storage_server`` contains information about the worst storage server durability lag. The ``versions`` subfield contains the maximum number of versions in a storage queue. Ideally, this should be near 5 million. The ``seconds`` subfield contains the maximum number of seconds of non-durable data in a storage queue. Ideally, this should be near 5 seconds. If a storage server is overwhelmed, the durability lag could rise, causing performance issues. +Transaction log queue ``cluster.qos.worst_queue_bytes_log_server`` contains the maximum size in bytes of the mutations stored on a transaction log that have not yet been popped by storage servers. A large transaction log queue size can potentially cause the ratekeeper to increase throttling. +====================== ============================================================================================================== + .. _administration_fdbmonitor: ``fdbmonitor`` and ``fdbserver`` @@ -621,6 +634,8 @@ To upgrade a FoundationDB cluster, you must install the updated version of Found .. warning:: |development-use-only-warning| +.. note:: For information about upgrading client application code to newer API versions, see the :doc:`api-version-upgrade-guide`. + Install updated client binaries ------------------------------- @@ -648,7 +663,7 @@ For **RHEL/CentOS**, perform the upgrade using the rpm command: user@host$ sudo rpm -Uvh |package-rpm-clients| \\ |package-rpm-server| -The ``foundationdb-clients`` package also installs the :doc:`Python ` and :doc:`C ` APIs. If your clients use :doc:`Ruby `, `Java `_, or `Go `_, follow the instructions in the corresponding language documentation to install the APIs. +The ``foundationdb-clients`` package also installs the :doc:`C ` API. If your clients use :doc:`Ruby `, :doc:`Python `, `Java `_, or `Go `_, follow the instructions in the corresponding language documentation to install the APIs. Test the database ----------------- @@ -678,12 +693,18 @@ Upgrades from 6.1.x will keep all your old data and configuration settings. Data Upgrading from 6.0.x -------------------- -Upgrades from 6.0.x will keep all your old data and configuration settings. Data distribution will slowly reorganize how data is spread across storage servers. +Upgrades from 6.0.x will keep all your old data and configuration settings. Upgrading from 5.2.x -------------------- -Upgrades from 5.2.x will keep all your old data and configuration settings. +Upgrades from 5.2.x will keep all your old data and configuration settings. Some affinities that certain roles have for running on processes that haven't set a process class have changed, which may result in these processes running in different locations after upgrading. To avoid this, set process classes as needed. The following changes were made: + +* The proxies and master no longer prefer ``resolution`` or ``transaction`` class processes to processes with unset class. +* The resolver no longer prefers ``transaction`` class processes to processes with unset class. +* The cluster controller no longer prefers ``master``, ``resolution`` or ``proxy`` class processes to processes with unset class. + +See :ref:`guidelines-process-class-config` for recommendations on setting process classes. All of the above roles will prefer ``stateless`` class processes to ones that don't set a class. Upgrading from 5.0.x - 5.1.x ---------------------------- diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index cb5b66755e..27924cf133 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -51,8 +51,6 @@ .. |timeout-database-option| replace:: FIXME .. |causal-read-risky-transaction-option| replace:: FIXME .. |causal-read-risky-database-option| replace:: FIXME -.. |include-port-in-address-database-option| replace:: FIXME -.. |include-port-in-address-transaction-option| replace:: FIXME .. |transaction-logging-max-field-length-database-option| replace:: FIXME .. |transaction-logging-max-field-length-transaction-option| replace:: FIXME @@ -135,7 +133,7 @@ API versioning Prior to including ``fdb_c.h``, you must define the ``FDB_API_VERSION`` macro. This, together with the :func:`fdb_select_api_version()` function, allows programs written against an older version of the API to compile and run with newer versions of the C library. The current version of the FoundationDB C API is |api-version|. :: - #define FDB_API_VERSION 620 + #define FDB_API_VERSION 630 #include .. function:: fdb_error_t fdb_select_api_version(int version) @@ -476,6 +474,11 @@ Applications must provide error handling and an appropriate retry loop around th ``snapshot`` |snapshot| +.. function:: FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length) + Returns an estimated byte size of the key range. + + |future-return0| the estimated size of the key range given. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2| + .. function:: FDBFuture* fdb_transaction_get_key(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t or_equal, int offset, fdb_bool_t snapshot) Resolves a :ref:`key selector ` against the keys in the database snapshot represented by ``transaction``. @@ -530,8 +533,7 @@ Applications must provide error handling and an appropriate retry loop around th |snapshot| ``reverse`` - - If non-zero, key-value pairs will be returned in reverse lexicographical order beginning at the end of the range. + If non-zero, key-value pairs will be returned in reverse lexicographical order beginning at the end of the range. Reading ranges in reverse is supported natively by the database and should have minimal extra cost. .. type:: FDBStreamingMode @@ -539,31 +541,31 @@ Applications must provide error handling and an appropriate retry loop around th ``FDB_STREAMING_MODE_ITERATOR`` - The caller is implementing an iterator (most likely in a binding to a higher level language). The amount of data returned depends on the value of the ``iteration`` parameter to :func:`fdb_transaction_get_range()`. + The caller is implementing an iterator (most likely in a binding to a higher level language). The amount of data returned depends on the value of the ``iteration`` parameter to :func:`fdb_transaction_get_range()`. ``FDB_STREAMING_MODE_SMALL`` - Data is returned in small batches (not much more expensive than reading individual key-value pairs). + Data is returned in small batches (not much more expensive than reading individual key-value pairs). ``FDB_STREAMING_MODE_MEDIUM`` - Data is returned in batches between _SMALL and _LARGE. + Data is returned in batches between _SMALL and _LARGE. ``FDB_STREAMING_MODE_LARGE`` - Data is returned in batches large enough to be, in a high-concurrency environment, nearly as efficient as possible. If the caller does not need the entire range, some disk and network bandwidth may be wasted. The batch size may be still be too small to allow a single client to get high throughput from the database. + Data is returned in batches large enough to be, in a high-concurrency environment, nearly as efficient as possible. If the caller does not need the entire range, some disk and network bandwidth may be wasted. The batch size may be still be too small to allow a single client to get high throughput from the database. ``FDB_STREAMING_MODE_SERIAL`` - Data is returned in batches large enough that an individual client can get reasonable read bandwidth from the database. If the caller does not need the entire range, considerable disk and network bandwidth may be wasted. + Data is returned in batches large enough that an individual client can get reasonable read bandwidth from the database. If the caller does not need the entire range, considerable disk and network bandwidth may be wasted. ``FDB_STREAMING_MODE_WANT_ALL`` - The caller intends to consume the entire range and would like it all transferred as early as possible. + The caller intends to consume the entire range and would like it all transferred as early as possible. ``FDB_STREAMING_MODE_EXACT`` - The caller has passed a specific row limit and wants that many rows delivered in a single batch. + The caller has passed a specific row limit and wants that many rows delivered in a single batch. .. function:: void fdb_transaction_set(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, uint8_t const* value, int value_length) diff --git a/documentation/sphinx/source/api-common.rst.inc b/documentation/sphinx/source/api-common.rst.inc index aced8ae7c7..39e09a83d0 100644 --- a/documentation/sphinx/source/api-common.rst.inc +++ b/documentation/sphinx/source/api-common.rst.inc @@ -142,7 +142,7 @@ A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database. The last 2 bytes are monotonic in the serialization order for transactions. .. |atomic-versionstamps-2| replace:: - A transaction is not permitted to read any transformed key or value previously set within that transaction, and an attempt to do so will result in an error. + A transaction is not permitted to read any transformed key or value previously set within that transaction, and an attempt to do so will result in an ``accessed_unreadable`` error. The range of keys marked unreadable when setting a versionstamped key begins at the transactions's read version if it is known, otherwise a versionstamp of all ``0x00`` bytes is conservatively assumed. The upper bound of the unreadable range is a versionstamp of all ``0xFF`` bytes. .. |atomic-versionstamps-tuple-warning-key| replace:: At this time, versionstamped keys are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped keys may not be used with the Subspace and Directory layers except in those languages. @@ -150,7 +150,7 @@ .. |atomic-versionstamps-tuple-warning-value| replace:: At this time, versionstamped values are not compatible with the Tuple layer except in Java, Python, and Go. Note that this implies versionstamped values may not be used with the Subspace and Directory layers except in those languages. -.. |api-version| replace:: 620 +.. |api-version| replace:: 630 .. |streaming-mode-blurb1| replace:: When using |get-range-func| and similar interfaces, API clients can request large ranges of the database to iterate over. Making such a request doesn't necessarily mean that the client will consume all of the data in the range - sometimes the client doesn't know how far it intends to iterate in advance. FoundationDB tries to balance latency and bandwidth by requesting data for iteration in batches. @@ -176,6 +176,9 @@ .. |transaction-get-committed-version-blurb| replace:: Gets the version number at which a successful commit modified the database. This must be called only after the successful (non-error) completion of a call to |commit-func| on this Transaction, or the behavior is undefined. Read-only transactions do not modify the database when committed and will have a committed version of -1. Keep in mind that a transaction which reads keys and then sets them to their current values may be optimized to a read-only transaction. +.. |transaction-get-approximate-size-blurb| replace:: + Gets the the approximate transaction size so far, which is the summation of the estimated size of mutations, read conflict ranges, and write conflict ranges. + .. |transaction-get-versionstamp-blurb| replace:: Returns a future which will contain the versionstamp which was used by any versionstamp operations in this transaction. This function must be called before a call to |commit-func| on this Transaction. The future will be ready only after the successful completion of a call to |commit-func| on this Transaction. Read-only transactions do not modify the database when committed and will result in the future completing with an error. Keep in mind that a transaction which reads keys and then sets them to their current values may be optimized to a read-only transaction. @@ -242,6 +245,9 @@ .. |option-trace-format-blurb| replace:: Select the format of the trace files for this FoundationDB client. xml (the default) and json are supported. +.. |option-trace-clock-source-blurb| replace:: + Select clock source for trace files. now (the default) or realtime are supported. + .. |network-options-warning| replace:: It is an error to set these options after the first call to |open-func| anywhere in your application. @@ -329,10 +335,6 @@ Transactions do not require the strict causal consistency guarantee that FoundationDB provides by default. The read version will be committed, and usually will be the latest committed, but might not be the latest committed in the event of a simultaneous fault and misbehaving clock. Enabling this option is equivalent to calling |causal-read-risky-transaction-option| on each transaction created by this database. -.. |option-db-include-port-in-address-blurb| replace:: - - Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated. Enabling this option is equivalent to calling |include-port-in-address-transaction-option| on each transaction created by this database. - .. |option-db-snapshot-ryw-enable-blurb| replace:: If this option has been set an equal or more times with this database than the disable option, snapshot reads *will* see the effects of prior writes in the same transaction. Enabling this option is equivalent to calling |snapshot-ryw-enable-transaction-option| on each transaction created by this database. @@ -372,10 +374,6 @@ This transaction does not require the strict causal consistency guarantee that FoundationDB provides by default. The read version will be committed, and usually will be the latest committed, but might not be the latest committed in the event of a simultaneous fault and misbehaving clock. One can set this for all transactions by calling |causal-read-risky-database-option|. -.. |option-include-port-in-address-blurb| replace:: - - Addresses returned by get_addresses_for_key include the port when enabled. This will be enabled by default in api version 700, and this option will be deprecated. One can set this for all transactions by calling |include-port-in-address-database-option|. - .. |option-causal-write-risky-blurb| replace:: The application either knows that this transaction will be self-conflicting (at least one read overlaps at least one set or clear), or is willing to accept a small risk that the transaction could be committed a second time after its commit apparently succeeds. This option provides a small performance benefit. diff --git a/documentation/sphinx/source/api-general.rst b/documentation/sphinx/source/api-general.rst index 8adc89b3be..81f981dc8b 100644 --- a/documentation/sphinx/source/api-general.rst +++ b/documentation/sphinx/source/api-general.rst @@ -11,6 +11,8 @@ Versioning FoundationDB supports a robust versioning system for both its API and binaries. This system allows clusters to be upgraded with minimal changes to both application code and FoundationDB binaries. The API and the FoundationDB binaries are each released in numbered versions. Each version of the binaries has a corresponding API version. +.. _api-versions: + API versions ------------ @@ -63,7 +65,7 @@ To install on **RHEL/CentOS** use the rpm command: To install on **macOS**, run the installer as in :doc:`getting-started-mac`, but deselect the "FoundationDB Server" feature. -The client binaries include the ``fdbcli`` tool and language bindings for C and Python. Other language bindings must be installed separately. +The client binaries include the ``fdbcli`` tool and language bindings for C. Other language bindings must be installed separately. Clients will also need a :ref:`cluster file ` to connect to a FoundationDB cluster. You should copy the ``fdb.cluster`` file from the :ref:`default location ` on one of your FoundationDB servers to the default location on the client machine. diff --git a/documentation/sphinx/source/api-python.rst b/documentation/sphinx/source/api-python.rst index c4273721e0..086eeb0bf0 100644 --- a/documentation/sphinx/source/api-python.rst +++ b/documentation/sphinx/source/api-python.rst @@ -26,7 +26,6 @@ .. |max-retry-delay-database-option| replace:: :func:`Database.options.set_transaction_max_retry_delay` .. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit` .. |causal-read-risky-database-option| replace:: :func:`Database.options.set_transaction_causal_read_risky` -.. |include-port-in-address-database-option| replace:: :func:`Database.options.set_transaction_include_port_in_address` .. |transaction-logging-max-field-length-database-option| replace:: :func:`Database.options.set_transaction_logging_max_field_length` .. |snapshot-ryw-enable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-database-option| replace:: :func:`Database.options.set_snapshot_ryw_disable` @@ -39,7 +38,6 @@ .. |snapshot-ryw-enable-transaction-option| replace:: :func:`Transaction.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-transaction-option| replace:: :func:`Transaction.options.set_snapshot_ryw_disable` .. |causal-read-risky-transaction-option| replace:: :func:`Transaction.options.set_causal_read_risky` -.. |include-port-in-address-transaction-option| replace:: :func:`Transaction.options.set_include_port_in_address` .. |transaction-logging-max-field-length-transaction-option| replace:: :func:`Transaction.options.set_transaction_logging_max_field_length` .. |lazy-iterator-object| replace:: generator .. |key-meth| replace:: :meth:`Subspace.key` @@ -73,9 +71,13 @@ Installation The FoundationDB Python API is compatible with Python 2.7 - 3.7. You will need to have a Python version within this range on your system before the FoundationDB Python API can be installed. Also please note that Python 3.7 no longer bundles a full copy of libffi, which is used for building the _ctypes module on non-macOS UNIX platforms. Hence, if you are using Python 3.7, you should make sure libffi is already installed on your system. -On macOS, the FoundationDB Python API is installed as part of the FoundationDB installation (see :ref:`installing-client-binaries`). On Ubuntu or RHEL/CentOS, you will need to install the FoundationDB Python API manually. +On macOS, the FoundationDB Python API is installed as part of the FoundationDB installation (see :ref:`installing-client-binaries`). On Ubuntu or RHEL/CentOS, you will need to install the FoundationDB Python API manually via Python's package manager ``pip``: -You can download the FoundationDB Python API source directly from :doc:`downloads`. +.. code-block:: none + + user@host$ pip install foundationdb + +You can also download the FoundationDB Python API source directly from :doc:`downloads`. .. note:: The Python language binding is compatible with FoundationDB client binaries of version 2.0 or higher. When used with version 2.0.x client binaries, the API version must be set to 200 or lower. @@ -98,7 +100,7 @@ When you import the ``fdb`` module, it exposes only one useful symbol: .. warning:: |api-version-multi-version-warning| -For API changes between version 13 and |api-version| (for the purpose of porting older programs), see :doc:`release-notes`. +For API changes between version 13 and |api-version| (for the purpose of porting older programs), see :doc:`release-notes` and :doc:`api-version-upgrade-guide`. Opening a database ================== @@ -106,7 +108,7 @@ Opening a database After importing the ``fdb`` module and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: import fdb - fdb.api_version(620) + fdb.api_version(630) db = fdb.open() .. function:: open( cluster_file=None, event_model=None ) @@ -141,6 +143,10 @@ After importing the ``fdb`` module and selecting an API version, you probably wa |option-trace-format-blurb| + .. method :: fdb.options.set_trace_clock_source(source) + + |option-trace-clock-source-blurb| + .. method :: fdb.options.set_disable_multi_version_client_api() |option-disable-multi-version-client-api| @@ -287,7 +293,7 @@ A |database-blurb1| |database-blurb2| If ``limit`` is specified, then only the first ``limit`` keys (and their values) in the range will be returned. - If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order. + If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost. If ``streaming_mode`` is specified, it must be a value from the :data:`StreamingMode` enumeration. It provides a hint to FoundationDB about how to retrieve the specified range. This option should generally not be specified, allowing FoundationDB to retrieve the full range very efficiently. @@ -396,10 +402,6 @@ Database options |option-db-causal-read-risky-blurb| -.. method:: Database.options.set_transaction_include_port_in_address() - - |option-db-include-port-in-address-blurb| - .. method:: Database.options.set_transaction_logging_max_field_length(size_limit) |option-db-tr-transaction-logging-max-field-length-blurb| @@ -503,7 +505,7 @@ Reading data If ``limit`` is specified, then only the first ``limit`` keys (and their values) in the range will be returned. - If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order. + If ``reverse`` is True, then the last ``limit`` keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost. If ``streaming_mode`` is specified, it must be a value from the :data:`StreamingMode` enumeration. It provides a hint to FoundationDB about how the returned container is likely to be used. The default is :data:`StreamingMode.iterator`. @@ -794,8 +796,22 @@ Most applications should use the read version that FoundationDB determines autom |infrequent| |transaction-get-versionstamp-blurb| +Transaction misc functions +-------------------------- + +.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) + + Get the estimated byte size of the given key range. Returns a :class:`FutureInt64`. + .. _api-python-transaction-options: +Transaction misc functions +-------------------------- + +.. method:: Transaction.get_approximate_size() + + |transaction-get-approximate-size-blurb|. Returns a :class:`FutureInt64`. + Transaction options ------------------- @@ -825,10 +841,6 @@ Transaction options |option-causal-read-risky-blurb| -.. method:: Transaction.options.set_include_port_in_address - - |option-include-port-in-address-blurb| - .. method:: Transaction.options.set_causal_write_risky |option-causal-write-risky-blurb| @@ -964,9 +976,9 @@ Asynchronous methods return one of the following subclasses of :class:`Future`: Represents a future string object and responds to the same methods as string in Python. They may be passed to FoundationDB methods that expect a string. -.. class:: FutureVersion +.. class:: FutureInt64 - Represents a future version (integer). You must call the :meth:`Future.wait()` method on this object to retrieve the version as an integer. + Represents a future integer. You must call the :meth:`Future.wait()` method on this object to retrieve the integer. .. class:: FutureStringArray diff --git a/documentation/sphinx/source/api-ruby.rst b/documentation/sphinx/source/api-ruby.rst index 3c075b6d0a..77ae67ceac 100644 --- a/documentation/sphinx/source/api-ruby.rst +++ b/documentation/sphinx/source/api-ruby.rst @@ -24,7 +24,6 @@ .. |max-retry-delay-database-option| replace:: :meth:`Database.options.set_transaction_max_retry_delay` .. |transaction-size-limit-database-option| replace:: :func:`Database.options.set_transaction_size_limit` .. |causal-read-risky-database-option| replace:: :meth:`Database.options.set_transaction_causal_read_risky` -.. |include-port-in-address-database-option| replace:: :meth:`Database.options.set_transaction_include_port_in_address` .. |snapshot-ryw-enable-database-option| replace:: :meth:`Database.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-database-option| replace:: :meth:`Database.options.set_snapshot_ryw_disable` .. |transaction-logging-max-field-length-database-option| replace:: :meth:`Database.options.set_transaction_logging_max_field_length` @@ -37,7 +36,6 @@ .. |snapshot-ryw-enable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_enable` .. |snapshot-ryw-disable-transaction-option| replace:: :meth:`Transaction.options.set_snapshot_ryw_disable` .. |causal-read-risky-transaction-option| replace:: :meth:`Transaction.options.set_causal_read_risky` -.. |include-port-in-address-transaction-option| replace:: :meth:`Transaction.options.set_include_port_in_address` .. |transaction-logging-max-field-length-transaction-option| replace:: :meth:`Transaction.options.set_transaction_logging_max_field_length` .. |lazy-iterator-object| replace:: :class:`Enumerator` .. |key-meth| replace:: :meth:`Subspace.key` @@ -87,7 +85,7 @@ When you require the ``FDB`` gem, it exposes only one useful method: .. warning:: |api-version-multi-version-warning| -For API changes between version 14 and |api-version| (for the purpose of porting older programs), see :doc:`release-notes`. +For API changes between version 14 and |api-version| (for the purpose of porting older programs), see :doc:`release-notes` and :doc:`api-version-upgrade-guide`. Opening a database ================== @@ -95,7 +93,7 @@ Opening a database After requiring the ``FDB`` gem and selecting an API version, you probably want to open a :class:`Database` using :func:`open`:: require 'fdb' - FDB.api_version 620 + FDB.api_version 630 db = FDB.open .. function:: open( cluster_file=nil ) -> Database @@ -128,6 +126,10 @@ After requiring the ``FDB`` gem and selecting an API version, you probably want |option-trace-format-blurb| + .. method:: FDB.options.set_trace_clock_source(source) -> nil + + |option-trace-clock-source-blurb| + .. method:: FDB.options.set_disable_multi_version_client_api() -> nil |option-disable-multi-version-client-api| @@ -211,21 +213,21 @@ Key selectors Creates a key selector with the given reference key, equality flag, and offset. It is usually more convenient to obtain a key selector with one of the following methods: - .. classmethod:: last_less_than(key) -> KeySelector + .. classmethod:: last_less_than(key) -> KeySelector - Returns a key selector referencing the last (greatest) key in the database less than the specified key. + Returns a key selector referencing the last (greatest) key in the database less than the specified key. - .. classmethod:: KeySelector.last_less_or_equal(key) -> KeySelector + .. classmethod:: KeySelector.last_less_or_equal(key) -> KeySelector - Returns a key selector referencing the last (greatest) key less than, or equal to, the specified key. + Returns a key selector referencing the last (greatest) key less than, or equal to, the specified key. - .. classmethod:: KeySelector.first_greater_than(key) -> KeySelector + .. classmethod:: KeySelector.first_greater_than(key) -> KeySelector - Returns a key selector referencing the first (least) key greater than the specified key. + Returns a key selector referencing the first (least) key greater than the specified key. - .. classmethod:: KeySelector.first_greater_or_equal(key) -> KeySelector + .. classmethod:: KeySelector.first_greater_or_equal(key) -> KeySelector - Returns a key selector referencing the first key greater than, or equal to, the specified key. + Returns a key selector referencing the first key greater than, or equal to, the specified key. .. method:: KeySelector.+(offset) -> KeySelector @@ -281,16 +283,16 @@ A |database-blurb1| |database-blurb2| The ``options`` hash accepts the following optional parameters: - ``:limit`` - Only the first ``limit`` keys (and their values) in the range will be returned. + ``:limit`` + Only the first ``limit`` keys (and their values) in the range will be returned. - ``:reverse`` - If ``true``, then the keys in the range will be returned in reverse order. + ``:reverse`` + If ``true``, then the keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost. - If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order. + If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order. - ``:streaming_mode`` - A valid |streaming-mode|, which provides a hint to FoundationDB about how to retrieve the specified range. This option should generally not be specified, allowing FoundationDB to retrieve the full range very efficiently. + ``:streaming_mode`` + A valid |streaming-mode|, which provides a hint to FoundationDB about how to retrieve the specified range. This option should generally not be specified, allowing FoundationDB to retrieve the full range very efficiently. .. method:: Database.get_range(begin, end, options={}) {|kv| block } -> nil @@ -392,10 +394,6 @@ Database options |option-db-causal-read-risky-blurb| -.. method:: Database.options.set_transaction_include_port_in_address() -> nil - - |option-db-include-port-in-address-blurb| - .. method:: Database.options.set_transaction_logging_max_field_length(size_limit) -> nil |option-db-tr-transaction-logging-max-field-length-blurb| @@ -459,16 +457,16 @@ Reading data The ``options`` hash accepts the following optional parameters: - ``:limit`` - Only the first ``limit`` keys (and their values) in the range will be returned. + ``:limit`` + Only the first ``limit`` keys (and their values) in the range will be returned. - ``:reverse`` - If true, then the keys in the range will be returned in reverse order. + ``:reverse`` + If ``true``, then the keys in the range will be returned in reverse order. Reading ranges in reverse is supported natively by the database and should have minimal extra cost. - If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order. + If ``:limit`` is also specified, the *last* ``limit`` keys in the range will be returned in reverse order. - ``:streaming_mode`` - A valid |streaming-mode|, which provides a hint to FoundationDB about how the returned enumerable is likely to be used. The default is ``:iterator``. + ``:streaming_mode`` + A valid |streaming-mode|, which provides a hint to FoundationDB about how the returned enumerable is likely to be used. The default is ``:iterator``. .. method:: Transaction.get_range(begin, end, options={}) {|kv| block } -> nil @@ -521,7 +519,7 @@ Snapshot reads Like :meth:`Transaction.get_range_start_with`, but as a snapshot read. -.. method:: Transaction.snapshot.get_read_version() -> Version +.. method:: Transaction.snapshot.get_read_version() -> Int64Future Identical to :meth:`Transaction.get_read_version` (since snapshot and strictly serializable reads use the same read version). @@ -730,7 +728,7 @@ Most applications should use the read version that FoundationDB determines autom |infrequent| Sets the database version that the transaction will read from the database. The database cannot guarantee causal consistency if this method is used (the transaction's reads will be causally consistent only if the provided read version has that property). -.. method:: Transaction.get_read_version() -> Version +.. method:: Transaction.get_read_version() -> Int64Future |infrequent| Returns the transaction's read version. @@ -738,10 +736,21 @@ Most applications should use the read version that FoundationDB determines autom |infrequent| |transaction-get-committed-version-blurb| -.. method:: Transaction.get_verionstamp() -> String +.. method:: Transaction.get_versionstamp() -> String |infrequent| |transaction-get-versionstamp-blurb| +Transaction misc functions +-------------------------- + +.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) + + Get the estimated byte size of the given key range. Returns a :class:`Int64Future`. + +.. method:: Transaction.get_approximate_size() -> Int64Future + + |transaction-get-approximate-size-blurb|. Returns a :class:`Int64Future`. + Transaction options ------------------- @@ -771,10 +780,6 @@ Transaction options |option-causal-read-risky-blurb| -.. method:: Transaction.options.set_include_port_in_address() -> nil - - |option-include-port-in-address-blurb| - .. method:: Transaction.options.set_causal_write_risky() -> nil |option-causal-write-risky-blurb| @@ -952,7 +957,7 @@ Asynchronous methods return one of the following subclasses of :class:`Future`: An implementation quirk of :class:`Value` is that it will never evaluate to ``false``, even if its value is ``nil``. It is important to use ``if value.nil?`` rather than ``if ~value`` when checking to see if a key was not present in the database. -.. class:: Version +.. class:: Int64Future This type is a future :class:`Integer` object. Objects of this type respond to the same methods as objects of type :class:`Integer`, and may be passed to any method that expects a :class:`Integer`. @@ -968,6 +973,7 @@ Asynchronous methods return one of the following subclasses of :class:`Future`: For a :class:`FutureNil` object returned by :meth:`Transaction.commit` or :meth:`Transaction.on_error`, you must call :meth:`FutureNil.wait`, which will return ``nil`` if the operation succeeds or raise an :exc:`FDB::Error` if an error occurred. Failure to call :meth:`FutureNil.wait` on a returned :class:`FutureNil` object means that any potential errors raised by the asynchronous operation that returned the object *will not be seen*, and represents a significant error in your code. + .. _ruby streaming mode: Streaming modes diff --git a/documentation/sphinx/source/api-version-upgrade-guide.rst b/documentation/sphinx/source/api-version-upgrade-guide.rst new file mode 100644 index 0000000000..35adaa3964 --- /dev/null +++ b/documentation/sphinx/source/api-version-upgrade-guide.rst @@ -0,0 +1,226 @@ +######################### +API Version Upgrade Guide +######################### + +Overview +======== + +This document provides an overview of changes that an application developer may need to make or effects that they should consider when upgrading the API version in their code. For each version, a list is provided that details the relevant changes when upgrading to that version from a prior version. To upgrade across multiple versions, make sure you apply changes from each version starting after your start version up to and including your target version. + +For more details about API versions, see :ref:`api-versions`. + +.. _api-version-upgrade-guide-630: + +API version 630 +=============== + +C bindings +---------- + +* The ``FDBKeyValue`` struct's ``key`` and ``value`` members have changed type from ``void*`` to ``uint8_t*``. + +.. _api-version-upgrade-guide-620: + +API version 620 +=============== + +C bindings +---------- + +* ``fdb_future_get_version`` has been renamed to ``fdb_future_get_int64``. + +.. _api-version-upgrade-guide-610: + +API version 610 +=============== + +General +------- + +* The concept of opening a cluster has been removed from the API. Instead, databases are opened directly. See binding specific notes for the details as they apply to your language binding. +* The ``TIMEOUT``, ``MAX_RETRY_DELAY``, and ``RETRY_LIMIT`` transaction options are no longer reset by calls to ``onError``. +* Calling ``onError`` with a non-retryable error will now put a transaction into an error state. Previously, this would partially reset the transaction. +* The ``TRANSACTION_LOGGING_ENABLE`` option has been deprecated. Its behavior can be replicated by setting the ``DEBUG_TRANSACTION_IDENTIFIER`` and ``LOG_TRANSACTION`` options. + +C bindings +---------- + +* Creating a database is now done by calling ``fdb_create_database``, which is a synchronous operation. +* The ``FDBCluster`` type has been eliminated and the following functions have been removed: ``fdb_create_cluster``, ``fdb_cluster_create_database``, ``fdb_cluster_set_option``, ``fdb_cluster_destroy``, ``fdb_future_get_cluster``, and ``fdb_future_get_database``. + +Python bindings +--------------- + +* ``fdb.open`` no longer accepts a ``database_name`` parameter. +* Removed ``fdb.init``, ``fdb.create_cluster``, and ``fdb.Cluster``. ``fdb.open`` should be used instead. + +Java bindings +------------- + +* ``FDB.createCluster`` and the ``Cluster`` class have been deprecated. ``FDB.open`` should be used instead. + +Ruby bindings +------------- + +* ``FDB.open`` no longer accepts a ``database_name`` parameter. +* Removed ``FDB.init``, ``FDB.create_cluster``, and ``FDB.Cluster``. ``FDB.open`` should be used instead. + +Go bindings +----------- + +* Added ``fdb.OpenDatabase`` and ``fdb.MustOpenDatabase`` to open a connection to the database by specifying a cluster file. +* Deprecated ``fdb.StartNetwork``, ``fdb.Open``, ``fdb.MustOpen``, and ``fdb.CreateCluster``. ``fdb.OpenDatabase`` or ``fdb.OpenDefault`` should be used instead. + +.. _api-version-upgrade-guide-600: + +API version 600 +=============== + +General +------- + +* The ``TLS_PLUGIN`` option is now a no-op and has been deprecated. TLS support is now included in the published binaries. + +.. _api-version-upgrade-guide-520: + +API version 520 +=============== + +General +------- + +* The ``SET_VERSIONSTAMPED_KEY`` atomic operation now uses four bytes instead of two to specify the versionstamp offset. +* The ``SET_VERSIONSTAMPED_VALUE`` atomic operation now requires a four byte versionstamp offset to be specified at the end of the value, similar to the behavior with ``SET_VERSIONSTAMPED_KEY``. +* The ``READ_AHEAD_DISABLE`` option has been deprecated. + +Java and Python bindings +------------------------ + +* Tuples packed with versionstamps will be encoded with four byte offsets instead of two. + +.. _api-version-upgrade-guide-510: + +API version 510 +=============== + +General +------- + +* The atomic operations ``AND`` and ``MIN`` have changed behavior when used on a key that isn't present in the database. Previously, these operations would set an unset key to a value of equal length with the specified value but containing all null bytes (0x00). Now, an unset key will be set with the value passed to the operation (equivalent to a set). + +Java bindings +------------- + +* Note: the Java bindings as of 5.1 no longer support API versions older that 510. +* The Java bindings have moved packages from ``com.apple.cie.foundationdb`` to ``com.apple.foundationdb``. +* The version of the Java bindings using our custom futures library has been deprecated and is no longer being maintained. The Java bindings using ``CompletableFuture`` are the only ones that remain. +* Finalizers now log a warning to ``stderr`` if an object with native resources is not closed. This can be disabled by calling ``FDB.setUnclosedWarning()``. +* Implementers of the ``Disposable`` interface now implement ``AutoCloseable`` instead, with ``close()`` replacing ``dispose()``. +* ``AutoCloseable`` objects will continue to be closed in object finalizers, but this behavior is being deprecated. All ``AutoCloseable`` objects should be explicitly closed. +* ``AsyncIterator`` is no longer closeable. +* ``getBoundaryKeys()`` now returns a ``CloseableAsyncIterable`` rather than an ``AsyncIterator``. + +.. _api-version-upgrade-guide-500: + +API version 500 +=============== + +Java bindings +------------- + +* Note: the Java bindings as of 5.0 no longer support API versions older than 500. +* ``FDB.open`` and ``Cluster.openDatabase`` no longer take a DB name parameter. +* ``Transaction.onError`` invalidates its transaction and asynchronously return a new replacement ``Transaction``. +* ``Transaction.reset`` has been removed. + +.. _api-version-upgrade-guide-460: + +API version 460 +=============== + +There are no behavior changes in this API version. + +.. _api-version-upgrade-guide-450: + +API version 450 +=============== + +There are no behavior changes in this API version. + +.. _api-version-upgrade-guide-440: + +API version 440 +=============== + +There are no behavior changes in this API version. + +.. _api-version-upgrade-guide-430: + +API version 430 +=============== + +There are no behavior changes in this API version. + +.. _api-version-upgrade-guide-420: + +API version 420 +=============== + +There are no behavior changes in this API version. + +.. _api-version-upgrade-guide-410: + +API version 410 +=============== + +General +------- + +* Transactions no longer reset after a successful commit. + +.. _api-version-upgrade-guide-400: + +API version 400 +=============== + +Java bindings +------------- + +* The Java bindings have moved packages from ``com.foundationdb`` to ``com.apple.cie.foundationdb``. + +.. _api-version-upgrade-guide-300: + +API version 300 +=============== + +General +------- + +* Snapshot reads now see the effects of prior writes within the same transaction. The previous behavior can be achieved using the ``SNAPSHOT_RYW_DISABLE`` transaction option. +* The transaction size limit now includes the size of conflict ranges in its calculation. The size of a conflict range is the sum of the lengths of its begin and end keys. +* Adding conflict ranges or watches in the system keyspace (beginning with ``\xFF``) now requires setting the ``READ_SYSTEM_KEYS`` or ``ACCESS_SYSTEM_KEYS`` option. + +.. _api-version-upgrade-guide-200: + +API version 200 +=============== + +General +------- + +* Read version requests will now fail when the transaction is reset or has experienced another error. + +.. _api-version-upgrade-guide-100: + +API version 100 +=============== + +Java bindings +------------- + +* ``Transaction.clearRangeStartsWith`` has been deprecated. ``Transaction.clear(Range)`` should be used instead. + +Older API versions +================== + +API versions from the beta and alpha releases of Foundationdb (pre-100) are not documented here. See :doc:`old-release-notes/release-notes-023` for details about changes in those releases. diff --git a/documentation/sphinx/source/backups.rst b/documentation/sphinx/source/backups.rst index 42a6ba9899..1a30a6e4b1 100644 --- a/documentation/sphinx/source/backups.rst +++ b/documentation/sphinx/source/backups.rst @@ -24,14 +24,14 @@ Backup vs DR FoundationDB can backup a database to local disks, a blob store (such as Amazon S3), or to another FoundationDB database. -Backing up one database to another is a special form of backup is called DR backup or just DR for short. DR stands for Disaster Recovery, as it can be used to keep two geographically separated databases in close synchronization to recover from a catastrophic disaster. Once a DR operation has reached 'differential' mode, the secondary database (the destination of the DR job) will always contains a *consistent* copy of the primary database (the source of the DR job) but it will be from some past point in time. If the primary database is lost and applications continue using the secondary database, the "ACI" in ACID is preserved but D (Durability) is lost for some amount of most recent changes. When DR is operating normally, the secondary database will lag behind the primary database by as little as a few seconds worth of database commits. +Backing up one database to another is a special form of backup is called DR backup or just DR for short. DR stands for Disaster Recovery, as it can be used to keep two geographically separated databases in close synchronization to recover from a catastrophic disaster. Once a DR operation has reached 'differential' mode, the secondary database (the destination of the DR job) will always contain a *consistent* copy of the primary database (the source of the DR job) but it will be from some past point in time. If the primary database is lost and applications continue using the secondary database, the "ACI" in ACID is preserved but D (Durability) is lost for some amount of most recent changes. When DR is operating normally, the secondary database will lag behind the primary database by as little as a few seconds worth of database commits. While a cluster is being used as the destination for a DR operation it will be locked to prevent accidental use or modification. Limitations =========== -Backup data is not encrypted on disk, in a blob store account, or in transit to a destination blob store account or database. +Backup data is not encrypted at rest on disk or in a blob store account. Tools =========== @@ -159,15 +159,14 @@ The Blob Credential File format is JSON with the following schema: } } -SSL Support +TLS Support =========== -By default, backup will communicate over https. To configure https, the following environment variables are used: +In-flight traffic for blob store or disaster recovery backups can be encrypted with the following environment variables. They are also offered as command-line flags or can be specified in ``foundationdb.conf`` for backup agents. ============================ ==================================================== Environment Variable Purpose ============================ ==================================================== -``FDB_TLS_PLUGIN`` Path to the file to be loaded as the TLS plugin ``FDB_TLS_CERTIFICATE_FILE`` Path to the file from which the local certificates can be loaded, used by the plugin ``FDB_TLS_KEY_FILE`` Path to the file from which to load the private @@ -177,8 +176,11 @@ Environment Variable Purpose ``FDB_TLS_CA_FILE`` Path to the file containing the CA certificates to trust. Specify to override the default openssl location. +``FDB_TLS_VERIFY_PEERS`` The byte-string for the verification of peer + certificates and sessions. ============================ ==================================================== +Blob store backups can be configured to use HTTPS/TLS by setting the ``secure_connection`` or ``sc`` backup URL option to ``1``, which is the default. Disaster recovery backups are secured by using TLS for both the source and target clusters and setting the TLS options for the ``fdbdr`` and ``dr_agent`` commands. ``fdbbackup`` command line tool =============================== diff --git a/documentation/sphinx/source/cap-theorem.rst b/documentation/sphinx/source/cap-theorem.rst index c5c3c64d55..42942d2f8c 100644 --- a/documentation/sphinx/source/cap-theorem.rst +++ b/documentation/sphinx/source/cap-theorem.rst @@ -9,9 +9,9 @@ What is the CAP Theorem? In 2000, Eric Brewer conjectured that a distributed system cannot simultaneously provide all three of the following desirable properties: - * Consistency: A read sees all previously completed writes. - * Availability: Reads and writes always succeed. - * Partition tolerance: Guaranteed properties are maintained even when network failures prevent some machines from communicating with others. +* Consistency: A read sees all previously completed writes. +* Availability: Reads and writes always succeed. +* Partition tolerance: Guaranteed properties are maintained even when network failures prevent some machines from communicating with others. In 2002, Gilbert and Lynch proved this in the asynchronous and partially synchronous network models, so it is now commonly called the `CAP Theorem `_. diff --git a/documentation/sphinx/source/class-scheduling-go.rst b/documentation/sphinx/source/class-scheduling-go.rst index a0d103769b..d8ea0a5b19 100644 --- a/documentation/sphinx/source/class-scheduling-go.rst +++ b/documentation/sphinx/source/class-scheduling-go.rst @@ -29,7 +29,7 @@ Before using the API, we need to specify the API version. This allows programs t .. code-block:: go - fdb.MustAPIVersion(620) + fdb.MustAPIVersion(630) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. @@ -78,7 +78,7 @@ If this is all working, it looks like we are ready to start building a real appl func main() { // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(620) + fdb.MustAPIVersion(630) // Open the default database from the system cluster db := fdb.MustOpenDefault() @@ -229,7 +229,7 @@ Furthermore, this version can only be called with a ``Database``, making it impo Note that by default, the operation will be retried an infinite number of times and the transaction will never time out. It is therefore recommended that the client choose a default transaction retry limit or timeout value that is suitable for their application. This can be set either at the transaction level using the ``SetRetryLimit`` or ``SetTimeout`` transaction options or at the database level with the ``SetTransactionRetryLimit`` or ``SetTransactionTimeout`` database options. For example, one can set a one minute timeout on each transaction and a default retry limit of 100 by calling:: db.Options().SetTransactionTimeout(60000) // 60,000 ms = 1 minute - db.Options().SetRetryLimit(100) + db.Options().SetTransactionRetryLimit(100) Making some sample classes -------------------------- @@ -666,7 +666,7 @@ Here's the code for the scheduling tutorial: } func main() { - fdb.MustAPIVersion(620) + fdb.MustAPIVersion(630) db := fdb.MustOpenDefault() db.Options().SetTransactionTimeout(60000) // 60,000 ms = 1 minute db.Options().SetTransactionRetryLimit(100) diff --git a/documentation/sphinx/source/class-scheduling-java.rst b/documentation/sphinx/source/class-scheduling-java.rst index 75db289e1a..c899c546dc 100644 --- a/documentation/sphinx/source/class-scheduling-java.rst +++ b/documentation/sphinx/source/class-scheduling-java.rst @@ -30,7 +30,7 @@ Before using the API, we need to specify the API version. This allows programs t private static final Database db; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); } @@ -66,7 +66,7 @@ If this is all working, it looks like we are ready to start building a real appl private static final Database db; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); } @@ -157,7 +157,7 @@ If instead you pass a :class:`Transaction` for the :class:`TransactionContext` p Note that by default, the operation will be retried an infinite number of times and the transaction will never time out. It is therefore recommended that the client choose a default transaction retry limit or timeout value that is suitable for their application. This can be set either at the transaction level using the ``setRetryLimit`` or ``setTimeout`` transaction options or at the database level with the ``setTransactionRetryLimit`` or ``setTransactionTimeout`` database options. For example, one can set a one minute timeout on each transaction and a default retry limit of 100 by calling:: db.options().setTransactionTimeout(60000); // 60,000 ms = 1 minute - db.options().setRetryLimit(100); + db.options().setTransactionRetryLimit(100); Making some sample classes -------------------------- @@ -441,10 +441,10 @@ Here's the code for the scheduling tutorial: private static final Database db; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); db.options().setTransactionTimeout(60000); // 60,000 ms = 1 minute - db.options().setRetryLimit(100); + db.options().setTransactionRetryLimit(100); } // Generate 1,620 classes like '9:00 chem for dummies' diff --git a/documentation/sphinx/source/class-scheduling-ruby.rst b/documentation/sphinx/source/class-scheduling-ruby.rst index 345678887c..d1f79c3725 100644 --- a/documentation/sphinx/source/class-scheduling-ruby.rst +++ b/documentation/sphinx/source/class-scheduling-ruby.rst @@ -23,7 +23,7 @@ Open a Ruby interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - > FDB.api_version 620 + > FDB.api_version 630 => nil Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -46,7 +46,7 @@ If this is all working, it looks like we are ready to start building a real appl .. code-block:: ruby require 'fdb' - FDB.api_version 620 + FDB.api_version 630 @db = FDB.open @db['hello'] = 'world' print 'hello ', @db['hello'] @@ -126,7 +126,7 @@ If instead you pass a :class:`Transaction` for the ``db_or_tr`` parameter, the t Note that by default, the operation will be retried an infinite number of times and the transaction will never time out. It is therefore recommended that the client choose a default transaction retry limit or timeout value that is suitable for their application. This can be set either at the transaction level using the ``set_retry_limit`` or ``set_timeout`` transaction options or at the database level with the ``set_transaction_retry_limit`` or ``set_transaction_timeout`` database options. For example, one can set a one minute timeout on each transaction and a default retry limit of 100 by calling:: @db.options.set_transaction_timeout(60000) # 60,000 ms = 1 minute - @db.options.set_retry_limit(100) + @db.options.set_transaction_retry_limit(100) Making some sample classes -------------------------- @@ -373,7 +373,7 @@ Here's the code for the scheduling tutorial: require 'fdb' - FDB.api_version 620 + FDB.api_version 630 #################################### ## Initialization ## diff --git a/documentation/sphinx/source/class-scheduling.rst b/documentation/sphinx/source/class-scheduling.rst index c13678a23f..b516bc9f7c 100644 --- a/documentation/sphinx/source/class-scheduling.rst +++ b/documentation/sphinx/source/class-scheduling.rst @@ -30,7 +30,7 @@ Open a Python interactive interpreter and import the FoundationDB API module:: Before using the API, we need to specify the API version. This allows programs to maintain compatibility even if the API is modified in future versions:: - >>> fdb.api_version(620) + >>> fdb.api_version(630) Next, we open a FoundationDB database. The API will connect to the FoundationDB cluster indicated by the :ref:`default cluster file `. :: @@ -48,7 +48,7 @@ When this command returns without exception, the modification is durably stored If this is all working, it looks like we are ready to start building a real application. For reference, here's the full code for "hello world":: import fdb - fdb.api_version(620) + fdb.api_version(630) db = fdb.open() db[b'hello'] = b'world' print 'hello', db[b'hello'] @@ -91,7 +91,7 @@ FoundationDB includes a few tools that make it easy to model data using this app opening a :ref:`directory ` in the database:: import fdb - fdb.api_version(620) + fdb.api_version(630) db = fdb.open() scheduling = fdb.directory.create_or_open(db, ('scheduling',)) @@ -136,7 +136,7 @@ If instead you pass a :class:`Transaction` for the ``tr`` parameter, the transac Note that by default, the operation will be retried an infinite number of times and the transaction will never time out. It is therefore recommended that the client choose a default transaction retry limit or timeout value that is suitable for their application. This can be set either at the transaction level using the ``set_retry_limit`` or ``set_timeout`` transaction options or at the database level with the ``set_transaction_retry_limit`` or ``set_transaction_timeout`` database options. For example, one can set a one minute timeout on each transaction and a default retry limit of 100 by calling:: db.options.set_transaction_timeout(60000) # 60,000 ms = 1 minute - db.options.set_retry_limit(100) + db.options.set_transaction_retry_limit(100) Making some sample classes -------------------------- @@ -337,7 +337,7 @@ Here's the code for the scheduling tutorial:: import fdb import fdb.tuple - fdb.api_version(620) + fdb.api_version(630) #################################### @@ -350,7 +350,7 @@ Here's the code for the scheduling tutorial:: db = fdb.open() db.options.set_transaction_timeout(60000) # 60,000 ms = 1 minute - db.options.set_retry_limit(100) + db.options.set_transaction_retry_limit(100) scheduling = fdb.directory.create_or_open(db, ('scheduling',)) course = scheduling['class'] attends = scheduling['attends'] diff --git a/documentation/sphinx/source/client-design.rst b/documentation/sphinx/source/client-design.rst index 0417704d15..c9706f0f46 100644 --- a/documentation/sphinx/source/client-design.rst +++ b/documentation/sphinx/source/client-design.rst @@ -18,6 +18,8 @@ FoundationDB supports language bindings for application development using the or * :doc:`api-general` contains information on FoundationDB clients applicable across all language bindings. +* :doc:`api-version-upgrade-guide` contains information about upgrading client code to a new API version. + * :doc:`known-limitations` describes both long-term design limitations of FoundationDB and short-term limitations applicable to the current version. .. toctree:: @@ -34,3 +36,4 @@ FoundationDB supports language bindings for application development using the or client-testing api-general known-limitations + api-version-upgrade-guide diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index 87cfba111a..0259525a7c 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -162,6 +162,16 @@ The ``getrangekeys`` command fetches keys in a range. Its syntax is ``getrangeke Note that :ref:`characters can be escaped ` when specifying keys (or values) in ``fdbcli``. +getversion +---------- + +The ``getversion`` command fetches the current read version of the cluster or currently running transaction. + +advanceversion +-------------- + +Forces the cluster to recover at the specified version. If the specified version is larger than the current version of the cluster, the cluster version is advanced to the specified version via a forced recovery. + help ---- @@ -211,7 +221,6 @@ The following options are available for use with the ``option`` command: ``TIMEOUT`` - Set a timeout in milliseconds which, when elapsed, will cause the transaction automatically to be cancelled. Valid parameter values are ``[0, INT_MAX]``. If set to 0, will disable all timeouts. All pending and any future uses of the transaction will throw an exception. The transaction can be used again after it is reset. Like all transaction options, a timeout must be reset after a call to ``onError``. This behavior allows the user to make the timeouts dynamic. - include ------- @@ -227,6 +236,11 @@ For each IP address or IP:port pair in ````, the command removes any For information on adding machines to a cluster, see :ref:`adding-machines-to-a-cluster`. +lock +---- + +The ``lock`` command locks the database with a randomly generated lockUID. + option ------ @@ -285,3 +299,8 @@ status json ^^^^^^^^^^^ ``status json`` will provide the cluster status in its JSON format. For a detailed description of this format, see :doc:`mr-status`. + +unlock +------ + +The ``unlock`` command unlocks the database with the specified lock UID. Because this is a potentially dangerous operation, users must copy a passphrase before the unlock command is executed. diff --git a/documentation/sphinx/source/configuration.rst b/documentation/sphinx/source/configuration.rst index 6c0b6e5cf0..671141313d 100644 --- a/documentation/sphinx/source/configuration.rst +++ b/documentation/sphinx/source/configuration.rst @@ -27,11 +27,11 @@ System requirements * Or, an unsupported Linux distribution with: * Kernel version between 2.6.33 and 3.0.x (inclusive) or 3.7 or greater - * Works with .deb or .rpm packages + * Preferably .deb or .rpm package support * Or, macOS 10.7 or later - .. warning:: The macOS version of the FoundationDB server is intended for use on locally accessible development machines only. Other uses are not supported. + .. warning:: The macOS and Windows versions of the FoundationDB server are intended for use on locally accessible development machines only. Other uses are not supported. * 4GB **ECC** RAM (per fdbserver process) * Storage @@ -387,6 +387,8 @@ FoundationDB will never use processes on the same machine for the replication of FoundationDB replicates data to three machines, and at least three available machines are required to make progress. This is the recommended mode for a cluster of five or more machines in a single datacenter. + .. note:: When running in cloud environments with managed disks that are already replicated and persistent, ``double`` replication may still be considered for 5+ machine clusters. This will result in lower availability fault tolerance for planned or unplanned failures and lower total read throughput, but offers a reasonable tradeoff for cost. + ``three_data_hall`` mode FoundationDB stores data in triplicate, with one copy on a storage server in each of three data halls. The transaction logs are replicated four times, with two data halls containing two replicas apiece. Four available machines (two in each of two data halls) are therefore required to make progress. This configuration enables the cluster to remain available after losing a single data hall and one machine in another data hall. @@ -395,7 +397,7 @@ Datacenter-aware mode In addition to the more commonly used modes listed above, this version of FoundationDB has support for redundancy across multiple datacenters. - .. note:: When using the datacenter-aware mode, all ``fdbserver`` processes should be passed a valid datacenter identifier on the command line. +.. note:: When using the datacenter-aware mode, all ``fdbserver`` processes should be passed a valid datacenter identifier on the command line. ``three_datacenter`` mode *(for 5+ machines in 3 datacenters)* @@ -622,23 +624,23 @@ The ``satellite_redundancy_mode`` is configured per region, and specifies how ma ``one_satellite_single`` mode - Keep one copy of the mutation log in the satellite datacenter with the highest priority. If the highest priority satellite is unavailable it will put the transaction log in the satellite datacenter with the next highest priority. +Keep one copy of the mutation log in the satellite datacenter with the highest priority. If the highest priority satellite is unavailable it will put the transaction log in the satellite datacenter with the next highest priority. ``one_satellite_double`` mode - Keep two copies of the mutation log in the satellite datacenter with the highest priority. +Keep two copies of the mutation log in the satellite datacenter with the highest priority. ``one_satellite_triple`` mode - Keep three copies of the mutation log in the satellite datacenter with the highest priority. +Keep three copies of the mutation log in the satellite datacenter with the highest priority. ``two_satellite_safe`` mode - Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. This mode will protect against the simultaneous loss of both the primary and one of the satellite datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter. +Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. This mode will protect against the simultaneous loss of both the primary and one of the satellite datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter. ``two_satellite_fast`` mode - Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. FoundationDB will only synchronously wait for one of the two satellite datacenters to make the mutations durable before considering a commit successful. This will reduce tail latencies caused by network issues between datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter. +Keep two copies of the mutation log in each of the two satellite datacenters with the highest priorities, for a total of four copies of each mutation. FoundationDB will only synchronously wait for one of the two satellite datacenters to make the mutations durable before considering a commit successful. This will reduce tail latencies caused by network issues between datacenters. If only one satellite is available, it will fall back to only storing two copies of the mutation log in the remaining datacenter. .. warning:: In release 6.0 this is implemented by waiting for all but 2 of the transaction logs. If ``satellite_logs`` is set to more than 4, FoundationDB will still need to wait for replies from both datacenters. @@ -696,17 +698,17 @@ Migrating a database to use a region configuration To configure an existing database to regions, do the following steps: - 1. Ensure all processes have their dcid locality set on the command line. All processes should exist in the same datacenter. If converting from a ``three_datacenter`` configuration, first configure down to using a single datacenter by changing the replication mode. Then exclude the machines in all datacenters but the one that will become the initial active region. +1. Ensure all processes have their dcid locality set on the command line. All processes should exist in the same datacenter. If converting from a ``three_datacenter`` configuration, first configure down to using a single datacenter by changing the replication mode. Then exclude the machines in all datacenters but the one that will become the initial active region. - 2. Configure the region configuration. The datacenter with all the existing processes should have a non-negative priority. The region which will eventually store the remote replica should be added with a negative priority. +2. Configure the region configuration. The datacenter with all the existing processes should have a non-negative priority. The region which will eventually store the remote replica should be added with a negative priority. - 3. Add processes to the cluster in the remote region. These processes will not take data yet, but need to be added to the cluster. If they are added before the region configuration is set they will be assigned data like any other FoundationDB process, which will lead to high latencies. +3. Add processes to the cluster in the remote region. These processes will not take data yet, but need to be added to the cluster. If they are added before the region configuration is set they will be assigned data like any other FoundationDB process, which will lead to high latencies. - 4. Configure ``usable_regions=2``. This will cause the cluster to start copying data between the regions. +4. Configure ``usable_regions=2``. This will cause the cluster to start copying data between the regions. - 5. Watch ``status`` and wait until data movement is complete. This will signal that the remote datacenter has a full replica of all of the data in the database. +5. Watch ``status`` and wait until data movement is complete. This will signal that the remote datacenter has a full replica of all of the data in the database. - 6. Change the region configuration to have a non-negative priority for the primary datacenters in both regions. This will enable automatic failover between regions. +6. Change the region configuration to have a non-negative priority for the primary datacenters in both regions. This will enable automatic failover between regions. Handling datacenter failures ---------------------------- @@ -717,9 +719,9 @@ When a primary datacenter fails, the cluster will go into a degraded state. It w To drop the dead datacenter do the following steps: - 1. Configure the region configuration so that the dead datacenter has a negative priority. +1. Configure the region configuration so that the dead datacenter has a negative priority. - 2. Configure ``usable_regions=1``. +2. Configure ``usable_regions=1``. If you are running in a configuration without a satellite datacenter, or you have lost all machines in a region simultaneously, the ``force_recovery_with_data_loss`` command from ``fdbcli`` allows you to force a recovery to the other region. This will discard the portion of the mutation log which did not make it across the WAN. Once the database has recovered, immediately follow the previous steps to drop the dead region the normal way. @@ -728,13 +730,10 @@ Region change safety The steps described above for both adding and removing replicas are enforced by ``fdbcli``. The following are the specific conditions checked by ``fdbcli``: - * You cannot change the ``regions`` configuration while also changing ``usable_regions``. - - * You can only change ``usable_regions`` when exactly one region has priority >= 0. - - * When ``usable_regions`` > 1, all regions with priority >= 0 must have a full replica of the data. - - * All storage servers must be in one of the regions specified by the region configuration. +* You cannot change the ``regions`` configuration while also changing ``usable_regions``. +* You can only change ``usable_regions`` when exactly one region has priority >= 0. +* When ``usable_regions`` > 1, all regions with priority >= 0 must have a full replica of the data. +* All storage servers must be in one of the regions specified by the region configuration. Monitoring ---------- @@ -768,13 +767,10 @@ Region configuration is better in almost all ways than the ``three_datacenter`` Known limitations ----------------- -The 6.0 release still has a number of rough edges related to region configuration. This is a collection of all the issues that have been pointed out in the sections above. These issues should be significantly improved in future releases of FoundationDB: +The 6.2 release still has a number of rough edges related to region configuration. This is a collection of all the issues that have been pointed out in the sections above. These issues should be significantly improved in future releases of FoundationDB: - * FoundationDB supports replicating data to at most two regions. - - * ``two_satellite_fast`` does not hide latency properly when configured with more than 4 satellite transaction logs. - - * While a datacenter has failed, the maximum write throughput of the cluster will be roughly 1/3 of normal performance. +* FoundationDB supports replicating data to at most two regions. +* ``two_satellite_fast`` does not hide latency properly when configured with more than 4 satellite transaction logs. .. _guidelines-process-class-config: diff --git a/documentation/sphinx/source/data-modeling.rst b/documentation/sphinx/source/data-modeling.rst index 24f3aad203..e039250f68 100644 --- a/documentation/sphinx/source/data-modeling.rst +++ b/documentation/sphinx/source/data-modeling.rst @@ -53,8 +53,6 @@ .. |timeout-database-option| replace:: FIXME .. |causal-read-risky-database-option| replace:: FIXME .. |causal-read-risky-transaction-option| replace:: FIXME -.. |include-port-in-address-database-option| replace:: FIXME -.. |include-port-in-address-transaction-option| replace:: FIXME .. |transaction-logging-max-field-length-transaction-option| replace:: FIXME .. |transaction-logging-max-field-length-database-option| replace:: FIXME @@ -269,6 +267,18 @@ Using the table name as the subspace, we could implement the common row-oriented cols[c] = v return cols + +Versionstamps +------------- + +A common data model is to index your data with a sequencing prefix to allow log scans or tails of recent data. This index requires a unique, monotonically increasing value, like an AUTO_INCREMENT PRIMARY KEY in SQL. This could be implemented at the client level by reading the value for conflict checks before every increment. A better solution is the versionstamp, which can be generated at commit-time with no read conflict ranges, providing a unique sequence ID in a single conflict-free write. + +Versioning commits provides FoundationDB with MVCC guarantees and transactional integrity. Versionstamps write the transaction's commit version as a value to an arbitrary key as part of the same transaction, allowing the client to leverage the version's unique and serial properties. Because the versionstamp is generated at commit-time, the versionstamped key cannot be read in the same transaction that it is written, and the versionstamp's value will be unknown until the transaction is committed. After the transaction is committed, the versionstamp can be obtained. + +The versionstamp guarantees uniqueness and monotonically increasing values for the entire lifetime of a single FDB cluster. This is even true if the cluster is restored from a backup, as a restored cluster will begin at a higher version than when the backup was taken. Special care must be taken when moving data between two FoundationDB clusters containing versionstamps, as the differing cluster versions might break the monotonicity. + +There are two concepts of versionstamp depending on your context. At the fdb_c client level, or any binding outside of the Tuple layer, the 'versionstamp' is 10 bytes: the transaction's commit version (8 bytes) and transaction batch order (2 bytes). The user can manually add 2 additional bytes to provide application level ordering. The tuple layer provides a useful api for getting and setting both the 10 byte system version and the 2 byte user version. In the context of the Tuple layer, the 'versionstamp' is all 12 bytes. For examples on how to use the versionstamp in the python binding, see the :doc:`api-python` documentation. + .. _data-modeling-entity-relationship: Entity-relationship models @@ -531,25 +541,25 @@ How you map your application data to keys and values can have a dramatic impact * Structure keys so that range reads can efficiently retrieve the most frequently accessed data. - * If you perform a range read that is, in total, much more than 1 kB, try to restrict your range as much as you can while still retrieving the needed data. + * If you perform a range read that is, in total, much more than 1 kB, try to restrict your range as much as you can while still retrieving the needed data. * Structure keys so that no single key needs to be updated too frequently, which can cause transaction conflicts. - * If a key is updated more than 10-100 times per second, try to split it into multiple keys. - * For example, if a key is storing a counter, split the counter into N separate counters that are randomly incremented by clients. The total value of the counter can then read by adding up the N individual ones. + * If a key is updated more than 10-100 times per second, try to split it into multiple keys. + * For example, if a key is storing a counter, split the counter into N separate counters that are randomly incremented by clients. The total value of the counter can then read by adding up the N individual ones. * Keep key sizes small. - * Try to keep key sizes below 1 kB. (Performance will be best with key sizes below 32 bytes and *cannot* be more than 10 kB.) - * When using the tuple layer to encode keys (as is recommended), select short strings or small integers for tuple elements. Small integers will encode to just two bytes. - * If your key sizes are above 1 kB, try either to move data from the key to the value, split the key into multiple keys, or encode the parts of the key more efficiently (remembering to preserve any important ordering). + * Try to keep key sizes below 1 kB. (Performance will be best with key sizes below 32 bytes and *cannot* be more than 10 kB.) + * When using the tuple layer to encode keys (as is recommended), select short strings or small integers for tuple elements. Small integers will encode to just two bytes. + * If your key sizes are above 1 kB, try either to move data from the key to the value, split the key into multiple keys, or encode the parts of the key more efficiently (remembering to preserve any important ordering). * Keep value sizes moderate. - * Try to keep value sizes below 10 kB. (Value sizes *cannot* be more than 100 kB.) - * If your value sizes are above 10 kB, consider splitting the value across multiple keys. - * If you read values with sizes above 1 kB but use only a part of each value, consider splitting the values using multiple keys. - * If you frequently perform individual reads on a set of values that total to fewer than 200 bytes, try either to combine the values into a single value or to store the values in adjacent keys and use a range read. + * Try to keep value sizes below 10 kB. (Value sizes *cannot* be more than 100 kB.) + * If your value sizes are above 10 kB, consider splitting the value across multiple keys. + * If you read values with sizes above 1 kB but use only a part of each value, consider splitting the values using multiple keys. + * If you frequently perform individual reads on a set of values that total to fewer than 200 bytes, try either to combine the values into a single value or to store the values in adjacent keys and use a range read. Large Values and Blobs ---------------------- diff --git a/documentation/sphinx/source/developer-guide.rst b/documentation/sphinx/source/developer-guide.rst index 5cda16c32a..ef1496f5c8 100644 --- a/documentation/sphinx/source/developer-guide.rst +++ b/documentation/sphinx/source/developer-guide.rst @@ -53,8 +53,6 @@ .. |timeout-database-option| replace:: FIXME .. |causal-read-risky-database-option| replace:: FIXME .. |causal-read-risky-transaction-option| replace:: FIXME -.. |include-port-in-address-database-option| replace:: FIXME -.. |include-port-in-address-transaction-option| replace:: FIXME .. |transaction-logging-max-field-length-transaction-option| replace:: FIXME .. |transaction-logging-max-field-length-database-option| replace:: FIXME diff --git a/documentation/sphinx/source/disk-snapshot-backup.rst b/documentation/sphinx/source/disk-snapshot-backup.rst new file mode 100644 index 0000000000..e5eccd8051 --- /dev/null +++ b/documentation/sphinx/source/disk-snapshot-backup.rst @@ -0,0 +1,323 @@ + +.. _disk-snapshot-backups: + +################################# +Disk snapshot backup and Restore +################################# + +This document covers disk snapshot based backup and restoration of a FoundationDB database. This tool leverages disk level snapshots and gets a point-in-time consistent copy of the database. The disk snapshot backup can be used for test and development purposes, for compliance reasons or to provide an additional level of protection in case of hardware or software failures. + +.. _disk-snapshot-backup-introduction: + +Introduction +============ + +FoundationDB's disk snapshot backup tool makes a consistent, point-in-time backup of FoundationDB database without downtime by taking crash consistent snapshot of all the disk stores that have persistent data. + +The prerequisite of this feature is to have crash consistent snapshot support on the filesystem (or the disks) on which FoundationDB is running. + +The disk snapshot backup tool orchestrates the snapshotting of all the disk images and ensures that they are restorable to a consistent point in time. + +Restore is achieved by copying or attaching the disk snapshot images to FoundationDB compute instances. Restore behaves as if the cluster were powered down and restarted. + +Backup vs Disk snapshot backup +============================== +Backup feature already exists in FoundationDB and is detailed here :ref:`backups`, any use of fdbbackup will refer to this feature. + +Both fdbbackup and Disk snapshot backup tools provide a point-in-time consistent backup of FoundationDB database, but they operate at different levels and there are differences in terms of performance, features and external dependency. + +fdbbackup operates at the key-value level. Backup involves copying of all the key-value pairs from the source cluster and restore involves applying all the key-value pairs to the destination database. Performance depends on the amount of data and the throughput with which the data can be read and written. This approach has no external dependency, there is no requirement for any snapshotting feature from the disk system. Additionally, it has an option for continuous backup with the flexibility to pick a restore point. + +Disk snapshot backup and restore are generally high performance because it operates at disk level and data is not read or written through the FoundationDB stack. In environments where disk snapshot and restore are highly performant this approach can be very fast. Frequent backups can be done as a substitute to continuous backup if the backups are performant. + +Limitations +=========== + +* No support for continuous backup +* Feature is not supported on Windows operating system +* Data encryption is dependent on the disk system +* Backup and restore involves tooling which are deployment and environment specific to be developed by operators +* ``snapshot`` command is a hidden fdbcli command in the current release and will be unhidden in a future patch release. + +Disk snapshot backup steps +========================== + +``snapshot`` + This command line tool is used to create the snapshot. It takes a full path to a ``snapshot create binary`` and reports the status. Optionally, it can take additional arguments to be passed down to the ``snapshot create binary``. It returns a unique identifier which can be used to identify all the disk snapshots of a backup. Even in case of failures the unique identifier is returned to identify and clear any partially create disk snapshots. + +In response to the snapshot request from the user, FoundationDB will run the user specified ``snapshot create binary`` on all processes which have persistent data, binary should call filesystem/disk system specific snapshot create API. + +Before using the ``snapshot`` command the following setup needs to be done + +* Write a program that will snapshot the local disk store when invoked by the ``fdbserver`` with the following arguments: + + - UID - 32 byte alpha-numeric unique identifier, the same identifier will be passed to all the nodes in the cluster, can be used to identify the set of disk snapshots associated with this backup + - Version - version string of the FoundationDB binary + - Path - path of the FoundationDB ``datadir`` to be snapshotted, ``datadir`` specified in :ref:`foundationdb-conf-fdbserver` + - Role - ``tlog``/``storage``/``coordinator``, identifies the role of the node on which the snapshot is being invoked + +* Install ``snapshot create binary`` on the FoundationDB instance in a secure path that can be invoked by the ``fdbserver`` +* Set a new config parameter ``whitelist_binpath`` in :ref:`foundationdb-conf-fdbserver`, whose value is the ``snapshot create binary`` absolute path. Running any ``snapshot`` command will validate that it is in the ``whitelist_binpath``. This is a security mechanism to stop running a random/insecure command on the cluster by a client using the ``snapshot`` command. Example configuration entry will look like:: + + whitelist_binpath = "/bin/snap_create.sh" + +* ``snapshot create binary`` should capture any additional data needed to restore the cluster. Additional data can be stored as tags in cloud environments or it can be stored in an additional file/directory in the ``datadir`` and then snapshotted. The section :ref:`disk-snapshot-backup-specification` describes the recommended specification of the list of things that can be gathered by the binary. +* Program should return a non-zero status for any failures and zero for success +* If the ``snapshot create binary`` process takes longer than 5 minutes to return a status then it will be killed and ``snapshot`` command will fail. Timeout of 5 minutes is configurable and can be set with ``SNAP_CREATE_MAX_TIMEOUT`` config parameter in :ref:`foundationdb-conf-fdbserver`. Since the default value is large enough, there should not be a need to modify this configuration. + +``snapshot`` is a synchronous command and when it returns successfully backup is considered complete and restorable. The time it takes to finish a backup is a function of the time it takes to snapshot the disk store. For example, if disk snapshot takes 1 second, time to finish backup should be less than < 10 seconds, this is general guidance and in some cases it may take longer. If the command is aborted by the user then the disk snapshots should not be used for restore, because the state of backup is undefined. If the command fails or aborts, operator can retry by issuing another ``snapshot`` command. + +Example ``snapshot`` command usage:: + + fdb> snapshot /bin/snap_create.sh --param1 param1-value --param2 param2-value + Snapshot command succeeded with UID c50263df28be44ebb596f5c2a849adbb + +will invoke the ``snapshot create binary`` on ``tlog`` role with the following arguments:: + + --param1 param1-value --param2 param2-value --path /mnt/circus/data/4502 --version 6.2.6 --role tlog --uid c50263df28be44ebb596f5c2a849adbb + + +.. _disk-snapshot-backup-specification: + +Disk snapshot backup specification +---------------------------------- + +Details the list of artifacts the ``snapshot create binary`` should gather to aid the restore. + +================================ ======================================================== ======================================================== +Field Name Description Source of information +================================ ======================================================== ======================================================== +``UID`` unique identifier passed with all the ``snapshot`` CLI command output contains the UID + snapshot create binary invocations associated with + a backup. Disk snapshots could be tagged with this UID. +``FoundationDB Server Version`` software version of the ``fdbserver`` command line argument to snap create binary +``CreationTime`` current system date and time time obtained by calling the system time +``FoundationDB Cluster File`` cluster file which has cluster-name, magic and read from the location of the cluster file location + the list of coordinators, cluster file is detailed mentioned in the command line arguments. Command + here :ref:`foundationdb-cluster-file` line arguments of ``fdbserver`` can be accessed from + /proc/$PPID/cmdline +``Config Knobs`` command line arguments passed to ``fdbserver`` available from command line arguments of ``fdbserver`` + or from foundationdb.conf +``IP Address + Port`` host address and port information of the ``fdbserver`` available from command line arguments of ``fdbserver`` + that is invoking the snapshot +``LocalityData`` machine id, zone id or any other locality information available from command line arguments of ``fdbserver`` +``Name for the snapshot file`` recommended name for the disk snapshot cluster-name:ip-addr:port:UID +================================ ======================================================== ======================================================== + +``snapshot create binary`` will not be invoked on processes which does not have any persistent data (for example, Cluster Controller or Master or MasterProxy). Since these processes are stateless, there is no need for a snapshot. Any specialized configuration knobs used for one of these stateless processes need to be copied and restored externally. + +Management of disk snapshots +---------------------------- + +Unused disk snapshots or disk snapshots that are part of failed backups have to deleted by the operator externally. + +Error codes +----------- + +Error codes returned by ``snapshot`` command + +======================================= ============ ============================= ============================================================= +Name Code Description Comments +======================================= ============ ============================= ============================================================= +snap_path_not_whitelisted 2505 Snapshot create binary path Whitelist the ``snap create binary`` path and retry the + not whitelisted operation. +snap_not_fully_recovered_unsupported 2506 Unsupported when the cluster Wait for the cluster to finish recovery and then retry the + is not fully recovered operation +snap_log_anti_quorum_unsupported 2507 Unsupported when log anti Feature is not supported when log anti quorum is configured + quorum is configured +snap_with_recovery_unsupported 2508 Cluster recovery during Recovery happened while snapshot operation was in progress, + snapshot operation not retry the operation. + supported +snap_storage_failed 2501 Failed to snapshot storage Verify that the ``snap create binary`` is installed and + nodes can be executed by the user running ``fdbserver`` +snap_tlog_failed 2502 Failed to snapshot TLog ,, + nodes +snap_coord_failed 2503 Failed to snapshot ,, + coordinator nodes +unknown_error 4000 An unknown error occurred ,, +snap_disable_tlog_pop_failed 2500 Disk Snapshot error No operator action is needed, retry the operation +snap_enable_tlog_pop_failed 2504 Disk Snapshot error ,, +======================================= ============ ============================= ============================================================= + + +Disk snapshot restore steps +=========================== + +Restore is the process of building up the cluster from the snapshotted disk images. There is no option to specify a restore version because there is no support for continuous backup. Here is the list of steps for the restore process: + +* Identify the snapshot disk images associated with the backup to be restored with the help of UID or creation time +* Group disk images of a backup by IP address and/or locality information +* Bring up a new cluster similar to the source cluster with FoundationDB services stopped and either attach the snapshot disk images or copy the snapshot disk images to the cluster in the following manner: + + * Map the old IP address to new IP address in a one to one fashion and use that mapping to guide the restoration of disk images +* Compute the new fdb.cluster file based on where the new ``coordinators`` disk stores are placed and push it to the all the instances in the new cluster +* Start the FoundationDB service on all the instances +* NOTE: Process can have multiple roles with persistent data which share the same ``datadir``. ``snapshot create binary`` will create multiple snapshots, one per role. In such case, snapshot disk images needs to go through additional processing before restore, if a snapshot image of a role has files that belongs to other roles then they need to be deleted. + +Cluster will start and get to healthy state indicating the completion of restore. Applications can optionally do any additional validations and use the cluster. + + +Example backup and restore steps +================================ + +Here are the backup and restore steps on an over simplified setup with a single node cluster and ``cp`` command to create snapshots and restore. This is purely for illustration, real world backup and restore scripts needs to follow all the steps detailed above. + + +* Create a single node cluster by following the steps here :ref:`building-cluster` + +* Check the status of the cluster and write a few sample keys:: + + fdb> status + + Using cluster file `/mnt/source/fdb.cluster'. + + Configuration: + Redundancy mode - single + Storage engine - ssd-2 + Coordinators - 1 + + Cluster: + FoundationDB processes - 1 + Zones - 1 + Machines - 1 + Memory availability - 30.6 GB per process on machine with least available + Fault Tolerance - 0 machines + Server time - 12/11/19 04:02:57 + + Data: + Replication health - Healthy + Moving data - 0.000 GB + Sum of key-value sizes - 0 MB + Disk space used - 210 MB + + Operating space: + Storage server - 72.6 GB free on most full server + Log server - 72.6 GB free on most full server + + Workload: + Read rate - 9 Hz + Write rate - 0 Hz + Transactions started - 5 Hz + Transactions committed - 0 Hz + Conflict rate - 0 Hz + + Backup and DR: + Running backups - 0 + Running DRs - 0 + + Client time: 12/11/19 04:02:57 + + fdb> writemode on + fdb> set key1 value1 + Committed (76339236) + fdb> set key2 value2 + Committed (80235963) + +* Write a ``snap create binary`` which copies the ``datadir`` to a user passed destination directory location:: + + #!/bin/sh + + while (( "$#" )); do + case "$1" in + --uid) + SNAPUID=$2 + shift 2 + ;; + --path) + DATADIR=$2 + shift 2 + ;; + --role) + ROLE=$2 + shift 2 + ;; + --destdir) + DESTDIR=$2 + shift 2 + ;; + *) + shift + ;; + esac + done + + mkdir -p "$DESTDIR/$SNAPUID/$ROLE" || exit 1 + cp "$DATADIR/"* "$DESTDIR/$SNAPUID/$ROLE/" || exit 1 + + exit 0 + +* Install the ``snap create binary`` as ``/bin/snap_create.sh``, add the entry for ``whitelist_binpath`` in :ref:`foundationdb-conf-fdbserver`, stop and start the foundationdb service for the configuration change to take effect +* Issue ``snapshot`` command as follows:: + + fdb> snapshot /bin/snap_create.sh --destdir /mnt/backup + Snapshot command succeeded with UID 69a5e0576621892f85f55b4ebfeb4312 + +* ``snapshot create binary`` gets invoked once for each role namely ``tlog``, ``storage`` and ``coordinator`` in this process with the following arguments:: + + --path /mnt/source/datadir --version 6.2.6 --role storage --uid 69a5e0576621892f85f55b4ebfeb4312 --destdir /mnt/backup + --path /mnt/source/datadir --version 6.2.6 --role tlog --uid 69a5e0576621892f85f55b4ebfeb4312 --destdir /mnt/backup + --path /mnt/source/datadir --version 6.2.6 --role coord --uid 69a5e0576621892f85f55b4ebfeb4312 --destdir /mnt/backup + +* Snapshot is successful and all the snapshot images are in ``destdir`` specified by the user in the command line argument to ``snapshot`` command, here is a sample directory listing of one of the coordinator backup directory:: + + $ ls /mnt/backup/69a5e0576621892f85f55b4ebfeb4312/coord/ + coordination-0.fdq log2-V_3_LS_2-b9990ae9bc00672f07264ad43d9d0792.sqlite-wal processId + coordination-1.fdq logqueue-V_3_LS_2-b9990ae9bc00672f07264ad43d9d0792-0.fdq storage-f0e72cdfed12a233e0e58291150ca597.sqlite + log2-V_3_LS_2-b9990ae9bc00672f07264ad43d9d0792.sqlite logqueue-V_3_LS_2-b9990ae9bc00672f07264ad43d9d0792-1.fdq storage-f0e72cdfed12a233e0e58291150ca597.sqlite-wal + +* To restore the ``coordinator`` backup image, setup a restore ``datadir`` and copy all the ``coordinator`` related files to it:: + + $ cp /mnt/backup/69a5e0576621892f85f55b4ebfeb4312/coord/coord* /mnt/restore/datadir/ + +* Repeat the above steps to restore ``storage`` and ``tlog`` backup images +* Prepare the ``fdb.cluster`` for the restore with new ``coordinator`` IP address, example:: + + znC1NC5b:iYHJLq7z@10.2.80.40:4500 -> znC1NC5b:iYHJLq7z@10.2.80.41:4500 +* ``foundationdb.conf`` can be exact same copy as the source cluster for this example +* Once all the backup images are restored, start a new fdbserver with the ``datadir`` pointing to ``/mnt/restore/datadir`` and the new ``fdb.cluster``. +* Verify the cluster is healthy and check the sample keys that we added are there:: + + fdb> status + + Using cluster file `/mnt/restore/fdb.cluster'. + + Configuration: + Redundancy mode - single + Storage engine - ssd-2 + Coordinators - 1 + + Cluster: + FoundationDB processes - 1 + Zones - 1 + Machines - 1 + Memory availability - 30.5 GB per process on machine with least available + Fault Tolerance - 0 machines + Server time - 12/11/19 09:04:53 + + Data: + Replication health - Healthy + Moving data - 0.000 GB + Sum of key-value sizes - 0 MB + Disk space used - 210 MB + + Operating space: + Storage server - 72.5 GB free on most full server + Log server - 72.5 GB free on most full server + + Workload: + Read rate - 7 Hz + Write rate - 0 Hz + Transactions started - 3 Hz + Transactions committed - 0 Hz + Conflict rate - 0 Hz + + Backup and DR: + Running backups - 0 + Running DRs - 0 + + Client time: 12/11/19 09:04:53 + + fdb> get key1 + `key1' is `value1' + fdb> get key2 + `key2' is `value2' diff --git a/documentation/sphinx/source/downloads.rst b/documentation/sphinx/source/downloads.rst index 8fe7e31338..65dd0713ef 100644 --- a/documentation/sphinx/source/downloads.rst +++ b/documentation/sphinx/source/downloads.rst @@ -10,38 +10,38 @@ macOS The macOS installation package is supported on macOS 10.7+. It includes the client and (optionally) the server. -* `FoundationDB-6.2.11.pkg `_ +* `FoundationDB-6.2.20.pkg `_ Ubuntu ------ The Ubuntu packages are supported on 64-bit Ubuntu 12.04+, but beware of the Linux kernel bug in Ubuntu 12.x. -* `foundationdb-clients-6.2.11-1_amd64.deb `_ -* `foundationdb-server-6.2.11-1_amd64.deb `_ (depends on the clients package) +* `foundationdb-clients-6.2.20-1_amd64.deb `_ +* `foundationdb-server-6.2.20-1_amd64.deb `_ (depends on the clients package) RHEL/CentOS EL6 --------------- The RHEL/CentOS EL6 packages are supported on 64-bit RHEL/CentOS 6.x. -* `foundationdb-clients-6.2.11-1.el6.x86_64.rpm `_ -* `foundationdb-server-6.2.11-1.el6.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.2.20-1.el6.x86_64.rpm `_ +* `foundationdb-server-6.2.20-1.el6.x86_64.rpm `_ (depends on the clients package) RHEL/CentOS EL7 --------------- The RHEL/CentOS EL7 packages are supported on 64-bit RHEL/CentOS 7.x. -* `foundationdb-clients-6.2.11-1.el7.x86_64.rpm `_ -* `foundationdb-server-6.2.11-1.el7.x86_64.rpm `_ (depends on the clients package) +* `foundationdb-clients-6.2.20-1.el7.x86_64.rpm `_ +* `foundationdb-server-6.2.20-1.el7.x86_64.rpm `_ (depends on the clients package) Windows ------- The Windows installer is supported on 64-bit Windows XP and later. It includes the client and (optionally) the server. -* `foundationdb-6.2.11-x64.msi `_ +* `foundationdb-6.2.20-x64.msi `_ API Language Bindings ===================== @@ -56,20 +56,20 @@ Python 2.7 - 3.5 On macOS and Windows, the FoundationDB Python API bindings are installed as part of your FoundationDB installation. -If you need to use the FoundationDB Python API from other Python installations or paths, download the Python package: +If you need to use the FoundationDB Python API from other Python installations or paths, use the Python package manager ``pip`` (``pip install foundationdb``) or download the Python package: -* `foundationdb-6.2.11.tar.gz `_ +* `foundationdb-6.2.20.tar.gz `_ Ruby 1.9.3/2.0.0+ ----------------- -* `fdb-6.2.11.gem `_ +* `fdb-6.2.20.gem `_ Java 8+ ------- -* `fdb-java-6.2.11.jar `_ -* `fdb-java-6.2.11-javadoc.jar `_ +* `fdb-java-6.2.20.jar `_ +* `fdb-java-6.2.20-javadoc.jar `_ Go 1.11+ -------- diff --git a/documentation/sphinx/source/flow.rst b/documentation/sphinx/source/flow.rst index 0bbc260d22..da6cf1fd4a 100644 --- a/documentation/sphinx/source/flow.rst +++ b/documentation/sphinx/source/flow.rst @@ -99,13 +99,13 @@ To write the equivalent code directly in C++, a developer would have to implemen Caveats ======= -Even though flow-code looks a lot like C++, it is not. It has different rules and the files are preprocessed. It is always important to keep this in mind when programming flow. +Even though Flow code looks a lot like C++, it is not. It has different rules and the files are preprocessed. It is always important to keep this in mind when programming flow. We still want to be able to use IDEs and modern editors (with language servers like cquery or clang-based completion engines like ycm). Because of this there is a header-file ``actorcompiler.h`` in flow which defines preprocessor definitions to make flow compile as normal C++ code. CMake even supports a special mode so that it doesn't preprocess flow files. This mode can be used by passing ``-DOPEN_FOR_IDE=ON`` to cmake. Additionally we generate a special ``compile_commands.json`` into the source-directory which will support opening the project in IDEs and editors that look for a compilation database. -Some preprocessor definitions will not fix all issues though. When programming flow the following things have to be taken care of by the programmer: +Some preprocessor definitions will not fix all issues though. When programming Flow the following things have to be taken care of by the programmer: -- Local variables don't survive a call to ``wait``. So this would be legal flow-code, but NOT legal C++-code: +- Local variables don't survive a call to ``wait``. So this would be legal Flow code, but NOT legal C++ code: .. code-block:: c @@ -133,8 +133,8 @@ Some preprocessor definitions will not fix all issues though. When programming f } } -- An ``ACTOR`` is compiled into a class internally. Which means that within an actor-function, ``this`` is a valid pointer to this class. But using them explicitely (or as described later implicitely) will break IDE support. One can use ``THIS`` and ``THIS_ADDR`` instead. But be careful as ``THIS`` will be of type ``nullptr_t`` in IDE-mode and of the actor-type in normal compilation mode. -- Lambdas and state variables are weird in a sense. After actorcompilation a state variable is a member of the compiled actor class. In IDE mode it is considered a normal local variable. This can result in some surprising side-effects. So the following code will only compile if the method ``Foo::bar`` is defined as ``const``: +- An ``ACTOR`` is compiled into a class internally. Which means that within an actor-function, ``this`` is a valid pointer to this class. But using them explicitly (or as described later implicitly) will break IDE support. One can use ``THIS`` and ``THIS_ADDR`` instead. But be careful as ``THIS`` will be of type ``nullptr_t`` in IDE-mode and of the actor-type in normal compilation mode. +- Lambdas and state variables are weird in a sense. After actor compilation, a state variable is a member of the compiled actor class. In IDE mode it is considered a normal local variable. This can result in some surprising side-effects. So the following code will only compile if the method ``Foo::bar`` is defined as ``const``: .. code-block:: c @@ -144,7 +144,7 @@ Some preprocessor definitions will not fix all issues though. When programming f } - If it is not, one has to pass the member explictely as reference: + If it is not, one has to pass the member explicitly as a reference: .. code-block:: c @@ -154,5 +154,5 @@ Some preprocessor definitions will not fix all issues though. When programming f foo([x]() { x->bar(); }) } -- state variables in flow don't follow the normal scoping rules. So in flow a state variable can be defined in a inner scope and later it can be used in the outer scope. In order to not break compilation in IDE-mode, always define state variables in the outermost scope they will be used. +- state variables in Flow don't follow the normal scoping rules. So in Flow a state variable can be defined in an inner scope and later it can be used in the outer scope. In order to not break compilation in IDE-mode, always define state variables in the outermost scope they will be used. diff --git a/documentation/sphinx/source/getting-started-linux.rst b/documentation/sphinx/source/getting-started-linux.rst index ff6759a3db..faafaf52be 100644 --- a/documentation/sphinx/source/getting-started-linux.rst +++ b/documentation/sphinx/source/getting-started-linux.rst @@ -110,7 +110,7 @@ Managing the FoundationDB service Next steps ========== -* Install the APIs for :doc:`Ruby `, `Java `_, or `Go `_ if you intend to use those languages. :doc:`Python ` and :doc:`C ` APIs were installed along with the ``foundationdb-clients`` package above. +* Install the APIs for :doc:`Ruby `, :doc:`Python `, `Java `_ or `Go `_ if you intend to use those languages. The :doc:`C ` API was installed along with the ``foundationdb-clients`` package above. * See :doc:`tutorials` for samples of developing applications with FoundationDB. * See :doc:`developer-guide` for information of interest to developers, including common design patterns and performance considerations. * See :doc:`administration` for detailed administration information. diff --git a/documentation/sphinx/source/hierarchical-documents-java.rst b/documentation/sphinx/source/hierarchical-documents-java.rst index 137dfa8f1f..c2631e5b36 100644 --- a/documentation/sphinx/source/hierarchical-documents-java.rst +++ b/documentation/sphinx/source/hierarchical-documents-java.rst @@ -69,7 +69,7 @@ Here’s a basic implementation of the recipe. private static final long EMPTY_ARRAY = -1; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); docSpace = new Subspace(Tuple.from("D")); } diff --git a/documentation/sphinx/source/kv-architecture.rst b/documentation/sphinx/source/kv-architecture.rst index d5670b1100..2bd5937cfe 100644 --- a/documentation/sphinx/source/kv-architecture.rst +++ b/documentation/sphinx/source/kv-architecture.rst @@ -20,7 +20,7 @@ The master is responsible for coordinating the transition of the write sub-syste Proxies ======= -The proxies are responsible for providing read versions, committing transactions, and tracking the storage servers responsible for each range of keys. To provide a read version, a proxy will ask all other proxies to see the largest committed version at this point in time, while simultaneously checking that the transaction logs have not been stopped. Ratekeeper will artificially slow down the rate at which the proxy provides read versions. +The proxies are responsible for providing read versions, committing transactions, and tracking the storage servers responsible for each range of keys. To provide a read version, a proxy will ask all other proxies to see the largest committed version at this point in time, while simultaneously checking that the transaction logs have not been stopped. Ratekeeper will artificially slow down the rate at which the proxy provides read versions. Commits are accomplished by: @@ -28,7 +28,7 @@ Commits are accomplished by: * Use the resolvers to determine if the transaction conflicts with previously committed transactions. * Make the transaction durable on the transaction logs. -The key space starting with the '\xff' byte is reserved for system metadata. All mutations committed into this key space are distributed to all of the proxies through the resolvers. This metadata includes a mapping between key ranges and the storage servers which have the data for that range of keys. The proxies provides this information to clients on-demand. The clients cache this mapping; if they ask a storage server for a key it does not have, they will clear their cache and get a more up-to-date list of servers from the proxies. +The key space starting with the '\xff' byte is reserved for system metadata. All mutations committed into this key space are distributed to all of the proxies through the resolvers. This metadata includes a mapping between key ranges and the storage servers which have the data for that range of keys. The proxies provide this information to clients on-demand. The clients cache this mapping; if they ask a storage server for a key it does not have, they will clear their cache and get a more up-to-date list of servers from the proxies. Transaction Logs ================ @@ -43,7 +43,7 @@ The resolvers are responsible determining conflicts between transactions. A tran Storage Servers =============== -The vast majority of processes in a cluster are storage servers. Storage servers are assigned ranges of key, and are responsible to storing all of the data for that range. They keep 5 seconds of mutations in memory, and an on disk copy of the data as of 5 second ago. Clients must read at a version within the last 5 seconds, or they will get a transaction_too_old error. The ssd storage engine stores the data in a b-tree. The memory storage engine store the data in memory with an append only log that is only read from disk if the process is rebooted. +The vast majority of processes in a cluster are storage servers. Storage servers are assigned ranges of keys, and are responsible for storing all of the data for that range. They keep 5 seconds of mutations in memory, and an on disk copy of the data as of 5 second ago. Clients must read at a version within the last 5 seconds, or they will get a transaction_too_old error. The ssd storage engine stores the data in a b-tree. The memory storage engine stores the data in memory with an append only log that is only read from disk if the process is rebooted. Clients ======= diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index c30f9932db..65bd004af7 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -190,6 +190,9 @@ }, "megabits_received":{ "hz":0.0 + }, + "tls_policy_failures":{ + "hz":0.0 } }, "run_loop_busy":0.2 // fraction of time the run loop was busy @@ -364,7 +367,9 @@ "layer_status_incomplete", "database_availability_timeout", "consistencycheck_suspendkey_fetch_timeout", - "consistencycheck_disabled" + "consistencycheck_disabled", + "primary_dc_missing", + "fetch_primary_dc_timeout" ] }, "issues":[ @@ -404,6 +409,7 @@ }, "required_logs":3, "missing_logs":"7f8d623d0cb9966e", + "active_generations":1, "description":"Recovery complete." }, "workload":{ @@ -422,6 +428,16 @@ "hz":0.0, "counter":0, "roughness":0.0 + }, + "location_requests":{ // measures number of outgoing key server location responses + "hz":0.0, + "counter":0, + "roughness":0.0 + }, + "memory_errors":{ // measures number of proxy_memory_limit_exceeded errors + "hz":0.0, + "counter":0, + "roughness":0.0 } }, "bytes":{ // measures number of logical bytes read/written (ignoring replication factor and overhead on disk). Perfectly spaced operations will have a roughness of 1.0. Randomly spaced (Poisson-distributed) operations will have a roughness of 2.0, with increased bunching resulting in increased values. Higher roughness can result in increased latency due to increased queuing. @@ -557,6 +573,7 @@ "auto_proxies":3, "auto_resolvers":1, "auto_logs":3, + "backup_worker_enabled":1, "proxies":5 // this field will be absent if a value has not been explicitly set }, "data":{ @@ -571,6 +588,7 @@ "missing_data", "healing", "optimizing_team_collections", + "healthy_populating_region", "healthy_repartitioning", "healthy_removing_server", "healthy_rebalancing", @@ -605,6 +623,7 @@ "missing_data", "healing", "optimizing_team_collections", + "healthy_populating_region", "healthy_repartitioning", "healthy_removing_server", "healthy_rebalancing", diff --git a/documentation/sphinx/source/mr-status.rst b/documentation/sphinx/source/mr-status.rst index 9e11906e71..6ba4537285 100644 --- a/documentation/sphinx/source/mr-status.rst +++ b/documentation/sphinx/source/mr-status.rst @@ -88,6 +88,8 @@ cluster.messages unreachable_ratekeeper_worker Unab cluster.messages unreachable_processes The cluster has some unreachable processes. cluster.messages unreadable_configuration Unable to read database configuration. cluster.messages layer_status_incomplete Some or all of the layers subdocument could not be read. +cluster.messages primary_dc_missing Unable to determine primary datacenter. +cluster.messages fetch_primary_dc_timeout Fetching primary DC timed out. cluster.processes..messages file_open_error Unable to open ‘’ (). cluster.processes..messages incorrect_cluster_file_contents Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally. cluster.processes..messages io_error occured in diff --git a/documentation/sphinx/source/multimaps-java.rst b/documentation/sphinx/source/multimaps-java.rst index 11febcb738..4ce8e1f3ba 100644 --- a/documentation/sphinx/source/multimaps-java.rst +++ b/documentation/sphinx/source/multimaps-java.rst @@ -74,7 +74,7 @@ Here’s a simple implementation of multimaps with multisets as described: private static final int N = 100; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); multi = new Subspace(Tuple.from("M")); } diff --git a/documentation/sphinx/source/old-release-notes/release-notes-014.rst b/documentation/sphinx/source/old-release-notes/release-notes-014.rst index 1ee2033c2a..4d4dfb04fa 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-014.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-014.rst @@ -1,12 +1,12 @@ -####################### -Release Notes (Alpha 5) -####################### +############# +Release Notes +############# FoundationDB Alpha 5 ==================== Language support -------------------------- +---------------- * FoundationDB now supports :doc:`Ruby ` @@ -17,7 +17,8 @@ Language support .. _alpha-5-rel-notes-features: Features ------------- +-------- + * A new :doc:`backup ` system allows scheduled backups of a snapshot of the FoundationDB database to an external filesystem. * :doc:`Integrated HTML documentation ` diff --git a/documentation/sphinx/source/old-release-notes/release-notes-016.rst b/documentation/sphinx/source/old-release-notes/release-notes-016.rst index 6fb4b0ea2a..8ec7fd1142 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-016.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-016.rst @@ -1,12 +1,12 @@ -####################### -Release Notes (Alpha 6) -####################### +############# +Release Notes +############# FoundationDB Alpha 6 ==================== Platform support -------------------------- +---------------- * FoundationDB now supports both clients and development servers on :doc:`Mac OS X `. @@ -15,7 +15,7 @@ Platform support * All language APIs are supported on Linux, Mac, and Windows (except for Ruby on Windows, because there is not a 64-bit Ruby for Windows.) Features ------------- +-------- * The set of coordination servers can be safely :ref:`changed ` on-the-fly via the CLI. @@ -34,14 +34,14 @@ Features * The database size estimate shown in the CLI status is much more accurate. Performance --------------- +----------- * Improved latency performance for intense workloads with range-read operations. * Improved performance and decreased memory usage for certain intense write workloads targeting a small set of keys (such as sequential insert). Fixes --------- +----- * An incorrect result could be returned by a range read when: (1) The range start was specified using a non-default "less than" type key selector; and (2) the range read started at the beginning of the database; and (3) the transaction also included a prior write to a key less than the key of the begin key selector. @@ -61,7 +61,7 @@ Changes to all APIs * Three new transaction options (:py:meth:`READ_AHEAD_DISABLE `, :py:meth:`READ_YOUR_WRITES_DISABLE `, and :py:meth:`ACCESS_SYSTEM_KEYS `) enable more control for advanced applications. Changes to the Java API ------------------------- +----------------------- * A new construct `AsyncUtil.whileTrue() <../javadoc/com/apple/cie/foundationdb/async/AsyncUtil.html#whileTrue-com.apple.foundationdb.async.Function->`_ simplifies writing loops using the asynchronous version of the Java FDB client. diff --git a/documentation/sphinx/source/old-release-notes/release-notes-021.rst b/documentation/sphinx/source/old-release-notes/release-notes-021.rst index 425452cdf7..f3aab022e6 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-021.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-021.rst @@ -1,6 +1,6 @@ -###################### -Release Notes (Beta 1) -###################### +############# +Release Notes +############# Beta 1 ====== diff --git a/documentation/sphinx/source/old-release-notes/release-notes-022.rst b/documentation/sphinx/source/old-release-notes/release-notes-022.rst index 9b9fa85a47..deb432424c 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-022.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-022.rst @@ -1,6 +1,6 @@ -###################### -Release Notes (Beta 2) -###################### +############# +Release Notes +############# Beta 2 ====== diff --git a/documentation/sphinx/source/old-release-notes/release-notes-023.rst b/documentation/sphinx/source/old-release-notes/release-notes-023.rst index 17008c7635..4f9368e78e 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-023.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-023.rst @@ -1,6 +1,6 @@ -###################### -Release Notes (Beta 3) -###################### +############# +Release Notes +############# Beta 3 ====== diff --git a/documentation/sphinx/source/old-release-notes/release-notes-100.rst b/documentation/sphinx/source/old-release-notes/release-notes-100.rst index 2146c6b67e..e82bf84069 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-100.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-100.rst @@ -1,11 +1,11 @@ -################### -Release Notes (1.0) -################### +############# +Release Notes +############# 1.0.1 ===== - * Fix segmentation fault in client when there are a very large number of dependent operations in a transaction and certain errors occur. +* Fix segmentation fault in client when there are a very large number of dependent operations in a transaction and certain errors occur. 1.0.0 ===== @@ -20,30 +20,35 @@ There are only minor technical differences between this release and the 0.3.0 re Java ---- - * ``clear(Range)`` replaces the now deprecated ``clearRangeStartsWith()``. + +* ``clear(Range)`` replaces the now deprecated ``clearRangeStartsWith()``. Python ------ - * Windows installer supports Python 3. + +* Windows installer supports Python 3. Node and Ruby ------------- - * String option parameters are converted to UTF-8. + +* String option parameters are converted to UTF-8. All --- - * API version changed to 100. Programs with lower versions continue to work. - * Runs on Mac OS X 10.7. - * Improvements to installation packages, including package paths and directory modes. - * Eliminated cases of excessive resource usage in the locality API. - * Watches are disabled when read-your-writes functionality is disabled. - * Fatal error paths now call ``_exit()`` instead instead of ``exit()``. + +* API version updated to 100. See the :ref:`API version upgrade guide ` for upgrade details. +* Runs on Mac OS X 10.7. +* Improvements to installation packages, including package paths and directory modes. +* Eliminated cases of excessive resource usage in the locality API. +* Watches are disabled when read-your-writes functionality is disabled. +* Fatal error paths now call ``_exit()`` instead instead of ``exit()``. Fixes ----- - * A few Python API entry points failed to respect the ``as_foundationdb_key()`` convenience interface. - * ``fdbcli`` could print commit version numbers incorrectly in Windows. - * Multiple watches set on the same key were not correctly triggered by a subsequent write in the same transaction. + +* A few Python API entry points failed to respect the ``as_foundationdb_key()`` convenience interface. +* ``fdbcli`` could print commit version numbers incorrectly in Windows. +* Multiple watches set on the same key were not correctly triggered by a subsequent write in the same transaction. Earlier release notes --------------------- diff --git a/documentation/sphinx/source/old-release-notes/release-notes-200.rst b/documentation/sphinx/source/old-release-notes/release-notes-200.rst index 5d5d465bf6..868aec6c5c 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-200.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-200.rst @@ -1,6 +1,6 @@ -################### -Release Notes (2.0) -################### +############# +Release Notes +############# 2.0.10 ====== @@ -49,12 +49,14 @@ Fixes PHP --- + * Package updated to support PHP 5.4+ (instead of 5.3+). * Fix: ``get_boundary_keys()`` could fail to complete successfully if certain retryable errors were encountered. * Fix: Bindings set error reporting level, which could interfere with clients that used alternate settings. Java ---- + * Fix: Calling ``getRange`` on a ``Transaction`` could leak memory. 2.0.7 @@ -110,10 +112,12 @@ Release 2.0.5 is protocol-compatible with 2.0.0, 2.0.1, 2.0.2, 2.0.3, and 2.0.4. Fixes ----- + * Clients and servers that specified a cluster file as a filename only (without path) could crash when the coordinators were changed. PHP --- + * Directory layer partitions created with the PHP bindings were incompatible with other language bindings. Contact us if you have data stored in a directory partition created by PHP that can't easily be restored and needs to be migrated. 2.0.4 @@ -123,11 +127,13 @@ Release 2.0.4 is protocol-compatible with 2.0.0, 2.0.1, 2.0.2, and 2.0.3. Users Fixes ----- + * Clearing a key larger than the legal limit of 10 kB caused the database to crash and become unreadable. * Explicitly added write conflict ranges were ignored when read-your-writes was disabled. Java ---- + * ``ByteArrayUtil.compareUnsigned()`` failed to return in some circumstances. 2.0.3 @@ -137,6 +143,7 @@ Release 2.0.3 is protocol-compatible with 2.0.0, 2.0.1, and 2.0.2. There are no Fixes ----- + * Updated FDBGnuTLS plugin with GnuTLS 3.2.12, incorporating fixes for `GNUTLS-SA-2014-1 `_ and `GNUTLS-SA-2014-2 `_. * When inserting a large number of keys close to the key size limit, server logs were unexpectedly verbose. @@ -147,6 +154,7 @@ Release 2.0.2 is protocol-compatible with 2.0.0 and 2.0.1. There are no updates Fixes ----- + * Windows: Possible database corruption when the FoundationDB service is stopped but unable to kill its child processes. 2.0.1 @@ -156,6 +164,7 @@ Release 2.0.1 is protocol-compatible with 2.0.0. There are no updates to the lan Fixes ----- + * In some cases, a server reincluded after previous exclusion would not participate in data distribution. * Clients could not reliably connect to multiple clusters. * The calculation of usable disk space on Linux and Mac OS X improperly included space reserved for superuser. @@ -165,24 +174,29 @@ Fixes New language support -------------------- + * `Go <../godoc/fdb.html>`_ * PHP New layers available in all languages ------------------------------------- + * The :ref:`Subspace ` layer provides a recommended way to define subspaces of keys by managing key prefixes. * The :ref:`Directory ` layer provides a tool to manage related subspaces as virtual directories. Recommended as a convenient and high-performance way to organize and layout different kinds of data within a single FoundationDB database. Security -------- + * Added certificate-based :doc:`Transport Layer Security ` to encrypt network traffic. Monitoring ---------- + * The ``fdbcli`` command-line interface reports information and warnings about available memory. Performance ----------- + * Improved client CPU performance overall. * Greatly improved client CPU performance for range-read operations. * Greatly improved concurrency when issuing writes between reads. @@ -192,6 +206,7 @@ Performance Fixes ----- + * In rare cases when many keys very close to the maximum key size are inserted, the database could become unavailable. * ``GetReadVersion`` did not properly throw ``transaction_cancelled`` when called on a transaction that had been cancelled. * When using the ``access_system_keys`` option, a ``get_range_startswith(\xff)`` would incorrectly return no results. @@ -204,14 +219,20 @@ Fixes Other changes ------------- + * To avoid confusing situations, any use of a transaction that is currently committing will cause both the commit and the use to throw a ``used_during_commit`` error. * The ``FDB_CLUSTER_FILE`` environment variable can point to a cluster file that takes precedence over both the current working directory and (e.g., in Linux) ``/etc/foundationdb/fdb.cluster``. * Disabled unloading the ``fdb_c`` library to prevent consequent unavoidable race conditions. * Discontinued testing and support for Ubuntu 11.04. We continue to support Ubuntu 11.10 and later. +Bindings +-------- + +* API version updated to 200. See the :ref:`API version upgrade guide ` for upgrade details. + Java ---- -* Support for API version 200 and backwards compatibility with previous API versions. + * New APIs for allocating and managing keyspace (:ref:`Directory `). * In most cases, exceptions thrown in synchronous-style Java programs will have the original calling line of code in the backtrace. * Native resources are handled in a safer and more efficient manner. @@ -221,7 +242,7 @@ Java Node ---- -* Support for API version 200 and backwards compatibility with previous API versions. + * New APIs for allocating and managing keyspace (:ref:`Directory `). * Support for the Promise/A+ specification with supporting utilities. * Futures can take multiple callbacks. Callbacks can be added if the original function was called with a callback. The Future type is exposed in our binding. @@ -235,7 +256,7 @@ Node Ruby ---- -* Support for API version 200 and backwards compatibility with previous API versions. + * New APIs for allocating and managing keyspace (:ref:`Directory `). * Tuple and subspace range assume the empty tuple if none is passed. * Added ``as_foundationdb_key`` and ``as_foundationdb_value`` support. @@ -245,7 +266,7 @@ Ruby Python ------ -* Support for API version 200 and backwards compatibility with previous API versions. + * New APIs for allocating and managing keyspace (:ref:`Directory `). * Snapshot transactions can be used in retry loops. * Support for gevent 1.0. @@ -255,15 +276,14 @@ Python C - + * Support for API version 200 and backwards compatibility with previous API versions. .NET ---- -* Support for API version 200 and backwards compatibility with previous API versions. + * New APIs for allocating and managing keyspace (:ref:`Directory `). - - Earlier release notes --------------------- * :doc:`1.0 (API Version 100) ` diff --git a/documentation/sphinx/source/old-release-notes/release-notes-300.rst b/documentation/sphinx/source/old-release-notes/release-notes-300.rst index 2fd0fd393c..289ad5c518 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-300.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-300.rst @@ -1,6 +1,6 @@ -################### -Release Notes (3.0) -################### +############# +Release Notes +############# 3.0.8 ===== @@ -13,6 +13,7 @@ Release 3.0.8 is protocol-compatible with all prior 3.0.x releases. All users sh Fixes ----- + * Backup: the backup agent could crash in some circumstances, preventing a backup from completing. * Linux: On some systems, disk space usage tracking could be inaccurate. * In rare cases, range reading could get stuck in an infinite past_version loop. @@ -20,6 +21,7 @@ Fixes Java ---- + * Fix: getBoundaryKeys could throw a NullPointerException. 3.0.7 @@ -32,6 +34,7 @@ Release 3.0.7 is protocol-compatible with all prior 3.0.x releases. All users sh Fixes ----- + * ``fdbcli`` would segmentation fault if there was a semicolon after a quoted string. * :ref:`Atomic operations ` performed on keys that had been :ref:`snapshot read ` would be converted into a set operation. * Reading a key to which an atomic operation had already been applied would cause the read to behave as a snapshot read. @@ -40,6 +43,7 @@ Fixes Ruby ---- + * Fix: ``FDB`` objects could not be garbage collected. 3.0.6 @@ -51,16 +55,19 @@ Release 3.0.6 is protocol-compatible with all prior 3.0.x releases. All users sh Fixes ----- + * Read-latency probes for status incorrectly returned zero. * Commit-latency probe for status included the time to acquire its read version. * Client and server could crash when experiencing problems with network connections. Node.js ------- + * Fix: npm source package did not compile on Mac OS X 10.9 or newer. Windows ------- + * Added registry key during installation. 3.0.5 @@ -72,6 +79,7 @@ Release 3.0.5 is protocol-compatible with all prior 3.0.x releases. This release Fixes ----- + * Windows: fix Visual Studio 2013 code generation bug on older processors or versions of Windows that don't support the AVX instruction set (see https://connect.microsoft.com/VisualStudio/feedback/details/811093). 3.0.4 @@ -83,6 +91,7 @@ Release 3.0.4 is protocol-compatible with all prior 3.0.x releases. Users should Fixes ----- + * Mac OS X: backup agent used 100% CPU even when idle. * Backups were inoperative on databases with greater than 32-bit versions. * Backup agents were not started on Windows. @@ -92,6 +101,7 @@ Fixes Node.js ------- + * Fixed a compilation problem on Linux and Mac OS X as distributed on ``npm``. (Note: The corrected binding is distributed as version 3.0.3.) 3.0.2 @@ -131,7 +141,7 @@ Fixes Client ------ -* Support for API version 300 and backwards compatible with previous API versions. +* API version updated to 300. See the :ref:`API version upgrade guide ` for upgrade details. * By default, :ref:`snapshot reads ` see writes within the same transaction. The previous behavior can be achieved using transaction options. * The :ref:`transaction size limit ` includes conflict ranges. * Explicitly added read or write :ref:`conflict ranges ` and :ref:`watches ` for keys that begin with ``\xFF`` require one of the transaction options ``access_system_keys`` or ``read_system_keys`` to be set. @@ -150,6 +160,7 @@ Java Node.js ------- + * Fix: ``fdb.open``, ``fdb.createCluster``, and ``cluster.openDatabase`` didn't use the callback in API versions 22 or lower. * Tuple performance is improved. diff --git a/documentation/sphinx/source/old-release-notes/release-notes-400.rst b/documentation/sphinx/source/old-release-notes/release-notes-400.rst index cff6ef38c9..93aac7d7dc 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-400.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-400.rst @@ -1,6 +1,6 @@ -################### -Release Notes (4.0) -################### +############# +Release Notes +############# 4.0.2 ===== @@ -41,6 +41,11 @@ Fixes * It was not safe to allocate multiple directories concurrently in the same transaction in the directory layer. +Bindings +-------- + +* API version updated to 400. See the :ref:`API version upgrade guide ` for upgrade details. + Java ---- diff --git a/documentation/sphinx/source/old-release-notes/release-notes-410.rst b/documentation/sphinx/source/old-release-notes/release-notes-410.rst index 776ccf90b0..e68330fed2 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-410.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-410.rst @@ -1,6 +1,6 @@ -################### -Release Notes (4.1) -################### +############# +Release Notes +############# 4.1.1 ===== @@ -40,6 +40,11 @@ Fixes * A rare scenario could cause a crash when a master is recovering metadata from the previous generation of logs. * Streaming mode ``EXACT`` was ignoring the ``target_bytes`` parameter. +Bindings +-------- + +* API version updated to 410. See the :ref:`API version upgrade guide ` for upgrade details. + Earlier release notes --------------------- * :doc:`4.0 (API Version 400) ` diff --git a/documentation/sphinx/source/old-release-notes/release-notes-420.rst b/documentation/sphinx/source/old-release-notes/release-notes-420.rst index 44b526e812..460f7f2ff0 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-420.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-420.rst @@ -1,6 +1,6 @@ -################### -Release Notes (4.2) -################### +############# +Release Notes +############# 4.2.1 ===== @@ -21,6 +21,11 @@ Features * Information on the versions of connected clients has been added to :doc:`Machine-Readable Status `. * Information on the status of running backups has been added to :doc:`Machine-Readable Status `. +Bindings +-------- + +* API version updated to 420. There are no behavior changes in this API version. See the :ref:`API version upgrade guide ` for upgrade details. + Earlier release notes --------------------- * :doc:`4.1 (API Version 410) ` diff --git a/documentation/sphinx/source/old-release-notes/release-notes-430.rst b/documentation/sphinx/source/old-release-notes/release-notes-430.rst index 83f04c5b4b..0d83be2450 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-430.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-430.rst @@ -1,6 +1,6 @@ -################### -Release Notes (4.3) -################### +############# +Release Notes +############# 4.3.0 ===== @@ -22,6 +22,11 @@ Fixes * Changed the blob restore read pattern to work around blob store issues. * External clients do not load environment variable options. +Bindings +-------- + +* API version updated to 430. There are no behavior changes in this API version. See the :ref:`API version upgrade guide ` for upgrade details. + Earlier release notes --------------------- * :doc:`4.2 (API Version 420) ` diff --git a/documentation/sphinx/source/old-release-notes/release-notes-440.rst b/documentation/sphinx/source/old-release-notes/release-notes-440.rst index f0acf88e8e..6f6c7cd993 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-440.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-440.rst @@ -1,6 +1,6 @@ -################### -Release Notes (4.4) -################### +############# +Release Notes +############# 4.4.2 ===== @@ -44,6 +44,11 @@ Fixes * DR errors were not being reported properly in DR status. * Backup and DR layer status expiration and cleanup now use database read version instead of time. +Bindings +-------- + +* API version updated to 440. There are no behavior changes in this API version. See the :ref:`API version upgrade guide ` for upgrade details. + Java ---- diff --git a/documentation/sphinx/source/old-release-notes/release-notes-450.rst b/documentation/sphinx/source/old-release-notes/release-notes-450.rst index 439777f620..e7ab354595 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-450.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-450.rst @@ -1,6 +1,6 @@ -################### -Release Notes (4.5) -################### +############# +Release Notes +############# 4.5.6 ===== @@ -115,6 +115,7 @@ Backup Bindings -------- +* API version updated to 450. There are no behavior changes in this API version. See the :ref:`API version upgrade guide ` for upgrade details. * Add error predicate testing to client bindings. This new functionality should help complex use cases write correct transaction retry loops where dispatching on error classes is needed. Other Changes diff --git a/documentation/sphinx/source/old-release-notes/release-notes-460.rst b/documentation/sphinx/source/old-release-notes/release-notes-460.rst index e4ce8e91f6..217ac141c9 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-460.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-460.rst @@ -107,6 +107,11 @@ Fixes * Java: fix race condition when removing an empty directory which could lead to a NoSuchElementException * Fixed a source of potential crashes in fdbcli +Bindings +-------- + +* API version updated to 460. There are no behavior changes in this API version. See the :ref:`API version upgrade guide ` for upgrade details. + Status ------ diff --git a/documentation/sphinx/source/old-release-notes/release-notes-500.rst b/documentation/sphinx/source/old-release-notes/release-notes-500.rst index 8babcbdaa6..600a8ff727 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-500.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-500.rst @@ -141,7 +141,7 @@ Status Bindings -------- -* API version updated to 500. +* API version updated to 500. See the :ref:`API version upgrade guide ` for upgrade details. * Tuples now support single- and double-precision floating point numbers, UUIDs, booleans, and nested tuples. * Add ``TRANSACTION_LOGGING_ENABLE`` transaction option that causes the details of a transaction's operations to be logged to the client trace logs. * Add ``USED_DURING_COMMIT_PROTECTION_DISABLE`` transaction option that prevents operations performed during that transaction's commit from causing the commit to fail. diff --git a/documentation/sphinx/source/old-release-notes/release-notes-510.rst b/documentation/sphinx/source/old-release-notes/release-notes-510.rst index 51f4b69d15..beff87cc3a 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-510.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-510.rst @@ -159,7 +159,7 @@ Status Bindings -------- -* API version updated to 510. +* API version updated to 510. See the :ref:`API version upgrade guide ` for upgrade details. * Add versionstamp support to the Tuple layer in Java and Python. Java diff --git a/documentation/sphinx/source/old-release-notes/release-notes-520.rst b/documentation/sphinx/source/old-release-notes/release-notes-520.rst index 712e290448..3ea8818aee 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-520.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-520.rst @@ -85,7 +85,7 @@ Status Bindings -------- -* API version updated to 520. +* API version updated to 520. See the :ref:`API version upgrade guide ` for upgrade details. * Java and Python: Versionstamp packing methods within tuple class now add four bytes for the offset instead of two if the API version is set to 520 or higher. `(Issue #148) `_ * Added convenience methods to determine if an API version has been set. `(PR #72) `_ * Go: Reduce memory allocations when packing tuples. `(PR #278) `_ diff --git a/documentation/sphinx/source/old-release-notes/release-notes-600.rst b/documentation/sphinx/source/old-release-notes/release-notes-600.rst index 28700cc075..a934dcae29 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-600.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-600.rst @@ -139,7 +139,7 @@ Status Bindings -------- -* API version updated to 600. There are no changes since API version 520. +* API version updated to 600. See the :ref:`API version upgrade guide ` for upgrade details. * Several cases where functions in go might previously cause a panic now return a non-``nil`` error. `(PR #532) `_ * C API calls made on the network thread could be reordered with calls made from other threads. [6.0.2] `(Issue #518) `_ * The TLS_PLUGIN option is now a no-op and has been deprecated. [6.0.10] `(PR #710) `_ @@ -156,6 +156,7 @@ Other Changes * Does not support upgrades from any version older than 5.0. * Normalized the capitalization of trace event names and attributes. `(PR #455) `_ +* Various stateless processes now have a higher affinity for running on processes with unset process class, which may result in those roles changing location upon upgrade. See :ref:`version-specific-upgrading` for details. `(PR #526) `_ * Increased the memory requirements of the transaction log by 400MB. [6.0.5] `(PR #673) `_ Earlier release notes diff --git a/documentation/sphinx/source/old-release-notes/release-notes-610.rst b/documentation/sphinx/source/old-release-notes/release-notes-610.rst index e473c940b4..6727195240 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-610.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-610.rst @@ -134,7 +134,7 @@ Status Bindings -------- -* API version updated to 610. +* API version updated to 610. See the :ref:`API version upgrade guide ` for upgrade details. * The API to create a database has been simplified across the bindings. All changes are backward compatible with previous API versions, with one exception in Java noted below. `(PR #942) `_ * C: ``FDBCluster`` objects and related methods (``fdb_create_cluster``, ``fdb_cluster_create_database``, ``fdb_cluster_set_option``, ``fdb_cluster_destroy``, ``fdb_future_get_cluster``) have been removed. `(PR #942) `_ * C: Added ``fdb_create_database`` that creates a new ``FDBDatabase`` object synchronously and removed ``fdb_future_get_database``. `(PR #942) `_ diff --git a/documentation/sphinx/source/old-release-notes/release-notes-620.rst b/documentation/sphinx/source/old-release-notes/release-notes-620.rst index e61b014033..e5dca521ce 100644 --- a/documentation/sphinx/source/old-release-notes/release-notes-620.rst +++ b/documentation/sphinx/source/old-release-notes/release-notes-620.rst @@ -2,13 +2,137 @@ Release Notes ############# -6.2.12 +6.2.19 ====== Fixes ----- +* Protect the proxies from running out of memory when bombarded with requests from clients. `(PR #2812) `_. +* One process with a ``proxy`` class would not become the first proxy when put with other ``stateless`` class processes. `(PR #2819) `_. +* If a transaction log stalled on a disk operation during recruitment the cluster would become unavailable until the process died. `(PR #2815) `_. +* Avoid recruiting satellite transaction logs when ``usable_regions=1``. `(PR #2813) `_. +* Prevent the cluster from having too many active generations as a safety measure against repeated failures. `(PR #2814) `_. +* ``fdbcli`` status JSON could become truncated because of unprintable characters. `(PR #2807) `_. +* The data distributor used too much CPU in large clusters (broken in 6.2.16). `(PR #2806) `_. + +Status +------ + +* Added ``cluster.workload.operations.memory_errors`` to measure the number of requests rejected by the proxies because the memory limit has been exceeded. `(PR #2812) `_. +* Added ``cluster.workload.operations.location_requests`` to measure the number of outgoing key server location responses from the proxies. `(PR #2812) `_. +* Added ``cluster.recovery_state.active_generations`` to track the number of generations for which the cluster still requires transaction logs. `(PR #2814) `_. +* Added ``network.tls_policy_failures`` to the ``processes`` section to record the number of TLS policy failures each process has observed. `(PR #2811) `_. + +Features +-------- + +* Added ``--debug-tls`` as a command line argument to ``fdbcli`` to help diagnose TLS issues. `(PR #2810) `_. + +6.2.18 +====== + +Fixes +----- + +* When configuring a cluster to usable_regions=2, data distribution would not react to machine failures while copying data to the remote region. `(PR #2774) `_. +* When a cluster is configured with usable_regions=2, data distribution could push a cluster into saturation by relocating too many shards simulatenously. `(PR #2776) `_. +* Do not allow the cluster controller to mark any process as failed within 30 seconds of startup. `(PR #2780) `_. +* Backup could not establish TLS connections (broken in 6.2.16). `(PR #2775) `_. +* Certificates were not refreshed automatically (broken in 6.2.16). `(PR #2781) `_. + +Performance +----------- + +* Improved the efficiency of establishing large numbers of network connections. `(PR #2777) `_. + +Features +-------- + +* Add support for setting knobs to modify the behavior of ``fdbcli``. `(PR #2773) `_. + +Other Changes +------------- + +* Setting invalid knobs in backup and DR binaries is now a warning instead of an error and will not result in the application being terminated. `(PR #2773) `_. + +6.2.17 +====== + +Fixes +----- + +* Restored the ability to set TLS configuration using environment variables (broken in 6.2.16). `(PR #2755) `_. + +6.2.16 +====== + +Performance +----------- + +* Reduced tail commit latencies by improving commit pipelining on the proxies. `(PR #2589) `_. +* Data distribution does a better job balancing data when disks are more than 70% full. `(PR #2722) `_. +* Reverse range reads could read too much data from disk, resulting in poor performance relative to forward range reads. `(PR #2650) `_. +* Switched from LibreSSL to OpenSSL to improve the speed of establishing connections. `(PR #2650) `_. +* The cluster controller does a better job avoiding multiple recoveries when first recruited. `(PR #2698) `_. + +Fixes +----- + +* Storage servers could fail to advance their version correctly in response to empty commits. `(PR #2617) `_. +* Status could not label more than 5 processes as proxies. `(PR #2653) `_. +* The ``TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER``, ``TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS``, ``TR_FLAG_DISABLE_SERVER_TEAM_REMOVER``, and ``BUGGIFY_ALL_COORDINATION`` knobs could not be set at runtime. `(PR #2661) `_. +* Backup container filename parsing was unnecessarily consulting the local filesystem which will error when permission is denied. `(PR #2693) `_. +* Rebalancing data movement could stop doing work even though the data in the cluster was not well balanced. `(PR #2703) `_. +* Data movement uses available space rather than free space when deciding how full a process is. `(PR #2708) `_. +* Fetching status attempts to reuse its connection with the cluster controller. `(PR #2583) `_. + +6.2.15 +====== + +Fixes +----- + +* TLS throttling could block legitimate connections. `(PR #2575) `_. + +6.2.14 +====== + +Fixes +----- + +* Data distribution was prioritizing shard merges too highly. `(PR #2562) `_. +* Status would incorrectly mark clusters as having no fault tolerance. `(PR #2562) `_. +* A proxy could run out of memory if disconnected from the cluster for too long. `(PR #2562) `_. + +6.2.13 +====== + +Performance +----------- + +* Optimized the commit path the proxies to significantly reduce commit latencies in large clusters. `(PR #2536) `_. +* Data distribution could create temporarily untrackable shards which could not be split if they became hot. `(PR #2546) `_. + +6.2.12 +====== + +Performance +----------- + +* Throttle TLS connect attempts from misconfigured clients. `(PR #2529) `_. +* Reduced master recovery times in large clusters. `(PR #2430) `_. +* Improved performance while a remote region is catching up. `(PR #2527) `_. +* The data distribution algorithm does a better job preventing hot shards while recovering from machine failures. `(PR #2526) `_. + +Fixes +----- + +* Improve the reliability of a ``kill`` command from ``fdbcli``. `(PR #2512) `_. +* The ``--traceclock`` parameter to fdbserver incorrectly had no effect. `(PR #2420) `_. * Clients could throw an internal error during ``commit`` if client buggification was enabled. `(PR #2427) `_. +* Backup and DR agent transactions which update and clean up status had an unnecessarily high conflict rate. `(PR #2483) `_. +* The slow task profiler used an unsafe call to get a timestamp in its signal handler that could lead to rare crashes. `(PR #2515) `_. 6.2.11 ====== @@ -129,6 +253,7 @@ Status Bindings -------- +* API version updated to 620. See the :ref:`API version upgrade guide ` for upgrade details. * Add a transaction size limit as both a database option and a transaction option. `(PR #1725) `_. * Added a new API to get the approximated transaction size before commit, e.g., ``fdb_transaction_get_approximate_size`` in the C binding. `(PR #1756) `_. * C: ``fdb_future_get_version`` has been renamed to ``fdb_future_get_int64``. `(PR #1756) `_. @@ -136,7 +261,7 @@ Bindings * Go: The Go bindings now require Go version 1.11 or later. * Go: Finalizers could run too early leading to undefined behavior. `(PR #1451) `_. * Added a transaction option to control the field length of keys and values in debug transaction logging in order to avoid truncation. `(PR #1844) `_. -* Added a transaction option to control the whether ``get_addresses_for_key`` includes a port in the address. This will be deprecated in api version 700, and addresses will include ports by default. [6.2.4] `(PR #2060) `_. +* Added a transaction option to control the whether ``get_addresses_for_key`` includes a port in the address. This will be deprecated in api version 630, and addresses will include ports by default. [6.2.4] `(PR #2060) `_. * Python: ``Versionstamp`` comparisons didn't work in Python 3. [6.2.4] `(PR #2089) `_. Features @@ -205,4 +330,4 @@ Earlier release notes * :doc:`Beta 2 (API Version 22) ` * :doc:`Beta 1 (API Version 21) ` * :doc:`Alpha 6 (API Version 16) ` -* :doc:`Alpha 5 (API Version 14) ` +* :doc:`Alpha 5 (API Version 14) ` \ No newline at end of file diff --git a/documentation/sphinx/source/operations.rst b/documentation/sphinx/source/operations.rst index 7c5b3628ef..bfdca2b45c 100644 --- a/documentation/sphinx/source/operations.rst +++ b/documentation/sphinx/source/operations.rst @@ -20,6 +20,8 @@ Ready to operate an externally accessible FoundationDB cluster? You'll find what * :doc:`backups` covers the FoundationDB backup tool, which provides an additional level of protection by supporting recovery from disasters or unintentional modification of the database. +* :doc:`disk-snapshot-backup` covers disk snapshot based FoundationDB backup tool, which is an alternate backup solution. + * :doc:`platforms` describes issues on particular platforms that affect the operation of FoundationDB. .. toctree:: @@ -34,4 +36,5 @@ Ready to operate an externally accessible FoundationDB cluster? You'll find what mr-status tls backups + disk-snapshot-backup platforms diff --git a/documentation/sphinx/source/priority-queues-java.rst b/documentation/sphinx/source/priority-queues-java.rst index 50a2b1e037..068349d680 100644 --- a/documentation/sphinx/source/priority-queues-java.rst +++ b/documentation/sphinx/source/priority-queues-java.rst @@ -74,7 +74,7 @@ Here's a basic implementation of the model: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); pq = new Subspace(Tuple.from("P")); diff --git a/documentation/sphinx/source/queues-java.rst b/documentation/sphinx/source/queues-java.rst index 7d39a27fce..1ed636146d 100644 --- a/documentation/sphinx/source/queues-java.rst +++ b/documentation/sphinx/source/queues-java.rst @@ -73,7 +73,7 @@ The following is a simple implementation of the basic pattern: private static final Random randno; static{ - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); queue = new Subspace(Tuple.from("Q")); randno = new Random(); diff --git a/documentation/sphinx/source/release-notes.rst b/documentation/sphinx/source/release-notes.rst index e09e8c7308..72e0df1b5c 100644 --- a/documentation/sphinx/source/release-notes.rst +++ b/documentation/sphinx/source/release-notes.rst @@ -2,12 +2,19 @@ Release Notes ############# -7.0.0 +6.3.0 ===== +Features +-------- +* Improved the slow task profiler to also report backtraces for periods when the run loop is saturated. `(PR #2608) `_ + Performance ----------- +* Improve GRV tail latencies, particularly as the transaction rate gets nearer the ratekeeper limit. `(PR #2735) `_ +* The proxies are now more responsive to changes in workload when unthrottling lower priority transactions. `(PR #2735) `_ + Fixes ----- @@ -17,11 +24,15 @@ Status Bindings -------- +* API version updated to 630. See the :ref:`API version upgrade guide ` for upgrade details. * Java: Introduced ``keyAfter`` utility function that can be used to create the immediate next key for a given byte array. `(PR #2458) `_ +* C: The ``FDBKeyValue`` struct's ``key`` and ``value`` members have changed type from ``void*`` to ``uint8_t*``. `(PR #2622) `_ +* Deprecated ``enable_slow_task_profiling`` transaction option and replaced it with ``enable_run_loop_profiling``. `(PR #2608) `_ Other Changes ------------- * Double the number of shard locations that the client will cache locally. `(PR #2198) `_ +* Add an option for transactions to report conflicting keys by calling getRange with the special key prefix \xff\xff/transaction/conflicting_keys/. `(PR 2257) `_ Earlier release notes --------------------- diff --git a/documentation/sphinx/source/simple-indexes-java.rst b/documentation/sphinx/source/simple-indexes-java.rst index 4bd1281221..709bc4bc7c 100644 --- a/documentation/sphinx/source/simple-indexes-java.rst +++ b/documentation/sphinx/source/simple-indexes-java.rst @@ -87,7 +87,7 @@ In this example, we’re storing user data based on user ID but sometimes need t private static final Subspace index; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); main = new Subspace(Tuple.from("user")); index = new Subspace(Tuple.from("zipcode_index")); diff --git a/documentation/sphinx/source/tables-java.rst b/documentation/sphinx/source/tables-java.rst index 0ca4add535..0f13cebd65 100644 --- a/documentation/sphinx/source/tables-java.rst +++ b/documentation/sphinx/source/tables-java.rst @@ -62,7 +62,7 @@ Here’s a simple implementation of the basic table pattern: private static final Subspace colIndex; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); table = new Subspace(Tuple.from("T")); rowIndex = table.subspace(Tuple.from("R")); diff --git a/documentation/sphinx/source/tls.rst b/documentation/sphinx/source/tls.rst index 02254989cd..bfdac3fc88 100644 --- a/documentation/sphinx/source/tls.rst +++ b/documentation/sphinx/source/tls.rst @@ -128,9 +128,9 @@ Certificate file default location The default behavior when the certificate or key file is not specified is to look for a file named ``fdb.pem`` in the current working directory. If this file is not present, an attempt is made to load a file from a system-dependent location as follows: - * Linux: ``/etc/foundationdb/fdb.pem`` - * macOS: ``/usr/local/etc/foundationdb/fdb.pem`` - * Windows: ``C:\ProgramData\foundationdb\fdb.pem`` +* Linux: ``/etc/foundationdb/fdb.pem`` +* macOS: ``/usr/local/etc/foundationdb/fdb.pem`` +* Windows: ``C:\ProgramData\foundationdb\fdb.pem`` Default Peer Verification ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -138,21 +138,23 @@ Default Peer Verification The default peer verification is ``Check.Valid=1``. Default Password -^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^ There is no default password. If no password is specified, it is assumed that the private key is unencrypted. -Parameters and client bindings ------------------------------- +Permissions +----------- + +All files used by TLS must have sufficient read permissions such that the user running the FoundationDB server or client process can access them. It may also be necessary to have similar read permissions on the parent directories of the files used in the TLS configuration. Automatic TLS certificate refresh --------------------------------- The TLS certificate will be automatically refreshed on a configurable cadence. The server will inspect the CA, certificate, and key files in the specified locations periodically, and will begin using the new versions if following criterion were met: - * They are changed, judging by the last modified time. - * They are valid certificates. - * The key file matches the certificate file. +* They are changed, judging by the last modified time. +* They are valid certificates. +* The key file matches the certificate file. The refresh rate is controlled by ``--knob_tls_cert_refresh_delay_seconds``. Setting it to 0 will disable the refresh. diff --git a/documentation/sphinx/source/vector-java.rst b/documentation/sphinx/source/vector-java.rst index 8b23b16d31..254ca26cc2 100644 --- a/documentation/sphinx/source/vector-java.rst +++ b/documentation/sphinx/source/vector-java.rst @@ -77,7 +77,7 @@ Here’s the basic pattern: private static final Subspace vector; static { - fdb = FDB.selectAPIVersion(620); + fdb = FDB.selectAPIVersion(630); db = fdb.open(); vector = new Subspace(Tuple.from("V")); } diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp index d0be6a3e2b..0d12e1c87d 100644 --- a/documentation/tutorial/tutorial.actor.cpp +++ b/documentation/tutorial/tutorial.actor.cpp @@ -24,6 +24,7 @@ #include "flow/DeterministicRandom.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" +#include "flow/TLSConfig.actor.h" #include #include #include @@ -439,7 +440,7 @@ int main(int argc, char* argv[]) { toRun.push_back(actor->second); } platformInit(); - g_network = newNet2(false, true); + g_network = newNet2(TLSConfig(), false, true); NetworkAddress publicAddress = NetworkAddress::parse("0.0.0.0:0"); if (isServer) { publicAddress = NetworkAddress::parse("0.0.0.0:" + port); diff --git a/fdbbackup/CMakeLists.txt b/fdbbackup/CMakeLists.txt index 830609e409..b9259935f3 100644 --- a/fdbbackup/CMakeLists.txt +++ b/fdbbackup/CMakeLists.txt @@ -4,6 +4,18 @@ set(FDBBACKUP_SRCS add_flow_target(EXECUTABLE NAME fdbbackup SRCS ${FDBBACKUP_SRCS}) target_link_libraries(fdbbackup PRIVATE fdbclient) +set(FDBCONVERT_SRCS + FileConverter.actor.cpp + FileConverter.h) +add_flow_target(EXECUTABLE NAME fdbconvert SRCS ${FDBCONVERT_SRCS}) +target_link_libraries(fdbconvert PRIVATE fdbclient) + +set(FDBDECODE_SRCS + FileDecoder.actor.cpp + FileConverter.h) +add_flow_target(EXECUTABLE NAME fdbdecode SRCS ${FDBDECODE_SRCS}) +target_link_libraries(fdbdecode PRIVATE fdbclient) + if(NOT OPEN_FOR_IDE) fdb_install(TARGETS fdbbackup DESTINATION bin COMPONENT clients) install_symlink( @@ -33,11 +45,11 @@ if(NOT OPEN_FOR_IDE) symlink_files( LOCATION packages/bin SOURCE fdbbackup - TARGETS fdbdr dr_agent backup_agent fdbrestore) + TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_agent) symlink_files( LOCATION bin SOURCE fdbbackup - TARGETS fdbdr dr_agent backup_agent fdbrestore) + TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_agent) endif() if (GPERFTOOLS_FOUND) diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp new file mode 100644 index 0000000000..67f0e3493d --- /dev/null +++ b/fdbbackup/FileConverter.actor.cpp @@ -0,0 +1,590 @@ +/* + * FileConverter.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbbackup/FileConverter.h" + +#include +#include +#include +#include +#include + +#include "fdbclient/BackupAgent.actor.h" +#include "fdbclient/BackupContainer.h" +#include "fdbclient/MutationList.h" +#include "flow/flow.h" +#include "flow/serialize.h" +#include "flow/actorcompiler.h" // has to be last include + +namespace file_converter { + +void printConvertUsage() { + std::cout << "\n" + << " -r, --container Container URL.\n" + << " -b, --begin BEGIN\n" + << " Begin version.\n" + << " -e, --end END End version.\n" + << " --log Enables trace file logging for the CLI session.\n" + << " --logdir PATH Specifes the output directory for trace files. If\n" + << " unspecified, defaults to the current directory. Has\n" + << " no effect unless --log is specified.\n" + << " --loggroup LOG_GROUP\n" + << " Sets the LogGroup field with the specified value for all\n" + << " events in the trace output (defaults to `default').\n" + << " --trace_format FORMAT\n" + << " Select the format of the trace files. xml (the default) and json are supported.\n" + << " Has no effect unless --log is specified.\n" + << " -h, --help Display this help and exit.\n" + << "\n"; + + return; +} + +void printLogFiles(std::string msg, const std::vector& files) { + std::cout << msg << " " << files.size() << " log files\n"; + for (const auto& file : files) { + std::cout << file.toString() << "\n"; + } + std::cout << std::endl; +} + +std::vector getRelevantLogFiles(const std::vector& files, Version begin, Version end) { + std::vector filtered; + for (const auto& file : files) { + if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0 && file.fileSize > 0) { + filtered.push_back(file); + } + } + std::sort(filtered.begin(), filtered.end()); + + // Remove duplicates. This is because backup workers may store the log for + // old epochs successfully, but do not update the progress before another + // recovery happened. As a result, next epoch will retry and creates + // duplicated log files. + std::vector sorted; + int i = 0; + for (int j = 1; j < filtered.size(); j++) { + if (!filtered[i].isSubset(filtered[j])) { + sorted.push_back(filtered[i]); + } + i = j; + } + if (i < filtered.size()) { + sorted.push_back(filtered[i]); + } + + return sorted; +} + +struct ConvertParams { + std::string container_url; + Version begin = invalidVersion; + Version end = invalidVersion; + bool log_enabled = false; + std::string log_dir, trace_format, trace_log_group; + + bool isValid() { return begin != invalidVersion && end != invalidVersion && !container_url.empty(); } + + std::string toString() { + std::string s; + s.append("ContainerURL:"); + s.append(container_url); + s.append(" Begin:"); + s.append(format("%" PRId64, begin)); + s.append(" End:"); + s.append(format("%" PRId64, end)); + if (log_enabled) { + if (!log_dir.empty()) { + s.append(" LogDir:").append(log_dir); + } + if (!trace_format.empty()) { + s.append(" Format:").append(trace_format); + } + if (!trace_log_group.empty()) { + s.append(" LogGroup:").append(trace_log_group); + } + } + return s; + } +}; + +struct VersionedData { + LogMessageVersion version; + StringRef message; // Serialized mutation. + Arena arena; // The arena that contains mutation. + + VersionedData() : version(invalidVersion, -1) {} + VersionedData(LogMessageVersion v, StringRef m, Arena a) : version(v), message(m), arena(a) {} +}; + +struct MutationFilesReadProgress : public ReferenceCounted { + MutationFilesReadProgress(std::vector& logs, Version begin, Version end) + : files(logs), beginVersion(begin), endVersion(end) {} + + struct FileProgress : public ReferenceCounted { + FileProgress(Reference f, int index) : fd(f), idx(index), offset(0), eof(false) {} + + bool operator<(const FileProgress& rhs) const { + if (rhs.mutations.empty()) return true; + if (mutations.empty()) return false; + return mutations[0].version < rhs.mutations[0].version; + } + bool operator<=(const FileProgress& rhs) const { + if (rhs.mutations.empty()) return true; + if (mutations.empty()) return false; + return mutations[0].version <= rhs.mutations[0].version; + } + bool empty() { return eof && mutations.empty(); } + + // Decodes the block into mutations and save them if >= minVersion and < maxVersion. + // Returns true if new mutations has been saved. + bool decodeBlock(const Standalone& buf, int len, Version minVersion, Version maxVersion) { + StringRef block(buf.begin(), len); + StringRefReader reader(block, restore_corrupted_data()); + int count = 0, inserted = 0; + Version msgVersion = invalidVersion; + + try { + // Read block header + if (reader.consume() != PARTITIONED_MLOG_VERSION) throw restore_unsupported_file_version(); + + while (1) { + // If eof reached or first key len bytes is 0xFF then end of block was reached. + if (reader.eof() || *reader.rptr == 0xFF) break; + + // Deserialize messages written in saveMutationsToFile(). + msgVersion = bigEndian64(reader.consume()); + uint32_t sub = bigEndian32(reader.consume()); + int msgSize = bigEndian32(reader.consume()); + const uint8_t* message = reader.consume(msgSize); + + ArenaReader rd(buf.arena(), StringRef(message, msgSize), AssumeVersion(currentProtocolVersion)); + MutationRef m; + rd >> m; + count++; + if (msgVersion >= maxVersion) { + TraceEvent("FileDecodeEnd") + .detail("MaxV", maxVersion) + .detail("Version", msgVersion) + .detail("File", fd->getFilename()); + eof = true; + break; // skip + } + if (msgVersion >= minVersion) { + mutations.emplace_back(LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), + buf.arena()); + inserted++; + } + } + offset += len; + + TraceEvent("Decoded") + .detail("Name", fd->getFilename()) + .detail("Count", count) + .detail("Insert", inserted) + .detail("BlockOffset", reader.rptr - buf.begin()) + .detail("Total", mutations.size()) + .detail("EOF", eof) + .detail("Version", msgVersion) + .detail("NewOffset", offset); + return inserted > 0; + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptLogFileBlock") + .error(e) + .detail("Filename", fd->getFilename()) + .detail("BlockOffset", offset) + .detail("BlockLen", len) + .detail("ErrorRelativeOffset", reader.rptr - buf.begin()) + .detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset); + throw; + } + } + + Reference fd; + int idx; // index in the MutationFilesReadProgress::files vector + int64_t offset; // offset of the file to be read + bool eof; // If EOF is seen so far or endVersion is encountered. If true, the file can't be read further. + std::vector mutations; // Buffered mutations read so far + }; + + bool hasMutations() { + for (const auto& fp : fileProgress) { + if (!fp->empty()) return true; + } + return false; + } + + void dumpProgress(std::string msg) { + std::cout << msg << "\n "; + for (const auto fp : fileProgress) { + std::cout << fp->fd->getFilename() << " " << fp->mutations.size() << " mutations"; + if (fp->mutations.size() > 0) { + std::cout << ", range " << fp->mutations[0].version.toString() << " " + << fp->mutations.back().version.toString() << "\n"; + } else { + std::cout << "\n\n"; + } + } + } + + // Sorts files according to their first mutation version and removes files without mutations. + void sortAndRemoveEmpty() { + std::sort(fileProgress.begin(), fileProgress.end(), + [](const Reference& a, const Reference& b) { return (*a) < (*b); }); + while (!fileProgress.empty() && fileProgress.back()->empty()) { + fileProgress.pop_back(); + } + } + + // Requires hasMutations() return true before calling this function. + // The caller must hold on the the arena associated with the mutation. + Future getNextMutation() { return getMutationImpl(this); } + + ACTOR static Future getMutationImpl(MutationFilesReadProgress* self) { + ASSERT(!self->fileProgress.empty() && !self->fileProgress[0]->mutations.empty()); + + state Reference fp = self->fileProgress[0]; + state VersionedData data = fp->mutations[0]; + fp->mutations.erase(fp->mutations.begin()); + if (fp->mutations.empty()) { + // decode one more block + wait(decodeToVersion(fp, /*version=*/0, self->endVersion, self->getLogFile(fp->idx))); + } + + if (fp->empty()) { + self->fileProgress.erase(self->fileProgress.begin()); + } else { + // Keep fileProgress sorted + for (int i = 1; i < self->fileProgress.size(); i++) { + if (*self->fileProgress[i - 1] <= *self->fileProgress[i]) { + break; + } + std::swap(self->fileProgress[i - 1], self->fileProgress[i]); + } + } + return data; + } + + LogFile& getLogFile(int index) { return files[index]; } + + Future openLogFiles(Reference container) { return openLogFilesImpl(this, container); } + + // Opens log files in the progress and starts decoding until the beginVersion is seen. + ACTOR static Future openLogFilesImpl(MutationFilesReadProgress* progress, + Reference container) { + state std::vector>> asyncFiles; + for (const auto& file : progress->files) { + asyncFiles.push_back(container->readFile(file.fileName)); + } + wait(waitForAll(asyncFiles)); // open all files + + // Attempt decode the first few blocks of log files until beginVersion is consumed + std::vector> fileDecodes; + for (int i = 0; i < asyncFiles.size(); i++) { + Reference fp(new FileProgress(asyncFiles[i].get(), i)); + progress->fileProgress.push_back(fp); + fileDecodes.push_back( + decodeToVersion(fp, progress->beginVersion, progress->endVersion, progress->getLogFile(i))); + } + + wait(waitForAll(fileDecodes)); + + progress->sortAndRemoveEmpty(); + + return Void(); + } + + // Decodes the file until EOF or an mutation >= minVersion and saves these mutations. + // Skip mutations >= maxVersion. + ACTOR static Future decodeToVersion(Reference fp, Version minVersion, Version maxVersion, + LogFile file) { + if (fp->empty()) return Void(); + + if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) return Void(); + + state int64_t len; + try { + // Read block by block until we see the minVersion + loop { + len = std::min(file.blockSize, file.fileSize - fp->offset); + if (len == 0) { + fp->eof = true; + return Void(); + } + + state Standalone buf = makeString(len); + int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset)); + if (len != rLen) throw restore_bad_read(); + + TraceEvent("ReadFile") + .detail("Name", fp->fd->getFilename()) + .detail("Length", rLen) + .detail("Offset", fp->offset); + if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) break; + } + return Void(); + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptedLogFileBlock") + .error(e) + .detail("Filename", fp->fd->getFilename()) + .detail("BlockOffset", fp->offset) + .detail("BlockLen", len); + throw; + } + } + + std::vector files; + const Version beginVersion, endVersion; + std::vector> fileProgress; +}; + +// Writes a log file in the old backup format, described in backup-dataFormat.md. +// This is similar to the LogFileWriter in FileBackupAgent.actor.cpp. +struct LogFileWriter { + LogFileWriter() : blockSize(-1) {} + LogFileWriter(Reference f, int bsize) : file(f), blockSize(bsize) {} + + // Returns the block key, i.e., `Param1`, in the back file. The format is + // `hash_value|commitVersion|part`. + static Standalone getBlockKey(Version commitVersion, int part) { + const int32_t version = commitVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; + + BinaryWriter wr(Unversioned()); + wr << (uint8_t)hashlittle(&version, sizeof(version), 0); + wr << bigEndian64(commitVersion); + wr << bigEndian32(part); + return wr.toValue(); + } + + // Start a new block if needed, then write the key and value + ACTOR static Future writeKV_impl(LogFileWriter* self, Key k, Value v) { + // If key and value do not fit in this block, end it and start a new one + int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size(); + if (self->file->size() + toWrite > self->blockEnd) { + // Write padding if needed + int bytesLeft = self->blockEnd - self->file->size(); + if (bytesLeft > 0) { + state Value paddingFFs = fileBackup::makePadding(bytesLeft); + wait(self->file->append(paddingFFs.begin(), bytesLeft)); + } + + // Set new blockEnd + self->blockEnd += self->blockSize; + + // write Header + wait(self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION))); + } + + wait(self->file->appendStringRefWithLen(k)); + wait(self->file->appendStringRefWithLen(v)); + + // At this point we should be in whatever the current block is or the block size is too small + if (self->file->size() > self->blockEnd) throw backup_bad_block_size(); + + return Void(); + } + + Future writeKV(Key k, Value v) { return writeKV_impl(this, k, v); } + + // Adds a new mutation to an interal buffer and writes out when encountering + // a new commitVersion or exceeding the block size. + ACTOR static Future addMutation(LogFileWriter* self, Version commitVersion, MutationListRef mutations) { + state Standalone value = BinaryWriter::toValue(mutations, IncludeVersion()); + + state int part = 0; + for (; part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE < value.size(); part++) { + StringRef partBuf = value.substr( + part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, + std::min(value.size() - part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, CLIENT_KNOBS->MUTATION_BLOCK_SIZE)); + Standalone key = getBlockKey(commitVersion, part); + wait(writeKV_impl(self, key, partBuf)); + } + return Void(); + } + +private: + Reference file; + int blockSize; + int64_t blockEnd = 0; +}; + +ACTOR Future convert(ConvertParams params) { + state Reference container = IBackupContainer::openContainer(params.container_url); + state BackupFileList listing = wait(container->dumpFileList()); + std::sort(listing.logs.begin(), listing.logs.end()); + TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size()); + state BackupDescription desc = wait(container->describeBackup()); + std::cout << "\n" << desc.toString() << "\n"; + + // std::cout << "Using Protocol Version: 0x" << std::hex << currentProtocolVersion.version() << std::dec << "\n"; + + std::vector logs = getRelevantLogFiles(listing.logs, params.begin, params.end); + printLogFiles("Range has", logs); + + state Reference progress(new MutationFilesReadProgress(logs, params.begin, params.end)); + + wait(progress->openLogFiles(container)); + + state int blockSize = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE; + state Reference outFile = wait(container->writeLogFile(params.begin, params.end, blockSize)); + state LogFileWriter logFile(outFile, blockSize); + std::cout << "Output file: " << outFile->getFileName() << "\n"; + + state MutationList list; + state Arena arena; + state Version version = invalidVersion; + while (progress->hasMutations()) { + state VersionedData data = wait(progress->getNextMutation()); + + // emit a mutation batch to file when encounter a new version + if (list.totalSize() > 0 && version != data.version.version) { + wait(LogFileWriter::addMutation(&logFile, version, list)); + list = MutationList(); + arena = Arena(); + } + + ArenaReader rd(data.arena, data.message, AssumeVersion(currentProtocolVersion)); + MutationRef m; + rd >> m; + std::cout << data.version.toString() << " m = " << m.toString() << "\n"; + list.push_back_deep(arena, m); + version = data.version.version; + } + if (list.totalSize() > 0) { + wait(LogFileWriter::addMutation(&logFile, version, list)); + } + + wait(outFile->finish()); + + return Void(); +} + +int parseCommandLine(ConvertParams* param, CSimpleOpt* args) { + while (args->Next()) { + auto lastError = args->LastError(); + switch (lastError) { + case SO_SUCCESS: + break; + + default: + std::cerr << "ERROR: argument given for option: " << args->OptionText() << "\n"; + return FDB_EXIT_ERROR; + break; + } + + int optId = args->OptionId(); + const char* arg = args->OptionArg(); + switch (optId) { + case OPT_HELP: + printConvertUsage(); + return FDB_EXIT_ERROR; + + case OPT_BEGIN_VERSION: + if (!sscanf(arg, "%" SCNd64, ¶m->begin)) { + std::cerr << "ERROR: could not parse begin version " << arg << "\n"; + printConvertUsage(); + return FDB_EXIT_ERROR; + } + break; + + case OPT_END_VERSION: + if (!sscanf(arg, "%" SCNd64, ¶m->end)) { + std::cerr << "ERROR: could not parse end version " << arg << "\n"; + printConvertUsage(); + return FDB_EXIT_ERROR; + } + break; + + case OPT_CONTAINER: + param->container_url = args->OptionArg(); + break; + + case OPT_TRACE: + param->log_enabled = true; + break; + + case OPT_TRACE_DIR: + param->log_dir = args->OptionArg(); + break; + + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args->OptionArg())) { + std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; + return FDB_EXIT_ERROR; + } + param->trace_format = args->OptionArg(); + break; + + case OPT_TRACE_LOG_GROUP: + param->trace_log_group = args->OptionArg(); + break; + } + } + return FDB_EXIT_SUCCESS; +} + +} // namespace file_converter + +int main(int argc, char** argv) { + try { + CSimpleOpt* args = new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT); + file_converter::ConvertParams param; + int status = file_converter::parseCommandLine(¶m, args); + std::cout << "Params: " << param.toString() << "\n"; + if (status != FDB_EXIT_SUCCESS || !param.isValid()) { + file_converter::printConvertUsage(); + return status; + } + + if (param.log_enabled) { + if (param.log_dir.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); + } else { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param.log_dir)); + } + if (!param.trace_format.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format)); + } + if (!param.trace_log_group.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group)); + } + } + + platformInit(); + Error::init(); + + StringRef url(param.container_url); + setupNetwork(0, true); + + TraceEvent::setNetworkThread(); + openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "convert", param.trace_log_group); + + auto f = stopAfter(convert(param)); + + runNetwork(); + return status; + } catch (Error& e) { + fprintf(stderr, "ERROR: %s\n", e.what()); + return FDB_EXIT_ERROR; + } catch (std::exception& e) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + return FDB_EXIT_MAIN_EXCEPTION; + } +} diff --git a/fdbbackup/FileConverter.h b/fdbbackup/FileConverter.h new file mode 100644 index 0000000000..e01566b889 --- /dev/null +++ b/fdbbackup/FileConverter.h @@ -0,0 +1,64 @@ +/* + * FileConverter.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FDBBACKUP_FILECONVERTER_H +#define FDBBACKUP_FILECONVERTER_H +#pragma once + +#include +#include "flow/SimpleOpt.h" + +namespace file_converter { + +// File format convertion constants +enum { + OPT_CONTAINER, + OPT_BEGIN_VERSION, + OPT_CRASHONERROR, + OPT_END_VERSION, + OPT_TRACE, + OPT_TRACE_DIR, + OPT_TRACE_FORMAT, + OPT_TRACE_LOG_GROUP, + OPT_INPUT_FILE, + OPT_HELP +}; + +CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP }, + { OPT_CONTAINER, "--container", SO_REQ_SEP }, + { OPT_BEGIN_VERSION, "-b", SO_REQ_SEP }, + { OPT_BEGIN_VERSION, "--begin", SO_REQ_SEP }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_END_VERSION, "-e", SO_REQ_SEP }, + { OPT_END_VERSION, "--end", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_INPUT_FILE, "-i", SO_REQ_SEP }, + { OPT_INPUT_FILE, "--input", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + SO_END_OF_OPTIONS }; + +} // namespace file_converter + +#endif // FDBBACKUP_FILECONVERTER_H diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp new file mode 100644 index 0000000000..f7de6d70ff --- /dev/null +++ b/fdbbackup/FileDecoder.actor.cpp @@ -0,0 +1,514 @@ +/* + * FileDecoder.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include "fdbclient/BackupAgent.actor.h" +#include "fdbclient/BackupContainer.h" +#include "fdbbackup/FileConverter.h" +#include "fdbclient/MutationList.h" +#include "flow/flow.h" +#include "flow/serialize.h" +#include "flow/actorcompiler.h" // has to be last include + +#define SevDecodeInfo SevVerbose + +extern bool g_crashOnError; + +namespace file_converter { + +void printDecodeUsage() { + std::cout << "\n" + " -r, --container Container URL.\n" + " -i, --input FILE Log file to be decoded.\n" + " --crash Crash on serious error.\n" + "\n"; + return; +} + +struct DecodeParams { + std::string container_url; + std::string file; + bool log_enabled = false; + std::string log_dir, trace_format, trace_log_group; + + std::string toString() { + std::string s; + s.append("ContainerURL: "); + s.append(container_url); + s.append(", File: "); + s.append(file); + if (log_enabled) { + if (!log_dir.empty()) { + s.append(" LogDir:").append(log_dir); + } + if (!trace_format.empty()) { + s.append(" Format:").append(trace_format); + } + if (!trace_log_group.empty()) { + s.append(" LogGroup:").append(trace_log_group); + } + } + return s; + } +}; + +int parseDecodeCommandLine(DecodeParams* param, CSimpleOpt* args) { + while (args->Next()) { + auto lastError = args->LastError(); + switch (lastError) { + case SO_SUCCESS: + break; + + default: + std::cerr << "ERROR: argument given for option: " << args->OptionText() << "\n"; + return FDB_EXIT_ERROR; + break; + } + int optId = args->OptionId(); + switch (optId) { + case OPT_HELP: + printDecodeUsage(); + return FDB_EXIT_ERROR; + + case OPT_CONTAINER: + param->container_url = args->OptionArg(); + break; + + case OPT_CRASHONERROR: + g_crashOnError = true; + break; + + case OPT_INPUT_FILE: + param->file = args->OptionArg(); + break; + + case OPT_TRACE: + param->log_enabled = true; + break; + + case OPT_TRACE_DIR: + param->log_dir = args->OptionArg(); + break; + + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args->OptionArg())) { + std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; + return FDB_EXIT_ERROR; + } + param->trace_format = args->OptionArg(); + break; + + case OPT_TRACE_LOG_GROUP: + param->trace_log_group = args->OptionArg(); + break; + } + } + return FDB_EXIT_SUCCESS; +} + +void printLogFiles(std::string msg, const std::vector& files) { + std::cout << msg << " " << files.size() << " log files\n"; + for (const auto& file : files) { + std::cout << file.toString() << "\n"; + } + std::cout << std::endl; +} + +std::vector getRelevantLogFiles(const std::vector& files, const DecodeParams& params) { + std::vector filtered; + for (const auto& file : files) { + if (file.fileName.find(params.file) != std::string::npos) { + filtered.push_back(file); + } + } + return filtered; +} + +std::pair decode_key(const StringRef& key) { + ASSERT(key.size() == sizeof(uint8_t) + sizeof(Version) + sizeof(int32_t)); + + uint8_t hash; + Version version; + int32_t part; + BinaryReader rd(key, Unversioned()); + rd >> hash >> version >> part; + version = bigEndian64(version); + part = bigEndian32(part); + + int32_t v = version / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; + ASSERT(((uint8_t)hashlittle(&v, sizeof(v), 0)) == hash); + + return std::make_pair(version, part); +} + +// Decodes an encoded list of mutations in the format of: +// [includeVersion:uint64_t][val_length:uint32_t][mutation_1][mutation_2]...[mutation_k], +// where a mutation is encoded as: +// [type:uint32_t][keyLength:uint32_t][valueLength:uint32_t][key][value] +std::vector decode_value(const StringRef& value) { + StringRefReader reader(value, restore_corrupted_data()); + + reader.consume(); // Consume the includeVersion + uint32_t val_length = reader.consume(); + if (val_length != value.size() - sizeof(uint64_t) - sizeof(uint32_t)) { + TraceEvent(SevError, "ValueError") + .detail("ValueLen", val_length) + .detail("ValueSize", value.size()) + .detail("Value", printable(value)); + } + + std::vector mutations; + while (1) { + if (reader.eof()) break; + + // Deserialization of a MutationRef, which was packed by MutationListRef::push_back_deep() + uint32_t type, p1len, p2len; + type = reader.consume(); + p1len = reader.consume(); + p2len = reader.consume(); + + const uint8_t* key = reader.consume(p1len); + const uint8_t* val = reader.consume(p2len); + + mutations.emplace_back((MutationRef::Type)type, StringRef(key, p1len), StringRef(val, p2len)); + } + return mutations; +} + +struct VersionedMutations { + Version version; + std::vector mutations; + Arena arena; // The arena that contains the mutations. +}; + +/* + * Model a decoding progress for a mutation file. Usage is: + * + * DecodeProgress progress(logfile); + * wait(progress->openFile(container)); + * while (!progress->finished()) { + * VersionedMutations m = wait(progress->getNextBatch()); + * ... + * } + * + * Internally, the decoding process is done block by block -- each block is + * decoded into a list of key/value pairs, which are then decoded into batches + * of mutations. Because a version's mutations can be split into many key/value + * pairs, the decoding of mutation batch needs to look ahead one more pair. So + * at any time this object might have two blocks of data in memory. + */ +struct DecodeProgress { + DecodeProgress() = default; + DecodeProgress(const LogFile& file, std::vector> values) + : file(file), keyValues(values) {} + + // If there are no more mutations to pull from the file. + // However, we could have unfinished version in the buffer when EOF is true, + // which means we should look for data in the next file. The caller + // should call getUnfinishedBuffer() to get these left data. + bool finished() { return (eof && keyValues.empty()) || (leftover && !keyValues.empty()); } + + std::vector>&& getUnfinishedBuffer() { return std::move(keyValues); } + + // Returns all mutations of the next version in a batch. + Future getNextBatch() { return getNextBatchImpl(this); } + + Future openFile(Reference container) { return openFileImpl(this, container); } + + // The following are private APIs: + + // Returns true if value contains complete data. + bool isValueComplete(StringRef value) { + StringRefReader reader(value, restore_corrupted_data()); + + reader.consume(); // Consume the includeVersion + uint32_t val_length = reader.consume(); + return val_length == value.size() - sizeof(uint64_t) - sizeof(uint32_t); + } + + // PRECONDITION: finished() must return false before calling this function. + // Returns the next batch of mutations along with the arena backing it. + // Note the returned batch can be empty when the file has unfinished + // version batch data that are in the next file. + ACTOR static Future getNextBatchImpl(DecodeProgress* self) { + ASSERT(!self->finished()); + + loop { + if (self->keyValues.size() <= 1) { + // Try to decode another block when less than one left + wait(readAndDecodeFile(self)); + } + + auto& tuple = self->keyValues[0]; + ASSERT(std::get<2>(tuple) == 0); // first part number must be 0. + + // decode next versions, check if they are continuous parts + int idx = 1; // next kv pair in "keyValues" + int bufSize = std::get<3>(tuple).size(); + for (int lastPart = 0; idx < self->keyValues.size(); idx++, lastPart++) { + if (idx == self->keyValues.size()) break; + + auto next_tuple = self->keyValues[idx]; + if (std::get<1>(tuple) != std::get<1>(next_tuple)) { + break; + } + + if (lastPart + 1 != std::get<2>(next_tuple)) { + TraceEvent("DecodeError").detail("Part1", lastPart).detail("Part2", std::get<2>(next_tuple)); + throw restore_corrupted_data(); + } + bufSize += std::get<3>(next_tuple).size(); + } + + VersionedMutations m; + m.version = std::get<1>(tuple); + TraceEvent("Decode").detail("Version", m.version).detail("Idx", idx).detail("Q", self->keyValues.size()); + StringRef value = std::get<3>(tuple); + if (idx > 1) { + // Stitch parts into one and then decode one by one + Standalone buf = self->combineValues(idx, bufSize); + value = buf; + m.arena = buf.arena(); + } + if (self->isValueComplete(value)) { + m.mutations = decode_value(value); + if (m.arena.getSize() == 0) { + m.arena = std::get<0>(tuple); + } + self->keyValues.erase(self->keyValues.begin(), self->keyValues.begin() + idx); + return m; + } else if (!self->eof) { + // Read one more block, hopefully the missing part of the value can be found. + wait(readAndDecodeFile(self)); + } else { + TraceEvent(SevWarn, "MissingValue").detail("Version", m.version); + self->leftover = true; + return m; // Empty mutations + } + } + } + + // Returns a buffer which stitches first "idx" values into one. + // "len" MUST equal the summation of these values. + Standalone combineValues(const int idx, const int len) { + ASSERT(idx <= keyValues.size() && idx > 1); + + Standalone buf = makeString(len); + int n = 0; + for (int i = 0; i < idx; i++) { + const auto& value = std::get<3>(keyValues[i]); + memcpy(mutateString(buf) + n, value.begin(), value.size()); + n += value.size(); + } + + ASSERT(n == len); + return buf; + } + + // Decodes a block into KeyValueRef stored in "keyValues". + void decode_block(const Standalone& buf, int len) { + StringRef block(buf.begin(), len); + StringRefReader reader(block, restore_corrupted_data()); + + try { + // Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION + if (reader.consume() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version(); + + // Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte. + while (1) { + // If eof reached or first key len bytes is 0xFF then end of block was reached. + if (reader.eof() || *reader.rptr == 0xFF) break; + + // Read key and value. If anything throws then there is a problem. + uint32_t kLen = reader.consumeNetworkUInt32(); + const uint8_t* k = reader.consume(kLen); + std::pair version_part = decode_key(StringRef(k, kLen)); + uint32_t vLen = reader.consumeNetworkUInt32(); + const uint8_t* v = reader.consume(vLen); + TraceEvent(SevDecodeInfo, "Block") + .detail("KeySize", kLen) + .detail("valueSize", vLen) + .detail("Offset", reader.rptr - buf.begin()) + .detail("Version", version_part.first) + .detail("Part", version_part.second); + keyValues.emplace_back(buf.arena(), version_part.first, version_part.second, StringRef(v, vLen)); + } + + // Make sure any remaining bytes in the block are 0xFF + for (auto b : reader.remainder()) { + if (b != 0xFF) throw restore_corrupted_data_padding(); + } + + // The (version, part) in a block can be out of order, i.e., (3, 0) + // can be followed by (4, 0), and then (3, 1). So we need to sort them + // first by version, and then by part number. + std::sort(keyValues.begin(), keyValues.end(), + [](const std::tuple& a, + const std::tuple& b) { + return std::get<1>(a) == std::get<1>(b) ? std::get<2>(a) < std::get<2>(b) + : std::get<1>(a) < std::get<1>(b); + }); + return; + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptBlock").error(e).detail("Offset", reader.rptr - buf.begin()); + throw; + } + } + + ACTOR static Future openFileImpl(DecodeProgress* self, Reference container) { + Reference fd = wait(container->readFile(self->file.fileName)); + self->fd = fd; + wait(readAndDecodeFile(self)); + return Void(); + } + + // Reads a file block, decodes it into key/value pairs, and stores these pairs. + ACTOR static Future readAndDecodeFile(DecodeProgress* self) { + try { + state int64_t len = std::min(self->file.blockSize, self->file.fileSize - self->offset); + if (len == 0) { + self->eof = true; + return Void(); + } + + state Standalone buf = makeString(len); + state int rLen = wait(self->fd->read(mutateString(buf), len, self->offset)); + TraceEvent("ReadFile") + .detail("Name", self->file.fileName) + .detail("Len", rLen) + .detail("Offset", self->offset); + if (rLen != len) { + throw restore_corrupted_data(); + } + self->decode_block(buf, rLen); + self->offset += rLen; + return Void(); + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptLogFileBlock") + .error(e) + .detail("Filename", self->file.fileName) + .detail("BlockOffset", self->offset) + .detail("BlockLen", self->file.blockSize); + throw; + } + } + + LogFile file; + Reference fd; + int64_t offset = 0; + bool eof = false; + bool leftover = false; // Done but has unfinished version batch data left + // A (version, part_number)'s mutations and memory arena. + std::vector> keyValues; +}; + +ACTOR Future decode_logs(DecodeParams params) { + state Reference container = IBackupContainer::openContainer(params.container_url); + + state BackupFileList listing = wait(container->dumpFileList()); + // remove partitioned logs + listing.logs.erase(std::remove_if(listing.logs.begin(), listing.logs.end(), + [](const LogFile& file) { + std::string prefix("plogs/"); + return file.fileName.substr(0, prefix.size()) == prefix; + }), + listing.logs.end()); + std::sort(listing.logs.begin(), listing.logs.end()); + TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size()); + + BackupDescription desc = wait(container->describeBackup()); + std::cout << "\n" << desc.toString() << "\n"; + + state std::vector logs = getRelevantLogFiles(listing.logs, params); + printLogFiles("Relevant files are: ", logs); + + state int i = 0; + // Previous file's unfinished version data + state std::vector> left; + for (; i < logs.size(); i++) { + if (logs[i].fileSize == 0) continue; + + state DecodeProgress progress(logs[i], left); + wait(progress.openFile(container)); + while (!progress.finished()) { + VersionedMutations vms = wait(progress.getNextBatch()); + for (const auto& m : vms.mutations) { + std::cout << vms.version << " " << m.toString() << "\n"; + } + } + left = progress.getUnfinishedBuffer(); + if (!left.empty()) { + TraceEvent("UnfinishedFile").detail("File", logs[i].fileName).detail("Q", left.size()); + } + } + return Void(); +} + +} // namespace file_converter + +int main(int argc, char** argv) { + try { + CSimpleOpt* args = new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT); + file_converter::DecodeParams param; + int status = file_converter::parseDecodeCommandLine(¶m, args); + std::cout << "Params: " << param.toString() << "\n"; + if (status != FDB_EXIT_SUCCESS) { + file_converter::printDecodeUsage(); + return status; + } + + if (param.log_enabled) { + if (param.log_dir.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); + } else { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param.log_dir)); + } + if (!param.trace_format.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format)); + } + if (!param.trace_log_group.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group)); + } + } + + platformInit(); + Error::init(); + + StringRef url(param.container_url); + setupNetwork(0, true); + + TraceEvent::setNetworkThread(); + openTraceFile(NetworkAddress(), 10 << 20, 10 << 20, param.log_dir, "decode", param.trace_log_group); + + auto f = stopAfter(decode_logs(param)); + + runNetwork(); + return status; + } catch (Error& e) { + fprintf(stderr, "ERROR: %s\n", e.what()); + return FDB_EXIT_ERROR; + } catch (std::exception& e) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + return FDB_EXIT_MAIN_EXCEPTION; + } +} diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 38e7d0fb73..60622981b4 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -26,6 +26,7 @@ #include "flow/serialize.h" #include "flow/IRandom.h" #include "flow/genericactors.actor.h" +#include "flow/TLSConfig.actor.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/BackupAgent.actor.h" @@ -36,8 +37,7 @@ #include "fdbclient/BlobStore.h" #include "fdbclient/json_spirit/json_spirit_writer_template.h" -#include "fdbrpc/Platform.h" -#include "fdbrpc/TLSConnection.h" +#include "flow/Platform.h" #include #include @@ -103,6 +103,7 @@ enum { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, OPT_EXPIRE_MIN_RESTORABLE_DAYS, OPT_BASEURL, OPT_BLOB_CREDENTIALS, OPT_DESCRIBE_DEEP, OPT_DESCRIBE_TIMESTAMPS, OPT_DUMP_BEGIN, OPT_DUMP_END, OPT_JSON, OPT_DELETE_DATA, OPT_MIN_CLEANUP_SECONDS, + OPT_USE_PARTITIONED_LOG, // Backup and Restore constants OPT_TAGNAME, OPT_BACKUPKEYS, OPT_WAITFORDONE, @@ -169,6 +170,8 @@ CSimpleOpt::SOption g_rgBackupStartOptions[] = { { OPT_NOSTOPWHENDONE, "--no-stop-when-done",SO_NONE }, { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_USE_PARTITIONED_LOG, "-p", SO_NONE }, + { OPT_USE_PARTITIONED_LOG, "--partitioned_log", SO_NONE }, { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, { OPT_SNAPSHOTINTERVAL, "--snapshot_interval", SO_REQ_SEP }, { OPT_TAGNAME, "-t", SO_REQ_SEP }, @@ -953,6 +956,7 @@ static void printBackupUsage(bool devhelp) { printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); printf(" -k KEYS List of key ranges to backup.\n" " If not specified, the entire database will be backed up.\n"); + printf(" -p, --partitioned_log Starts with new type of backup system using partitioned logs.\n"); printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); printf(" --log Enables trace file logging for the CLI session.\n" " --logdir PATH Specifes the output directory for trace files. If\n" @@ -1342,7 +1346,7 @@ enumDBType getDBType(std::string dbType) return enBackupType; } -ACTOR Future getLayerStatus(Reference tr, std::string name, std::string id, enumProgramExe exe, Database dest) { +ACTOR Future getLayerStatus(Reference tr, std::string name, std::string id, enumProgramExe exe, Database dest, bool snapshot = false) { // This process will write a document that looks like this: // { backup : { $expires : {}, version: } // so that the value under 'backup' will eventually expire to null and thus be ignored by @@ -1393,28 +1397,28 @@ ACTOR Future getLayerStatus(Reference tr totalBlobStats.create(p.first + ".$sum") = p.second; state FileBackupAgent fba; - state std::vector backupTags = wait(getAllBackupTags(tr)); + state std::vector backupTags = wait(getAllBackupTags(tr, snapshot)); state std::vector> tagLastRestorableVersions; state std::vector> tagStates; state std::vector>> tagContainers; state std::vector> tagRangeBytes; state std::vector> tagLogBytes; - state Future> fBackupPaused = tr->get(fba.taskBucket->getPauseKey()); + state Future> fBackupPaused = tr->get(fba.taskBucket->getPauseKey(), snapshot); tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state std::vector::iterator tag; state std::vector backupTagUids; for (tag = backupTags.begin(); tag != backupTags.end(); tag++) { - UidAndAbortedFlagT uidAndAbortedFlag = wait(tag->getOrThrow(tr)); + UidAndAbortedFlagT uidAndAbortedFlag = wait(tag->getOrThrow(tr, snapshot)); BackupConfig config(uidAndAbortedFlag.first); backupTagUids.push_back(config.getUid()); - tagStates.push_back(config.stateEnum().getOrThrow(tr)); - tagRangeBytes.push_back(config.rangeBytesWritten().getD(tr, false, 0)); - tagLogBytes.push_back(config.logBytesWritten().getD(tr, false, 0)); - tagContainers.push_back(config.backupContainer().getOrThrow(tr)); - tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName))); + tagStates.push_back(config.stateEnum().getOrThrow(tr, snapshot)); + tagRangeBytes.push_back(config.rangeBytesWritten().getD(tr, snapshot, 0)); + tagLogBytes.push_back(config.logBytesWritten().getD(tr, snapshot, 0)); + tagContainers.push_back(config.backupContainer().getOrThrow(tr, snapshot)); + tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName), snapshot)); } wait( waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); @@ -1451,21 +1455,21 @@ ACTOR Future getLayerStatus(Reference tr state Reference tr2(new ReadYourWritesTransaction(dest)); tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr2->setOption(FDBTransactionOptions::LOCK_AWARE); - state Standalone tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000)); + state Standalone tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000, snapshot)); state std::vector>> backupVersion; state std::vector> backupStatus; state std::vector> tagRangeBytesDR; state std::vector> tagLogBytesDR; - state Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey()); + state Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot); state std::vector drTagUids; for(int i = 0; i < tagNames.size(); i++) { - backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin))); + backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin), snapshot)); UID tagUID = BinaryReader::fromStringRef(tagNames[i].value, Unversioned()); drTagUids.push_back(tagUID); - backupStatus.push_back(dba.getStateValue(tr2, tagUID)); - tagRangeBytesDR.push_back(dba.getRangeBytesWritten(tr2, tagUID)); - tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID)); + backupStatus.push_back(dba.getStateValue(tr2, tagUID, snapshot)); + tagRangeBytesDR.push_back(dba.getRangeBytesWritten(tr2, tagUID, snapshot)); + tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID, snapshot)); } wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && waitForAll(tagLogBytesDR) && success(fDRPaused)); @@ -1618,7 +1622,7 @@ ACTOR Future statusUpdateActor(Database statusUpdateDest, std::string name try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state Future futureStatusDoc = getLayerStatus(tr, name, id, exe, taskDest); + state Future futureStatusDoc = getLayerStatus(tr, name, id, exe, taskDest, true); wait(cleanupStatus(tr, rootKey, name, id)); std::string statusdoc = wait(futureStatusDoc); tr->set(instanceKey, statusdoc); @@ -1744,9 +1748,10 @@ ACTOR Future submitDBBackup(Database src, Database dest, Standalone submitBackup(Database db, std::string url, int snapshotIntervalSeconds, Standalone> backupRanges, std::string tagName, bool dryRun, bool waitForCompletion, bool stopWhenDone) { - try - { +ACTOR Future submitBackup(Database db, std::string url, int snapshotIntervalSeconds, + Standalone> backupRanges, std::string tagName, bool dryRun, + bool waitForCompletion, bool stopWhenDone, bool usePartitionedLog) { + try { state FileBackupAgent backupAgent; // Backup everything, if no ranges were specified @@ -1789,7 +1794,8 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv } else { - wait(backupAgent.submitBackup(db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone)); + wait(backupAgent.submitBackup(db, KeyRef(url), snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, + usePartitionedLog)); // Wait for the backup to complete, if requested if (waitForCompletion) { @@ -1811,8 +1817,7 @@ ACTOR Future submitBackup(Database db, std::string url, int snapshotInterv } } } - } - catch (Error& e) { + } catch (Error& e) { if(e.code() == error_code_actor_cancelled) throw; switch (e.code()) @@ -2046,8 +2051,8 @@ ACTOR Future discontinueBackup(Database db, std::string tagName, bool wait ACTOR Future changeBackupResumed(Database db, bool pause) { try { - state FileBackupAgent backupAgent; - wait(backupAgent.taskBucket->changePause(db, pause)); + FileBackupAgent backupAgent; + wait(backupAgent.changePause(db, pause)); printf("All backup agents have been %s.\n", pause ? "paused" : "resumed"); } catch (Error& e) { @@ -2187,23 +2192,25 @@ ACTOR Future runRestore(Database db, std::string originalClusterFile, std: // Fast restore agent that kicks off the restore: send restore requests to restore workers. ACTOR Future runFastRestoreAgent(Database db, std::string tagName, std::string container, Standalone> ranges, Version dbVersion, - bool performRestore, bool verbose, bool waitForDone, std::string addPrefix, - std::string removePrefix) { + bool performRestore, bool verbose, bool waitForDone) { try { state FileBackupAgent backupAgent; state Version restoreVersion = invalidVersion; if (ranges.size() > 1) { - fprintf(stderr, "Currently only a single restore range is supported!\n"); - throw restore_error(); + fprintf(stdout, "[WARNING] Currently only a single restore range is tested!\n"); } - state KeyRange range = (ranges.size() == 0) ? normalKeys : ranges.front(); + if (ranges.size() == 0) { + ranges.push_back(ranges.arena(), normalKeys); + } - printf("[INFO] runFastRestoreAgent: num_ranges:%d restore_range:%s\n", ranges.size(), range.toString().c_str()); + printf("[INFO] runFastRestoreAgent: restore_ranges:%d first range:%s\n", ranges.size(), + ranges.front().toString().c_str()); if (performRestore) { if (dbVersion == invalidVersion) { + TraceEvent("FastRestoreAgent").detail("TargetRestoreVersion", "Largest restorable version"); BackupDescription desc = wait(IBackupContainer::openContainer(container)->describeBackup()); if (!desc.maxRestorableVersion.present()) { fprintf(stderr, "The specified backup is not restorable to any version.\n"); @@ -2211,10 +2218,28 @@ ACTOR Future runFastRestoreAgent(Database db, std::string tagName, std::st } dbVersion = desc.maxRestorableVersion.get(); + TraceEvent("FastRestoreAgent").detail("TargetRestoreVersion", dbVersion); } - Version _restoreVersion = wait(fastRestore(db, KeyRef(tagName), KeyRef(container), waitForDone, dbVersion, - verbose, range, KeyRef(addPrefix), KeyRef(removePrefix))); - restoreVersion = _restoreVersion; + state UID randomUID = deterministicRandom()->randomUniqueID(); + TraceEvent("FastRestoreAgent") + .detail("SubmitRestoreRequests", ranges.size()) + .detail("RestoreUID", randomUID); + wait(backupAgent.submitParallelRestore(db, KeyRef(tagName), ranges, KeyRef(container), dbVersion, true, + randomUID)); + if (waitForDone) { + // Wait for parallel restore to finish and unlock DB after that + TraceEvent("FastRestoreAgent").detail("BackupAndParallelRestore", "WaitForRestoreToFinish"); + wait(backupAgent.parallelRestoreFinish(db, randomUID)); + TraceEvent("FastRestoreAgent").detail("BackupAndParallelRestore", "RestoreFinished"); + } else { + TraceEvent("FastRestoreAgent") + .detail("RestoreUID", randomUID) + .detail("OperationGuide", "Manually unlock DB when restore finishes"); + printf("WARNING: DB will be in locked state after restore. Need UID:%s to unlock DB\n", + randomUID.toString().c_str()); + } + + restoreVersion = dbVersion; } else { state Reference bc = IBackupContainer::openContainer(container); state BackupDescription description = wait(bc->describeBackup()); @@ -2906,6 +2931,7 @@ int main(int argc, char* argv[]) { std::string restoreTimestamp; bool waitForDone = false; bool stopWhenDone = true; + bool usePartitionedLog = false; // Set to true to use new backup system bool forceAction = false; bool trace = false; bool quietDisplay = false; @@ -3151,6 +3177,9 @@ int main(int argc, char* argv[]) { case OPT_NOSTOPWHENDONE: stopWhenDone = false; break; + case OPT_USE_PARTITIONED_LOG: + usePartitionedLog = true; + break; case OPT_RESTORECONTAINER: restoreContainer = args->OptionArg(); // If the url starts with '/' then prepend "file://" for backwards compatibility @@ -3223,22 +3252,22 @@ int main(int argc, char* argv[]) { blobCredentials.push_back(args->OptionArg()); break; #ifndef TLS_DISABLED - case TLSOptions::OPT_TLS_PLUGIN: + case TLSConfig::OPT_TLS_PLUGIN: args->OptionArg(); break; - case TLSOptions::OPT_TLS_CERTIFICATES: + case TLSConfig::OPT_TLS_CERTIFICATES: tlsCertPath = args->OptionArg(); break; - case TLSOptions::OPT_TLS_PASSWORD: + case TLSConfig::OPT_TLS_PASSWORD: tlsPassword = args->OptionArg(); break; - case TLSOptions::OPT_TLS_CA_FILE: + case TLSConfig::OPT_TLS_CA_FILE: tlsCAPath = args->OptionArg(); break; - case TLSOptions::OPT_TLS_KEY: + case TLSConfig::OPT_TLS_KEY: tlsKeyPath = args->OptionArg(); break; - case TLSOptions::OPT_TLS_VERIFY_PEERS: + case TLSConfig::OPT_TLS_VERIFY_PEERS: tlsVerifyPeers = args->OptionArg(); break; #endif @@ -3343,11 +3372,11 @@ int main(int argc, char* argv[]) { } delete FLOW_KNOBS; - FlowKnobs* flowKnobs = new FlowKnobs(true); + FlowKnobs* flowKnobs = new FlowKnobs; FLOW_KNOBS = flowKnobs; delete CLIENT_KNOBS; - ClientKnobs* clientKnobs = new ClientKnobs(true); + ClientKnobs* clientKnobs = new ClientKnobs; CLIENT_KNOBS = clientKnobs; for(auto k=knobs.begin(); k!=knobs.end(); ++k) { @@ -3355,18 +3384,26 @@ int main(int argc, char* argv[]) { if (!flowKnobs->setKnob( k->first, k->second ) && !clientKnobs->setKnob( k->first, k->second )) { - fprintf(stderr, "Unrecognized knob option '%s'\n", k->first.c_str()); - return FDB_EXIT_ERROR; + fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str()); + TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first)); } } catch (Error& e) { if (e.code() == error_code_invalid_option_value) { - fprintf(stderr, "Invalid value '%s' for option '%s'\n", k->second.c_str(), k->first.c_str()); - return FDB_EXIT_ERROR; + fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", k->second.c_str(), k->first.c_str()); + TraceEvent(SevWarnAlways, "InvalidKnobValue").detail("Knob", printable(k->first)).detail("Value", printable(k->second)); + } + else { + fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what()); + TraceEvent(SevError, "FailedToSetKnob").detail("Knob", printable(k->first)).detail("Value", printable(k->second)).error(e); + throw; } - throw; } } + // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs + flowKnobs->initialize(true); + clientKnobs->initialize(true); + if (trace) { if(!traceLogGroup.empty()) setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup)); @@ -3558,7 +3595,8 @@ int main(int argc, char* argv[]) { return FDB_EXIT_ERROR; // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually writeable. openBackupContainer(argv[0], destinationContainer); - f = stopAfter( submitBackup(db, destinationContainer, snapshotIntervalSeconds, backupKeys, tagName, dryRun, waitForDone, stopWhenDone) ); + f = stopAfter(submitBackup(db, destinationContainer, snapshotIntervalSeconds, backupKeys, tagName, + dryRun, waitForDone, stopWhenDone, usePartitionedLog)); break; } @@ -3720,7 +3758,7 @@ int main(int argc, char* argv[]) { switch (restoreType) { case RESTORE_START: f = stopAfter(runFastRestoreAgent(db, tagName, restoreContainer, backupKeys, restoreVersion, !dryRun, - !quietDisplay, waitForDone, addPrefix, removePrefix)); + !quietDisplay, waitForDone)); break; case RESTORE_WAIT: printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n"); @@ -3853,6 +3891,13 @@ int main(int argc, char* argv[]) { } catch (Error& e) { TraceEvent(SevError, "MainError").error(e); status = FDB_EXIT_MAIN_ERROR; + } catch (boost::system::system_error& e) { + if (g_network) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + } else { + fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code().value()); + } + status = FDB_EXIT_MAIN_EXCEPTION; } catch (std::exception& e) { TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); status = FDB_EXIT_MAIN_EXCEPTION; @@ -3860,100 +3905,3 @@ int main(int argc, char* argv[]) { flushAndExit(status); } - -//------Restore Agent: Kick off the restore by sending the restore requests -ACTOR static Future waitFastRestore(Database cx, Key tagName, bool verbose) { - // We should wait on all restore to finish before proceeds - TraceEvent("FastRestore").detail("Progress", "WaitForRestoreToFinish"); - state ReadYourWritesTransaction tr(cx); - state Future watchForRestoreRequestDone; - state bool restoreRequestDone = false; - - loop { - try { - tr.reset(); - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - // In case restoreRequestDoneKey is already set before we set watch on it - Optional restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey)); - if (restoreRequestDoneKeyValue.present()) { - restoreRequestDone = true; - tr.clear(restoreRequestDoneKey); - wait(tr.commit()); - break; - } else { - watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey); - wait(tr.commit()); - } - // The clear transaction may fail in uncertain state, which may already clear the restoreRequestDoneKey - if (restoreRequestDone) break; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - - TraceEvent("FastRestore").detail("Progress", "RestoreFinished"); - - return FileBackupAgent::ERestoreState::COMPLETED; -} - -ACTOR static Future _fastRestore(Database cx, Key tagName, Key url, bool waitForComplete, - Version targetVersion, bool verbose, KeyRange range, Key addPrefix, - Key removePrefix) { - state Reference bc = IBackupContainer::openContainer(url.toString()); - state BackupDescription desc = wait(bc->describeBackup()); - wait(desc.resolveVersionTimes(cx)); - - if (targetVersion == invalidVersion && desc.maxRestorableVersion.present()) - targetVersion = desc.maxRestorableVersion.get(); - - Optional restoreSet = wait(bc->getRestoreSet(targetVersion)); - TraceEvent("FastRestore").detail("BackupDesc", desc.toString()).detail("TargetVersion", targetVersion); - - if (!restoreSet.present()) { - TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible") - .detail("BackupContainer", bc->getURL()) - .detail("TargetVersion", targetVersion); - throw restore_invalid_version(); - } - - // NOTE: The restore agent makes sure we only support 1 restore range for each restore request for now! - // The simulation test did test restoring multiple restore ranges in one restore request though. - state Reference tr(new ReadYourWritesTransaction(cx)); - state int restoreIndex = 0; - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - Standalone restoreTag(tagName.toString() + "_" + std::to_string(restoreIndex)); - bool locked = true; - struct RestoreRequest restoreRequest(restoreIndex, restoreTag, KeyRef(bc->getURL()), true, targetVersion, - true, range, Key(), Key(), locked, - deterministicRandom()->randomUniqueID()); - tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest)); - // backupRanges.size = 1 because we only support restoring 1 range in real mode for now - tr->set(restoreRequestTriggerKey, restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(),1)); - wait(tr->commit()); // Trigger fast restore - break; - } catch (Error& e) { - if (e.code() != error_code_restore_duplicate_tag) { - wait(tr->onError(e)); - } - } - } - - if (waitForComplete) { - FileBackupAgent::ERestoreState finalState = wait(waitFastRestore(cx, tagName, verbose)); - if (finalState != FileBackupAgent::ERestoreState::COMPLETED) throw restore_error(); - } - - return targetVersion; -} - -ACTOR Future fastRestore(Database cx, Standalone tagName, Standalone url, - bool waitForComplete, long targetVersion, bool verbose, Standalone range, - Standalone addPrefix, Standalone removePrefix) { - Version result = - wait(_fastRestore(cx, tagName, url, waitForComplete, targetVersion, verbose, range, addPrefix, removePrefix)); - return result; -} diff --git a/fdbbackup/fdbbackup.vcxproj b/fdbbackup/fdbbackup.vcxproj deleted file mode 100644 index e936c24ec8..0000000000 --- a/fdbbackup/fdbbackup.vcxproj +++ /dev/null @@ -1,137 +0,0 @@ - - - - - -PRERELEASE - - - - - FDB_CLEAN_BUILD;%(PreprocessorDefinitions) - - - - Debug - X64 - - - Release - X64 - - - - - - - {8E959DA5-5925-45CE-BFC4-C84EB632A29B} - v4.5 - Win32Proj - flow - - - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - $(IntDir)\$(MSBuildProjectName).log - - - - Application - MultiByte - v141 - - - Application - MultiByte - v141 - - - - - - - - - - true - $(IncludePath);../;C:\Program Files\boost_1_67_0 - - - false - $(IncludePath);../;C:\Program Files\boost_1_67_0 - PreBuildEvent - - - - $(TargetDir)fdbclient.lib - - - FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions) - stdcpp17 - - - - - - - Level3 - false - ProgramDatabase - Disabled - EnableFastChecks - MultiThreadedDebug - TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - %(AdditionalIncludeDirectories) - true - /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - Console - true - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;Advapi32.lib - - - - - Level3 - - - ProgramDatabase - Full - MultiThreaded - true - TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - %(AdditionalIncludeDirectories) - NotSet - false - /bigobj @../flow/no_intellisense.opt %(AdditionalOptions) - true - Speed - false - false - stdcpp17 - - - Console - true - false - false - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;Advapi32.lib - /LTCG %(AdditionalOptions) - - - - - - - - - - - - - - - - - diff --git a/fdbbackup/local.mk b/fdbbackup/local.mk deleted file mode 100644 index 1c717db8c0..0000000000 --- a/fdbbackup/local.mk +++ /dev/null @@ -1,58 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdbbackup_CFLAGS := $(fdbclient_CFLAGS) -fdbbackup_LDFLAGS := $(fdbrpc_LDFLAGS) -fdbbackup_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(FDB_TLS_LIB) -fdbbackup_STATIC_LIBS := $(TLS_LIBS) - -ifeq ($(PLATFORM),linux) - fdbbackup_LDFLAGS += -static-libstdc++ -static-libgcc -ldl -lpthread -lrt - - # GPerfTools profiler (uncomment to use) - # fdbbackup_CFLAGS += -I/opt/gperftools/include -DUSE_GPERFTOOLS=1 -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free - # fdbbackup_LDFLAGS += -L/opt/gperftools/lib - # fdbbackup_STATIC_LIBS += -ltcmalloc -lunwind -lprofiler -else ifeq ($(PLATFORM),osx) - fdbbackup_LDFLAGS += -lc++ -endif - -fdbbackup_GENERATED_SOURCES += versions.h - -#ifeq ($(WORKLOADS),false) -# fdbbackup_ALL_SOURCES := $(filter-out fdbbackup/workloads/%,$(fdbbackup_ALL_SOURCES)) -# fdbbackup_BUILD_SOURCES := $(filter-out fdbbackup/workloads/%,$(fdbbackup_BUILD_SOURCES)) -#endif - -bin/fdbbackup: bin/coverage.fdbbackup.xml - -bin/fdbbackup.debug: bin/fdbbackup - -BACKUP_ALIASES = fdbrestore fdbdr dr_agent backup_agent - -$(addprefix bin/, $(BACKUP_ALIASES)): bin/fdbbackup - @[ -f $@ ] || (echo "SymLinking $@" && ln -s fdbbackup $@) - -$(addprefix bin/, $(addsuffix .debug, $(BACKUP_ALIASES))): bin/fdbbackup.debug - @[ -f $@ ] || (echo "SymLinking $@" && ln -s fdbbackup.debug $@) - -FORCE: diff --git a/fdbcli/FlowLineNoise.actor.cpp b/fdbcli/FlowLineNoise.actor.cpp index 85ae2c0bfb..6c101ca666 100644 --- a/fdbcli/FlowLineNoise.actor.cpp +++ b/fdbcli/FlowLineNoise.actor.cpp @@ -113,7 +113,7 @@ LineNoise::LineNoise( for( auto const& c : completions ) linenoiseAddCompletion( lc, c.c_str() ); }); - /*linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> const char* { + linenoiseSetHintsCallback( [](const char* line, int* color, int*bold) -> char* { Hint h = onMainThread( [line]() -> Future { return hint_callback(line); }).getBlocking(); @@ -122,7 +122,7 @@ LineNoise::LineNoise( *bold = h.bold; return strdup( h.text.c_str() ); }); - linenoiseSetFreeHintsCallback( free );*/ + linenoiseSetFreeHintsCallback( free ); #endif threadPool->addThread(reader); diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 3807ab7cf9..bd2ac38913 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -32,9 +32,9 @@ #include "fdbclient/FDBOptions.g.h" #include "flow/DeterministicRandom.h" -#include "fdbrpc/TLSConnection.h" -#include "fdbrpc/Platform.h" +#include "flow/Platform.h" +#include "flow/TLSConfig.actor.h" #include "flow/SimpleOpt.h" #include "fdbcli/FlowLineNoise.h" @@ -67,9 +67,12 @@ enum { OPT_TIMEOUT, OPT_EXEC, OPT_NO_STATUS, + OPT_NO_HINTS, OPT_STATUS_FROM_JSON, OPT_VERSION, - OPT_TRACE_FORMAT + OPT_TRACE_FORMAT, + OPT_KNOB, + OPT_DEBUG_TLS }; CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, @@ -80,6 +83,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, { OPT_TIMEOUT, "--timeout", SO_REQ_SEP }, { OPT_EXEC, "--exec", SO_REQ_SEP }, { OPT_NO_STATUS, "--no-status", SO_NONE }, + { OPT_NO_HINTS, "--no-hints", SO_NONE }, { OPT_HELP, "-?", SO_NONE }, { OPT_HELP, "-h", SO_NONE }, { OPT_HELP, "--help", SO_NONE }, @@ -87,12 +91,14 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, { OPT_VERSION, "--version", SO_NONE }, { OPT_VERSION, "-v", SO_NONE }, { OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP }, + { OPT_KNOB, "--knob_", SO_REQ_SEP }, + { OPT_DEBUG_TLS, "--debug-tls", SO_NONE }, #ifndef TLS_DISABLED TLS_OPTION_FLAGS #endif - SO_END_OF_OPTIONS }; + SO_END_OF_OPTIONS }; void printAtCol(const char* text, int col) { const char* iter = text; @@ -423,6 +429,10 @@ static void printProgramUsage(const char* name) { #ifndef TLS_DISABLED TLS_HELP #endif + " --knob_KNOBNAME KNOBVALUE\n" + " Changes a knob option. KNOBNAME should be lowercase.\n" + " --debug-tls Prints the TLS configuration and certificate chain, then exits.\n" + " Useful in reporting and diagnosing TLS issues.\n" " -v, --version Print FoundationDB CLI version information and exit.\n" " -h, --help Display this help and exit.\n"); } @@ -460,7 +470,7 @@ void initHelp() { "clear a range of keys from the database", "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); helpMap["configure"] = CommandHelp( - "configure [new] |logs=|resolvers=>*", + "configure [new] |logs=|resolvers=>*", "change the database configuration", "The `new' option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When used, both a redundancy mode and a storage engine must be specified.\n\nRedundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. Must be at least 1, or set to -1 which restores the number of proxies to the default value.\n\nlogs=: Sets the desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\nSee the FoundationDB Administration Guide for more information."); helpMap["fileconfigure"] = CommandHelp( @@ -489,7 +499,7 @@ void initHelp() { "change the class of a process", "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to `default' resets the process class to the class specified on the command line."); helpMap["status"] = CommandHelp( - "status [minimal] [details] [json]", + "status [minimal|details|json]", "get the status of a FoundationDB cluster", "If the cluster is down, this command will print a diagnostic which may be useful in figuring out what is wrong. If the cluster is running, this command will print cluster statistics.\n\nSpecifying 'minimal' will provide a minimal description of the status of your database.\n\nSpecifying 'details' will provide load information for individual workers.\n\nSpecifying 'json' will provide status information in a machine readable JSON format."); helpMap["exit"] = CommandHelp("exit", "exit the CLI", ""); @@ -512,6 +522,14 @@ void initHelp() { "getrangekeys [ENDKEY] [LIMIT]", "fetch keys in a range of keys", "Displays up to LIMIT keys for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); + helpMap["getversion"] = + CommandHelp("getversion", "Fetch the current read version", + "Displays the current read version of the database or currently running transaction."); + helpMap["advanceversion"] = CommandHelp( + "advanceversion ", "Force the cluster to recover at the specified version", + "Forces the cluster to recover at the specified version. If the specified version is larger than the current " + "version of the cluster, the cluster version is advanced " + "to the specified version via a forced recovery."); helpMap["reset"] = CommandHelp( "reset", "reset the current transaction", @@ -540,7 +558,7 @@ void initHelp() { "attempts to kill one or more processes in the cluster", "If no addresses are specified, populates the list of processes which can be killed. Processes cannot be killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is unresponsive.\n\nFor each IP:port pair in
*, attempt to kill the specified process."); helpMap["profile"] = CommandHelp( - " ", + "profile ", "namespace for all the profiling-related commands.", "Different types support different actions. Run `profile` to get a list of types, and iteratively explore the help.\n"); helpMap["force_recovery_with_data_loss"] = CommandHelp( @@ -555,6 +573,14 @@ void initHelp() { "consistencycheck [on|off]", "permits or prevents consistency checking", "Calling this command with `on' permits consistency check processes to run and `off' will halt their checking. Calling this command with no arguments will display if consistency checking is currently allowed.\n"); + helpMap["lock"] = CommandHelp( + "lock", + "lock the database with a randomly generated lockUID", + "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database."); + helpMap["unlock"] = + CommandHelp("unlock ", "unlock the database with the provided lockUID", + "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the " + "user will be asked to enter a passphrase to confirm their intent."); hiddenCommands.insert("expensive_data_check"); hiddenCommands.insert("datadistribution"); @@ -923,7 +949,11 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, StatusObjectReader statusObjConfig; StatusArray excludedServersArr; + Optional activePrimaryDC; + if (statusObjCluster.has("active_primary_dc")) { + activePrimaryDC = statusObjCluster["active_primary_dc"].get_str(); + } if (statusObjCluster.get("configuration", statusObjConfig)) { if (statusObjConfig.has("excluded_servers")) excludedServersArr = statusObjConfig.last().get_array(); @@ -979,6 +1009,73 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, if (statusObjConfig.get("log_routers", intVal)) outputString += format("\n Desired Log Routers - %d", intVal); + + outputString += "\n Usable Regions - "; + if (statusObjConfig.get("usable_regions", intVal)) { + outputString += std::to_string(intVal); + } else { + outputString += "unknown"; + } + + StatusArray regions; + if (statusObjConfig.has("regions")) { + outputString += "\n Regions: "; + regions = statusObjConfig["regions"].get_array(); + bool isPrimary = false; + std::vector regionSatelliteDCs; + std::string regionDC; + for (StatusObjectReader region : regions) { + for (StatusObjectReader dc : region["datacenters"].get_array()) { + if (!dc.has("satellite")) { + regionDC = dc["id"].get_str(); + if (activePrimaryDC.present() && dc["id"].get_str() == activePrimaryDC.get()) { + isPrimary = true; + } + } else if (dc["satellite"].get_int() == 1) { + regionSatelliteDCs.push_back(dc["id"].get_str()); + } + } + if (activePrimaryDC.present()) { + if (isPrimary) { + outputString += "\n Primary -"; + } else { + outputString += "\n Remote -"; + } + } else { + outputString += "\n Region -"; + } + outputString += format("\n Datacenter - %s", regionDC.c_str()); + if (regionSatelliteDCs.size() > 0) { + outputString += "\n Satellite datacenters - "; + for (int i = 0; i < regionSatelliteDCs.size(); i++) { + if (i != regionSatelliteDCs.size() - 1) { + outputString += format("%s, ", regionSatelliteDCs[i].c_str()); + } else { + outputString += format("%s", regionSatelliteDCs[i].c_str()); + } + } + } + isPrimary = false; + if (region.get("satellite_redundancy_mode", strVal)) { + outputString += format("\n Satellite Redundancy Mode - %s", strVal.c_str()); + } + if (region.get("satellite_anti_quorum", intVal)) { + outputString += format("\n Satellite Anti Quorum - %d", intVal); + } + if (region.get("satellite_logs", intVal)) { + outputString += format("\n Satellite Logs - %d", intVal); + } + if (region.get("satellite_log_policy", strVal)) { + outputString += format("\n Satellite Log Policy - %s", strVal.c_str()); + } + if (region.get("satellite_log_replicas", intVal)) { + outputString += format("\n Satellite Log Replicas - %d", intVal); + } + if (region.get("satellite_usable_dcs", intVal)) { + outputString += format("\n Satellite Usable DCs - %d", intVal); + } + } + } } catch (std::runtime_error& ) { outputString = outputStringCache; @@ -1383,7 +1480,7 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level, NetworkAddress parsedAddress; try { parsedAddress = NetworkAddress::parse(address); - } catch (Error& e) { + } catch (Error&) { // Groups all invalid IP address/port pair in the end of this detail group. line = format(" %-22s (invalid IP address or port)", address.c_str()); IPAddress::IPAddressStore maxIp; @@ -1602,9 +1699,9 @@ ACTOR Future timeWarning( double when, const char* msg ) { return Void(); } -ACTOR Future checkStatus(Future f, Reference clusterFile, bool displayDatabaseAvailable = true) { +ACTOR Future checkStatus(Future f, Database db, bool displayDatabaseAvailable = true) { wait(f); - StatusObject s = wait(StatusClient::statusFetcher(clusterFile)); + StatusObject s = wait(StatusClient::statusFetcher(db)); printf("\n"); printStatus(s, StatusClient::MINIMAL, displayDatabaseAvailable); printf("\n"); @@ -1646,7 +1743,7 @@ ACTOR Future configure( Database db, std::vector tokens, Refere state Optional conf; if( tokens[startToken] == LiteralStringRef("auto") ) { - StatusObject s = wait( makeInterruptable(StatusClient::statusFetcher( ccf )) ); + StatusObject s = wait( makeInterruptable(StatusClient::statusFetcher( db )) ); if(warn.isValid()) warn.cancel(); @@ -1776,6 +1873,10 @@ ACTOR Future configure( Database db, std::vector tokens, Refere printf("Configuration changed\n"); ret=false; break; + case ConfigurationResult::LOCKED_NOT_NEW: + printf("ERROR: `only new databases can be configured as locked`\n"); + ret = true; + break; default: ASSERT(false); ret=true; @@ -1916,10 +2017,10 @@ ACTOR Future fileConfigure(Database db, std::string filePath, bool isNewDa ACTOR Future coordinators( Database db, std::vector tokens, bool isClusterTLS ) { state StringRef setName; StringRef nameTokenBegin = LiteralStringRef("description="); - for(auto t = tokens.begin()+1; t != tokens.end(); ++t) - if (t->startsWith(nameTokenBegin)) { - setName = t->substr(nameTokenBegin.size()); - std::copy( t+1, tokens.end(), t ); + for(auto tok = tokens.begin()+1; tok != tokens.end(); ++tok) + if (tok->startsWith(nameTokenBegin)) { + setName = tok->substr(nameTokenBegin.size()); + std::copy( tok+1, tokens.end(), tok ); tokens.resize( tokens.size()-1 ); break; } @@ -2091,7 +2192,7 @@ ACTOR Future exclude( Database db, std::vector tokens, Referenc return true; } } - StatusObject status = wait( makeInterruptable( StatusClient::statusFetcher( ccf ) ) ); + StatusObject status = wait( makeInterruptable( StatusClient::statusFetcher( db ) ) ); state std::string errorString = "ERROR: Could not calculate the impact of this exclude on the total free space in the cluster.\n" "Please try the exclude again in 30 seconds.\n" @@ -2194,36 +2295,44 @@ ACTOR Future exclude( Database db, std::vector tokens, Referenc workerPorts[addr.address.ip].insert(addr.address.port); // Print a list of all excluded addresses that don't have a corresponding worker - std::vector absentExclusions; + std::set absentExclusions; for(auto addr : addresses) { auto worker = workerPorts.find(addr.ip); if(worker == workerPorts.end()) - absentExclusions.push_back(addr); + absentExclusions.insert(addr); else if(addr.port > 0 && worker->second.count(addr.port) == 0) - absentExclusions.push_back(addr); + absentExclusions.insert(addr); } - if(!absentExclusions.empty()) { - printf("\nWARNING: the following servers were not present in the cluster. Be sure that you\n" - "excluded the correct machines or processes before removing them from the cluster:\n"); - for(auto addr : absentExclusions) { + for (auto addr : addresses) { + NetworkAddress _addr(addr.ip, addr.port); + if (absentExclusions.find(addr) != absentExclusions.end()) { if(addr.port == 0) - printf(" %s\n", addr.ip.toString().c_str()); + printf(" %s(Whole machine) ---- WARNING: Missing from cluster!Be sure that you excluded the " + "correct machines before removing them from the cluster!\n", + addr.ip.toString().c_str()); else - printf(" %s\n", addr.toString().c_str()); - } - - printf("\n"); - } else if (notExcludedServers.empty()) { - printf("\nIt is now safe to remove these machines or processes from the cluster.\n"); - } else { - printf("\nWARNING: Exclusion in progress. It is not safe to remove the following machines\n" - "or processes from the cluster:\n"); - for (auto addr : notExcludedServers) { + printf(" %s ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes " + "before removing them from the cluster!\n", + addr.toString().c_str()); + } else if (notExcludedServers.find(_addr) != notExcludedServers.end()) { if (addr.port == 0) - printf(" %s\n", addr.ip.toString().c_str()); + printf(" %s(Whole machine) ---- WARNING: Exclusion in progress! It is not safe to remove this " + "machine from the cluster\n", + addr.ip.toString().c_str()); else - printf(" %s\n", addr.toString().c_str()); + printf(" %s ---- WARNING: Exclusion in progress! It is not safe to remove this process from the " + "cluster\n", + addr.toString().c_str()); + } else { + if (addr.port == 0) + printf(" %s(Whole machine) ---- Successfully excluded. It is now safe to remove this machine " + "from the cluster.\n", + addr.ip.toString().c_str()); + else + printf( + " %s ---- Successfully excluded. It is now safe to remove this process from the cluster.\n", + addr.toString().c_str()); } } @@ -2377,7 +2486,7 @@ void onoff_generator(const char* text, const char *line, std::vector& lc) { - const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "proxies=", "logs=", "resolvers=", NULL}; + const char* opts[] = {"new", "single", "double", "triple", "three_data_hall", "three_datacenter", "ssd", "ssd-1", "ssd-2", "memory", "memory-1", "memory-2", "memory-radixtree-beta", "proxies=", "logs=", "resolvers=", NULL}; array_generator(text, line, opts, lc); } @@ -2458,28 +2567,28 @@ void LogCommand(std::string line, UID randomID, std::string errMsg) { struct CLIOptions { std::string program_name; - int exit_code; + int exit_code = -1; std::string commandLine; std::string clusterFile; - bool trace; + bool trace = false; std::string traceDir; std::string traceFormat; - int exit_timeout; + int exit_timeout = 0; Optional exec; - bool initialStatusCheck; + bool initialStatusCheck = true; + bool cliHints = true; + bool debugTLS = false; std::string tlsCertPath; std::string tlsKeyPath; std::string tlsVerifyPeers; std::string tlsCAPath; std::string tlsPassword; + std::vector> knobs; + CLIOptions( int argc, char* argv[] ) - : trace(false), - exit_timeout(0), - initialStatusCheck(true), - exit_code(-1) { program_name = argv[0]; for (int a = 0; asetKnob( k->first, k->second ) && + !clientKnobs->setKnob( k->first, k->second )) + { + fprintf(stderr, "WARNING: Unrecognized knob option '%s'\n", k->first.c_str()); + TraceEvent(SevWarnAlways, "UnrecognizedKnobOption").detail("Knob", printable(k->first)); + } + } catch (Error& e) { + if (e.code() == error_code_invalid_option_value) { + fprintf(stderr, "WARNING: Invalid value '%s' for knob option '%s'\n", k->second.c_str(), k->first.c_str()); + TraceEvent(SevWarnAlways, "InvalidKnobValue").detail("Knob", printable(k->first)).detail("Value", printable(k->second)); + } + else { + fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", k->first.c_str(), e.what()); + TraceEvent(SevError, "FailedToSetKnob").detail("Knob", printable(k->first)).detail("Value", printable(k->second)).error(e); + exit_code = FDB_EXIT_ERROR; + } + } + } + + // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs + flowKnobs->initialize(true); + clientKnobs->initialize(true); } int processArg(CSimpleOpt& args) { @@ -2534,44 +2676,59 @@ struct CLIOptions { case OPT_NO_STATUS: initialStatusCheck = false; break; + case OPT_NO_HINTS: + cliHints = false; #ifndef TLS_DISABLED // TLS Options - case TLSOptions::OPT_TLS_PLUGIN: - args.OptionArg(); - break; - case TLSOptions::OPT_TLS_CERTIFICATES: - tlsCertPath = args.OptionArg(); - break; - case TLSOptions::OPT_TLS_CA_FILE: - tlsCAPath = args.OptionArg(); - break; - case TLSOptions::OPT_TLS_KEY: - tlsKeyPath = args.OptionArg(); - break; - case TLSOptions::OPT_TLS_PASSWORD: - tlsPassword = args.OptionArg(); - break; - case TLSOptions::OPT_TLS_VERIFY_PEERS: - tlsVerifyPeers = args.OptionArg(); - break; + case TLSConfig::OPT_TLS_PLUGIN: + args.OptionArg(); + break; + case TLSConfig::OPT_TLS_CERTIFICATES: + tlsCertPath = args.OptionArg(); + break; + case TLSConfig::OPT_TLS_CA_FILE: + tlsCAPath = args.OptionArg(); + break; + case TLSConfig::OPT_TLS_KEY: + tlsKeyPath = args.OptionArg(); + break; + case TLSConfig::OPT_TLS_PASSWORD: + tlsPassword = args.OptionArg(); + break; + case TLSConfig::OPT_TLS_VERIFY_PEERS: + tlsVerifyPeers = args.OptionArg(); + break; #endif - case OPT_HELP: - printProgramUsage(program_name.c_str()); - return 0; - case OPT_STATUS_FROM_JSON: - return printStatusFromJSON(args.OptionArg()); - case OPT_TRACE_FORMAT: - if (!validateTraceFormat(args.OptionArg())) { - fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg()); - } - traceFormat = args.OptionArg(); - break; - case OPT_VERSION: - printVersion(); - return FDB_EXIT_SUCCESS; - } - return -1; + case OPT_HELP: + printProgramUsage(program_name.c_str()); + return 0; + case OPT_STATUS_FROM_JSON: + return printStatusFromJSON(args.OptionArg()); + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args.OptionArg())) { + fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg()); + } + traceFormat = args.OptionArg(); + break; + case OPT_KNOB: { + std::string syn = args.OptionSyntax(); + if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) { + fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str()); + return FDB_EXIT_ERROR; + } + syn = syn.substr(7); + knobs.push_back( std::make_pair( syn, args.OptionArg() ) ); + break; + } + case OPT_DEBUG_TLS: + debugTLS = true; + break; + case OPT_VERSION: + printVersion(); + return FDB_EXIT_SUCCESS; + } + return -1; } }; @@ -2587,6 +2744,27 @@ Future stopNetworkAfter( Future what ) { } } +ACTOR Future addInterface( std::map>* address_interface, Reference connectLock, KeyValue kv) { + wait(connectLock->take()); + state FlowLock::Releaser releaser(*connectLock); + state ClientWorkerInterface workerInterf = BinaryReader::fromStringRef(kv.value, IncludeVersion()); + state ClientLeaderRegInterface leaderInterf(workerInterf.address()); + choose { + when( Optional rep = wait( brokenPromiseToNever(leaderInterf.getLeader.getReply(GetLeaderRequest())) ) ) { + StringRef ip_port = kv.key.endsWith(LiteralStringRef(":tls")) ? kv.key.removeSuffix(LiteralStringRef(":tls")) : kv.key; + (*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf); + + if(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) { + Key full_ip_port2 = StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString()); + StringRef ip_port2 = full_ip_port2.endsWith(LiteralStringRef(":tls")) ? full_ip_port2.removeSuffix(LiteralStringRef(":tls")) : full_ip_port2; + (*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf); + } + } + when( wait(delay(CLIENT_KNOBS->CLI_CONNECT_TIMEOUT)) ) {} + } + return Void(); +} + ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { state LineNoise& linenoise = *plinenoise; state bool intrans = false; @@ -2597,7 +2775,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { state bool writeMode = false; state std::string clusterConnectString; - state std::map address_interface; + state std::map> address_interface; state FdbOptions globalOptions; state FdbOptions activeOptions; @@ -2645,7 +2823,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { if (!opt.exec.present()) { if(opt.initialStatusCheck) { - Future checkStatusF = checkStatus(Void(), db->getConnectionFile()); + Future checkStatusF = checkStatus(Void(), db); wait(makeInterruptable(success(checkStatusF))); } else { @@ -2679,11 +2857,12 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; // Don't put dangerous commands in the command history - if (line.find("writemode") == std::string::npos && line.find("expensive_data_check") == std::string::npos) + if (line.find("writemode") == std::string::npos && line.find("expensive_data_check") == std::string::npos && + line.find("unlock") == std::string::npos) linenoise.historyAdd(line); } - warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db->getConnectionFile()); + warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db); try { state UID randomID = deterministicRandom()->randomUniqueID(); @@ -2828,7 +3007,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } - StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db->getConnectionFile()))); + StatusObject s = wait(makeInterruptable(StatusClient::statusFetcher(db))); if (!opt.exec.present()) printf("\n"); printStatus(s, level); @@ -2894,6 +3073,52 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } + if (tokencmp(tokens[0], "lock")) { + if (tokens.size() != 1) { + printUsage(tokens[0]); + is_error = true; + } else { + state UID lockUID = deterministicRandom()->randomUniqueID(); + printf("Locking database with lockUID: %s\n", lockUID.toString().c_str()); + wait(makeInterruptable(lockDatabase(db, lockUID))); + printf("Database locked.\n"); + } + continue; + } + + if (tokencmp(tokens[0], "unlock")) { + if ((tokens.size() != 2) || (tokens[1].size() != 32) || + !std::all_of(tokens[1].begin(), tokens[1].end(), &isxdigit)) { + printUsage(tokens[0]); + is_error = true; + } else { + state std::string passPhrase = deterministicRandom()->randomAlphaNumeric(10); + warn.cancel(); // don't warn while waiting on user input + printf("Unlocking the database is a potentially dangerous operation.\n"); + Optional input = wait(linenoise.read( + format("Repeat the following passphrase if you would like to proceed (%s) : ", + passPhrase.c_str()))); + warn = checkStatus(timeWarning(5.0, "\nWARNING: Long delay (Ctrl-C to interrupt)\n"), db); + if (input.present() && input.get() == passPhrase) { + UID unlockUID = UID::fromString(tokens[1].toString()); + try { + wait(makeInterruptable(unlockDatabase(db, unlockUID))); + printf("Database unlocked.\n"); + } catch (Error& e) { + if (e.code() == error_code_database_locked) { + printf( + "Unable to unlock database. Make sure to unlock with the correct lock UID.\n"); + } + throw e; + } + } else { + printf("ERROR: Incorrect passphrase entered.\n"); + is_error = true; + } + } + continue; + } + if (tokencmp(tokens[0], "setclass")) { if (tokens.size() != 3 && tokens.size() != 1) { printUsage(tokens[0]); @@ -2986,14 +3211,44 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { continue; } + if (tokencmp(tokens[0], "getversion")) { + if (tokens.size() != 1) { + printUsage(tokens[0]); + is_error = true; + } else { + Version v = wait(makeInterruptable(getTransaction(db, tr, options, intrans)->getReadVersion())); + printf("%ld\n", v); + } + continue; + } + + if (tokencmp(tokens[0], "advanceversion")) { + if (tokens.size() != 2) { + printUsage(tokens[0]); + is_error = true; + } else { + Version v; + int n = 0; + if (sscanf(tokens[1].toString().c_str(), "%ld%n", &v, &n) != 1 || n != tokens[1].size()) { + printUsage(tokens[0]); + is_error = true; + } else { + wait(makeInterruptable(advanceVersion(db, v))); + } + } + continue; + } + if (tokencmp(tokens[0], "kill")) { getTransaction(db, tr, options, intrans); if (tokens.size() == 1) { Standalone kvs = wait( makeInterruptable( tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces"), LiteralStringRef("\xff\xff\xff")), 1) ) ); + Reference connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM)); + std::vector> addInterfs; for( auto it : kvs ) { - auto ip_port = it.key.endsWith(LiteralStringRef(":tls")) ? it.key.removeSuffix(LiteralStringRef(":tls")) : it.key; - address_interface[ip_port] = it.value; + addInterfs.push_back(addInterface(&address_interface, connectLock, it)); } + wait( waitForAll(addInterfs) ); } if (tokens.size() == 1 || tokencmp(tokens[1], "list")) { if(address_interface.size() == 0) { @@ -3009,7 +3264,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { printf("\n"); } else if (tokencmp(tokens[1], "all")) { for( auto it : address_interface ) { - tr->set(LiteralStringRef("\xff\xff/reboot_worker"), it.second); + tr->set(LiteralStringRef("\xff\xff/reboot_worker"), it.second.first); } if (address_interface.size() == 0) { printf("ERROR: no processes to kill. You must run the `kill’ command before running `kill all’.\n"); @@ -3027,7 +3282,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { if(!is_error) { for(int i = 1; i < tokens.size(); i++) { - tr->set(LiteralStringRef("\xff\xff/reboot_worker"), address_interface[tokens[i]]); + tr->set(LiteralStringRef("\xff\xff/reboot_worker"), address_interface[tokens[i]].first); } printf("Attempted to kill %zu processes\n", tokens.size() - 1); } @@ -3302,9 +3557,12 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { getTransaction(db, tr, options, intrans); if (tokens.size() == 1) { Standalone kvs = wait( makeInterruptable( tr->getRange(KeyRangeRef(LiteralStringRef("\xff\xff/worker_interfaces"), LiteralStringRef("\xff\xff\xff")), 1) ) ); + Reference connectLock(new FlowLock(CLIENT_KNOBS->CLI_CONNECT_PARALLELISM)); + std::vector> addInterfs; for( auto it : kvs ) { - address_interface[it.key] = it.value; + addInterfs.push_back(addInterface(&address_interface, connectLock, it)); } + wait( waitForAll(addInterfs) ); } if (tokens.size() == 1 || tokencmp(tokens[1], "list")) { if(address_interface.size() == 0) { @@ -3320,7 +3578,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { printf("\n"); } else if (tokencmp(tokens[1], "all")) { for( auto it : address_interface ) { - tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), it.second); + tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), it.second.first); } if (address_interface.size() == 0) { printf("ERROR: no processes to check. You must run the `expensive_data_check’ command before running `expensive_data_check all’.\n"); @@ -3338,7 +3596,7 @@ ACTOR Future cli(CLIOptions opt, LineNoise* plinenoise) { if(!is_error) { for(int i = 1; i < tokens.size(); i++) { - tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), address_interface[tokens[i]]); + tr->set(LiteralStringRef("\xff\xff/reboot_and_check_worker"), address_interface[tokens[i]].first); } printf("Attempted to kill and check %zu processes\n", tokens.size() - 1); } @@ -3627,8 +3885,37 @@ ACTOR Future runCli(CLIOptions opt) { [](std::string const& line, std::vector& completions) { fdbcli_comp_cmd(line, completions); }, - [](std::string const& line)->LineNoise::Hint { - return LineNoise::Hint(); + [enabled=opt.cliHints](std::string const& line)->LineNoise::Hint { + if (!enabled) { + return LineNoise::Hint(); + } + + bool error = false; + bool partial = false; + std::string linecopy = line; + std::vector> parsed = parseLine(linecopy, error, partial); + if (parsed.size() == 0 || parsed.back().size() == 0) return LineNoise::Hint(); + StringRef command = parsed.back().front(); + int finishedParameters = parsed.back().size() + error; + + // As a user is typing an escaped character, e.g. \", after the \ and before the " is typed + // the string will be a parse error. Ignore this parse error to avoid flipping the hint to + // {malformed escape sequence} and back to the original hint for the span of one character + // being entered. + if (error && line.back() != '\\') return LineNoise::Hint(std::string(" {malformed escape sequence}"), 90, false); + + auto iter = helpMap.find(command.toString()); + if (iter != helpMap.end()) { + std::string helpLine = iter->second.usage; + std::vector> parsedHelp = parseLine(helpLine, error, partial); + std::string hintLine = (*(line.end() - 1) == ' ' ? "" : " "); + for (int i = finishedParameters; i < parsedHelp.back().size(); i++) { + hintLine = hintLine + parsedHelp.back()[i].toString() + " "; + } + return LineNoise::Hint(hintLine, 90, false); + } else { + return LineNoise::Hint(); + } }, 1000, false); @@ -3745,6 +4032,30 @@ int main(int argc, char **argv) { return 1; } + if (opt.debugTLS) { +#ifndef TLS_DISABLED + // Backdoor into NativeAPI's tlsConfig, which is where the above network option settings ended up. + extern TLSConfig tlsConfig; + printf("TLS Configuration:\n"); + printf("\tCertificate Path: %s\n", tlsConfig.getCertificatePathSync().c_str()); + printf("\tKey Path: %s\n", tlsConfig.getKeyPathSync().c_str()); + printf("\tCA Path: %s\n", tlsConfig.getCAPathSync().c_str()); + try { + LoadedTLSConfig loaded = tlsConfig.loadSync(); + printf("\tPassword: %s\n", loaded.getPassword().empty() ? "Not configured" : "Exists, but redacted"); + printf("\n"); + loaded.print(stdout); + } catch (Error& e) { + printf("ERROR: %s (%d)\n", e.what(), e.code()); + printf("Use --log and look at the trace logs for more detailed information on the failure.\n"); + return 1; + } +#else + printf("This fdbcli was built with TLS disabled.\n"); +#endif + return 0; + } + try { setupNetwork(); Future cliFuture = runCli(opt); diff --git a/fdbcli/fdbcli.vcxproj b/fdbcli/fdbcli.vcxproj deleted file mode 100644 index 2c22697ee6..0000000000 --- a/fdbcli/fdbcli.vcxproj +++ /dev/null @@ -1,137 +0,0 @@ - - - - - -PRERELEASE - - - - - - - - Debug - x64 - - - Release - x64 - - - - - - - - - - - - - - - {4631CC93-52A3-4537-9BE9-6B237A3AC6B2} - Win32Proj - fdbcli - - - - Application - true - MultiByte - v141 - - - Application - false - false - MultiByte - v141 - - - - - - - - - - - - - true - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - $(IncludePath);../;C:\Program Files\boost_1_67_0 - - - false - $(SolutionDir)bin\$(Configuration)\ - $(SystemDrive)\temp\msvcfdb\$(Platform)$(Configuration)\$(MSBuildProjectName)\ - $(IncludePath);../;C:\Program Files\boost_1_67_0 - - - - FDB_VT_VERSION="$(Version)$(PreReleaseDecoration)";FDB_VT_PACKAGE_NAME="$(PackageName)";%(PreprocessorDefinitions) - stdcpp17 - - - - - - - Level3 - Disabled - TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;_DEBUG;_HAS_ITERATOR_DEBUGGING=0;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - ..\zookeeper\win32;..\zookeeper\generated;..\zookeeper\include;%(AdditionalIncludeDirectories) - true - false - MultiThreadedDebug - @../flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - Console - true - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;Advapi32.lib - - - - - - - - - Level3 - - - Full - true - TLS_DISABLED;WIN32;_WIN32_WINNT=0x0502;WINVER=0x0502;BOOST_ALL_NO_LIB;NTDDI_VERSION=0x05020000;NDEBUG;_CONSOLE;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - ..\zookeeper\win32;..\zookeeper\generated;..\zookeeper\include;%(AdditionalIncludeDirectories) - true - Speed - MultiThreaded - false - StreamingSIMDExtensions2 - @../flow/no_intellisense.opt %(AdditionalOptions) - stdcpp17 - - - Console - true - false - false - $(SolutionDir)bin\$(Configuration)\fdbclient.lib;Advapi32.lib - Default - - - - - - - - - - - diff --git a/fdbcli/fdbcli.vcxproj.filters b/fdbcli/fdbcli.vcxproj.filters deleted file mode 100644 index e4363c462f..0000000000 --- a/fdbcli/fdbcli.vcxproj.filters +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/fdbcli/linenoise/linenoise.c b/fdbcli/linenoise/linenoise.c index 30d64ececf..10ffd71c35 100644 --- a/fdbcli/linenoise/linenoise.c +++ b/fdbcli/linenoise/linenoise.c @@ -111,6 +111,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,8 @@ #define LINENOISE_MAX_LINE 4096 static char *unsupported_term[] = {"dumb","cons25","emacs",NULL}; static linenoiseCompletionCallback *completionCallback = NULL; +static linenoiseHintsCallback *hintsCallback = NULL; +static linenoiseFreeHintsCallback *freeHintsCallback = NULL; static struct termios orig_termios; /* In order to restore at exit.*/ static int rawmode = 0; /* For atexit() function to check if restore is needed*/ @@ -407,6 +410,18 @@ void linenoiseSetCompletionCallback(linenoiseCompletionCallback *fn) { completionCallback = fn; } +/* Register a hits function to be called to show hits to the user at the + * right of the prompt. */ +void linenoiseSetHintsCallback(linenoiseHintsCallback *fn) { + hintsCallback = fn; +} + +/* Register a function to free the hints returned by the hints callback + * registered with linenoiseSetHintsCallback(). */ +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *fn) { + freeHintsCallback = fn; +} + /* This function is used by the callback function registered by the user * in order to add completion options given the input string when the * user typed . See the example.c source code for a very easy to @@ -456,6 +471,32 @@ static void abFree(struct abuf *ab) { free(ab->b); } +/* Helper of refreshSingleLine() and refreshMultiLine() to show hints + * to the right of the prompt. */ +void refreshShowHints(struct abuf *ab, struct linenoiseState *l, int plen) { + char seq[64]; + if (hintsCallback && plen+l->len < l->cols) { + int color = -1, bold = 0; + char *hint = hintsCallback(l->buf,&color,&bold); + if (hint) { + int hintlen = strlen(hint); + int hintmaxlen = l->cols-(plen+l->len); + if (hintlen > hintmaxlen) hintlen = hintmaxlen; + if (bold == 1 && color == -1) color = 37; + if (color != -1 || bold != 0) + snprintf(seq,64,"\033[%d;%d;49m",bold,color); + else + seq[0] = '\0'; + abAppend(ab,seq,strlen(seq)); + abAppend(ab,hint,hintlen); + if (color != -1 || bold != 0) + abAppend(ab,"\033[0m",4); + /* Call the function to free the hint returned. */ + if (freeHintsCallback) freeHintsCallback(hint); + } + } +} + /* Single line low level line refresh. * * Rewrite the currently edited line accordingly to the buffer content, @@ -485,6 +526,8 @@ static void refreshSingleLine(struct linenoiseState *l) { /* Write the prompt and the current buffer content */ abAppend(&ab,l->prompt,strlen(l->prompt)); abAppend(&ab,buf,len); + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); /* Erase to right */ snprintf(seq,64,"\x1b[0K"); abAppend(&ab,seq,strlen(seq)); @@ -538,6 +581,9 @@ static void refreshMultiLine(struct linenoiseState *l) { abAppend(&ab,l->prompt,strlen(l->prompt)); abAppend(&ab,l->buf,l->len); + /* Show hits if any. */ + refreshShowHints(&ab,l,plen); + /* If we are at the very end of the screen with our prompt, we need to * emit a newline and move the prompt to the first column. */ if (l->pos && @@ -598,7 +644,7 @@ int linenoiseEditInsert(struct linenoiseState *l, char c) { l->pos++; l->len++; l->buf[l->len] = '\0'; - if ((!mlmode && l->plen+l->len < l->cols) /* || mlmode */) { + if ((!mlmode && l->plen+l->len < l->cols && !hintsCallback)) { /* Avoid a full update of the line in the * trivial case. */ if (write(l->ofd,&c,1) == -1) return -1; @@ -772,6 +818,14 @@ static int linenoiseEdit(int stdin_fd, int stdout_fd, char *buf, size_t buflen, history_len--; free(history[history_len]); if (mlmode) linenoiseEditMoveEnd(&l); + if (hintsCallback) { + /* Force a refresh without hints to leave the previous + * line as the user typed it after a newline. */ + linenoiseHintsCallback *hc = hintsCallback; + hintsCallback = NULL; + refreshLine(&l); + hintsCallback = hc; + } return (int)l.len; case CTRL_C: /* ctrl-c */ errno = EAGAIN; @@ -1010,6 +1064,14 @@ char *linenoise(const char *prompt) { } } +/* This is just a wrapper the user may want to call in order to make sure + * the linenoise returned buffer is freed with the same allocator it was + * created with. Useful when the main program is using an alternative + * allocator. */ +void linenoiseFree(void *ptr) { + free(ptr); +} + /* ================================ History ================================= */ /* Free the history, but does not reset it. Only used when we have to @@ -1101,10 +1163,14 @@ int linenoiseHistorySetMaxLen(int len) { /* Save the history in the specified file. On success 0 is returned * otherwise -1 is returned. */ int linenoiseHistorySave(const char *filename) { - FILE *fp = fopen(filename,"w"); + mode_t old_umask = umask(S_IXUSR|S_IRWXG|S_IRWXO); + FILE *fp; int j; + fp = fopen(filename,"w"); + umask(old_umask); if (fp == NULL) return -1; + chmod(filename,S_IRUSR|S_IWUSR); for (j = 0; j < history_len; j++) fprintf(fp,"%s\n",history[j]); fclose(fp); diff --git a/fdbcli/linenoise/linenoise.h b/fdbcli/linenoise/linenoise.h index fbb01cfaad..c388e25a4f 100644 --- a/fdbcli/linenoise/linenoise.h +++ b/fdbcli/linenoise/linenoise.h @@ -39,6 +39,8 @@ #ifndef __LINENOISE_H #define __LINENOISE_H +#include + #ifdef __cplusplus extern "C" { #endif @@ -49,10 +51,15 @@ typedef struct linenoiseCompletions { } linenoiseCompletions; typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *); +typedef char*(linenoiseHintsCallback)(const char *, int *color, int *bold); +typedef void(linenoiseFreeHintsCallback)(void *); void linenoiseSetCompletionCallback(linenoiseCompletionCallback *); +void linenoiseSetHintsCallback(linenoiseHintsCallback *); +void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *); void linenoiseAddCompletion(linenoiseCompletions *, const char *); char *linenoise(const char *prompt); +void linenoiseFree(void *ptr); int linenoiseHistoryAdd(const char *line); int linenoiseHistorySetMaxLen(int len); int linenoiseHistorySave(const char *filename); diff --git a/fdbcli/local.mk b/fdbcli/local.mk deleted file mode 100644 index 3af026b911..0000000000 --- a/fdbcli/local.mk +++ /dev/null @@ -1,39 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdbcli_CFLAGS := $(fdbclient_CFLAGS) -fdbcli_LDFLAGS := $(fdbrpc_LDFLAGS) -fdbcli_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a $(FDB_TLS_LIB) -fdbcli_STATIC_LIBS := $(TLS_LIBS) - -fdbcli_GENERATED_SOURCES += versions.h - -ifeq ($(PLATFORM),linux) - fdbcli_LDFLAGS += -static-libstdc++ -static-libgcc -lpthread -lrt -ldl -else ifeq ($(PLATFORM),osx) - fdbcli_LDFLAGS += -lc++ -endif - -test_fdbcli_status: fdbcli - python scripts/test_status.py - -bin/fdbcli.debug: bin/fdbcli diff --git a/fdbclient/Atomic.h b/fdbclient/Atomic.h index d9aecbe8a3..f32950cc70 100644 --- a/fdbclient/Atomic.h +++ b/fdbclient/Atomic.h @@ -24,15 +24,15 @@ #include "fdbclient/CommitTransaction.h" -static ValueRef doLittleEndianAdd(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doLittleEndianAdd(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); if(!existingValue.size()) return otherOperand; if(!otherOperand.size()) return otherOperand; - + uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; int i = 0; int carry = 0; - + for(i = 0; i& existingValueOptiona carry = sum >> 8; } - return StringRef(buf, i); + return StringRef(buf, i); } -static ValueRef doAnd(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doAnd(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); if(!otherOperand.size()) return otherOperand; - + uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; int i = 0; - + for(i = 0; i& existingValueOptional, const Val return StringRef(buf, i); } -static ValueRef doAndV2(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doAndV2(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { if (!existingValueOptional.present()) return otherOperand; return doAnd(existingValueOptional, otherOperand, ar); } -static ValueRef doOr(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doOr(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); if(!existingValue.size()) return otherOperand; if(!otherOperand.size()) return otherOperand; uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; int i = 0; - + for(i = 0; i& existingValueOptional, const Valu return StringRef(buf, i); } -static ValueRef doXor(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doXor(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); if(!existingValue.size()) return otherOperand; if(!otherOperand.size()) return otherOperand; - + uint8_t* buf = new (ar) uint8_t [otherOperand.size()]; int i = 0; - + for(i = 0; i& existingValueOptional, const Val return StringRef(buf, i); } -static ValueRef doAppendIfFits(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doAppendIfFits(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); if(!existingValue.size()) return otherOperand; if(!otherOperand.size()) return existingValue; @@ -123,7 +123,7 @@ static ValueRef doAppendIfFits(const Optional& existingValueOptional, return StringRef(buf, i+j); } -static ValueRef doMax(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doMax(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); if (!existingValue.size()) return otherOperand; if (!otherOperand.size()) return otherOperand; @@ -155,7 +155,7 @@ static ValueRef doMax(const Optional& existingValueOptional, const Val return otherOperand; } -static ValueRef doByteMax(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doByteMax(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { if (!existingValueOptional.present()) return otherOperand; const ValueRef& existingValue = existingValueOptional.get(); @@ -165,7 +165,7 @@ static ValueRef doByteMax(const Optional& existingValueOptional, const return otherOperand; } -static ValueRef doMin(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doMin(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { if (!otherOperand.size()) return otherOperand; const ValueRef& existingValue = existingValueOptional.present() ? existingValueOptional.get() : StringRef(); @@ -203,16 +203,16 @@ static ValueRef doMin(const Optional& existingValueOptional, const Val return otherOperand; } -static ValueRef doMinV2(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doMinV2(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { if (!existingValueOptional.present()) return otherOperand; return doMin(existingValueOptional, otherOperand, ar); } -static ValueRef doByteMin(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { +inline ValueRef doByteMin(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { if (!existingValueOptional.present()) return otherOperand; - + const ValueRef& existingValue = existingValueOptional.get(); if (existingValue < otherOperand) return existingValue; @@ -220,7 +220,7 @@ static ValueRef doByteMin(const Optional& existingValueOptional, const return otherOperand; } -static Optional doCompareAndClear(const Optional& existingValueOptional, +inline Optional doCompareAndClear(const Optional& existingValueOptional, const ValueRef& otherOperand, Arena& ar) { if (!existingValueOptional.present() || existingValueOptional.get() == otherOperand) { // Clear the value. @@ -229,10 +229,19 @@ static Optional doCompareAndClear(const Optional& existingVa return existingValueOptional; // No change required. } +static void placeVersionstamp( uint8_t* destination, Version version, uint16_t transactionNumber ) { + version = bigEndian64(version); + transactionNumber = bigEndian16(transactionNumber); + static_assert( sizeof(version) == 8, "version size mismatch" ); + memcpy( destination, &version, sizeof(version) ); + static_assert( sizeof(transactionNumber) == 2, "txn num size mismatch"); + memcpy( destination + sizeof(version), &transactionNumber, sizeof(transactionNumber) ); +} + /* * Returns the range corresponding to the specified versionstamp key. */ -static KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, const KeyRef &maxKey) { +inline KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, Version minVersion, const KeyRef &maxKey) { KeyRef begin(arena, key); KeyRef end(arena, key); @@ -249,22 +258,26 @@ static KeyRangeRef getVersionstampKeyRange(Arena& arena, const KeyRef &key, cons if (pos < 0 || pos + 10 > begin.size()) throw client_invalid_operation(); - memset(mutateString(begin) + pos, 0, 10); + placeVersionstamp(mutateString(begin)+pos, minVersion, 0); memset(mutateString(end) + pos, '\xff', 10); return KeyRangeRef(begin, std::min(end, maxKey)); } -static void placeVersionstamp( uint8_t* destination, Version version, uint16_t transactionNumber ) { - version = bigEndian64(version); - transactionNumber = bigEndian16(transactionNumber); - static_assert( sizeof(version) == 8, "version size mismatch" ); - memcpy( destination, &version, sizeof(version) ); - static_assert( sizeof(transactionNumber) == 2, "txn num size mismatch"); - memcpy( destination + sizeof(version), &transactionNumber, sizeof(transactionNumber) ); +inline void transformVersionstampKey( StringRef& key, Version version, uint16_t transactionNumber ) { + if (key.size() < 4) + throw client_invalid_operation(); + + int32_t pos; + memcpy(&pos, key.end() - sizeof(int32_t), sizeof(int32_t)); + pos = littleEndian32(pos); + if (pos < 0 || pos + 10 > key.size()) + throw client_invalid_operation(); + + placeVersionstamp( mutateString(key) + pos, version, transactionNumber ); } -static void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) { +inline void transformVersionstampMutation( MutationRef& mutation, StringRef MutationRef::* param, Version version, uint16_t transactionNumber ) { if ((mutation.*param).size() >= 4) { int32_t pos; memcpy(&pos, (mutation.*param).end() - sizeof(int32_t), sizeof(int32_t)); diff --git a/fdbclient/BackupAgent.actor.h b/fdbclient/BackupAgent.actor.h index 725ab2e59c..962d37b10b 100644 --- a/fdbclient/BackupAgent.actor.h +++ b/fdbclient/BackupAgent.actor.h @@ -275,6 +275,13 @@ public: enum ERestoreState { UNITIALIZED = 0, QUEUED = 1, STARTING = 2, RUNNING = 3, COMPLETED = 4, ABORTED = 5 }; static StringRef restoreStateText(ERestoreState id); + // parallel restore + Future parallelRestoreFinish(Database cx, UID randomUID); + Future submitParallelRestore(Database cx, Key backupTag, Standalone> backupRanges, + Key bcUrl, Version targetVersion, bool lockDB, UID randomUID); + Future atomicParallelRestore(Database cx, Key tagName, Standalone> ranges, + Key addPrefix, Key removePrefix); + // restore() will // - make sure that url is readable and appears to be a complete backup // - make sure the requested TargetVersion is valid @@ -308,9 +315,16 @@ public: /** BACKUP METHODS **/ - Future submitBackup(Reference tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone> backupRanges, bool stopWhenDone = true); - Future submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone> backupRanges, bool stopWhenDone = true) { - return runRYWTransactionFailIfLocked(cx, [=](Reference tr){ return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone); }); + Future submitBackup(Reference tr, Key outContainer, int snapshotIntervalSeconds, + std::string tagName, Standalone> backupRanges, + bool stopWhenDone = true, bool partitionedLog = false); + Future submitBackup(Database cx, Key outContainer, int snapshotIntervalSeconds, std::string tagName, + Standalone> backupRanges, bool stopWhenDone = true, + bool partitionedLog = false) { + return runRYWTransactionFailIfLocked(cx, [=](Reference tr) { + return submitBackup(tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone, + partitionedLog); + }); } Future discontinueBackup(Reference tr, Key tagName); @@ -333,7 +347,7 @@ public: Future getStatus(Database cx, bool showErrors, std::string tagName); Future getStatusJSON(Database cx, std::string tagName); - Future getLastRestorable(Reference tr, Key tagName); + Future getLastRestorable(Reference tr, Key tagName, bool snapshot = false); void setLastRestorable(Reference tr, Key tagName, Version version); // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it @@ -348,6 +362,9 @@ public: Future checkActive(Database cx) { return taskBucket->checkActive(cx); } + // If "pause" is true, pause all backups; otherwise, resume all. + Future changePause(Database db, bool pause); + friend class FileBackupAgentImpl; static const int dataFooterSize; @@ -414,23 +431,23 @@ public: Future getStatus(Database cx, int errorLimit, Key tagName); - Future getStateValue(Reference tr, UID logUid); + Future getStateValue(Reference tr, UID logUid, bool snapshot = false); Future getStateValue(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr){ return getStateValue(tr, logUid); }); } - Future getDestUid(Reference tr, UID logUid); + Future getDestUid(Reference tr, UID logUid, bool snapshot = false); Future getDestUid(Database cx, UID logUid) { return runRYWTransaction(cx, [=](Reference tr){ return getDestUid(tr, logUid); }); } - Future getLogUid(Reference tr, Key tagName); + Future getLogUid(Reference tr, Key tagName, bool snapshot = false); Future getLogUid(Database cx, Key tagName) { return runRYWTransaction(cx, [=](Reference tr){ return getLogUid(tr, tagName); }); } - Future getRangeBytesWritten(Reference tr, UID logUid); - Future getLogBytesWritten(Reference tr, UID logUid); + Future getRangeBytesWritten(Reference tr, UID logUid, bool snapshot = false); + Future getLogBytesWritten(Reference tr, UID logUid, bool snapshot = false); // stopWhenDone will return when the backup is stopped, if enabled. Otherwise, it // will return when the backup directory is restorable. @@ -487,7 +504,7 @@ Standalone> getLogRanges(Version beginVersion, Version en Standalone> getApplyRanges(Version beginVersion, Version endVersion, Key backupUid); Future eraseLogData(Reference tr, Key logUidValue, Key destUidValue, Optional endVersion = Optional(), bool checkBackupUid = false, Version backupUid = 0); Key getApplyKey( Version version, Key backupUid ); -std::pair decodeBKMutationLogKey(Key key); +std::pair decodeBKMutationLogKey(Key key); Standalone> decodeBackupLogValue(StringRef value); void decodeBackupLogValue(Arena& arena, VectorRef& result, int64_t& mutationSize, StringRef value, StringRef addPrefix = StringRef(), StringRef removePrefix = StringRef()); Future logError(Database cx, Key keyErrors, const std::string& message); @@ -543,11 +560,10 @@ class TagUidMap : public KeyBackedMap { public: TagUidMap(const StringRef & prefix) : TagMap(LiteralStringRef("tag->uid/").withPrefix(prefix)), prefix(prefix) {} - ACTOR static Future> getAll_impl(TagUidMap* tagsMap, - Reference tr); + ACTOR static Future> getAll_impl(TagUidMap* tagsMap, Reference tr, bool snapshot); - Future> getAll(Reference tr) { - return getAll_impl(this, tr); + Future> getAll(Reference tr, bool snapshot = false) { + return getAll_impl(this, tr, snapshot); } Key prefix; @@ -561,12 +577,12 @@ static inline KeyBackedTag makeBackupTag(std::string tagName) { return KeyBackedTag(tagName, fileBackupPrefixRange.begin); } -static inline Future> getAllRestoreTags(Reference tr) { - return TagUidMap(fileRestorePrefixRange.begin).getAll(tr); +static inline Future> getAllRestoreTags(Reference tr, bool snapshot = false) { + return TagUidMap(fileRestorePrefixRange.begin).getAll(tr, snapshot); } -static inline Future> getAllBackupTags(Reference tr) { - return TagUidMap(fileBackupPrefixRange.begin).getAll(tr); +static inline Future> getAllBackupTags(Reference tr, bool snapshot = false) { + return TagUidMap(fileBackupPrefixRange.begin).getAll(tr, snapshot); } class KeyBackedConfig { @@ -783,6 +799,31 @@ public: return configSpace.pack(LiteralStringRef(__FUNCTION__)); } + // Set to true when all backup workers for saving mutation logs have been started. + KeyBackedProperty allWorkerStarted() { + return configSpace.pack(LiteralStringRef(__FUNCTION__)); + } + + // Each backup worker adds its (epoch, tag.id) to this property. + KeyBackedProperty>> startedBackupWorkers() { + return configSpace.pack(LiteralStringRef(__FUNCTION__)); + } + + // Set to true if backup worker is enabled. + KeyBackedProperty backupWorkerEnabled() { + return configSpace.pack(LiteralStringRef(__FUNCTION__)); + } + + // Set to true if partitioned log is enabled (only useful if backup worker is also enabled). + KeyBackedProperty partitionedLogEnabled() { + return configSpace.pack(LiteralStringRef(__FUNCTION__)); + } + + // Latest version for which all prior versions have saved by backup workers. + KeyBackedProperty latestBackupWorkerSavedVersion() { + return configSpace.pack(LiteralStringRef(__FUNCTION__)); + } + // Stop differntial logging if already started or don't start after completing KV ranges KeyBackedProperty stopWhenDone() { return configSpace.pack(LiteralStringRef(__FUNCTION__)); @@ -812,10 +853,17 @@ public: tr->setOption(FDBTransactionOptions::READ_LOCK_AWARE); auto lastLog = latestLogEndVersion().get(tr); auto firstSnapshot = firstSnapshotEndVersion().get(tr); - return map(success(lastLog) && success(firstSnapshot), [=](Void) -> Optional { + auto workerEnabled = backupWorkerEnabled().get(tr); + auto plogEnabled = partitionedLogEnabled().get(tr); + auto workerVersion = latestBackupWorkerSavedVersion().get(tr); + return map(success(lastLog) && success(firstSnapshot) && success(workerEnabled) && success(plogEnabled) && success(workerVersion), [=](Void) -> Optional { // The latest log greater than the oldest snapshot is the restorable version - if(lastLog.get().present() && firstSnapshot.get().present() && lastLog.get().get() > firstSnapshot.get().get()) { - return std::max(lastLog.get().get() - 1, firstSnapshot.get().get()); + Optional logVersion = workerEnabled.get().present() && workerEnabled.get().get() && + plogEnabled.get().present() && plogEnabled.get().get() + ? workerVersion.get() + : lastLog.get(); + if (logVersion.present() && firstSnapshot.get().present() && logVersion.get() > firstSnapshot.get().get()) { + return std::max(logVersion.get() - 1, firstSnapshot.get().get()); } return {}; }); @@ -845,9 +893,50 @@ public: } }; -ACTOR Future fastRestore(Database cx, Standalone tagName, Standalone url, - bool waitForComplete, long targetVersion, bool verbose, Standalone range, - Standalone addPrefix, Standalone removePrefix); +// Helper class for reading restore data from a buffer and throwing the right errors. +struct StringRefReader { + StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {} + + // Return remainder of data as a StringRef + StringRef remainder() { return StringRef(rptr, end - rptr); } + + // Return a pointer to len bytes at the current read position and advance read pos + const uint8_t* consume(unsigned int len) { + if (rptr == end && len != 0) throw end_of_stream(); + const uint8_t* p = rptr; + rptr += len; + if (rptr > end) throw failure_error; + return p; + } + + // Return a T from the current read position and advance read pos + template + const T consume() { + return *(const T*)consume(sizeof(T)); + } + + // Functions for consuming big endian (network byte order) integers. + // Consumes a big endian number, swaps it to little endian, and returns it. + const int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume()); } + const uint32_t consumeNetworkUInt32() { return bigEndian32(consume()); } + + // Convert big Endian value (e.g., encoded in log file) into a littleEndian uint64_t value. + int64_t consumeNetworkInt64() { return (int64_t)bigEndian64((uint32_t)consume()); } + uint64_t consumeNetworkUInt64() { return bigEndian64(consume()); } + + bool eof() { return rptr == end; } + + const uint8_t *rptr, *end; + Error failure_error; +}; + +namespace fileBackup { +ACTOR Future>> decodeRangeFileBlock(Reference file, int64_t offset, + int len); + +// Return a block of contiguous padding bytes "\0xff" for backup files, growing if needed. +Value makePadding(int size); +} #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/BackupAgentBase.actor.cpp b/fdbclient/BackupAgentBase.actor.cpp index 6a02bac4b3..8a034bd3d3 100644 --- a/fdbclient/BackupAgentBase.actor.cpp +++ b/fdbclient/BackupAgentBase.actor.cpp @@ -131,7 +131,7 @@ bool copyParameter(Reference source, Reference dest, Key key) { } Version getVersionFromString(std::string const& value) { - Version version(-1); + Version version = invalidVersion; int n = 0; if (sscanf(value.c_str(), "%lld%n", (long long*)&version, &n) != 1 || n != value.size()) { TraceEvent(SevWarnAlways, "GetVersionFromString").detail("InvalidVersion", value); @@ -204,7 +204,7 @@ Key getApplyKey( Version version, Key backupUid ) { //returns(version, part) where version is the database version number of //the transaction log data in the value, and part is 0 for the first such //data for a given version, 1 for the second block of data, etc. -std::pair decodeBKMutationLogKey(Key key) { +std::pair decodeBKMutationLogKey(Key key) { return std::make_pair(bigEndian64(*(int64_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t))), bigEndian32(*(int32_t*)(key.begin() + backupLogPrefixBytes + sizeof(UID) + sizeof(uint8_t) + sizeof(int64_t)))); } @@ -379,7 +379,9 @@ void decodeBackupLogValue(Arena& arena, VectorRef& result, int& mut throw; } } + static double lastErrorTime = 0; + void logErrorWorker(Reference tr, Key keyErrors, std::string message) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp index 09fca80b16..6bc176df08 100644 --- a/fdbclient/BackupContainer.actor.cpp +++ b/fdbclient/BackupContainer.actor.cpp @@ -20,12 +20,13 @@ #include "fdbclient/BackupContainer.h" #include "fdbclient/BackupAgent.actor.h" +#include "fdbclient/FDBTypes.h" #include "fdbclient/JsonBuilder.h" #include "flow/Trace.h" #include "flow/UnitTest.h" #include "flow/Hash3.h" #include "fdbrpc/AsyncFileReadAhead.actor.h" -#include "fdbrpc/Platform.h" +#include "flow/Platform.h" #include "fdbclient/AsyncFileBlobStore.actor.h" #include "fdbclient/Status.h" #include "fdbclient/SystemData.h" @@ -39,7 +40,7 @@ namespace IBackupFile_impl { - ACTOR Future appendStringRefWithLen(Reference file, Standalone s) { +ACTOR Future appendStringRefWithLen(Reference file, Standalone s) { state uint32_t lenBuf = bigEndian32((uint32_t)s.size()); wait(file->append(&lenBuf, sizeof(lenBuf))); wait(file->append(s.begin(), s.size())); @@ -115,6 +116,7 @@ std::string BackupDescription::toString() const { info.append(format("URL: %s\n", url.c_str())); info.append(format("Restorable: %s\n", maxRestorableVersion.present() ? "true" : "false")); + info.append(format("Partitioned logs: %s\n", partitioned ? "true" : "false")); auto formatVersion = [&](Version v) { std::string s; @@ -169,6 +171,7 @@ std::string BackupDescription::toJSON() const { doc.setKey("SchemaVersion", "1.0.0"); doc.setKey("URL", url.c_str()); doc.setKey("Restorable", maxRestorableVersion.present()); + doc.setKey("Partitioned", partitioned); auto formatVersion = [&](Version v) { JsonBuilderObject doc; @@ -228,28 +231,33 @@ std::string BackupDescription::toJSON() const { * Snapshot manifests (a complete set of files constituting a database snapshot for the backup's target ranges) * are stored as JSON files at paths like * /snapshots/snapshot,minVersion,maxVersion,totalBytes - * + * * Key range files for snapshots are stored at paths like * /kvranges/snapshot,startVersion/N/range,version,uid,blockSize * where startVersion is the version at which the backup snapshot execution began and N is a number - * that is increased as key range files are generated over time (at varying rates) such that there + * that is increased as key range files are generated over time (at varying rates) such that there * are around 5,000 key range files in each folder. * - * Note that startVersion will NOT correspond to the minVersion of a snapshot manifest because + * Note that startVersion will NOT correspond to the minVersion of a snapshot manifest because * snapshot manifest min/max versions are based on the actual contained data and the first data * file written will be after the start version of the snapshot's execution. - * + * * Log files are at file paths like - * /logs/.../log,startVersion,endVersion,blockSize + * /plogs/...log,startVersion,endVersion,UID,tagID-of-N,blocksize + * /logs/.../log,startVersion,endVersion,UID,blockSize * where ... is a multi level path which sorts lexically into version order and results in approximately 1 - * unique folder per day containing about 5,000 files. + * unique folder per day containing about 5,000 files. Logs after FDB 6.3 are stored in "plogs" + * directory and are partitioned according to tagIDs (0, 1, 2, ...) and the total number partitions is N. + * Old backup logs FDB 6.2 and earlier are stored in "logs" directory and are not partitioned. + * After FDB 6.3, users can choose to use the new partitioned logs or old logs. + * * * BACKWARD COMPATIBILITY * * Prior to FDB version 6.0.16, key range files were stored using a different folder scheme. Newer versions * still support this scheme for all restore and backup management operations but key range files generated - * by backup using version 6.0.16 or later use the scheme describe above. - * + * by backup using version 6.0.16 or later use the scheme describe above. + * * The old format stored key range files at paths like * /ranges/.../range,version,uid,blockSize * where ... is a multi level path with sorts lexically into version order and results in up to approximately @@ -258,15 +266,15 @@ std::string BackupDescription::toJSON() const { */ class BackupContainerFileSystem : public IBackupContainer { public: - virtual void addref() = 0; - virtual void delref() = 0; + void addref() override = 0; + void delref() override = 0; BackupContainerFileSystem() {} virtual ~BackupContainerFileSystem() {} // Create the container - virtual Future create() = 0; - virtual Future exists() = 0; + Future create() override = 0; + Future exists() override = 0; // Get a list of fileNames and their sizes in the container under the given path // Although not required, an implementation can avoid traversing unwanted subfolders @@ -275,7 +283,7 @@ public: virtual Future listFiles(std::string path = "", std::function folderPathFilter = nullptr) = 0; // Open a file for read by fileName - virtual Future> readFile(std::string fileName) = 0; + Future> readFile(std::string fileName) override = 0; // Open a file for write by fileName virtual Future> writeFile(std::string fileName) = 0; @@ -285,7 +293,7 @@ public: // Delete entire container. During the process, if pNumDeleted is not null it will be // updated with the count of deleted files so that progress can be seen. - virtual Future deleteContainer(int *pNumDeleted) = 0; + Future deleteContainer(int* pNumDeleted) override = 0; // Creates a 2-level path (x/y) where v should go such that x/y/* contains (10^smallestBucket) possible versions static std::string versionFolderString(Version v, int smallestBucket) { @@ -329,15 +337,25 @@ public: } // The innermost folder covers 100,000 seconds (1e11 versions) which is 5,000 mutation log files at current settings. - static std::string logVersionFolderString(Version v) { - return format("logs/%s/", versionFolderString(v, 11).c_str()); + static std::string logVersionFolderString(Version v, bool partitioned) { + return format("%s/%s/", (partitioned ? "plogs" : "logs"), versionFolderString(v, 11).c_str()); } - Future> writeLogFile(Version beginVersion, Version endVersion, int blockSize) { - return writeFile(logVersionFolderString(beginVersion) + format("log,%lld,%lld,%s,%d", beginVersion, endVersion, deterministicRandom()->randomUniqueID().toString().c_str(), blockSize)); + Future> writeLogFile(Version beginVersion, Version endVersion, int blockSize) final { + return writeFile(logVersionFolderString(beginVersion, false) + + format("log,%lld,%lld,%s,%d", beginVersion, endVersion, + deterministicRandom()->randomUniqueID().toString().c_str(), blockSize)); } - Future> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) { + Future> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize, + uint16_t tagId, int totalTags) final { + return writeFile(logVersionFolderString(beginVersion, true) + + format("log,%lld,%lld,%s,%d-of-%d,%d", beginVersion, endVersion, + deterministicRandom()->randomUniqueID().toString().c_str(), tagId, totalTags, + blockSize)); + } + + Future> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) override { std::string fileName = format("range,%" PRId64 ",%s,%d", fileVersion, deterministicRandom()->randomUniqueID().toString().c_str(), blockSize); // In order to test backward compatibility in simulation, sometimes write to the old path format @@ -348,8 +366,23 @@ public: return writeFile(snapshotFolderString(snapshotBeginVersion) + format("/%d/", snapshotFileCount / (BUGGIFY ? 1 : 5000)) + fileName); } + // Find what should be the filename of a path by finding whatever is after the last forward or backward slash, or failing to find those, the whole string. + static std::string fileNameOnly(std::string path) { + // Find the last forward slash position, defaulting to 0 if not found + int pos = path.find_last_of('/'); + if(pos == std::string::npos) { + pos = 0; + } + // Find the last backward slash position after pos, and update pos if found + int b = path.find_last_of('\\', pos); + if(b != std::string::npos) { + pos = b; + } + return path.substr(pos + 1); + } + static bool pathToRangeFile(RangeFile &out, std::string path, int64_t size) { - std::string name = basename(path); + std::string name = fileNameOnly(path); RangeFile f; f.fileName = path; f.fileSize = size; @@ -362,7 +395,7 @@ public: } static bool pathToLogFile(LogFile &out, std::string path, int64_t size) { - std::string name = basename(path); + std::string name = fileNameOnly(path); LogFile f; f.fileName = path; f.fileSize = size; @@ -370,12 +403,17 @@ public: if(sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size()) { out = f; return true; + } else if (sscanf(name.c_str(), "log,%" SCNd64 ",%" SCNd64 ",%*[^,],%d-of-%d,%u%n", &f.beginVersion, + &f.endVersion, &f.tagId, &f.totalTags, &f.blockSize, &len) == 5 && + len == name.size() && f.tagId >= 0) { + out = f; + return true; } return false; } static bool pathToKeyspaceSnapshotFile(KeyspaceSnapshotFile &out, std::string path) { - std::string name = basename(path); + std::string name = fileNameOnly(path); KeyspaceSnapshotFile f; f.fileName = path; int len; @@ -387,9 +425,11 @@ public: } // TODO: Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of megabytes. - ACTOR static Future> readKeyspaceSnapshot_impl(Reference bc, KeyspaceSnapshotFile snapshot) { + ACTOR static Future, std::map>> readKeyspaceSnapshot_impl( + Reference bc, KeyspaceSnapshotFile snapshot) { // Read the range file list for the specified version range, and then index them by fileName. - // This is so we can verify that each of the files listed in the manifest file are also in the container at this time. + // This is so we can verify that each of the files listed in the manifest file are also in the container at this + // time. std::vector files = wait(bc->listRangeFiles(snapshot.beginVersion, snapshot.endVersion)); state std::map rangeIndex; for(auto &f : files) @@ -445,16 +485,38 @@ public: throw restore_missing_data(); } - return results; + // Check key ranges for files + std::map fileKeyRanges; + JSONDoc ranges = doc.subDoc("keyRanges"); // Create an empty doc if not existed + for (auto i : ranges.obj()) { + const std::string& filename = i.first; + JSONDoc fields(i.second); + std::string begin, end; + if (fields.tryGet("beginKey", begin) && fields.tryGet("endKey", end)) { + TraceEvent("ManifestFields") + .detail("File", filename) + .detail("Begin", printable(StringRef(begin))) + .detail("End", printable(StringRef(end))); + fileKeyRanges.emplace(filename, KeyRange(KeyRangeRef(StringRef(begin), StringRef(end)))); + } else { + TraceEvent("MalFormattedManifest").detail("Key", filename); + throw restore_corrupted_data(); + } + } + + return std::make_pair(results, fileKeyRanges); } - Future> readKeyspaceSnapshot(KeyspaceSnapshotFile snapshot) { + Future, std::map>> readKeyspaceSnapshot( + KeyspaceSnapshotFile snapshot) { return readKeyspaceSnapshot_impl(Reference::addRef(this), snapshot); } - ACTOR static Future writeKeyspaceSnapshotFile_impl(Reference bc, std::vector fileNames, int64_t totalBytes) { - ASSERT(!fileNames.empty()); - + ACTOR static Future writeKeyspaceSnapshotFile_impl(Reference bc, + std::vector fileNames, + std::vector> beginEndKeys, + int64_t totalBytes) { + ASSERT(!fileNames.empty() && fileNames.size() == beginEndKeys.size()); state Version minVer = std::numeric_limits::max(); state Version maxVer = 0; @@ -485,6 +547,13 @@ public: doc.create("beginVersion") = minVer; doc.create("endVersion") = maxVer; + auto ranges = doc.subDoc("keyRanges"); + for (int i = 0; i < beginEndKeys.size(); i++) { + auto fileDoc = ranges.subDoc(fileNames[i], /*split=*/false); + fileDoc.create("beginKey") = beginEndKeys[i].first.toString(); + fileDoc.create("endKey") = beginEndKeys[i].second.toString(); + } + wait(yield()); state std::string docString = json_spirit::write_string(json); @@ -495,20 +564,26 @@ public: return Void(); } - Future writeKeyspaceSnapshotFile(std::vector fileNames, int64_t totalBytes) { - return writeKeyspaceSnapshotFile_impl(Reference::addRef(this), fileNames, totalBytes); + Future writeKeyspaceSnapshotFile(const std::vector& fileNames, + const std::vector>& beginEndKeys, + int64_t totalBytes) final { + return writeKeyspaceSnapshotFile_impl(Reference::addRef(this), fileNames, + beginEndKeys, totalBytes); }; - // List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion - Future> listLogFiles(Version beginVersion = 0, Version targetVersion = std::numeric_limits::max()) { - // The first relevant log file could have a begin version less than beginVersion based on the knobs which determine log file range size, - // so start at an earlier version adjusted by how many versions a file could contain. + // List log files, unsorted, which contain data at any version >= beginVersion and <= targetVersion. + // "partitioned" flag indicates if new partitioned mutation logs or old logs should be listed. + Future> listLogFiles(Version beginVersion, Version targetVersion, bool partitioned) { + // The first relevant log file could have a begin version less than beginVersion based on the knobs which + // determine log file range size, so start at an earlier version adjusted by how many versions a file could + // contain. // // Get the cleaned (without slashes) first and last folders that could contain relevant results. - std::string firstPath = cleanFolderString(logVersionFolderString( - std::max(0, beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES * CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE) - )); - std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion)); + std::string firstPath = cleanFolderString( + logVersionFolderString(std::max(0, beginVersion - CLIENT_KNOBS->BACKUP_MAX_LOG_RANGES * + CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE), + partitioned)); + std::string lastPath = cleanFolderString(logVersionFolderString(targetVersion, partitioned)); std::function pathFilter = [=](const std::string &folderPath) { // Remove slashes in the given folder path so that the '/' positions in the version folder string do not matter @@ -518,7 +593,7 @@ public: || (cleaned > firstPath && cleaned < lastPath); }; - return map(listFiles("logs/", pathFilter), [=](const FilesAndSizesT &files) { + return map(listFiles((partitioned ? "plogs/" : "logs/"), pathFilter), [=](const FilesAndSizesT& files) { std::vector results; LogFile lf; for(auto &f : files) { @@ -605,14 +680,18 @@ public: ACTOR static Future dumpFileList_impl(Reference bc, Version begin, Version end) { state Future> fRanges = bc->listRangeFiles(begin, end); state Future> fSnapshots = bc->listKeyspaceSnapshots(begin, end); - state Future> fLogs = bc->listLogFiles(begin, end); + state std::vector logs; + state std::vector pLogs; - wait(success(fRanges) && success(fSnapshots) && success(fLogs)); + wait(success(fRanges) && success(fSnapshots) && + store(logs, bc->listLogFiles(begin, end, false)) && + store(pLogs, bc->listLogFiles(begin, end, true))); + logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end())); - return BackupFileList({fRanges.get(), fLogs.get(), fSnapshots.get()}); + return BackupFileList({ fRanges.get(), std::move(logs), fSnapshots.get() }); } - Future dumpFileList(Version begin, Version end) { + Future dumpFileList(Version begin, Version end) override { return dumpFileList_impl(Reference::addRef(this), begin, end); } @@ -631,7 +710,28 @@ public: return v; } - ACTOR static Future describeBackup_impl(Reference bc, bool deepScan, Version logStartVersionOverride) { + // Computes the continuous end version for non-partitioned mutation logs up to + // the "targetVersion". If "outLogs" is not nullptr, it will be updated with + // continuous log files. "*end" is updated with the continuous end version. + static void computeRestoreEndVersion(const std::vector& logs, std::vector* outLogs, Version* end, + Version targetVersion) { + auto i = logs.begin(); + if (outLogs != nullptr) outLogs->push_back(*i); + + // Add logs to restorable logs set until continuity is broken OR we reach targetVersion + while (++i != logs.end()) { + if (i->beginVersion > *end || i->beginVersion > targetVersion) break; + + // If the next link in the log chain is found, update the end + if (i->beginVersion == *end) { + if (outLogs != nullptr) outLogs->push_back(*i); + *end = i->endVersion; + } + } + } + + ACTOR static Future describeBackup_impl(Reference bc, bool deepScan, + Version logStartVersionOverride) { state BackupDescription desc; desc.url = bc->getURL(); @@ -650,7 +750,8 @@ public: // This could be handled more efficiently without recursion but it's tricky, this will do for now. if(logStartVersionOverride != invalidVersion && logStartVersionOverride < 0) { BackupDescription tmp = wait(bc->describeBackup(false, invalidVersion)); - logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride, "LogStartVersionOverride", invalid_option_value()); + logStartVersionOverride = resolveRelativeVersion(tmp.maxLogEnd, logStartVersionOverride, + "LogStartVersionOverride", invalid_option_value()); } // Get metadata versions @@ -658,10 +759,12 @@ public: state Optional metaLogEnd; state Optional metaExpiredEnd; state Optional metaUnreliableEnd; + state Optional metaLogType; std::vector> metaReads; metaReads.push_back(store(metaExpiredEnd, bc->expiredEndVersion().get())); metaReads.push_back(store(metaUnreliableEnd, bc->unreliableEndVersion().get())); + metaReads.push_back(store(metaLogType, bc->logType().get())); // Only read log begin/end versions if not doing a deep scan, otherwise scan files and recalculate them. if(!deepScan) { @@ -672,12 +775,13 @@ public: wait(waitForAll(metaReads)); TraceEvent("BackupContainerDescribe2") - .detail("URL", bc->getURL()) - .detail("LogStartVersionOverride", logStartVersionOverride) - .detail("ExpiredEndVersion", metaExpiredEnd.orDefault(invalidVersion)) - .detail("UnreliableEndVersion", metaUnreliableEnd.orDefault(invalidVersion)) - .detail("LogBeginVersion", metaLogBegin.orDefault(invalidVersion)) - .detail("LogEndVersion", metaLogEnd.orDefault(invalidVersion)); + .detail("URL", bc->getURL()) + .detail("LogStartVersionOverride", logStartVersionOverride) + .detail("ExpiredEndVersion", metaExpiredEnd.orDefault(invalidVersion)) + .detail("UnreliableEndVersion", metaUnreliableEnd.orDefault(invalidVersion)) + .detail("LogBeginVersion", metaLogBegin.orDefault(invalidVersion)) + .detail("LogEndVersion", metaLogEnd.orDefault(invalidVersion)) + .detail("LogType", metaLogType.orDefault(-1)); // If the logStartVersionOverride is positive (not relative) then ensure that unreliableEndVersion is equal or greater if(logStartVersionOverride != invalidVersion && metaUnreliableEnd.orDefault(invalidVersion) < logStartVersionOverride) { @@ -736,31 +840,41 @@ public: } state std::vector logs; - wait(store(logs, bc->listLogFiles(scanBegin, scanEnd)) && store(desc.snapshots, bc->listKeyspaceSnapshots())); + state std::vector plogs; + wait(store(logs, bc->listLogFiles(scanBegin, scanEnd, false)) && + store(plogs, bc->listLogFiles(scanBegin, scanEnd, true)) && + store(desc.snapshots, bc->listKeyspaceSnapshots())); + + if (plogs.size() > 0) { + desc.partitioned = true; + logs.swap(plogs); + } else { + desc.partitioned = metaLogType.present() && metaLogType.get() == PARTITIONED_MUTATION_LOG; + } // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); - if(!logs.empty()) { + // Find out contiguous log end version + if (!logs.empty()) { desc.maxLogEnd = logs.rbegin()->endVersion; - - auto i = logs.begin(); // If we didn't get log versions above then seed them using the first log file - if(!desc.contiguousLogEnd.present()) { - desc.minLogBegin = i->beginVersion; - desc.contiguousLogEnd = i->endVersion; - ++i; + if (!desc.contiguousLogEnd.present()) { + desc.minLogBegin = logs.begin()->beginVersion; + if (desc.partitioned) { + // Cannot use the first file's end version, which may not be contiguous + // for other partitions. Set to its beginVersion to be safe. + desc.contiguousLogEnd = logs.begin()->beginVersion; + } else { + desc.contiguousLogEnd = logs.begin()->endVersion; + } } - auto &end = desc.contiguousLogEnd.get(); // For convenience to make loop cleaner - // Advance until continuity is broken - while(i != logs.end()) { - if(i->beginVersion > end) - break; - // If the next link in the log chain is found, update the end - if(i->beginVersion == end) - end = i->endVersion; - ++i; + if (desc.partitioned) { + updatePartitionedLogsContinuousEnd(&desc, logs, scanBegin, scanEnd); + } else { + Version& end = desc.contiguousLogEnd.get(); + computeRestoreEndVersion(logs, nullptr, &end, std::numeric_limits::max()); } } @@ -782,6 +896,11 @@ public: updates = updates && bc->logEndVersion().set(desc.contiguousLogEnd.get()); } + if (!metaLogType.present()) { + updates = updates && bc->logType().set(desc.partitioned ? PARTITIONED_MUTATION_LOG + : NON_PARTITIONED_MUTATION_LOG); + } + wait(updates); } catch(Error &e) { if(e.code() == error_code_actor_cancelled) @@ -829,8 +948,9 @@ public: } // Uses the virtual methods to describe the backup contents - Future describeBackup(bool deepScan, Version logStartVersionOverride) { - return describeBackup_impl(Reference::addRef(this), deepScan, logStartVersionOverride); + Future describeBackup(bool deepScan, Version logStartVersionOverride) final { + return describeBackup_impl(Reference::addRef(this), deepScan, + logStartVersionOverride); } ACTOR static Future expireData_impl(Reference bc, Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) { @@ -848,8 +968,10 @@ public: state BackupDescription desc = wait(bc->describeBackup(false, expireEndVersion)); // Resolve relative versions using max log version - expireEndVersion = resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value()); - restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion, "RestorableBeginVersion", invalid_option_value()); + expireEndVersion = + resolveRelativeVersion(desc.maxLogEnd, expireEndVersion, "ExpireEndVersion", invalid_option_value()); + restorableBeginVersion = resolveRelativeVersion(desc.maxLogEnd, restorableBeginVersion, + "RestorableBeginVersion", invalid_option_value()); // It would be impossible to have restorability to any version < expireEndVersion after expiring to that version if(restorableBeginVersion < expireEndVersion) @@ -890,13 +1012,17 @@ public: .detail("ScanBeginVersion", scanBegin); state std::vector logs; + state std::vector pLogs; // partitioned mutation logs state std::vector ranges; if(progress != nullptr) { progress->step = "Listing files"; } // Get log files or range files that contain any data at or before expireEndVersion - wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1)) && store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1))); + wait(store(logs, bc->listLogFiles(scanBegin, expireEndVersion - 1, false)) && + store(pLogs, bc->listLogFiles(scanBegin, expireEndVersion - 1, true)) && + store(ranges, bc->listRangeFiles(scanBegin, expireEndVersion - 1))); + logs.insert(logs.end(), std::make_move_iterator(pLogs.begin()), std::make_move_iterator(pLogs.end())); // The new logBeginVersion will be taken from the last log file, if there is one state Optional newLogBeginVersion; @@ -1009,10 +1135,226 @@ public: } // Delete all data up to (but not including endVersion) - Future expireData(Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) { + Future expireData(Version expireEndVersion, bool force, ExpireProgress* progress, + Version restorableBeginVersion) final { return expireData_impl(Reference::addRef(this), expireEndVersion, force, progress, restorableBeginVersion); } + // For a list of log files specified by their indices (of the same tag), + // returns if they are continous in the range [begin, end]. If "tags" is not + // nullptr, then it will be populated with [begin, end] -> tags, where next + // pair's begin <= previous pair's end + 1. On return, the last pair's end + // version (inclusive) gives the continuous range from begin. + static bool isContinuous(const std::vector& files, const std::vector& indices, Version begin, + Version end, std::map, int>* tags) { + Version lastBegin = invalidVersion; + Version lastEnd = invalidVersion; + int lastTags = -1; + + ASSERT(tags == nullptr || tags->empty()); + for (int idx : indices) { + const LogFile& file = files[idx]; + if (lastEnd == invalidVersion) { + if (file.beginVersion > begin) return false; + if (file.endVersion > begin) { + lastBegin = begin; + lastTags = file.totalTags; + } else { + continue; + } + } else if (lastEnd < file.beginVersion) { + if (tags != nullptr) { + tags->emplace(std::make_pair(lastBegin, lastEnd - 1), lastTags); + } + return false; + } + + if (lastTags != file.totalTags) { + if (tags != nullptr) { + tags->emplace(std::make_pair(lastBegin, file.beginVersion - 1), lastTags); + } + lastBegin = file.beginVersion; + lastTags = file.totalTags; + } + lastEnd = file.endVersion; + if (lastEnd > end) break; + } + if (tags != nullptr && lastBegin != invalidVersion) { + tags->emplace(std::make_pair(lastBegin, std::min(end, lastEnd - 1)), lastTags); + } + return lastBegin != invalidVersion && lastEnd > end; + } + + // Returns true if logs are continuous in the range [begin, end]. + // "files" should be pre-sorted according to version order. + static bool isPartitionedLogsContinuous(const std::vector& files, Version begin, Version end) { + std::map> tagIndices; // tagId -> indices in files + for (int i = 0; i < files.size(); i++) { + ASSERT(files[i].tagId >= 0 && files[i].tagId < files[i].totalTags); + auto& indices = tagIndices[files[i].tagId]; + indices.push_back(i); + } + + // check partition 0 is continuous and create a map of ranges to tags + std::map, int> tags; // range [begin, end] -> tags + if (!isContinuous(files, tagIndices[0], begin, end, &tags)) { + TraceEvent(SevWarn, "BackupFileNotContinuous") + .detail("Partition", 0) + .detail("RangeBegin", begin) + .detail("RangeEnd", end); + return false; + } + + // for each range in tags, check all tags from 1 are continouous + for (const auto [beginEnd, count] : tags) { + for (int i = 1; i < count; i++) { + if (!isContinuous(files, tagIndices[i], beginEnd.first, std::min(beginEnd.second - 1, end), nullptr)) { + TraceEvent(SevWarn, "BackupFileNotContinuous") + .detail("Partition", i) + .detail("RangeBegin", beginEnd.first) + .detail("RangeEnd", beginEnd.second); + return false; + } + } + } + return true; + } + + // Returns log files that are not duplicated, or subset of another log. + // If a log file's progress is not saved, a new log file will be generated + // with the same begin version. So we can have a file that contains a subset + // of contents in another log file. + // PRE-CONDITION: logs are already sorted by (tagId, beginVersion, endVersion). + static std::vector filterDuplicates(const std::vector& logs) { + std::vector filtered; + int i = 0; + for (int j = 1; j < logs.size(); j++) { + if (logs[j].isSubset(logs[i])) { + ASSERT(logs[j].fileSize <= logs[i].fileSize); + continue; + } + + if (!logs[i].isSubset(logs[j])) { + filtered.push_back(logs[i]); + } + i = j; + } + if (i < logs.size()) filtered.push_back(logs[i]); + return filtered; + } + + // Analyze partitioned logs and set contiguousLogEnd for "desc" if larger + // than the "scanBegin" version. + static void updatePartitionedLogsContinuousEnd(BackupDescription* desc, const std::vector& logs, + const Version scanBegin, const Version scanEnd) { + if (logs.empty()) return; + + Version snapshotBeginVersion = desc->snapshots.size() > 0 ? desc->snapshots[0].beginVersion : invalidVersion; + Version begin = std::max(scanBegin, desc->minLogBegin.get()); + TraceEvent("ContinuousLogEnd") + .detail("ScanBegin", scanBegin) + .detail("ScanEnd", scanEnd) + .detail("Begin", begin) + .detail("ContiguousLogEnd", desc->contiguousLogEnd.get()); + for (const auto& file : logs) { + if (file.beginVersion > begin) { + if (scanBegin > 0) return; + + // scanBegin is 0 + desc->minLogBegin = file.beginVersion; + begin = file.beginVersion; + } + + Version ver = getPartitionedLogsContinuousEndVersion(logs, begin); + if (ver >= desc->contiguousLogEnd.get()) { + // contiguousLogEnd is not inclusive, so +1 here. + desc->contiguousLogEnd.get() = ver + 1; + TraceEvent("UpdateContinuousLogEnd").detail("Version", ver + 1); + if (ver > snapshotBeginVersion) return; + } + } + } + + // Returns the end version such that [begin, end] is continuous. + // "logs" should be already sorted. + static Version getPartitionedLogsContinuousEndVersion(const std::vector& logs, Version begin) { + Version end = 0; + + std::map> tagIndices; // tagId -> indices in files + for (int i = 0; i < logs.size(); i++) { + ASSERT(logs[i].tagId >= 0); + ASSERT(logs[i].tagId < logs[i].totalTags); + auto& indices = tagIndices[logs[i].tagId]; + // filter out if indices.back() is subset of files[i] or vice versa + if (!indices.empty()) { + if (logs[indices.back()].isSubset(logs[i])) { + ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize); + indices.back() = i; + } else if (!logs[i].isSubset(logs[indices.back()])) { + indices.push_back(i); + } + } else { + indices.push_back(i); + } + end = std::max(end, logs[i].endVersion - 1); + } + TraceEvent("ContinuousLogEnd").detail("Begin", begin).detail("InitVersion", end); + + // check partition 0 is continuous in [begin, end] and create a map of ranges to partitions + std::map, int> tags; // range [start, end] -> partitions + isContinuous(logs, tagIndices[0], begin, end, &tags); + if (tags.empty() || end <= begin) return 0; + end = std::min(end, tags.rbegin()->first.second); + TraceEvent("ContinuousLogEnd").detail("Partition", 0).detail("EndVersion", end).detail("Begin", begin); + + // for each range in tags, check all partitions from 1 are continouous + Version lastEnd = begin; + for (const auto [beginEnd, count] : tags) { + Version tagEnd = beginEnd.second; // This range's minimum continous partition version + for (int i = 1; i < count; i++) { + std::map, int> rangeTags; + isContinuous(logs, tagIndices[i], beginEnd.first, beginEnd.second, &rangeTags); + tagEnd = rangeTags.empty() ? 0 : std::min(tagEnd, rangeTags.rbegin()->first.second); + TraceEvent("ContinuousLogEnd") + .detail("Partition", i) + .detail("EndVersion", tagEnd) + .detail("RangeBegin", beginEnd.first) + .detail("RangeEnd", beginEnd.second); + if (tagEnd == 0) return lastEnd == begin ? 0 : lastEnd; + } + if (tagEnd < beginEnd.second) { + return tagEnd; + } + lastEnd = beginEnd.second; + } + + return end; + } + + ACTOR static Future getSnapshotFileKeyRange_impl(Reference bc, + RangeFile file) { + state Reference inFile = wait(bc->readFile(file.fileName)); + state bool beginKeySet = false; + state Key beginKey; + state Key endKey; + state int64_t j = 0; + for (; j < file.fileSize; j += file.blockSize) { + int64_t len = std::min(file.blockSize, file.fileSize - j); + Standalone> blockData = wait(fileBackup::decodeRangeFileBlock(inFile, j, len)); + if (!beginKeySet) { + beginKey = blockData.front().key; + beginKeySet = true; + } + endKey = blockData.back().key; + } + return KeyRange(KeyRangeRef(beginKey, endKey)); + } + + Future getSnapshotFileKeyRange(const RangeFile& file) final { + ASSERT(g_network->isSimulated()); + return getSnapshotFileKeyRange_impl(Reference::addRef(this), file); + } + ACTOR static Future> getRestoreSet_impl(Reference bc, Version targetVersion) { // Find the most recent keyrange snapshot to end at or before targetVersion state Optional snapshot; @@ -1027,38 +1369,68 @@ public: restorable.snapshot = snapshot.get(); restorable.targetVersion = targetVersion; - std::vector ranges = wait(bc->readKeyspaceSnapshot(snapshot.get())); - restorable.ranges = ranges; + std::pair, std::map> results = + wait(bc->readKeyspaceSnapshot(snapshot.get())); + restorable.ranges = std::move(results.first); + restorable.keyRanges = std::move(results.second); + // TODO: Reenable the sanity check after TooManyFiles error is resolved + if (false && g_network->isSimulated()) { + // Sanity check key ranges + state std::map::iterator rit; + for (rit = restorable.keyRanges.begin(); rit != restorable.keyRanges.end(); rit++) { + auto it = std::find_if(restorable.ranges.begin(), restorable.ranges.end(), + [file = rit->first](const RangeFile f) { return f.fileName == file; }); + ASSERT(it != restorable.ranges.end()); + KeyRange result = wait(bc->getSnapshotFileKeyRange(*it)); + ASSERT(rit->second.begin <= result.begin && rit->second.end >= result.end); + } + } // No logs needed if there is a complete key space snapshot at the target version. if (snapshot.get().beginVersion == snapshot.get().endVersion && snapshot.get().endVersion == targetVersion) { + restorable.continuousBeginVersion = restorable.continuousEndVersion = invalidVersion; return Optional(restorable); } - state std::vector logs = wait(bc->listLogFiles(snapshot.get().beginVersion, targetVersion)); + // FIXME: check if there are tagged logs. for each tag, there is no version gap. + state std::vector logs; + state std::vector plogs; + wait(store(logs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, false)) && + store(plogs, bc->listLogFiles(snapshot.get().beginVersion, targetVersion, true))); + + if (plogs.size() > 0) { + logs.swap(plogs); + // sort by tag ID so that filterDuplicates works. + std::sort(logs.begin(), logs.end(), [](const LogFile& a, const LogFile& b) { + return std::tie(a.tagId, a.beginVersion, a.endVersion) < + std::tie(b.tagId, b.beginVersion, b.endVersion); + }); + + // Remove duplicated log files that can happen for old epochs. + std::vector filtered = filterDuplicates(logs); + + restorable.logs.swap(filtered); + // sort by version order again for continuous analysis + std::sort(restorable.logs.begin(), restorable.logs.end()); + if (isPartitionedLogsContinuous(restorable.logs, snapshot.get().beginVersion, targetVersion)) { + restorable.continuousBeginVersion = snapshot.get().beginVersion; + restorable.continuousEndVersion = targetVersion + 1; // not inclusive + return Optional(restorable); + } + return Optional(); + } // List logs in version order so log continuity can be analyzed std::sort(logs.begin(), logs.end()); // If there are logs and the first one starts at or before the snapshot begin version then proceed if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) { - auto i = logs.begin(); - Version end = i->endVersion; - restorable.logs.push_back(*i); - - // Add logs to restorable logs set until continuity is broken OR we reach targetVersion - while(++i != logs.end()) { - if(i->beginVersion > end || i->beginVersion > targetVersion) - break; - // If the next link in the log chain is found, update the end - if(i->beginVersion == end) { - restorable.logs.push_back(*i); - end = i->endVersion; - } - } - - if(end >= targetVersion) { + Version end = logs.begin()->endVersion; + computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion); + if (end >= targetVersion) { + restorable.continuousBeginVersion = logs.begin()->beginVersion; + restorable.continuousEndVersion = end; return Optional(restorable); } } @@ -1067,7 +1439,7 @@ public: return Optional(); } - Future> getRestoreSet(Version targetVersion){ + Future> getRestoreSet(Version targetVersion) final { return getRestoreSet_impl(Reference::addRef(this), targetVersion); } @@ -1103,6 +1475,11 @@ public: VersionProperty expiredEndVersion() { return {Reference::addRef(this), "expired_end_version"}; } VersionProperty unreliableEndVersion() { return {Reference::addRef(this), "unreliable_end_version"}; } + // Backup log types + const static Version NON_PARTITIONED_MUTATION_LOG = 0; + const static Version PARTITIONED_MUTATION_LOG = 1; + VersionProperty logType() { return { Reference::addRef(this), "mutation_log_type" }; } + ACTOR static Future writeVersionProperty(Reference bc, std::string path, Version v) { try { state Reference f = wait(bc->writeFile(path)); @@ -1152,8 +1529,8 @@ public: class BackupContainerLocalDirectory : public BackupContainerFileSystem, ReferenceCounted { public: - void addref() { return ReferenceCounted::addref(); } - void delref() { return ReferenceCounted::delref(); } + void addref() final { return ReferenceCounted::addref(); } + void delref() final { return ReferenceCounted::delref(); } static std::string getURLFormat() { return "file://"; } @@ -1202,7 +1579,7 @@ public: return results; } - Future create() { + Future create() final { // Nothing should be done here because create() can be called by any process working with the container URL, such as fdbbackup. // Since "local directory" containers are by definition local to the machine they are accessed from, // the container's creation (in this case the creation of a directory) must be ensured prior to every file creation, @@ -1212,11 +1589,11 @@ public: } // The container exists if the folder it resides in exists - Future exists() { + Future exists() final { return directoryExists(m_path); } - Future> readFile(std::string path) { + Future> readFile(std::string path) final { int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED; // Simulation does not properly handle opening the same file from multiple machines using a shared filesystem, // so create a symbolic link to make each file opening appear to be unique. This could also work in production @@ -1241,15 +1618,16 @@ public: int blockSize = 0; // Extract block size from the filename, if present size_t lastComma = path.find_last_of(','); - if(lastComma != path.npos) { + if (lastComma != path.npos) { blockSize = atoi(path.substr(lastComma + 1).c_str()); } - if(blockSize <= 0) { + if (blockSize <= 0) { blockSize = deterministicRandom()->randomInt(1e4, 1e6); } if(deterministicRandom()->random01() < .01) { blockSize /= deterministicRandom()->randomInt(1, 3); } + ASSERT(blockSize > 0); return map(f, [=](Reference fr) { int readAhead = deterministicRandom()->randomInt(0, 3); @@ -1285,15 +1663,15 @@ public: return finish_impl(Reference::addRef(this)); } - void addref() { return ReferenceCounted::addref(); } - void delref() { return ReferenceCounted::delref(); } + void addref() override { return ReferenceCounted::addref(); } + void delref() override { return ReferenceCounted::delref(); } private: Reference m_file; std::string m_finalFullPath; }; - Future> writeFile(std::string path) { + Future> writeFile(std::string path) final { int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; std::string fullPath = joinPath(m_path, path); platform::createDirectory(parentDirectory(fullPath)); @@ -1304,12 +1682,12 @@ public: }); } - Future deleteFile(std::string path) { + Future deleteFile(std::string path) final { ::deleteFile(joinPath(m_path, path)); return Void(); } - Future listFiles(std::string path, std::function) { + Future listFiles(std::string path, std::function) final { FilesAndSizesT results; std::vector files; @@ -1329,7 +1707,7 @@ public: return results; } - Future deleteContainer(int *pNumDeleted) { + Future deleteContainer(int* pNumDeleted) final { // In order to avoid deleting some random directory due to user error, first describe the backup // and make sure it has something in it. return map(describeBackup(false, invalidVersion), [=](BackupDescription const &desc) { @@ -1389,8 +1767,8 @@ public: } } - void addref() { return ReferenceCounted::addref(); } - void delref() { return ReferenceCounted::delref(); } + void addref() final { return ReferenceCounted::addref(); } + void delref() final { return ReferenceCounted::delref(); } static std::string getURLFormat() { return BlobStoreEndpoint::getURLFormat(true) + " (Note: The 'bucket' parameter is required.)"; @@ -1398,16 +1776,16 @@ public: virtual ~BackupContainerBlobStore() {} - Future> readFile(std::string path) { - return Reference( - new AsyncFileReadAheadCache( - Reference(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))), - m_bstore->knobs.read_block_size, - m_bstore->knobs.read_ahead_blocks, - m_bstore->knobs.concurrent_reads_per_file, - m_bstore->knobs.read_cache_blocks_per_file - ) - ); + Future> readFile(std::string path) final { + return Reference( + new AsyncFileReadAheadCache( + Reference(new AsyncFileBlobStoreRead(m_bstore, m_bucket, dataPath(path))), + m_bstore->knobs.read_block_size, + m_bstore->knobs.read_ahead_blocks, + m_bstore->knobs.concurrent_reads_per_file, + m_bstore->knobs.read_cache_blocks_per_file + ) + ); } ACTOR static Future> listURLs(Reference bstore, std::string bucket) { @@ -1435,17 +1813,18 @@ public: return map(m_file->sync(), [=](Void _) { self->m_file.clear(); return Void(); }); } - void addref() { return ReferenceCounted::addref(); } - void delref() { return ReferenceCounted::delref(); } + void addref() final { return ReferenceCounted::addref(); } + void delref() final { return ReferenceCounted::delref(); } + private: Reference m_file; }; - Future> writeFile(std::string path) { + Future> writeFile(std::string path) final { return Reference(new BackupFile(path, Reference(new AsyncFileBlobStoreWrite(m_bstore, m_bucket, dataPath(path))))); } - Future deleteFile(std::string path) { + Future deleteFile(std::string path) final { return m_bstore->deleteObject(m_bucket, dataPath(path)); } @@ -1467,7 +1846,7 @@ public: return files; } - Future listFiles(std::string path, std::function pathFilter) { + Future listFiles(std::string path, std::function pathFilter) final { return listFiles_impl(Reference::addRef(this), path, pathFilter); } @@ -1483,12 +1862,12 @@ public: return Void(); } - Future create() { + Future create() final { return create_impl(Reference::addRef(this)); } // The container exists if the index entry in the blob bucket exists - Future exists() { + Future exists() final { return m_bstore->objectExists(m_bucket, indexEntry()); } @@ -1508,7 +1887,7 @@ public: return Void(); } - Future deleteContainer(int *pNumDeleted) { + Future deleteContainer(int* pNumDeleted) final { return deleteContainer_impl(Reference::addRef(this), pNumDeleted); } @@ -1718,6 +2097,8 @@ ACTOR Future> timeKeeperEpochsFromVersion(Version v, Reference return found.first + (v - found.second) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; } +namespace backup_test { + int chooseFileSize(std::vector &sizes) { int size = 1000; if(!sizes.empty()) { @@ -1755,7 +2136,30 @@ Version nextVersion(Version v) { return v + increment; } -ACTOR Future testBackupContainer(std::string url) { +// Write a snapshot file with only begin & end key +ACTOR static Future testWriteSnapshotFile(Reference file, Key begin, Key end, uint32_t blockSize) { + ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size()); + + uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION; + // write Header + wait(file->append((uint8_t*)&fileVersion, sizeof(fileVersion))); + + // write begin key length and key + wait(file->appendStringRefWithLen(begin)); + + // write end key length and key + wait(file->appendStringRefWithLen(end)); + + int bytesLeft = blockSize - file->size(); + if (bytesLeft > 0) { + Value paddings = fileBackup::makePadding(bytesLeft); + wait(file->append(paddings.begin(), bytesLeft)); + } + wait(file->finish()); + return Void(); +} + +ACTOR static Future testBackupContainer(std::string url) { printf("BackupContainerTest URL %s\n", url.c_str()); state Reference c = IBackupContainer::openContainer(url); @@ -1773,6 +2177,7 @@ ACTOR Future testBackupContainer(std::string url) { state std::vector> writes; state std::map> snapshots; state std::map snapshotSizes; + state std::map>> snapshotBeginEndKeys; state int nRangeFiles = 0; state std::map logs; state Version v = deterministicRandom()->randomInt64(0, std::numeric_limits::max() / 2); @@ -1783,27 +2188,36 @@ ACTOR Future testBackupContainer(std::string url) { loop { state Version logStart = v; state int kvfiles = deterministicRandom()->randomInt(0, 3); + state Key begin = LiteralStringRef(""); + state Key end = LiteralStringRef(""); + state int blockSize = 3 * sizeof(uint32_t) + begin.size() + end.size() + 8; while(kvfiles > 0) { if(snapshots.empty()) { snapshots[v] = {}; + snapshotBeginEndKeys[v] = {}; snapshotSizes[v] = 0; if(deterministicRandom()->coinflip()) { v = nextVersion(v); } } - Reference range = wait(c->writeRangeFile(snapshots.rbegin()->first, 0, v, 10)); + Reference range = wait(c->writeRangeFile(snapshots.rbegin()->first, 0, v, blockSize)); ++nRangeFiles; v = nextVersion(v); snapshots.rbegin()->second.push_back(range->getFileName()); + snapshotBeginEndKeys.rbegin()->second.emplace_back(begin, end); int size = chooseFileSize(fileSizes); snapshotSizes.rbegin()->second += size; - writes.push_back(writeAndVerifyFile(c, range, size)); + // Write in actual range file format, instead of random data. + // writes.push_back(writeAndVerifyFile(c, range, size)); + wait(testWriteSnapshotFile(range, begin, end, blockSize)); if(deterministicRandom()->random01() < .2) { - writes.push_back(c->writeKeyspaceSnapshotFile(snapshots.rbegin()->second, snapshotSizes.rbegin()->second)); + writes.push_back(c->writeKeyspaceSnapshotFile( + snapshots.rbegin()->second, snapshotBeginEndKeys.rbegin()->second, snapshotSizes.rbegin()->second)); snapshots[v] = {}; + snapshotBeginEndKeys[v] = {}; snapshotSizes[v] = 0; break; } @@ -1932,3 +2346,67 @@ TEST_CASE("/backup/time") { return Void(); } + +TEST_CASE("/backup/continuous") { + std::vector files; + + // [0, 100) 2 tags + files.push_back({ 0, 100, 10, "file1", 100, 0, 2 }); // Tag 0: 0-100 + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 0); + + files.push_back({ 0, 100, 10, "file2", 200, 1, 2 }); // Tag 1: 0-100 + std::sort(files.begin(), files.end()); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99)); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99); + + // [100, 300) 3 tags + files.push_back({ 100, 200, 10, "file3", 200, 0, 3 }); // Tag 0: 100-200 + files.push_back({ 100, 250, 10, "file4", 200, 1, 3 }); // Tag 1: 100-250 + std::sort(files.begin(), files.end()); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 99)); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 100)); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 99); + + files.push_back({ 100, 300, 10, "file5", 200, 2, 3 }); // Tag 2: 100-300 + std::sort(files.begin(), files.end()); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 150)); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 200)); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 10, 199)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 199); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 100) == 199); + + files.push_back({ 250, 300, 10, "file6", 200, 0, 3 }); // Tag 0: 250-300, missing 200-250 + std::sort(files.begin(), files.end()); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 50, 240)); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 99) == 199); + + files.push_back({ 250, 300, 10, "file7", 200, 1, 3 }); // Tag 1: 250-300 + std::sort(files.begin(), files.end()); + ASSERT(!BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280)); + + files.push_back({ 200, 250, 10, "file8", 200, 0, 3 }); // Tag 0: 200-250 + std::sort(files.begin(), files.end()); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 299)); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 280)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 150) == 299); + + // [300, 400) 1 tag + // files.push_back({200, 250, 10, "file9", 200, 0, 3}); // Tag 0: 200-250, duplicate file + files.push_back({ 300, 400, 10, "file10", 200, 0, 1 }); // Tag 1: 300-400 + std::sort(files.begin(), files.end()); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 0, 399)); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 100, 399)); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 150, 399)); + ASSERT(BackupContainerFileSystem::isPartitionedLogsContinuous(files, 250, 399)); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 0) == 399); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 99) == 399); + ASSERT(BackupContainerFileSystem::getPartitionedLogsContinuousEndVersion(files, 250) == 399); + + return Void(); +} + +} // namespace backup_test diff --git a/fdbclient/BackupContainer.h b/fdbclient/BackupContainer.h index b14ce7e37c..8ac79937dd 100644 --- a/fdbclient/BackupContainer.h +++ b/fdbclient/BackupContainer.h @@ -62,23 +62,45 @@ protected: // Structures for various backup components +// Mutation log version written by old FileBackupAgent +static const uint32_t BACKUP_AGENT_MLOG_VERSION = 2001; + +// Mutation log version written by BackupWorker +static const uint32_t PARTITIONED_MLOG_VERSION = 4110; + +// Snapshot file version written by FileBackupAgent +static const uint32_t BACKUP_AGENT_SNAPSHOT_FILE_VERSION = 1001; + struct LogFile { Version beginVersion; Version endVersion; uint32_t blockSize; std::string fileName; int64_t fileSize; + int tagId = -1; // Log router tag. Non-negative for new backup format. + int totalTags = -1; // Total number of log router tags. // Order by beginVersion, break ties with endVersion bool operator< (const LogFile &rhs) const { return beginVersion == rhs.beginVersion ? endVersion < rhs.endVersion : beginVersion < rhs.beginVersion; } + // Returns if this log file contains a subset of content of the given file + // by comparing version range and tag ID. + bool isSubset(const LogFile& rhs) const { + return beginVersion >= rhs.beginVersion && endVersion <= rhs.endVersion && tagId == rhs.tagId; + } + + bool isPartitionedLog() const { + return tagId >= 0 && tagId < totalTags; + } + std::string toString() const { std::stringstream ss; - ss << "beginVersion:" << std::to_string(beginVersion) << " endVersion:" << std::to_string(endVersion) << - " blockSize:" << std::to_string(blockSize) << " filename:" << fileName << - " fileSize:" << std::to_string(fileSize); + ss << "beginVersion:" << std::to_string(beginVersion) << " endVersion:" << std::to_string(endVersion) + << " blockSize:" << std::to_string(blockSize) << " filename:" << fileName + << " fileSize:" << std::to_string(fileSize) + << " tagId: " << (tagId >= 0 ? std::to_string(tagId) : std::string("(None)")); return ss.str(); } }; @@ -159,6 +181,7 @@ struct BackupDescription { // The minimum version which this backup can be used to restore to Optional minRestorableVersion; std::string extendedDetail; // Freeform container-specific info. + bool partitioned; // If this backup contains partitioned mutation logs. // Resolves the versions above to timestamps using a given database's TimeKeeper data. // toString will use this information if present. @@ -173,6 +196,14 @@ struct RestorableFileSet { Version targetVersion; std::vector logs; std::vector ranges; + + // Range file's key ranges. Can be empty for backups generated before 6.3. + std::map keyRanges; + + // Mutation logs continuous range [begin, end). Both can be invalidVersion + // when the entire key space snapshot is at the target version. + Version continuousBeginVersion, continuousEndVersion; + KeyspaceSnapshotFile snapshot; // Info. for debug purposes }; @@ -205,13 +236,23 @@ public: virtual Future> writeLogFile(Version beginVersion, Version endVersion, int blockSize) = 0; virtual Future> writeRangeFile(Version snapshotBeginVersion, int snapshotFileCount, Version fileVersion, int blockSize) = 0; + // Open a tagged log file for writing, where tagId is the log router tag's id. + virtual Future> writeTaggedLogFile(Version beginVersion, Version endVersion, int blockSize, + uint16_t tagId, int totalTags) = 0; + // Write a KeyspaceSnapshotFile of range file names representing a full non overlapping // snapshot of the key ranges this backup is targeting. - virtual Future writeKeyspaceSnapshotFile(std::vector fileNames, int64_t totalBytes) = 0; + virtual Future writeKeyspaceSnapshotFile(const std::vector& fileNames, + const std::vector>& beginEndKeys, + int64_t totalBytes) = 0; // Open a file for read by name virtual Future> readFile(std::string name) = 0; + // Returns the key ranges in the snapshot file. This is an expensive function + // and should only be used in simulation for sanity check. + virtual Future getSnapshotFileKeyRange(const RangeFile& file) = 0; + struct ExpireProgress { std::string step; int total; diff --git a/fdbclient/BlobStore.actor.cpp b/fdbclient/BlobStore.actor.cpp index 571a92693c..604c8e1ed6 100644 --- a/fdbclient/BlobStore.actor.cpp +++ b/fdbclient/BlobStore.actor.cpp @@ -510,6 +510,7 @@ ACTOR Future connect_impl(Referenceknobs.secure_connection ? "https" : "http"; state Reference conn = wait(INetworkConnections::net()->connect(b->host, service, b->knobs.secure_connection ? true : false)); + wait(conn->connectHandshake()); TraceEvent("BlobStoreEndpointNewConnection").suppressFor(60) .detail("RemoteEndpoint", conn->getPeerAddress()) diff --git a/fdbclient/CMakeLists.txt b/fdbclient/CMakeLists.txt index da58789a11..0782c65360 100644 --- a/fdbclient/CMakeLists.txt +++ b/fdbclient/CMakeLists.txt @@ -18,8 +18,6 @@ set(FDBCLIENT_SRCS DatabaseConfiguration.h DatabaseContext.h EventTypes.actor.h - FailureMonitorClient.actor.cpp - FailureMonitorClient.h FDBOptions.h FDBTypes.h FileBackupAgent.actor.cpp @@ -46,6 +44,8 @@ set(FDBCLIENT_SRCS NativeAPI.actor.cpp NativeAPI.actor.h Notified.h + SpecialKeySpace.actor.cpp + SpecialKeySpace.actor.h ReadYourWrites.actor.cpp ReadYourWrites.h RestoreWorkerInterface.actor.h diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h index b0724e2b57..8e2839cfbb 100644 --- a/fdbclient/ClusterInterface.h +++ b/fdbclient/ClusterInterface.h @@ -93,6 +93,7 @@ struct ClientVersionRef { } ClientVersionRef(Arena &arena, ClientVersionRef const& cv) : clientVersion(arena, cv.clientVersion), sourceVersion(arena, cv.sourceVersion), protocolVersion(arena, cv.protocolVersion) {} + ClientVersionRef(StringRef clientVersion, StringRef sourceVersion, StringRef protocolVersion) : clientVersion(clientVersion), sourceVersion(sourceVersion), protocolVersion(protocolVersion) {} ClientVersionRef(std::string versionString) { size_t index = versionString.find(","); if(index == versionString.npos) { diff --git a/fdbclient/CommitTransaction.h b/fdbclient/CommitTransaction.h index 540157e5c9..234a869731 100644 --- a/fdbclient/CommitTransaction.h +++ b/fdbclient/CommitTransaction.h @@ -23,6 +23,7 @@ #pragma once #include "fdbclient/FDBTypes.h" +#include "fdbserver/Knobs.h" // The versioned message has wire format : -1, version, messages static const int32_t VERSION_HEADER = -1; @@ -47,9 +48,10 @@ static const char* typeString[] = { "SetValue", "ByteMax", "MinV2", "AndV2", - "CompareAndClear"}; + "CompareAndClear", + "MAX_ATOMIC_OP" }; -struct MutationRef { +struct MutationRef { static const int OVERHEAD_BYTES = 12; //12 is the size of Header in MutationList entries enum Type : uint8_t { SetValue = 0, @@ -82,8 +84,18 @@ struct MutationRef { MutationRef() {} MutationRef( Type t, StringRef a, StringRef b ) : type(t), param1(a), param2(b) {} MutationRef( Arena& to, const MutationRef& from ) : type(from.type), param1( to, from.param1 ), param2( to, from.param2 ) {} - int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); } + int totalSize() const { return OVERHEAD_BYTES + param1.size() + param2.size(); } int expectedSize() const { return param1.size() + param2.size(); } + int weightedTotalSize() const { + // AtomicOp can cause more workload to FDB cluster than the same-size set mutation; + // Amplify atomicOp size to consider such extra workload. + // A good value for FASTRESTORE_ATOMICOP_WEIGHT needs experimental evaluations. + if (isAtomicOp()) { + return totalSize() * SERVER_KNOBS->FASTRESTORE_ATOMICOP_WEIGHT; + } else { + return totalSize(); + } + } std::string toString() const { if (type < MutationRef::MAX_ATOMIC_OP) { @@ -94,9 +106,21 @@ struct MutationRef { } } + bool isAtomicOp() const { return (ATOMIC_MASK & (1 << type)) != 0; } + template void serialize( Ar& ar ) { - serializer(ar, type, param1, param2); + if (!ar.isDeserializing && type == ClearRange && equalsKeyAfter(param1, param2)) { + StringRef empty; + serializer(ar, type, param2, empty); + } else { + serializer(ar, type, param1, param2); + } + if (ar.isDeserializing && type == ClearRange && param2 == StringRef() && param1 != StringRef()) { + ASSERT(param1[param1.size()-1] == '\x00'); + param2 = param1; + param1 = param2.substr(0, param2.size()-1); + } } // These masks define which mutation types have particular properties (they are used to implement isSingleKeyMutation() etc) @@ -111,6 +135,10 @@ struct MutationRef { }; }; +static inline std::string getTypeString(MutationRef::Type type) { + return type < MutationRef::MAX_ATOMIC_OP ? typeString[(int)type] : "Unset"; +} + // A 'single key mutation' is one which affects exactly the value of the key specified by its param1 static inline bool isSingleKeyMutation(MutationRef::Type type) { return (MutationRef::SINGLE_KEY_MASK & (1< read_conflict_ranges; VectorRef< KeyRangeRef > write_conflict_ranges; VectorRef< MutationRef > mutations; Version read_snapshot; + bool report_conflicting_keys; template - force_inline void serialize( Ar& ar ) { - serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot); + force_inline void serialize(Ar& ar) { + if constexpr (is_fb_function) { + serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot, + report_conflicting_keys); + } else { + serializer(ar, read_conflict_ranges, write_conflict_ranges, mutations, read_snapshot); + if (ar.protocolVersion().hasReportConflictingKeys()) { + serializer(ar, report_conflicting_keys); + } + } } // Convenience for internal code required to manipulate these without the Native API diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp index ca91e8b8b3..1da07379e7 100644 --- a/fdbclient/DatabaseBackupAgent.actor.cpp +++ b/fdbclient/DatabaseBackupAgent.actor.cpp @@ -1490,6 +1490,12 @@ namespace dbBackup { Version bVersion = wait(srcTr->getReadVersion()); beginVersionKey = BinaryWriter::toValue(bVersion, Unversioned()); + state Key versionKey = logUidValue.withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix); + Optional versionRecord = wait( srcTr->get(versionKey) ); + if(!versionRecord.present()) { + srcTr->set(versionKey, beginVersionKey); + } + task->params[BackupAgentBase::destUid] = destUidValue; wait(srcTr->commit()); @@ -1539,9 +1545,6 @@ namespace dbBackup { if(v.present() && BinaryReader::fromStringRef(v.get(), Unversioned()) >= BinaryReader::fromStringRef(task->params[DatabaseBackupAgent::keyFolderId], Unversioned())) return Void(); - Key versionKey = logUidValue.withPrefix(destUidValue).withPrefix(backupLatestVersionsPrefix); - srcTr2->set(versionKey, beginVersionKey); - srcTr2->set( Subspace(databaseBackupPrefixRange.begin).get(BackupAgentBase::keySourceTagName).pack(task->params[BackupAgentBase::keyTagName]), logUidValue ); srcTr2->set( sourceStates.pack(DatabaseBackupAgent::keyFolderId), task->params[DatabaseBackupAgent::keyFolderId] ); srcTr2->set( sourceStates.pack(DatabaseBackupAgent::keyStateStatus), StringRef(BackupAgentBase::getStateText(BackupAgentBase::STATE_RUNNING))); @@ -1840,6 +1843,9 @@ public: tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + //This commit must happen on the first proxy to ensure that the applier has flushed all mutations from previous DRs + tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); // We will use the global status for now to ensure that multiple backups do not start place with different tags state int status = wait(backupAgent->getStateValue(tr, logUidCurrent)); @@ -1959,8 +1965,8 @@ public: } if (!g_network->isSimulated() && !forceAction) { - state StatusObject srcStatus = wait(StatusClient::statusFetcher(backupAgent->taskBucket->src->getConnectionFile())); - StatusObject destStatus = wait(StatusClient::statusFetcher(dest->getConnectionFile())); + state StatusObject srcStatus = wait(StatusClient::statusFetcher(backupAgent->taskBucket->src)); + StatusObject destStatus = wait(StatusClient::statusFetcher(dest)); checkAtomicSwitchOverConfig(srcStatus, destStatus, tagName); } @@ -2274,6 +2280,7 @@ public: state Reference tr(new ReadYourWritesTransaction(cx)); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state std::string statusText; + state int retries = 0; loop{ try { @@ -2291,27 +2298,33 @@ public: tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Future> fPaused = tr->get(backupAgent->taskBucket->getPauseKey()); + state Future> fErrorValues = errorLimit > 0 ? tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(), errorLimit, false, true) : Future>(); + state Future> fBackupUid = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId)); + state Future> fBackupVerison = tr->get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin)); + state Future> fTagName = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupTag)); + state Future> fStopVersionKey = tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyStateStop)); + state Future> fBackupKeysPacked = tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupRanges)); + int backupStateInt = wait(backupAgent->getStateValue(tr, logUid)); state BackupAgentBase::enumState backupState = (BackupAgentBase::enumState)backupStateInt; - + if (backupState == DatabaseBackupAgent::STATE_NEVERRAN) { statusText += "No previous backups found.\n"; } else { state std::string tagNameDisplay; - Optional tagName = wait(tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupTag))); + Optional tagName = wait(fTagName); // Define the display tag name if (tagName.present()) { tagNameDisplay = tagName.get().toString(); } - state Optional uid = wait(tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyFolderId))); - state Optional stopVersionKey = wait(tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyStateStop))); + state Optional stopVersionKey = wait(fStopVersionKey); + + Optional backupKeysPacked = wait(fBackupKeysPacked); state Standalone> backupRanges; - Optional backupKeysPacked = wait(tr->get(backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::keyConfigBackupRanges))); - if (backupKeysPacked.present()) { BinaryReader br(backupKeysPacked.get(), IncludeVersion()); br >> backupRanges; @@ -2347,7 +2360,7 @@ public: // Append the errors, if requested if (errorLimit > 0) { - Standalone values = wait(tr->getRange(backupAgent->errors.get(BinaryWriter::toValue(logUid, Unversioned())).range(), errorLimit, false, true)); + Standalone values = wait( fErrorValues ); // Display the errors, if any if (values.size() > 0) { @@ -2364,10 +2377,9 @@ public: //calculate time differential - state Optional backupUid = wait(tr->get(backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyFolderId))); + Optional backupUid = wait(fBackupUid); if(backupUid.present()) { - Optional v = wait(tr->get(BinaryWriter::toValue(logUid, Unversioned()).withPrefix(applyMutationsBeginRange.begin))); - + Optional v = wait(fBackupVerison); if (v.present()) { state Version destApplyBegin = BinaryReader::fromStringRef(v.get(), Unversioned()); Version sourceVersion = wait(srcReadVersion); @@ -2384,6 +2396,11 @@ public: break; } catch (Error &e) { + retries++; + if(retries > 5) { + statusText += format("\nWARNING: Could not fetch full DR status: %s\n", e.name()); + return statusText; + } wait(tr->onError(e)); } } @@ -2391,28 +2408,28 @@ public: return statusText; } - ACTOR static Future getStateValue(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid) { + ACTOR static Future getStateValue(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, bool snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key statusKey = backupAgent->states.get(BinaryWriter::toValue(logUid, Unversioned())).pack(DatabaseBackupAgent::keyStateStatus); - Optional status = wait(tr->get(statusKey)); + Optional status = wait(tr->get(statusKey, snapshot)); return (!status.present()) ? DatabaseBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(status.get().toString()); } - ACTOR static Future getDestUid(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid) { + ACTOR static Future getDestUid(DatabaseBackupAgent* backupAgent, Reference tr, UID logUid, bool snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); state Key destUidKey = backupAgent->config.get(BinaryWriter::toValue(logUid, Unversioned())).pack(BackupAgentBase::destUid); - Optional destUid = wait(tr->get(destUidKey)); + Optional destUid = wait(tr->get(destUidKey, snapshot)); return (destUid.present()) ? BinaryReader::fromStringRef(destUid.get(), Unversioned()) : UID(); } - ACTOR static Future getLogUid(DatabaseBackupAgent* backupAgent, Reference tr, Key tagName) { + ACTOR static Future getLogUid(DatabaseBackupAgent* backupAgent, Reference tr, Key tagName, bool snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state Optional logUid = wait(tr->get(backupAgent->tagNames.pack(tagName))); + state Optional logUid = wait(tr->get(backupAgent->tagNames.pack(tagName), snapshot)); return (logUid.present()) ? BinaryReader::fromStringRef(logUid.get(), Unversioned()) : UID(); } @@ -2442,16 +2459,16 @@ Future DatabaseBackupAgent::getStatus(Database cx, int errorLimit, return DatabaseBackupAgentImpl::getStatus(this, cx, errorLimit, tagName); } -Future DatabaseBackupAgent::getStateValue(Reference tr, UID logUid) { - return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid); +Future DatabaseBackupAgent::getStateValue(Reference tr, UID logUid, bool snapshot) { + return DatabaseBackupAgentImpl::getStateValue(this, tr, logUid, snapshot); } -Future DatabaseBackupAgent::getDestUid(Reference tr, UID logUid) { - return DatabaseBackupAgentImpl::getDestUid(this, tr, logUid); +Future DatabaseBackupAgent::getDestUid(Reference tr, UID logUid, bool snapshot) { + return DatabaseBackupAgentImpl::getDestUid(this, tr, logUid, snapshot); } -Future DatabaseBackupAgent::getLogUid(Reference tr, Key tagName) { - return DatabaseBackupAgentImpl::getLogUid(this, tr, tagName); +Future DatabaseBackupAgent::getLogUid(Reference tr, Key tagName, bool snapshot) { + return DatabaseBackupAgentImpl::getLogUid(this, tr, tagName, snapshot); } Future DatabaseBackupAgent::waitUpgradeToLatestDrVersion(Database cx, Key tagName) { @@ -2466,10 +2483,10 @@ Future DatabaseBackupAgent::waitSubmitted(Database cx, Key tagName) { return DatabaseBackupAgentImpl::waitSubmitted(this, cx, tagName); } -Future DatabaseBackupAgent::getRangeBytesWritten(Reference tr, UID logUid) { - return DRConfig(logUid).rangeBytesWritten().getD(tr); +Future DatabaseBackupAgent::getRangeBytesWritten(Reference tr, UID logUid, bool snapshot) { + return DRConfig(logUid).rangeBytesWritten().getD(tr, snapshot); } -Future DatabaseBackupAgent::getLogBytesWritten(Reference tr, UID logUid) { - return DRConfig(logUid).logBytesWritten().getD(tr); +Future DatabaseBackupAgent::getLogBytesWritten(Reference tr, UID logUid, bool snapshot) { + return DRConfig(logUid).logBytesWritten().getD(tr, snapshot); } diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp index e432dee63c..d7b8468f25 100644 --- a/fdbclient/DatabaseConfiguration.cpp +++ b/fdbclient/DatabaseConfiguration.cpp @@ -41,6 +41,7 @@ void DatabaseConfiguration::resetInternal() { tLogPolicy = storagePolicy = remoteTLogPolicy = Reference(); remoteDesiredTLogCount = -1; remoteTLogReplicationFactor = repopulateRegionAntiQuorum = 0; + backupWorkerEnabled = false; } void parse( int* i, ValueRef const& v ) { @@ -269,6 +270,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { result["storage_engine"] = "ssd-redwood-experimental"; } else if( tLogDataStoreType == KeyValueStoreType::MEMORY && storageServerStoreType == KeyValueStoreType::MEMORY ) { result["storage_engine"] = "memory-1"; + } else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::MEMORY_RADIXTREE ) { + result["storage_engine"] = "memory-radixtree-beta"; } else if( tLogDataStoreType == KeyValueStoreType::SSD_BTREE_V2 && storageServerStoreType == KeyValueStoreType::MEMORY ) { result["storage_engine"] = "memory-2"; } else { @@ -320,6 +323,8 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { if (autoDesiredTLogCount != CLIENT_KNOBS->DEFAULT_AUTO_LOGS) { result["auto_logs"] = autoDesiredTLogCount; } + + result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled; } return result; @@ -409,10 +414,17 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { type = std::min((int)TLogVersion::MAX_SUPPORTED, type); tLogVersion = (TLogVersion::Version)type; } - else if (ck == LiteralStringRef("log_engine")) { parse((&type), value); tLogDataStoreType = (KeyValueStoreType::StoreType)type; + else if (ck == LiteralStringRef("log_engine")) { + parse((&type), value); + tLogDataStoreType = (KeyValueStoreType::StoreType)type; // TODO: Remove this once Redwood works as a log engine - if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) + if(tLogDataStoreType == KeyValueStoreType::SSD_REDWOOD_V1) { tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2; + } + // TODO: Remove this once memroy radix tree works as a log engine + if(tLogDataStoreType == KeyValueStoreType::MEMORY_RADIXTREE) { + tLogDataStoreType = KeyValueStoreType::SSD_BTREE_V2; + } } else if (ck == LiteralStringRef("log_spill")) { parse((&type), value); tLogSpillType = (TLogSpillType::SpillType)type; } else if (ck == LiteralStringRef("storage_engine")) { parse((&type), value); storageServerStoreType = (KeyValueStoreType::StoreType)type; } @@ -425,6 +437,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { else if (ck == LiteralStringRef("remote_logs")) parse(&remoteDesiredTLogCount, value); else if (ck == LiteralStringRef("remote_log_replicas")) parse(&remoteTLogReplicationFactor, value); else if (ck == LiteralStringRef("remote_log_policy")) parseReplicationPolicy(&remoteTLogPolicy, value); + else if (ck == LiteralStringRef("backup_worker_enabled")) { parse((&type), value); backupWorkerEnabled = (type != 0); } else if (ck == LiteralStringRef("usable_regions")) parse(&usableRegions, value); else if (ck == LiteralStringRef("repopulate_anti_quorum")) parse(&repopulateRegionAntiQuorum, value); else if (ck == LiteralStringRef("regions")) parse(®ions, value); @@ -481,11 +494,16 @@ Optional DatabaseConfiguration::get( KeyRef key ) const { } } -bool DatabaseConfiguration::isExcludedServer( NetworkAddress a ) const { - return get( encodeExcludedServersKey( AddressExclusion(a.ip, a.port) ) ).present() || - get( encodeExcludedServersKey( AddressExclusion(a.ip) ) ).present() || - get( encodeFailedServersKey( AddressExclusion(a.ip, a.port) ) ).present() || - get( encodeFailedServersKey( AddressExclusion(a.ip) ) ).present(); +bool DatabaseConfiguration::isExcludedServer( NetworkAddressList a ) const { + return get( encodeExcludedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() || + get( encodeExcludedServersKey( AddressExclusion(a.address.ip) ) ).present() || + get( encodeFailedServersKey( AddressExclusion(a.address.ip, a.address.port) ) ).present() || + get( encodeFailedServersKey( AddressExclusion(a.address.ip) ) ).present() || + ( a.secondaryAddress.present() && ( + get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() || + get( encodeExcludedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() || + get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip, a.secondaryAddress.get().port) ) ).present() || + get( encodeFailedServersKey( AddressExclusion(a.secondaryAddress.get().ip) ) ).present() ) ); } std::set DatabaseConfiguration::getExcludedServers() const { const_cast(this)->makeConfigurationImmutable(); diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index 0fdae09956..46e0fbfc1f 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -107,7 +107,7 @@ struct DatabaseConfiguration { int expectedLogSets( Optional dcId ) const { int result = 1; - if(dcId.present() && getRegion(dcId.get()).satelliteTLogReplicationFactor > 0) { + if(dcId.present() && getRegion(dcId.get()).satelliteTLogReplicationFactor > 0 && usableRegions > 1) { result++; } @@ -178,13 +178,16 @@ struct DatabaseConfiguration { int32_t remoteTLogReplicationFactor; Reference remoteTLogPolicy; + // Backup Workers + bool backupWorkerEnabled; + //Data centers int32_t usableRegions; int32_t repopulateRegionAntiQuorum; std::vector regions; // Excluded servers (no state should be here) - bool isExcludedServer( NetworkAddress ) const; + bool isExcludedServer( NetworkAddressList ) const; std::set getExcludedServers() const; int32_t getDesiredProxies() const { if(masterProxyCount == -1) return autoMasterProxyCount; return masterProxyCount; } diff --git a/fdbclient/DatabaseContext.h b/fdbclient/DatabaseContext.h index 0f714c758b..8082f4200a 100644 --- a/fdbclient/DatabaseContext.h +++ b/fdbclient/DatabaseContext.h @@ -25,6 +25,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/KeyRangeMap.h" #include "fdbclient/MasterProxyInterface.h" +#include "fdbclient/SpecialKeySpace.actor.h" #include "fdbrpc/QueueModel.h" #include "fdbrpc/MultiInterface.h" #include "flow/TDMetric.actor.h" @@ -160,18 +161,41 @@ public: CounterCollection cc; Counter transactionReadVersions; + Counter transactionReadVersionsCompleted; + Counter transactionReadVersionBatches; + Counter transactionBatchReadVersions; + Counter transactionDefaultReadVersions; + Counter transactionImmediateReadVersions; + Counter transactionBatchReadVersionsCompleted; + Counter transactionDefaultReadVersionsCompleted; + Counter transactionImmediateReadVersionsCompleted; Counter transactionLogicalReads; Counter transactionPhysicalReads; + Counter transactionPhysicalReadsCompleted; + Counter transactionGetKeyRequests; + Counter transactionGetValueRequests; + Counter transactionGetRangeRequests; + Counter transactionWatchRequests; + Counter transactionGetAddressesForKeyRequests; + Counter transactionBytesRead; + Counter transactionKeysRead; + Counter transactionMetadataVersionReads; Counter transactionCommittedMutations; Counter transactionCommittedMutationBytes; + Counter transactionSetMutations; + Counter transactionClearMutations; + Counter transactionAtomicMutations; Counter transactionsCommitStarted; Counter transactionsCommitCompleted; + Counter transactionKeyServerLocationRequests; + Counter transactionKeyServerLocationRequestsCompleted; Counter transactionsTooOld; Counter transactionsFutureVersions; Counter transactionsNotCommitted; Counter transactionsMaybeCommitted; Counter transactionsResourceConstrained; Counter transactionsProcessBehind; + Counter transactionsThrottled; ContinuousSample latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit, bytesPerCommit; @@ -191,6 +215,10 @@ public: Future clientInfoMonitor; Future connected; + Reference>> statusClusterInterface; + Future statusLeaderMon; + double lastStatusFetch; + int apiVersion; int mvCacheInsertLocation; @@ -201,6 +229,8 @@ public: double detailedHealthMetricsLastUpdated; UniqueOrderedOptionList transactionDefaults; + std::shared_ptr specialKeySpace; + std::shared_ptr cKImpl; }; #endif diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index 4358e83887..21b5d00dc5 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -42,11 +42,16 @@ enum { tagLocalityRemoteLog = -3, tagLocalityUpgraded = -4, tagLocalitySatellite = -5, - tagLocalityLogRouterMapped = -6, + tagLocalityLogRouterMapped = -6, // used by log router to pop from TLogs tagLocalityTxs = -7, + tagLocalityBackup = -8, // used by backup role to pop from TLogs tagLocalityInvalid = -99 }; //The TLog and LogRouter require these number to be as compact as possible +inline bool isPseudoLocality(int8_t locality) { + return locality == tagLocalityLogRouterMapped || locality == tagLocalityBackup; +} + #pragma pack(push, 1) struct Tag { int8_t locality; @@ -68,7 +73,7 @@ struct Tag { } template - force_inline void serialize_unversioned(Ar& ar) { + force_inline void serialize_unversioned(Ar& ar) { serializer(ar, locality, id); } }; @@ -157,11 +162,11 @@ void uniquify( Collection& c ) { c.resize( std::unique(c.begin(), c.end()) - c.begin() ); } -static std::string describe( const Tag item ) { +inline std::string describe( const Tag item ) { return format("%d:%d", item.locality, item.id); } -static std::string describe( const int item ) { +inline std::string describe( const int item ) { return format("%d", item); } @@ -171,17 +176,17 @@ static std::string describe(const std::string& s) { } template -static std::string describe( Reference const& item ) { +std::string describe( Reference const& item ) { return item->toString(); } template -static std::string describe( T const& item ) { +std::string describe( T const& item ) { return item.toString(); } template -static std::string describe( std::map const& items, int max_items = -1 ) { +std::string describe( std::map const& items, int max_items = -1 ) { if(!items.size()) return "[no items]"; @@ -197,7 +202,7 @@ static std::string describe( std::map const& items, int max_items = -1 ) { } template -static std::string describeList( T const& items, int max_items ) { +std::string describeList( T const& items, int max_items ) { if(!items.size()) return "[no items]"; @@ -213,12 +218,12 @@ static std::string describeList( T const& items, int max_items ) { } template -static std::string describe( std::vector const& items, int max_items = -1 ) { +std::string describe( std::vector const& items, int max_items = -1 ) { return describeList(items, max_items); } template -static std::string describe( std::set const& items, int max_items = -1 ) { +std::string describe( std::set const& items, int max_items = -1 ) { return describeList(items, max_items); } @@ -277,8 +282,20 @@ struct KeyRangeRef { template force_inline void serialize(Ar& ar) { - serializer(ar, const_cast(begin), const_cast(end)); + if (!ar.isDeserializing && equalsKeyAfter(begin, end)) { + StringRef empty; + serializer(ar, const_cast(end), empty); + } else { + serializer(ar, const_cast(begin), const_cast(end)); + } + if (ar.isDeserializing && end == StringRef() && begin != StringRef()) { + ASSERT(begin[begin.size()-1] == '\x00'); + const_cast(end) = begin; + const_cast(begin) = end.substr(0, end.size()-1); + } + if( begin > end ) { + TraceEvent("InvertedRange").detail("Begin", begin).detail("End", end); throw inverted_range(); }; } @@ -409,9 +426,9 @@ typedef Standalone Key; typedef Standalone Value; typedef Standalone KeyRange; typedef Standalone KeyValue; -typedef Standalone KeySelector; +typedef Standalone KeySelector; -enum { invalidVersion = -1, latestVersion = -2 }; +enum { invalidVersion = -1, latestVersion = -2, MAX_VERSION = std::numeric_limits::max() }; inline Key keyAfter( const KeyRef& key ) { if(key == LiteralStringRef("\xff\xff")) @@ -573,7 +590,7 @@ struct KeyRangeWith : KeyRange { } }; template -static inline KeyRangeWith keyRangeWith( const KeyRangeRef& range, const Val& value ) { +KeyRangeWith keyRangeWith( const KeyRangeRef& range, const Val& value ) { return KeyRangeWith(range, value); } @@ -648,6 +665,7 @@ struct KeyValueStoreType { MEMORY, SSD_BTREE_V2, SSD_REDWOOD_V1, + MEMORY_RADIXTREE, END }; @@ -657,6 +675,7 @@ struct KeyValueStoreType { this->type = END; } operator StoreType() const { return StoreType(type); } + StoreType storeType() const { return StoreType(type); } template void serialize(Ar& ar) { serializer(ar, type); } @@ -667,6 +686,7 @@ struct KeyValueStoreType { case SSD_BTREE_V2: return "ssd-2"; case SSD_REDWOOD_V1: return "ssd-redwood-experimental"; case MEMORY: return "memory"; + case MEMORY_RADIXTREE: return "memory-radixtree-beta"; default: return "unknown"; } } @@ -687,6 +707,9 @@ struct TLogVersion { UNSET = 0, // Everything between BEGIN and END should be densely packed, so that we // can iterate over them easily. + // V3 was the introduction of spill by reference; + // V4 changed how data gets written to satellite TLogs so that we can peek from them; + // V5 merged reference and value spilling // V1 = 1, // 4.6 is dispatched to via 6.0 V2 = 2, // 6.0 V3 = 3, // 6.1 @@ -809,6 +832,11 @@ struct LogMessageVersion { explicit LogMessageVersion(Version version) : version(version), sub(0) {} LogMessageVersion() : version(0), sub(0) {} bool empty() const { return (version == 0) && (sub == 0); } + + template + void serialize(Ar& ar) { + serializer(ar, version, sub); + } }; struct AddressExclusion { @@ -849,7 +877,7 @@ struct AddressExclusion { } }; -static bool addressExcluded( std::set const& exclusions, NetworkAddress const& addr ) { +inline bool addressExcluded( std::set const& exclusions, NetworkAddress const& addr ) { return exclusions.count( AddressExclusion(addr.ip, addr.port) ) || exclusions.count( AddressExclusion(addr.ip) ); } @@ -966,4 +994,19 @@ struct HealthMetrics { } }; +struct WorkerBackupStatus { + LogEpoch epoch; + Version version; + Tag tag; + int32_t totalTags; + + WorkerBackupStatus() : epoch(0), version(invalidVersion) {} + WorkerBackupStatus(LogEpoch e, Version v, Tag t, int32_t total) : epoch(e), version(v), tag(t), totalTags(total) {} + + template + void serialize(Ar& ar) { + serializer(ar, epoch, version, tag, totalTags); + } +}; + #endif diff --git a/fdbclient/FailureMonitorClient.actor.cpp b/fdbclient/FailureMonitorClient.actor.cpp deleted file mode 100644 index 7cb1a3144e..0000000000 --- a/fdbclient/FailureMonitorClient.actor.cpp +++ /dev/null @@ -1,186 +0,0 @@ -/* - * FailureMonitorClient.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbclient/FailureMonitorClient.h" -#include "fdbrpc/FailureMonitor.h" -#include "fdbclient/ClusterInterface.h" -#include "flow/actorcompiler.h" // has to be last include -#include - -struct FailureMonitorClientState : ReferenceCounted { - std::unordered_set knownAddrs; - double serverFailedTimeout; - - FailureMonitorClientState() { - serverFailedTimeout = CLIENT_KNOBS->FAILURE_TIMEOUT_DELAY; - } -}; - -ACTOR Future failureMonitorClientLoop( - SimpleFailureMonitor* monitor, - ClusterInterface controller, - Reference fmState, - bool trackMyStatus) -{ - state Version version = 0; - state Future request = Never(); - state Future nextRequest = delay(0, TaskPriority::FailureMonitor); - state Future requestTimeout = Never(); - state double before = now(); - state double waitfor = 0; - - state NetworkAddressList controlAddr = controller.failureMonitoring.getEndpoint().addresses; - monitor->setStatus(controlAddr.address, FailureStatus(false)); - fmState->knownAddrs.insert(controlAddr.address); - if(controlAddr.secondaryAddress.present()) { - monitor->setStatus(controlAddr.secondaryAddress.get(), FailureStatus(false)); - fmState->knownAddrs.insert(controlAddr.secondaryAddress.get()); - } - - //The cluster controller's addresses (controller.failureMonitoring.getEndpoint().addresses) are treated specially because we can declare that it is down independently - //of the response from the cluster controller. It still needs to be in knownAddrs in case the cluster controller changes, so the next cluster controller resets its state - - try { - loop { - choose { - when( FailureMonitoringReply reply = wait( request ) ) { - g_network->setCurrentTask(TaskPriority::DefaultDelay); - request = Never(); - requestTimeout = Never(); - if (reply.allOthersFailed) { - // Reset all systems *not* mentioned in the reply to the default (failed) state - fmState->knownAddrs.erase( controller.failureMonitoring.getEndpoint().addresses.address ); - if(controller.failureMonitoring.getEndpoint().addresses.secondaryAddress.present()) { - fmState->knownAddrs.erase( controller.failureMonitoring.getEndpoint().addresses.secondaryAddress.get() ); - } - - std::set changedAddresses; - for(int c=0; cknownAddrs) - if (!changedAddresses.count( it )) - monitor->setStatus( it, FailureStatus() ); - fmState->knownAddrs.clear(); - } else { - ASSERT( version != 0 ); - } - - if( monitor->getState( controller.failureMonitoring.getEndpoint() ).isFailed() ) - TraceEvent("FailureMonitoringServerUp").detail("OldServer",controller.id()); - - monitor->setStatus(controlAddr.address, FailureStatus(false)); - fmState->knownAddrs.insert(controlAddr.address); - if(controlAddr.secondaryAddress.present()) { - monitor->setStatus(controlAddr.secondaryAddress.get(), FailureStatus(false)); - fmState->knownAddrs.insert(controlAddr.secondaryAddress.get()); - } - - //if (version != reply.failureInformationVersion) - // printf("Client '%s': update from %lld to %lld (%d changes, aof=%d)\n", g_network->getLocalAddress().toString().c_str(), version, reply.failureInformationVersion, reply.changes.size(), reply.allOthersFailed); - - version = reply.failureInformationVersion; - fmState->serverFailedTimeout = reply.considerServerFailedTimeoutMS * .001; - for(int c=0; cgetLocalAddress().toString().c_str(), reply.changes[c].address.toString().c_str(), reply.changes[c].status.failed ? "Failed" : "OK"); - auto& addrList = reply.changes[c].addresses; - monitor->setStatus( addrList.address, reply.changes[c].status ); - if(addrList.secondaryAddress.present()) { - monitor->setStatus( addrList.secondaryAddress.get(), reply.changes[c].status ); - } - if (reply.changes[c].status != FailureStatus()) { - fmState->knownAddrs.insert( addrList.address ); - if(addrList.secondaryAddress.present()) { - fmState->knownAddrs.insert( addrList.secondaryAddress.get() ); - } - } else { - fmState->knownAddrs.erase( addrList.address ); - if(addrList.secondaryAddress.present()) { - fmState->knownAddrs.erase( addrList.secondaryAddress.get() ); - } - } - } - before = now(); - waitfor = reply.clientRequestIntervalMS * .001; - nextRequest = delayJittered( waitfor, TaskPriority::FailureMonitor ); - } - when( wait( requestTimeout ) ) { - g_network->setCurrentTask(TaskPriority::DefaultDelay); - requestTimeout = Never(); - TraceEvent(SevWarn, "FailureMonitoringServerDown").detail("OldServerID",controller.id()); - monitor->setStatus(controlAddr.address, FailureStatus(true)); - fmState->knownAddrs.erase(controlAddr.address); - if(controlAddr.secondaryAddress.present()) { - monitor->setStatus(controlAddr.secondaryAddress.get(), FailureStatus(true)); - fmState->knownAddrs.erase(controlAddr.secondaryAddress.get()); - } - } - when( wait( nextRequest ) ) { - g_network->setCurrentTask(TaskPriority::DefaultDelay); - nextRequest = Never(); - - double elapsed = now() - before; - double slowThreshold = .200 + waitfor + FLOW_KNOBS->MAX_BUGGIFIED_DELAY; - double warnAlwaysThreshold = CLIENT_KNOBS->FAILURE_MIN_DELAY/2; - - if (elapsed > slowThreshold && deterministicRandom()->random01() < elapsed / warnAlwaysThreshold) { - TraceEvent(elapsed > warnAlwaysThreshold ? SevWarnAlways : SevWarn, "FailureMonitorClientSlow").detail("Elapsed", elapsed).detail("Expected", waitfor); - } - - FailureMonitoringRequest req; - req.failureInformationVersion = version; - req.addresses = g_network->getLocalAddresses(); - if (trackMyStatus) - req.senderStatus = FailureStatus(false); - request = controller.failureMonitoring.getReply( req, TaskPriority::FailureMonitor ); - if(!controller.failureMonitoring.getEndpoint().isLocal()) - requestTimeout = delay( fmState->serverFailedTimeout, TaskPriority::FailureMonitor ); - } - } - } - } catch (Error& e) { - if (e.code() == error_code_broken_promise) // broken promise from clustercontroller means it has died (and hopefully will be replaced) - return Void(); - TraceEvent(SevError, "FailureMonitorClientError").error(e); - throw; // goes nowhere - } -} - -ACTOR Future failureMonitorClient( Reference>> ci, bool trackMyStatus ) { - TraceEvent("FailureMonitorStart").detail("IsClient", FlowTransport::transport().isClient()); - if (FlowTransport::transport().isClient()) { - wait(Never()); - } - - state SimpleFailureMonitor* monitor = static_cast( &IFailureMonitor::failureMonitor() ); - state Reference fmState = Reference(new FailureMonitorClientState()); - auto localAddr = g_network->getLocalAddresses(); - monitor->setStatus(localAddr.address, FailureStatus(false)); - if(localAddr.secondaryAddress.present()) { - monitor->setStatus(localAddr.secondaryAddress.get(), FailureStatus(false)); - } - loop { - state Future client = ci->get().present() ? failureMonitorClientLoop(monitor, ci->get().get(), fmState, trackMyStatus) : Void(); - wait( ci->onChange() ); - } -} diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp index ac4f6f6e17..49bd98816d 100644 --- a/fdbclient/FileBackupAgent.actor.cpp +++ b/fdbclient/FileBackupAgent.actor.cpp @@ -21,8 +21,10 @@ #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" #include "fdbclient/DatabaseContext.h" +#include "fdbclient/Knobs.h" #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/Status.h" +#include "fdbclient/SystemData.h" #include "fdbclient/KeyBackedTypes.h" #include "fdbclient/JsonBuilder.h" @@ -97,9 +99,9 @@ StringRef FileBackupAgent::restoreStateText(ERestoreState id) { template<> Tuple Codec::pack(ERestoreState const &val) { return Tuple().append(val); } template<> ERestoreState Codec::unpack(Tuple const &val) { return (ERestoreState)val.getInt(0); } -ACTOR Future> TagUidMap::getAll_impl(TagUidMap *tagsMap, Reference tr) { +ACTOR Future> TagUidMap::getAll_impl(TagUidMap *tagsMap, Reference tr, bool snapshot) { state Key prefix = tagsMap->prefix; // Copying it here as tagsMap lifetime is not tied to this actor - TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6)); + TagMap::PairsType tagPairs = wait(tagsMap->getRange(tr, std::string(), {}, 1e6, snapshot)); std::vector results; for(auto &p : tagPairs) results.push_back(KeyBackedTag(p.first, prefix)); @@ -459,7 +461,8 @@ namespace fileBackup { // then the space after the final key to the next 1MB boundary would // just be padding anyway. struct RangeFileWriter { - RangeFileWriter(Reference file = Reference(), int blockSize = 0) : file(file), blockSize(blockSize), blockEnd(0), fileVersion(1001) {} + RangeFileWriter(Reference file = Reference(), int blockSize = 0) + : file(file), blockSize(blockSize), blockEnd(0), fileVersion(BACKUP_AGENT_SNAPSHOT_FILE_VERSION) {} // Handles the first block and internal blocks. Ends current block if needed. // The final flag is used in simulation to pad the file's final block to a whole block size @@ -545,42 +548,6 @@ namespace fileBackup { Key lastValue; }; - // Helper class for reading restore data from a buffer and throwing the right errors. - struct StringRefReader { - StringRefReader(StringRef s = StringRef(), Error e = Error()) : rptr(s.begin()), end(s.end()), failure_error(e) {} - - // Return remainder of data as a StringRef - StringRef remainder() { - return StringRef(rptr, end - rptr); - } - - // Return a pointer to len bytes at the current read position and advance read pos - const uint8_t * consume(unsigned int len) { - if(rptr == end && len != 0) - throw end_of_stream(); - const uint8_t *p = rptr; - rptr += len; - if(rptr > end) - throw failure_error; - return p; - } - - // Return a T from the current read position and advance read pos - template const T consume() { - return *(const T *)consume(sizeof(T)); - } - - // Functions for consuming big endian (network byte order) integers. - // Consumes a big endian number, swaps it to little endian, and returns it. - int32_t consumeNetworkInt32() { return (int32_t)bigEndian32((uint32_t)consume< int32_t>());} - uint32_t consumeNetworkUInt32() { return bigEndian32( consume());} - - bool eof() { return rptr == end; } - - const uint8_t *rptr, *end; - Error failure_error; - }; - ACTOR Future>> decodeRangeFileBlock(Reference file, int64_t offset, int len) { state Standalone buf = makeString(len); int rLen = wait(file->read(mutateString(buf), len, offset)); @@ -591,8 +558,8 @@ namespace fileBackup { state StringRefReader reader(buf, restore_corrupted_data()); try { - // Read header, currently only decoding version 1001 - if(reader.consume() != 1001) + // Read header, currently only decoding BACKUP_AGENT_SNAPSHOT_FILE_VERSION + if(reader.consume() != BACKUP_AGENT_SNAPSHOT_FILE_VERSION) throw restore_unsupported_file_version(); // Read begin key, if this fails then block was invalid. @@ -647,7 +614,8 @@ namespace fileBackup { struct LogFileWriter { static const std::string &FFs; - LogFileWriter(Reference file = Reference(), int blockSize = 0) : file(file), blockSize(blockSize), blockEnd(0), fileVersion(2001) {} + LogFileWriter(Reference file = Reference(), int blockSize = 0) + : file(file), blockSize(blockSize), blockEnd(0) {} // Start a new block if needed, then write the key and value ACTOR static Future writeKV_impl(LogFileWriter *self, Key k, Value v) { @@ -664,8 +632,8 @@ namespace fileBackup { // Set new blockEnd self->blockEnd += self->blockSize; - // write Header - wait(self->file->append((uint8_t *)&self->fileVersion, sizeof(self->fileVersion))); + // write the block header + wait(self->file->append((uint8_t *)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION))); } wait(self->file->appendStringRefWithLen(k)); @@ -685,7 +653,6 @@ namespace fileBackup { private: int64_t blockEnd; - uint32_t fileVersion; }; ACTOR Future>> decodeLogFileBlock(Reference file, int64_t offset, int len) { @@ -698,8 +665,8 @@ namespace fileBackup { state StringRefReader reader(buf, restore_corrupted_data()); try { - // Read header, currently only decoding version 2001 - if(reader.consume() != 2001) + // Read header, currently only decoding version BACKUP_AGENT_MLOG_VERSION + if(reader.consume() != BACKUP_AGENT_MLOG_VERSION) throw restore_unsupported_file_version(); // Read k/v pairs. Block ends either at end of last value exactly or with 0xFF as first key len byte. @@ -935,6 +902,29 @@ namespace fileBackup { return LiteralStringRef("OnSetAddTask"); } + // Clears the backup ID from "backupStartedKey" to pause backup workers. + ACTOR static Future clearBackupStartID(Reference tr, UID backupUid) { + // If backup worker is not enabled, exit early. + Optional started = wait(tr->get(backupStartedKey)); + std::vector> ids; + if (started.present()) { + ids = decodeBackupStartedValue(started.get()); + } + auto it = std::find_if(ids.begin(), ids.end(), + [=](const std::pair& p) { return p.first == backupUid; }); + if (it != ids.end()) { + ids.erase(it); + } + + if (ids.empty()) { + TraceEvent("ClearBackup").detail("BackupID", backupUid); + tr->clear(backupStartedKey); + } else { + tr->set(backupStartedKey, encodeBackupStartedValue(ids)); + } + return Void(); + } + // Backup and Restore taskFunc definitions will inherit from one of the following classes which // servers to catch and log to the appropriate config any error that execute/finish didn't catch and log. struct RestoreTaskFuncBase : TaskFuncBase { @@ -989,7 +979,7 @@ namespace fileBackup { } Params; std::string toString(Reference task) { - return format("beginKey '%s' endKey '%s' addTasks %d", + return format("beginKey '%s' endKey '%s' addTasks %d", Params.beginKey().get(task).printable().c_str(), Params.endKey().get(task).printable().c_str(), Params.addBackupRangeTasks().get(task) @@ -1001,7 +991,7 @@ namespace fileBackup { Future execute(Database cx, Reference tb, Reference fb, Reference task) { return _execute(cx, tb, fb, task); }; Future finish(Reference tr, Reference tb, Reference fb, Reference task) { return _finish(tr, tb, fb, task); }; - // Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable. + // Finish (which flushes/syncs) the file, and then in a single transaction, make some range backup progress durable. // This means: // - increment the backup config's range bytes written // - update the range file map @@ -1576,7 +1566,7 @@ namespace fileBackup { } // The number of shards 'behind' the snapshot is the count of how may additional shards beyond normal are being dispatched, if any. - int countShardsBehind = std::max(0, countShardsToDispatch + snapshotBatchSize.get() - countShardsExpectedPerNormalWindow); + int countShardsBehind = std::max(0, countShardsToDispatch + snapshotBatchSize.get() - countShardsExpectedPerNormalWindow); Params.shardsBehind().set(task, countShardsBehind); TraceEvent("FileBackupSnapshotDispatchStats") @@ -1627,7 +1617,7 @@ namespace fileBackup { state int64_t oldBatchSize = snapshotBatchSize.get(); state int64_t newBatchSize = oldBatchSize + rangesToAdd.size(); - // Now add the selected ranges in a single transaction. + // Now add the selected ranges in a single transaction. tr->reset(); loop { try { @@ -1880,7 +1870,7 @@ namespace fileBackup { for (auto &range : ranges) { rc.push_back(readCommitted(cx, results, lock, range, false, true, true)); } - + state Future sendEOS = map(errorOr(waitForAll(rc)), [=](ErrorOr const &result) { if(result.isError()) results.sendError(result.getError()); @@ -1982,7 +1972,6 @@ namespace fileBackup { if (Params.addBackupLogRangeTasks().get(task)) { wait(startBackupLogRangeInternal(tr, taskBucket, futureBucket, task, taskFuture, beginVersion, endVersion)); - endVersion = beginVersion; } else { wait(taskFuture->set(tr, taskBucket)); } @@ -2085,12 +2074,14 @@ namespace fileBackup { state EBackupState backupState; state Optional tag; state Optional latestSnapshotEndVersion; + state Optional partitionedLog; - wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) + wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) && store(restorableVersion, config.getLatestRestorableVersion(tr)) && store(backupState, config.stateEnum().getOrThrow(tr)) && store(tag, config.tag().get(tr)) - && store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr))); + && store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) + && store(partitionedLog, config.partitionedLogEnabled().get(tr))); // If restorable, update the last restorable version for this tag if(restorableVersion.present() && tag.present()) { @@ -2126,14 +2117,20 @@ namespace fileBackup { // If a snapshot has ended for this backup then mutations are higher priority to reduce backup lag state int priority = latestSnapshotEndVersion.present() ? 1 : 0; - // Add the initial log range task to read/copy the mutations and the next logs dispatch task which will run after this batch is done - wait(success(BackupLogRangeTaskFunc::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::joinWith(logDispatchBatchFuture)))); - wait(success(BackupLogsDispatchTask::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::signal(onDone), logDispatchBatchFuture))); + if (!partitionedLog.present() || !partitionedLog.get()) { + // Add the initial log range task to read/copy the mutations and the next logs dispatch task which will run after this batch is done + wait(success(BackupLogRangeTaskFunc::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::joinWith(logDispatchBatchFuture)))); + wait(success(BackupLogsDispatchTask::addTask(tr, taskBucket, task, priority, beginVersion, endVersion, TaskCompletionKey::signal(onDone), logDispatchBatchFuture))); - // Do not erase at the first time - if (prevBeginVersion > 0) { - state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); - wait( eraseLogData(tr, config.getUidAsKey(), destUidValue, Optional(beginVersion)) ); + // Do not erase at the first time + if (prevBeginVersion > 0) { + state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); + wait( eraseLogData(tr, config.getUidAsKey(), destUidValue, Optional(beginVersion)) ); + } + } else { + // Skip mutation copy and erase backup mutations. Just check back periodically. + Version scheduledVersion = tr->getReadVersion().get() + CLIENT_KNOBS->BACKUP_POLL_PROGRESS_SECONDS * CLIENT_KNOBS->VERSIONS_PER_SECOND; + wait(success(BackupLogsDispatchTask::addTask(tr, taskBucket, task, 1, beginVersion, endVersion, TaskCompletionKey::signal(onDone), Reference(), scheduledVersion))); } wait(taskBucket->finish(tr, task)); @@ -2147,7 +2144,7 @@ namespace fileBackup { return Void(); } - ACTOR static Future addTask(Reference tr, Reference taskBucket, Reference parentTask, int priority, Version prevBeginVersion, Version beginVersion, TaskCompletionKey completionKey, Reference waitFor = Reference()) { + ACTOR static Future addTask(Reference tr, Reference taskBucket, Reference parentTask, int priority, Version prevBeginVersion, Version beginVersion, TaskCompletionKey completionKey, Reference waitFor = Reference(), Version scheduledVersion = invalidVersion) { Key key = wait(addBackupTask(BackupLogsDispatchTask::name, BackupLogsDispatchTask::version, tr, taskBucket, completionKey, @@ -2156,6 +2153,9 @@ namespace fileBackup { [=](Reference task) { Params.prevBeginVersion().set(task, prevBeginVersion); Params.beginVersion().set(task, beginVersion); + if (scheduledVersion != invalidVersion) { + ReservedTaskParams::scheduledVersion().set(task, scheduledVersion); + } }, priority)); return key; @@ -2184,8 +2184,9 @@ namespace fileBackup { tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); state Key destUidValue = wait(backup.destUidValue().getOrThrow(tr)); - wait( eraseLogData(tr, backup.getUidAsKey(), destUidValue) ); - + + wait(eraseLogData(tr, backup.getUidAsKey(), destUidValue) && clearBackupStartID(tr, uid)); + backup.stateEnum().set(tr, EBackupState::STATE_COMPLETED); wait(taskBucket->finish(tr, task)); @@ -2257,6 +2258,7 @@ namespace fileBackup { } std::vector files; + std::vector> beginEndKeys; state Version maxVer = 0; state Version minVer = std::numeric_limits::max(); state int64_t totalBytes = 0; @@ -2272,6 +2274,9 @@ namespace fileBackup { // Add file to final file list files.push_back(r.fileName); + // Add (beginKey, endKey) pairs to the list + beginEndKeys.emplace_back(i->second.begin, i->first); + // Update version range seen if(r.version < minVer) minVer = r.version; @@ -2293,7 +2298,7 @@ namespace fileBackup { } Params.endVersion().set(task, maxVer); - wait(bc->writeKeyspaceSnapshotFile(files, totalBytes)); + wait(bc->writeKeyspaceSnapshotFile(files, beginEndKeys, totalBytes)); TraceEvent(SevInfo, "FileBackupWroteSnapshotManifest") .detail("BackupUID", config.getUid()) @@ -2318,7 +2323,7 @@ namespace fileBackup { state Optional firstSnapshotEndVersion; state Optional tag; - wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) + wait(store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) && store(backupState, config.stateEnum().getOrThrow(tr)) && store(restorableVersion, config.getLatestRestorableVersion(tr)) && store(firstSnapshotEndVersion, config.firstSnapshotEndVersion().get(tr)) @@ -2382,8 +2387,8 @@ namespace fileBackup { ACTOR static Future _execute(Database cx, Reference taskBucket, Reference futureBucket, Reference task) { wait(checkTaskVersion(cx, task, StartFullBackupTaskFunc::name, StartFullBackupTaskFunc::version)); + state Reference tr(new ReadYourWritesTransaction(cx)); loop{ - state Reference tr(new ReadYourWritesTransaction(cx)); try { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); @@ -2397,7 +2402,66 @@ namespace fileBackup { } } - return Void(); + // Check if backup worker is enabled + DatabaseConfiguration dbConfig = wait(getDatabaseConfiguration(cx)); + state bool backupWorkerEnabled = dbConfig.backupWorkerEnabled; + if (!backupWorkerEnabled) { + wait(success(changeConfig(cx, "backup_worker_enabled:=1", true))); + backupWorkerEnabled = true; + } + + // Set the "backupStartedKey" and wait for all backup worker started + tr->reset(); + state BackupConfig config(task); + loop { + state Future watchFuture; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + state Future keepRunning = taskBucket->keepRunning(tr, task); + + state Future> started = tr->get(backupStartedKey); + state Future> taskStarted = tr->get(config.allWorkerStarted().key); + state Future> partitionedLog = config.partitionedLogEnabled().get(tr); + wait(success(started) && success(taskStarted) && success(partitionedLog)); + + if (!partitionedLog.get().present() || !partitionedLog.get().get()) { + return Void(); // Skip if not using partitioned logs + } + + std::vector> ids; + if (started.get().present()) { + ids = decodeBackupStartedValue(started.get().get()); + } + const UID uid = config.getUid(); + auto it = std::find_if(ids.begin(), ids.end(), + [uid](const std::pair& p) { return p.first == uid; }); + if (it == ids.end()) { + ids.emplace_back(uid, Params.beginVersion().get(task)); + } else { + Params.beginVersion().set(task, it->second); + } + + tr->set(backupStartedKey, encodeBackupStartedValue(ids)); + if (backupWorkerEnabled) { + config.backupWorkerEnabled().set(tr, true); + } + + // The task may be restarted. Set the watch if started key has NOT been set. + if (!taskStarted.get().present()) { + watchFuture = tr->watch(config.allWorkerStarted().key); + } + + wait(keepRunning); + wait(tr->commit()); + if (!taskStarted.get().present()) { + wait(watchFuture); + } + return Void(); + } catch (Error &e) { + wait(tr->onError(e)); + } + } } ACTOR static Future _finish(Reference tr, Reference taskBucket, Reference futureBucket, Reference task) { @@ -2406,13 +2470,16 @@ namespace fileBackup { state Future> backupRangesFuture = config.backupRanges().getOrThrow(tr); state Future destUidValueFuture = config.destUidValue().getOrThrow(tr); - wait(success(backupRangesFuture) && success(destUidValueFuture)); + state Future> partitionedLog = config.partitionedLogEnabled().get(tr); + wait(success(backupRangesFuture) && success(destUidValueFuture) && success(partitionedLog)); std::vector backupRanges = backupRangesFuture.get(); Key destUidValue = destUidValueFuture.get(); - // Start logging the mutations for the specified ranges of the tag - for (auto &backupRange : backupRanges) { - config.startMutationLogs(tr, backupRange, destUidValue); + // Start logging the mutations for the specified ranges of the tag if needed + if (!partitionedLog.get().present() || !partitionedLog.get().get()) { + for (auto& backupRange : backupRanges) { + config.startMutationLogs(tr, backupRange, destUidValue); + } } config.stateEnum().set(tr, EBackupState::STATE_RUNNING); @@ -2432,6 +2499,7 @@ namespace fileBackup { wait(success(FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished))); wait(taskBucket->finish(tr, task)); + return Void(); } @@ -2513,12 +2581,12 @@ namespace fileBackup { std::string toString(Reference task) { return format("fileName '%s' readLen %lld readOffset %lld", - Params.inputFile().get(task).fileName.c_str(), + Params.inputFile().get(task).fileName.c_str(), Params.readLen().get(task), Params.readOffset().get(task)); } }; - + struct RestoreRangeTaskFunc : RestoreFileTaskFuncBase { static struct : InputParams { // The range of data that the (possibly empty) data represented, which is set if it intersects the target restore range @@ -2743,7 +2811,7 @@ namespace fileBackup { // Create a restore config from the current task and bind it to the new task. wait(RestoreConfig(parentTask).toTask(tr, task)); - + Params.inputFile().set(task, rf); Params.readOffset().set(task, offset); Params.readLen().set(task, len); @@ -3055,7 +3123,7 @@ namespace fileBackup { } // Start moving through the file list and queuing up blocks. Only queue up to RESTORE_DISPATCH_ADDTASK_SIZE blocks per Dispatch task - // and target batchSize total per batch but a batch must end on a complete version boundary so exceed the limit if necessary + // and target batchSize total per batch but a batch must end on a complete version boundary so exceed the limit if necessary // to reach the end of a version of files. state std::vector> addTaskFutures; state Version endVersion = files[0].version; @@ -3103,12 +3171,12 @@ namespace fileBackup { ++blocksDispatched; --remainingInBatch; } - + // Stop if we've reached the addtask limit if(blocksDispatched == taskBatchSize) break; - // We just completed an entire file so the next task should start at the file after this one within endVersion (or later) + // We just completed an entire file so the next task should start at the file after this one within endVersion (or later) // if this iteration ends up being the last for this task beginFile = beginFile + '\x00'; beginBlock = 0; @@ -3146,7 +3214,7 @@ namespace fileBackup { .detail("RemainingInBatch", remainingInBatch); wait(success(RestoreDispatchTaskFunc::addTask(tr, taskBucket, task, endVersion, beginFile, beginBlock, batchSize, remainingInBatch, TaskCompletionKey::joinWith((allPartsDone))))); - + // If adding to existing batch then task is joined with a batch future so set done future. // Note that this must be done after joining at least one task with the batch future in case all other blockers already finished. Future setDone = addingToExistingBatch ? onDone->set(tr, taskBucket) : Void(); @@ -3159,7 +3227,7 @@ namespace fileBackup { // Increment the number of blocks dispatched in the restore config restore.filesBlocksDispatched().atomicOp(tr, blocksDispatched, MutationRef::Type::AddValue); - // If beginFile is not empty then we had to stop in the middle of a version (possibly within a file) so we cannot end + // If beginFile is not empty then we had to stop in the middle of a version (possibly within a file) so we cannot end // the batch here because we do not know if we got all of the files and blocks from the last version queued, so // make sure remainingInBatch is at least 1. if(!beginFile.empty()) @@ -3296,7 +3364,7 @@ namespace fileBackup { wait( tr->onError(e) ); } } - + tr = Reference( new ReadYourWritesTransaction(cx) ); //Commit a dummy transaction before returning success, to ensure the mutation applier has stopped submitting mutations @@ -3509,6 +3577,136 @@ class FileBackupAgentImpl { public: static const int MAX_RESTORABLE_FILE_METASECTION_BYTES = 1024 * 8; + // Parallel restore + ACTOR static Future parallelRestoreFinish(Database cx, UID randomUID) { + state ReadYourWritesTransaction tr(cx); + state Future watchForRestoreRequestDone; + state bool restoreDone = false; + TraceEvent("FastRestoreAgentWaitForRestoreToFinish").detail("DBLock", randomUID); + loop { + try { + tr.reset(); + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + Optional restoreRequestDoneKeyValue = wait(tr.get(restoreRequestDoneKey)); + // Restore may finish before restoreAgent waits on the restore finish event. + if (restoreRequestDoneKeyValue.present()) { + restoreDone = true; // In case commit clears the key but in unknown_state + tr.clear(restoreRequestDoneKey); + wait(tr.commit()); + break; + } else if (!restoreDone) { + watchForRestoreRequestDone = tr.watch(restoreRequestDoneKey); + wait(tr.commit()); + wait(watchForRestoreRequestDone); + } else { + break; + } + } catch (Error& e) { + wait(tr.onError(e)); + } + } + TraceEvent("FastRestoreAgentRestoreFinished").detail("UnlockDBStart", randomUID); + try { + wait(unlockDatabase(cx, randomUID)); + } catch (Error& e) { + if (e.code() == error_code_operation_cancelled) { // Should only happen in simulation + TraceEvent(SevWarnAlways, "FastRestoreAgentOnCancelingActor") + .detail("DBLock", randomUID) + .detail("ManualCheck", "Is DB locked"); + } else { + TraceEvent(SevError, "FastRestoreAgentUnlockDBFailed") + .detail("DBLock", randomUID) + .detail("ErrorCode", e.code()) + .detail("Error", e.what()); + ASSERT_WE_THINK(false); // This unlockDatabase should always succeed, we think. + } + } + TraceEvent("FastRestoreAgentRestoreFinished").detail("UnlockDBFinish", randomUID); + return Void(); + } + + ACTOR static Future submitParallelRestore(Database cx, Key backupTag, + Standalone> backupRanges, Key bcUrl, + Version targetVersion, bool lockDB, UID randomUID) { + // Sanity check backup is valid + state Reference bc = IBackupContainer::openContainer(bcUrl.toString()); + state BackupDescription desc = wait(bc->describeBackup()); + wait(desc.resolveVersionTimes(cx)); + + if (targetVersion == invalidVersion && desc.maxRestorableVersion.present()) { + targetVersion = desc.maxRestorableVersion.get(); + TraceEvent(SevWarn, "FastRestoreSubmitRestoreRequestWithInvalidTargetVersion") + .detail("OverrideTargetVersion", targetVersion); + } + + Optional restoreSet = wait(bc->getRestoreSet(targetVersion)); + + if (!restoreSet.present()) { + TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible") + .detail("BackupContainer", bc->getURL()) + .detail("TargetVersion", targetVersion); + throw restore_invalid_version(); + } + + TraceEvent("FastRestoreSubmitRestoreRequest") + .detail("BackupDesc", desc.toString()) + .detail("TargetVersion", targetVersion); + + state Reference tr(new ReadYourWritesTransaction(cx)); + state int restoreIndex = 0; + state int numTries = 0; + // lock DB for restore + loop { + try { + if (lockDB) { + wait(lockDatabase(cx, randomUID)); + } + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + wait(checkDatabaseLock(tr, randomUID)); + + TraceEvent("FastRestoreAgentSubmitRestoreRequests").detail("DBIsLocked", randomUID); + break; + } catch (Error& e) { + TraceEvent("FastRestoreAgentSubmitRestoreRequests").detail("CheckLockError", e.what()); + TraceEvent(numTries > 50 ? SevError : SevWarnAlways, "FastRestoreMayFail") + .detail("Reason", "DB is not properly locked") + .detail("ExpectedLockID", randomUID); + numTries++; + wait(delay(5.0)); + } + } + + // set up restore request + loop { + tr->reset(); + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + try { + // Note: we always lock DB here in case DB is modified at the bacupRanges boundary. + for (restoreIndex = 0; restoreIndex < backupRanges.size(); restoreIndex++) { + auto range = backupRanges[restoreIndex]; + Standalone restoreTag(backupTag.toString() + "_" + std::to_string(restoreIndex)); + // Register the request request in DB, which will be picked up by restore worker leader + struct RestoreRequest restoreRequest(restoreIndex, restoreTag, bcUrl, true, targetVersion, true, + range, Key(), Key(), lockDB, + deterministicRandom()->randomUniqueID()); + tr->set(restoreRequestKeyFor(restoreRequest.index), restoreRequestValue(restoreRequest)); + } + tr->set(restoreRequestTriggerKey, + restoreRequestTriggerValue(deterministicRandom()->randomUniqueID(), backupRanges.size())); + wait(tr->commit()); // Trigger restore + break; + } catch (Error& e) { + wait(tr->onError(e)); + } + } + return Void(); + } + // This method will return the final status of the backup at tag, and return the URL that was used on the tag // when that status value was read. ACTOR static Future waitBackup(FileBackupAgent* backupAgent, Database cx, std::string tagName, bool stopWhenDone, Reference *pContainer = nullptr, UID *pUID = nullptr) { @@ -3556,14 +3754,19 @@ public: } } - ACTOR static Future submitBackup(FileBackupAgent* backupAgent, Reference tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone> backupRanges, bool stopWhenDone) { + ACTOR static Future submitBackup(FileBackupAgent* backupAgent, Reference tr, + Key outContainer, int snapshotIntervalSeconds, std::string tagName, + Standalone> backupRanges, bool stopWhenDone, + bool partitionedLog) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY); TraceEvent(SevInfo, "FBA_SubmitBackup") - .detail("TagName", tagName.c_str()) - .detail("StopWhenDone", stopWhenDone) - .detail("OutContainer", outContainer.toString()); + .detail("TagName", tagName.c_str()) + .detail("StopWhenDone", stopWhenDone) + .detail("UsePartitionedLog", partitionedLog) + .detail("OutContainer", outContainer.toString()); state KeyBackedTag tag = makeBackupTag(tagName); Optional uidAndAbortedFlag = wait(tag.get(tr)); @@ -3656,6 +3859,7 @@ public: config.stopWhenDone().set(tr, stopWhenDone); config.backupRanges().set(tr, normalizedRanges); config.snapshotIntervalSeconds().set(tr, snapshotIntervalSeconds); + config.partitionedLogEnabled().set(tr, partitionedLog); Key taskKey = wait(fileBackup::StartFullBackupTaskFunc::addTask(tr, backupAgent->taskBucket, uid, TaskCompletionKey::noSignal())); @@ -3809,7 +4013,7 @@ public: throw backup_unneeded(); } - // If the backup is already restorable then 'mostly' abort it - cancel all tasks via the tag + // If the backup is already restorable then 'mostly' abort it - cancel all tasks via the tag // and clear the mutation logging config and data - but set its state as COMPLETED instead of ABORTED. state Optional latestRestorableVersion = wait(config.getLatestRestorableVersion(tr)); @@ -3826,7 +4030,8 @@ public: state Key destUidValue = wait(config.destUidValue().getOrThrow(tr)); wait(success(tr->getReadVersion())); - wait( eraseLogData(tr, config.getUidAsKey(), destUidValue) ); + wait(eraseLogData(tr, config.getUidAsKey(), destUidValue) && + fileBackup::clearBackupStartID(tr, config.getUid())); config.stateEnum().set(tr, EBackupState::STATE_COMPLETED); @@ -3865,14 +4070,37 @@ public: // Cancel backup task through tag wait(tag.cancel(tr)); - - wait(eraseLogData(tr, config.getUidAsKey(), destUidValue)); + + wait(eraseLogData(tr, config.getUidAsKey(), destUidValue) && + fileBackup::clearBackupStartID(tr, config.getUid())); config.stateEnum().set(tr, EBackupState::STATE_ABORTED); return Void(); } + ACTOR static Future changePause(FileBackupAgent* backupAgent, Database db, bool pause) { + state Reference tr(new ReadYourWritesTransaction(db)); + state Future change = backupAgent->taskBucket->changePause(db, pause); + + loop { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + try { + tr->set(backupPausedKey, pause ? LiteralStringRef("1") : LiteralStringRef("0")); + wait(tr->commit()); + break; + } catch (Error& e) { + wait(tr->onError(e)); + } + } + wait(change); + TraceEvent("FileBackupAgentChangePaused").detail("Action", pause ? "Paused" : "Resumed"); + return Void(); + } + struct TimestampedVersion { Optional version; Optional epochs; @@ -3976,7 +4204,7 @@ public: wait( store(snapshotInterval, config.snapshotIntervalSeconds().getOrThrow(tr)) && store(logBytesWritten, config.logBytesWritten().getD(tr)) && store(rangeBytesWritten, config.rangeBytesWritten().getD(tr)) - && store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) + && store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) && store(snapshotBegin, getTimestampedVersion(tr, config.snapshotBeginVersion().get(tr))) && store(snapshotTargetEnd, getTimestampedVersion(tr, config.snapshotTargetEndVersion().get(tr))) && store(latestLogEnd, getTimestampedVersion(tr, config.latestLogEndVersion().get(tr))) @@ -4075,7 +4303,7 @@ public: state Reference bc; state Optional latestRestorableVersion; state Version recentReadVersion; - + wait( store(latestRestorableVersion, config.getLatestRestorableVersion(tr)) && store(bc, config.backupContainer().getOrThrow(tr)) && store(recentReadVersion, tr->getReadVersion()) @@ -4126,7 +4354,7 @@ public: && store(rangeBytesWritten, config.rangeBytesWritten().get(tr)) && store(latestLogEndVersion, config.latestLogEndVersion().get(tr)) && store(latestSnapshotEndVersion, config.latestSnapshotEndVersion().get(tr)) - && store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) + && store(stopWhenDone, config.stopWhenDone().getOrThrow(tr)) ); wait( store(latestSnapshotEndVersionTimestamp, getTimestampFromVersion(latestSnapshotEndVersion, tr)) @@ -4140,7 +4368,7 @@ public: statusText += format("Current snapshot progress target is %3.2f%% (>100%% means the snapshot is supposed to be done)\n", 100.0 * (recentReadVersion - snapshotBeginVersion) / (snapshotTargetEndVersion - snapshotBeginVersion)) ; else statusText += "The initial snapshot is still running.\n"; - + statusText += format("\nDetails:\n LogBytes written - %ld\n RangeBytes written - %ld\n " "Last complete log version and timestamp - %s, %s\n " "Last complete snapshot version and timestamp - %s, %s\n " @@ -4202,10 +4430,10 @@ public: return statusText; } - ACTOR static Future getLastRestorable(FileBackupAgent* backupAgent, Reference tr, Key tagName) { + ACTOR static Future getLastRestorable(FileBackupAgent* backupAgent, Reference tr, Key tagName, bool snapshot) { tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state Optional version = wait(tr->get(backupAgent->lastRestorable.pack(tagName))); + state Optional version = wait(tr->get(backupAgent->lastRestorable.pack(tagName), snapshot)); return (version.present()) ? BinaryReader::fromStringRef(version.get(), Unversioned()) : 0; } @@ -4270,7 +4498,9 @@ public: //used for correctness only, locks the database before discontinuing the backup and that same lock is then used while doing the restore. //the tagname of the backup must be the same as the restore. - ACTOR static Future atomicRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, Standalone> ranges, Key addPrefix, Key removePrefix) { + ACTOR static Future atomicRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, + Standalone> ranges, Key addPrefix, + Key removePrefix, bool fastRestore) { state Reference ryw_tr = Reference(new ReadYourWritesTransaction(cx)); state BackupConfig backupConfig; loop { @@ -4291,7 +4521,7 @@ public: wait( ryw_tr->onError(e) ); } } - + //Lock src, record commit version state Transaction tr(cx); state Version commitVersion; @@ -4347,6 +4577,7 @@ public: ryw_tr->reset(); loop { + try { ryw_tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); ryw_tr->setOption(FDBTransactionOptions::LOCK_AWARE); @@ -4364,9 +4595,30 @@ public: Reference bc = wait(backupConfig.backupContainer().getOrThrow(cx)); - TraceEvent("AS_StartRestore"); - Version ver = wait( restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true, addPrefix, removePrefix, true, randomUid) ); - return ver; + if (fastRestore) { + TraceEvent("AtomicParallelRestoreStartRestore"); + Version targetVersion = -1; + bool lockDB = true; + wait(submitParallelRestore(cx, tagName, ranges, KeyRef(bc->getURL()), targetVersion, lockDB, randomUid)); + TraceEvent("AtomicParallelRestoreWaitForRestoreFinish"); + wait(parallelRestoreFinish(cx, randomUid)); + return -1; + } else { + TraceEvent("AS_StartRestore"); + Version ver = wait(restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true, + addPrefix, removePrefix, true, randomUid)); + return ver; + } + } + + // Similar to atomicRestore, only used in simulation test. + // locks the database before discontinuing the backup and that same lock is then used while doing the restore. + // the tagname of the backup must be the same as the restore. + ACTOR static Future atomicParallelRestore(FileBackupAgent* backupAgent, Database cx, Key tagName, + Standalone> ranges, Key addPrefix, + Key removePrefix) { + Version ver = wait(atomicRestore(backupAgent, cx, tagName, ranges, addPrefix, removePrefix, true)); + return Void(); } }; @@ -4374,12 +4626,29 @@ const std::string BackupAgentBase::defaultTagName = "default"; const int BackupAgentBase::logHeaderSize = 12; const int FileBackupAgent::dataFooterSize = 20; +// Return if parallel restore has finished +Future FileBackupAgent::parallelRestoreFinish(Database cx, UID randomUID) { + return FileBackupAgentImpl::parallelRestoreFinish(cx, randomUID); +} + +Future FileBackupAgent::submitParallelRestore(Database cx, Key backupTag, + Standalone> backupRanges, Key bcUrl, + Version targetVersion, bool lockDB, UID randomUID) { + return FileBackupAgentImpl::submitParallelRestore(cx, backupTag, backupRanges, bcUrl, targetVersion, lockDB, + randomUID); +} + +Future FileBackupAgent::atomicParallelRestore(Database cx, Key tagName, Standalone> ranges, + Key addPrefix, Key removePrefix) { + return FileBackupAgentImpl::atomicParallelRestore(this, cx, tagName, ranges, addPrefix, removePrefix); +} + Future FileBackupAgent::restore(Database cx, Optional cxOrig, Key tagName, Key url, Standalone> ranges, bool waitForComplete, Version targetVersion, bool verbose, Key addPrefix, Key removePrefix, bool lockDB) { return FileBackupAgentImpl::restore(this, cx, cxOrig, tagName, url, ranges, waitForComplete, targetVersion, verbose, addPrefix, removePrefix, lockDB, deterministicRandom()->randomUniqueID()); } Future FileBackupAgent::atomicRestore(Database cx, Key tagName, Standalone> ranges, Key addPrefix, Key removePrefix) { - return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix); + return FileBackupAgentImpl::atomicRestore(this, cx, tagName, ranges, addPrefix, removePrefix, false); } Future FileBackupAgent::abortRestore(Reference tr, Key tagName) { @@ -4398,8 +4667,12 @@ Future FileBackupAgent::waitRestore(Database cx, Key tagName, boo return FileBackupAgentImpl::waitRestore(cx, tagName, verbose); }; -Future FileBackupAgent::submitBackup(Reference tr, Key outContainer, int snapshotIntervalSeconds, std::string tagName, Standalone> backupRanges, bool stopWhenDone) { - return FileBackupAgentImpl::submitBackup(this, tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, stopWhenDone); +Future FileBackupAgent::submitBackup(Reference tr, Key outContainer, + int snapshotIntervalSeconds, std::string tagName, + Standalone> backupRanges, bool stopWhenDone, + bool partitionedLog) { + return FileBackupAgentImpl::submitBackup(this, tr, outContainer, snapshotIntervalSeconds, tagName, backupRanges, + stopWhenDone, partitionedLog); } Future FileBackupAgent::discontinueBackup(Reference tr, Key tagName){ @@ -4418,8 +4691,8 @@ Future FileBackupAgent::getStatusJSON(Database cx, std::string tagN return FileBackupAgentImpl::getStatusJSON(this, cx, tagName); } -Future FileBackupAgent::getLastRestorable(Reference tr, Key tagName) { - return FileBackupAgentImpl::getLastRestorable(this, tr, tagName); +Future FileBackupAgent::getLastRestorable(Reference tr, Key tagName, bool snapshot) { + return FileBackupAgentImpl::getLastRestorable(this, tr, tagName, snapshot); } void FileBackupAgent::setLastRestorable(Reference tr, Key tagName, Version version) { @@ -4432,3 +4705,6 @@ Future FileBackupAgent::waitBackup(Database cx, std::string tagName, bool s return FileBackupAgentImpl::waitBackup(this, cx, tagName, stopWhenDone, pContainer, pUID); } +Future FileBackupAgent::changePause(Database db, bool pause) { + return FileBackupAgentImpl::changePause(this, db, pause); +} diff --git a/fdbclient/IClientApi.h b/fdbclient/IClientApi.h index b3e6217054..154ac9723f 100644 --- a/fdbclient/IClientApi.h +++ b/fdbclient/IClientApi.h @@ -48,6 +48,7 @@ public: virtual ThreadFuture> getVersionstamp() = 0; virtual void addReadConflictRange(const KeyRangeRef& keys) = 0; + virtual ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) = 0; virtual void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) = 0; virtual void set(const KeyRef& key, const ValueRef& value) = 0; diff --git a/fdbclient/JSONDoc.h b/fdbclient/JSONDoc.h index 70c05375aa..aafd1bb87f 100644 --- a/fdbclient/JSONDoc.h +++ b/fdbclient/JSONDoc.h @@ -193,7 +193,7 @@ struct JSONDoc { return v.get_value(); } - // Ensures that a an Object exists at path and returns a JSONDoc that writes to it. + // Ensures that an Object exists at path and returns a JSONDoc that writes to it. JSONDoc subDoc(std::string path, bool split=true) { json_spirit::mValue &v = create(path, split); if(v.type() != json_spirit::obj_type) diff --git a/fdbclient/KeyRangeMap.actor.cpp b/fdbclient/KeyRangeMap.actor.cpp index d3cb36f833..39697334cc 100644 --- a/fdbclient/KeyRangeMap.actor.cpp +++ b/fdbclient/KeyRangeMap.actor.cpp @@ -150,7 +150,9 @@ ACTOR Future krmSetRange( Reference tr, Key map //Sets a range of keys in a key range map, coalescing with adjacent regions if the values match //Ranges outside of maxRange will not be coalesced //CAUTION: use care when attempting to coalesce multiple ranges in the same prefix in a single transaction -ACTOR Future krmSetRangeCoalescing( Transaction *tr, Key mapPrefix, KeyRange range, KeyRange maxRange, Value value ) { +ACTOR template +static Future krmSetRangeCoalescing_(Transaction* tr, Key mapPrefix, KeyRange range, KeyRange maxRange, + Value value) { ASSERT(maxRange.contains(range)); state KeyRange withPrefix = KeyRangeRef( mapPrefix.toString() + range.begin.toString(), mapPrefix.toString() + range.end.toString() ); @@ -216,3 +218,11 @@ ACTOR Future krmSetRangeCoalescing( Transaction *tr, Key mapPrefix, KeyRan return Void(); } +Future krmSetRangeCoalescing(Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, + KeyRange const& maxRange, Value const& value) { + return krmSetRangeCoalescing_(tr, mapPrefix, range, maxRange, value); +} +Future krmSetRangeCoalescing(Reference const& tr, Key const& mapPrefix, + KeyRange const& range, KeyRange const& maxRange, Value const& value) { + return holdWhile(tr, krmSetRangeCoalescing_(tr.getPtr(), mapPrefix, range, maxRange, value)); +} diff --git a/fdbclient/KeyRangeMap.h b/fdbclient/KeyRangeMap.h index aafd92cb69..94493053ff 100644 --- a/fdbclient/KeyRangeMap.h +++ b/fdbclient/KeyRangeMap.h @@ -103,6 +103,8 @@ void krmSetPreviouslyEmptyRange( struct CommitTransactionRef& tr, Arena& trArena Future krmSetRange( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value ); Future krmSetRange( Reference const& tr, Key const& mapPrefix, KeyRange const& range, Value const& value ); Future krmSetRangeCoalescing( Transaction* const& tr, Key const& mapPrefix, KeyRange const& range, KeyRange const& maxRange, Value const& value ); +Future krmSetRangeCoalescing(Reference const& tr, Key const& mapPrefix, + KeyRange const& range, KeyRange const& maxRange, Value const& value); Standalone krmDecodeRanges( KeyRef mapPrefix, KeyRange keys, Standalone kv ); template diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index 824b122564..33bafcdf93 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -21,12 +21,18 @@ #include "fdbclient/Knobs.h" #include "fdbclient/FDBTypes.h" #include "fdbclient/SystemData.h" +#include "flow/UnitTest.h" ClientKnobs const* CLIENT_KNOBS = new ClientKnobs(); #define init( knob, value ) initKnob( knob, value, #knob ) -ClientKnobs::ClientKnobs(bool randomize) { +ClientKnobs::ClientKnobs() { + initialize(); +} + +void ClientKnobs::initialize(bool randomize) { + // clang-format off // FIXME: These are not knobs, get them out of ClientKnobs! BYTE_LIMIT_UNLIMITED = GetRangeLimits::BYTE_LIMIT_UNLIMITED; ROW_LIMIT_UNLIMITED = GetRangeLimits::ROW_LIMIT_UNLIMITED; @@ -41,11 +47,16 @@ ClientKnobs::ClientKnobs(bool randomize) { init( CLIENT_FAILURE_TIMEOUT_DELAY, FAILURE_MIN_DELAY ); init( FAILURE_EMERGENCY_DELAY, 30.0 ); init( FAILURE_MAX_GENERATIONS, 10 ); + init( RECOVERY_DELAY_START_GENERATION, 70 ); + init( RECOVERY_DELAY_SECONDS_PER_GENERATION, 60.0 ); + init( MAX_GENERATIONS, 100 ); + init( MAX_GENERATIONS_OVERRIDE, 0 ); init( COORDINATOR_RECONNECTION_DELAY, 1.0 ); init( CLIENT_EXAMPLE_AMOUNT, 20 ); init( MAX_CLIENT_STATUS_AGE, 1.0 ); - init( MAX_CLIENT_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_CLIENT_PROXY_CONNECTIONS = 1; + init( MAX_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_PROXY_CONNECTIONS = 1; + init( STATUS_IDLE_TIMEOUT, 120.0 ); // wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin @@ -76,6 +87,7 @@ ClientKnobs::ClientKnobs(bool randomize) { init( GET_RANGE_SHARD_LIMIT, 2 ); init( WARM_RANGE_SHARD_LIMIT, 100 ); init( STORAGE_METRICS_SHARD_LIMIT, 100 ); if( randomize && BUGGIFY ) STORAGE_METRICS_SHARD_LIMIT = 3; + init( SHARD_COUNT_LIMIT, 80 ); if( randomize && BUGGIFY ) SHARD_COUNT_LIMIT = 3; init( STORAGE_METRICS_UNFAIR_SPLIT_LIMIT, 2.0/3.0 ); init( STORAGE_METRICS_TOO_MANY_SHARDS_DELAY, 15.0 ); init( AGGREGATE_HEALTH_METRICS_MAX_STALENESS, 0.5 ); @@ -132,6 +144,8 @@ ClientKnobs::ClientKnobs(bool randomize) { init( BACKUP_COPY_TASKS, 90 ); init( BACKUP_BLOCK_SIZE, LOG_RANGE_BLOCK_SIZE/10 ); init( BACKUP_TASKS_PER_AGENT, 10 ); + init( BACKUP_POLL_PROGRESS_SECONDS, 10 ); + init( VERSIONS_PER_SECOND, 1e6 ); // Must be the same as SERVER_KNOBS->VERSIONS_PER_SECOND init( SIM_BACKUP_TASKS_PER_AGENT, 10 ); init( BACKUP_RANGEFILE_BLOCK_SIZE, 1024 * 1024); init( BACKUP_LOGFILE_BLOCK_SIZE, 1024 * 1024); @@ -197,6 +211,28 @@ ClientKnobs::ClientKnobs(bool randomize) { } init(CSI_STATUS_DELAY, 10.0 ); - init( CONSISTENCY_CHECK_RATE_LIMIT_MAX, 50e6 ); // Limit in per sec + init( CONSISTENCY_CHECK_RATE_LIMIT_MAX, 50e6 ); // Limit in per sec init( CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME, 7 * 24 * 60 * 60 ); // 7 days + + //fdbcli + init( CLI_CONNECT_PARALLELISM, 400 ); + init( CLI_CONNECT_TIMEOUT, 10.0 ); + + // trace + init( TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH, 50 ); + // clang-format on +} + +TEST_CASE("/fdbclient/knobs/initialize") { + // This test depends on TASKBUCKET_TIMEOUT_VERSIONS being defined as a constant multiple of CORE_VERSIONSPERSECOND + ClientKnobs clientKnobs; + int initialCoreVersionsPerSecond = clientKnobs.CORE_VERSIONSPERSECOND; + int initialTaskBucketTimeoutVersions = clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS; + clientKnobs.setKnob("core_versionspersecond", format("%ld", initialCoreVersionsPerSecond * 2)); + ASSERT(clientKnobs.CORE_VERSIONSPERSECOND == initialCoreVersionsPerSecond * 2); + ASSERT(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS == initialTaskBucketTimeoutVersions); + clientKnobs.initialize(); + ASSERT(clientKnobs.CORE_VERSIONSPERSECOND == initialCoreVersionsPerSecond * 2); + ASSERT(clientKnobs.TASKBUCKET_TIMEOUT_VERSIONS == initialTaskBucketTimeoutVersions * 2); + return Void(); } diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h index 6e1a50ed4a..5b1784f94b 100644 --- a/fdbclient/Knobs.h +++ b/fdbclient/Knobs.h @@ -40,11 +40,16 @@ public: double CLIENT_FAILURE_TIMEOUT_DELAY; double FAILURE_EMERGENCY_DELAY; double FAILURE_MAX_GENERATIONS; + double RECOVERY_DELAY_START_GENERATION; + double RECOVERY_DELAY_SECONDS_PER_GENERATION; + double MAX_GENERATIONS; + double MAX_GENERATIONS_OVERRIDE; double COORDINATOR_RECONNECTION_DELAY; int CLIENT_EXAMPLE_AMOUNT; double MAX_CLIENT_STATUS_AGE; - int MAX_CLIENT_PROXY_CONNECTIONS; + int MAX_PROXY_CONNECTIONS; + double STATUS_IDLE_TIMEOUT; // wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test) @@ -75,6 +80,7 @@ public: int GET_RANGE_SHARD_LIMIT; int WARM_RANGE_SHARD_LIMIT; int STORAGE_METRICS_SHARD_LIMIT; + int SHARD_COUNT_LIMIT; double STORAGE_METRICS_UNFAIR_SPLIT_LIMIT; double STORAGE_METRICS_TOO_MANY_SHARDS_DELAY; double AGGREGATE_HEALTH_METRICS_MAX_STALENESS; @@ -133,6 +139,8 @@ public: int BACKUP_COPY_TASKS; int BACKUP_BLOCK_SIZE; int BACKUP_TASKS_PER_AGENT; + int BACKUP_POLL_PROGRESS_SECONDS; + int64_t VERSIONS_PER_SECOND; // Copy of SERVER_KNOBS, as we can't link with it int SIM_BACKUP_TASKS_PER_AGENT; int BACKUP_RANGEFILE_BLOCK_SIZE; int BACKUP_LOGFILE_BLOCK_SIZE; @@ -190,7 +198,15 @@ public: int CONSISTENCY_CHECK_RATE_LIMIT_MAX; int CONSISTENCY_CHECK_ONE_ROUND_TARGET_COMPLETION_TIME; - ClientKnobs(bool randomize = false); + // fdbcli + int CLI_CONNECT_PARALLELISM; + double CLI_CONNECT_TIMEOUT; + + // trace + int TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH; + + ClientKnobs(); + void initialize(bool randomize = false); }; extern ClientKnobs const* CLIENT_KNOBS; diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index a133203ee4..06aff74ff2 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -52,6 +52,13 @@ std::map configForToken( std::string const& mode ) { return out; } + if (mode == "locked") { + // Setting this key is interpreted as an instruction to use the normal version-stamp-based mechanism for locking + // the database. + out[databaseLockedKey.toString()] = deterministicRandom()->randomUniqueID().toString(); + return out; + } + size_t pos; // key:=value is unvalidated and unchecked @@ -100,12 +107,15 @@ std::map configForToken( std::string const& mode ) { } else if (mode == "memory-1") { logType = KeyValueStoreType::MEMORY; storeType= KeyValueStoreType::MEMORY; + } else if (mode == "memory-radixtree-beta") { + logType = KeyValueStoreType::SSD_BTREE_V2; + storeType= KeyValueStoreType::MEMORY_RADIXTREE; } // Add any new store types to fdbserver/workloads/ConfigureDatabase, too if (storeType.present()) { - out[p+"log_engine"] = format("%d", logType.get()); - out[p+"storage_engine"] = format("%d", storeType.get()); + out[p+"log_engine"] = format("%d", logType.get().storeType()); + out[p+"storage_engine"] = format("%d", KeyValueStoreType::StoreType(storeType.get())); return out; } @@ -297,6 +307,17 @@ ACTOR Future changeConfig( Database cx, std::map locked; + { + auto iter = m.find(databaseLockedKey.toString()); + if (iter != m.end()) { + if (!creating) { + return ConfigurationResult::LOCKED_NOT_NEW; + } + locked = UID::fromString(iter->second); + m.erase(iter); + } + } if (creating) { m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString(); if (!isCompleteConfiguration(m)) { @@ -474,7 +495,6 @@ ACTOR Future changeConfig( Database cx, std::map changeConfig( Database cx, std::mapfirst) ); } + if (locked.present()) { + ASSERT(creating); + tr.atomicOp(databaseLockedKey, + BinaryWriter::toValue(locked.get(), Unversioned()) + .withPrefix(LiteralStringRef("0123456789")) + .withSuffix(LiteralStringRef("\x00\x00\x00\x00")), + MutationRef::SetVersionstampedValue); + } + for (auto i = m.begin(); i != m.end(); ++i) { tr.set( StringRef(i->first), StringRef(i->second) ); } @@ -956,9 +985,13 @@ ACTOR Future changeQuorum( Database cx, ReferenceisSimulated()) { for(int i = 0; i < (desiredCoordinators.size()/2)+1; i++) { - auto address = NetworkAddress(desiredCoordinators[i].ip,desiredCoordinators[i].port,true,false); - g_simulator.protectedAddresses.insert(address); - TraceEvent("ProtectCoordinator").detail("Address", address).backtrace(); + auto addresses = g_simulator.getProcessByAddress(desiredCoordinators[i])->addresses; + + g_simulator.protectedAddresses.insert(addresses.address); + if(addresses.secondaryAddress.present()) { + g_simulator.protectedAddresses.insert(addresses.secondaryAddress.get()); + } + TraceEvent("ProtectCoordinator").detail("Address", desiredCoordinators[i]).backtrace(); } } @@ -1117,19 +1150,33 @@ struct AutoQuorumChange : IQuorumChange { *err = CoordinatorsResult::NOT_ENOUGH_MACHINES; return vector(); } - desiredCount = std::max(oldCoordinators.size(), (workers.size() - 1) | 1); - chosen.resize(desiredCount); + chosen.resize((chosen.size() - 1) | 1); } return chosen; } + // Select a desired set of workers such that + // (1) the number of workers at each locality type (e.g., dcid) <= desiredCount; and + // (2) prefer workers at a locality where less workers has been chosen than other localities: evenly distribute workers. void addDesiredWorkers(vector& chosen, const vector& workers, int desiredCount, const std::set& excluded) { vector remainingWorkers(workers); deterministicRandom()->randomShuffle(remainingWorkers); std::partition(remainingWorkers.begin(), remainingWorkers.end(), [](const ProcessData& data) { return (data.processClass == ProcessClass::CoordinatorClass); }); + TraceEvent(SevDebug, "AutoSelectCoordinators").detail("CandidateWorkers", remainingWorkers.size()); + for (auto worker = remainingWorkers.begin(); worker != remainingWorkers.end(); worker++) { + TraceEvent(SevDebug, "AutoSelectCoordinators") + .detail("Worker", worker->processClass.toString()) + .detail("Address", worker->address.toString()) + .detail("Locality", worker->locality.toString()); + } + TraceEvent(SevDebug, "AutoSelectCoordinators").detail("ExcludedAddress", excluded.size()); + for (auto& excludedAddr : excluded) { + TraceEvent(SevDebug, "AutoSelectCoordinators").detail("ExcludedAddress", excludedAddr.toString()); + } + std::map maxCounts; std::map> currentCounts; std::map hardLimits; @@ -1156,6 +1203,12 @@ struct AutoQuorumChange : IQuorumChange { if(addressExcluded(excluded, worker->address)) { continue; } + // Exclude faulty node due to machine assassination + if (g_network->isSimulated() && g_simulator.protectedAddresses.count(worker->address) && + !g_simulator.getProcessByAddress(worker->address)->isReliable()) { + TraceEvent("AutoSelectCoordinators").detail("SkipUnreliableWorker", worker->address.toString()); + continue; + } bool valid = true; for(auto field = fields.begin(); field != fields.end(); field++) { if(maxCounts[*field] == 0) { @@ -1382,6 +1435,7 @@ ACTOR Future printHealthyZone( Database cx ) { ACTOR Future clearHealthyZone(Database cx, bool printWarning, bool clearSSFailureZoneString) { state Transaction tr(cx); + TraceEvent("ClearHealthyZone").detail("ClearSSFailureZoneString", clearSSFailureZoneString); loop { try { tr.setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1407,6 +1461,7 @@ ACTOR Future clearHealthyZone(Database cx, bool printWarning, bool clearSS ACTOR Future setHealthyZone(Database cx, StringRef zoneId, double seconds, bool printWarning) { state Transaction tr(cx); + TraceEvent("SetHealthyZone").detail("Zone", zoneId).detail("DurationSeconds", seconds); loop { try { tr.setOption(FDBTransactionOptions::LOCK_AWARE); @@ -1514,10 +1569,14 @@ ACTOR Future> checkForExcludingServers(Database cx, vec state bool ok = true; inProgressExclusion.clear(); for(auto& s : serverList) { - auto addr = decodeServerListValue( s.value ).address(); - if ( addressExcluded(exclusions, addr) ) { + auto addresses = decodeServerListValue( s.value ).getKeyValues.getEndpoint().addresses; + if ( addressExcluded(exclusions, addresses.address) ) { ok = false; - inProgressExclusion.insert(addr); + inProgressExclusion.insert(addresses.address); + } + if ( addresses.secondaryAddress.present() && addressExcluded(exclusions, addresses.secondaryAddress.get()) ) { + ok = false; + inProgressExclusion.insert(addresses.secondaryAddress.get()); } } @@ -1744,6 +1803,26 @@ ACTOR Future checkDatabaseLock( Reference tr, U return Void(); } +ACTOR Future advanceVersion(Database cx, Version v) { + state Transaction tr(cx); + loop { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + try { + Version rv = wait(tr.getReadVersion()); + if (rv <= v) { + tr.set(minRequiredCommitVersionKey, BinaryWriter::toValue(v + 1, Unversioned())); + wait(tr.commit()); + } else { + printf("Current read version is %ld\n", rv); + return Void(); + } + } catch (Error& e) { + wait(tr.onError(e)); + } + } +} + ACTOR Future forceRecovery( Reference clusterFile, Key dcId ) { state Reference>> clusterInterface(new AsyncVar>); state Future leaderMon = monitorLeader(clusterFile, clusterInterface); diff --git a/fdbclient/ManagementAPI.actor.h b/fdbclient/ManagementAPI.actor.h index d5934f274e..a024f596c8 100644 --- a/fdbclient/ManagementAPI.actor.h +++ b/fdbclient/ManagementAPI.actor.h @@ -61,7 +61,8 @@ public: NOT_ENOUGH_WORKERS, REGION_REPLICATION_MISMATCH, DCID_MISSING, - SUCCESS + LOCKED_NOT_NEW, + SUCCESS, }; }; @@ -177,6 +178,8 @@ ACTOR Future unlockDatabase( Database cx, UID id ); ACTOR Future checkDatabaseLock( Transaction* tr, UID id ); ACTOR Future checkDatabaseLock( Reference tr, UID id ); +ACTOR Future advanceVersion(Database cx, Version v); + ACTOR Future setDDMode( Database cx, int mode ); ACTOR Future forceRecovery( Reference clusterFile, Standalone dcId ); diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h index 20065027ee..ff1273add5 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/MasterProxyInterface.h @@ -68,11 +68,11 @@ struct MasterProxyInterface { } void initEndpoints() { - getConsistentReadVersion.getEndpoint(TaskPriority::ProxyGetConsistentReadVersion); + getConsistentReadVersion.getEndpoint(TaskPriority::ReadSocket); getRawCommittedVersion.getEndpoint(TaskPriority::ProxyGetRawCommittedVersion); - commit.getEndpoint(TaskPriority::ProxyCommitDispatcher); + commit.getEndpoint(TaskPriority::ReadSocket); getStorageServerRejoinInfo.getEndpoint(TaskPriority::ProxyStorageRejoin); - //getKeyServersLocations.getEndpoint(TaskProxyGetKeyServersLocations); //do not increase the priority of these requests, because clients cans bring down the cluster with too many of these messages. + getKeyServersLocations.getEndpoint(TaskPriority::ReadSocket); //priority lowered to TaskPriority::DefaultEndpoint on the proxy } }; @@ -82,6 +82,7 @@ struct ClientDBInfo { constexpr static FileIdentifier file_identifier = 5355080; UID id; // Changes each time anything else changes vector< MasterProxyInterface > proxies; + Optional firstProxy; //not serialized, used for commitOnFirstProxy when the proxies vector has been shrunk double clientTxnInfoSampleRate; int64_t clientTxnInfoSizeLimit; Optional forward; @@ -104,14 +105,18 @@ struct CommitID { Version version; // returns invalidVersion if transaction conflicts uint16_t txnBatchId; Optional metadataVersion; + Optional>> conflictingKRIndices; template void serialize(Ar& ar) { - serializer(ar, version, txnBatchId, metadataVersion); + serializer(ar, version, txnBatchId, metadataVersion, conflictingKRIndices); } CommitID() : version(invalidVersion), txnBatchId(0) {} - CommitID( Version version, uint16_t txnBatchId, const Optional& metadataVersion ) : version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion) {} + CommitID(Version version, uint16_t txnBatchId, const Optional& metadataVersion, + const Optional>>& conflictingKRIndices = Optional>>()) + : version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion), + conflictingKRIndices(conflictingKRIndices) {} }; struct CommitTransactionRequest : TimedRequest { @@ -173,6 +178,7 @@ struct GetReadVersionRequest : TimedRequest { PRIORITY_BATCH = 1 << 24 }; enum { + FLAG_USE_MIN_KNOWN_COMMITTED_VERSION = 4, FLAG_USE_PROVISIONAL_PROXIES = 2, FLAG_CAUSAL_READ_RISKY = 1, FLAG_PRIORITY_MASK = PRIORITY_SYSTEM_IMMEDIATE, @@ -272,11 +278,12 @@ struct TxnStateRequest { VectorRef data; Sequence sequence; bool last; + std::vector broadcastInfo; ReplyPromise reply; template void serialize(Ar& ar) { - serializer(ar, data, sequence, last, reply, arena); + serializer(ar, data, sequence, last, broadcastInfo, reply, arena); } }; diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index df3a6737ab..20317168e0 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -23,7 +23,7 @@ #include "flow/ActorCollection.h" #include "flow/UnitTest.h" #include "fdbrpc/genericactors.actor.h" -#include "fdbrpc/Platform.h" +#include "flow/Platform.h" #include "flow/actorcompiler.h" // has to be last include std::pair< std::string, bool > ClusterConnectionFile::lookupClusterFileName( std::string const& filename ) { @@ -511,7 +511,7 @@ ACTOR Future asyncDeserializeClusterInterface(Reference> s Reference>> outKnownLeader) { state Reference>> knownLeader( new AsyncVar>{}); - state Future deserializer = asyncDeserialize(serializedInfo, knownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER); + state Future deserializer = asyncDeserialize(serializedInfo, knownLeader); loop { choose { when(wait(deserializer)) { UNSTOPPABLE_ASSERT(false); } @@ -655,23 +655,38 @@ ACTOR Future monitorLeaderForProxies( Key clusterKey, vectorUSE_OBJECT_SERIALIZER) { - ObjectReader reader(leader.get().first.serializedInfo.begin(), IncludeVersion()); - ClusterControllerClientInterface res; - reader.deserialize(res); - knownLeader->set(res); - } else { - ClusterControllerClientInterface res = BinaryReader::fromStringRef( leader.get().first.serializedInfo, IncludeVersion() ); - knownLeader->set(res); - } + ObjectReader reader(leader.get().first.serializedInfo.begin(), IncludeVersion()); + ClusterControllerClientInterface res; + reader.deserialize(res); + knownLeader->set(res); } } wait( nomineeChange.onTrigger() || allActors ); } } +void shrinkProxyList( ClientDBInfo& ni, std::vector& lastProxyUIDs, std::vector& lastProxies ) { + if(ni.proxies.size() > CLIENT_KNOBS->MAX_PROXY_CONNECTIONS) { + std::vector proxyUIDs; + for(auto& proxy : ni.proxies) { + proxyUIDs.push_back(proxy.id()); + } + if(proxyUIDs != lastProxyUIDs) { + lastProxyUIDs = proxyUIDs; + lastProxies = ni.proxies; + deterministicRandom()->randomShuffle(lastProxies); + lastProxies.resize(CLIENT_KNOBS->MAX_PROXY_CONNECTIONS); + for(int i = 0; i < lastProxies.size(); i++) { + TraceEvent("ConnectedProxy").detail("Proxy", lastProxies[i].id()); + } + } + ni.firstProxy = ni.proxies[0]; + ni.proxies = lastProxies; + } +} + // Leader is the process that will be elected by coordinators as the cluster controller -ACTOR Future monitorProxiesOneGeneration( Reference connFile, Reference> clientInfo, MonitorLeaderInfo info, Standalone> supportedVersions, Key traceLogGroup) { +ACTOR Future monitorProxiesOneGeneration( Reference connFile, Reference> clientInfo, MonitorLeaderInfo info, Reference>>> supportedVersions, Key traceLogGroup) { state ClusterConnectionString cs = info.intermediateConnFile->getConnectionString(); state vector addrs = cs.coordinators(); state int idx = 0; @@ -687,7 +702,7 @@ ACTOR Future monitorProxiesOneGeneration( Referenceget().id; - req.supportedVersions = supportedVersions; + req.supportedVersions = supportedVersions->get(); req.traceLogGroup = traceLogGroup; ClusterConnectionString fileConnectionString; @@ -730,24 +745,8 @@ ACTOR Future monitorProxiesOneGeneration( ReferencenotifyConnected(); auto& ni = rep.get().mutate(); - if(ni.proxies.size() > CLIENT_KNOBS->MAX_CLIENT_PROXY_CONNECTIONS) { - std::vector proxyUIDs; - for(auto& proxy : ni.proxies) { - proxyUIDs.push_back(proxy.id()); - } - if(proxyUIDs != lastProxyUIDs) { - lastProxyUIDs = proxyUIDs; - lastProxies = ni.proxies; - deterministicRandom()->randomShuffle(lastProxies); - lastProxies.resize(CLIENT_KNOBS->MAX_CLIENT_PROXY_CONNECTIONS); - for(int i = 0; i < lastProxies.size(); i++) { - TraceEvent("ClientConnectedProxy").detail("Proxy", lastProxies[i].id()); - } - } - ni.proxies = lastProxies; - } - - clientInfo->set( rep.get().read() ); + shrinkProxyList(ni, lastProxyUIDs, lastProxies); + clientInfo->set( ni ); successIdx = idx; } else if(idx == successIdx) { wait(delay(CLIENT_KNOBS->COORDINATOR_RECONNECTION_DELAY)); @@ -756,7 +755,7 @@ ACTOR Future monitorProxiesOneGeneration( Reference monitorProxies( Reference>> connFile, Reference> clientInfo, Standalone> supportedVersions, Key traceLogGroup ) { +ACTOR Future monitorProxies( Reference>> connFile, Reference> clientInfo, Reference>>> supportedVersions, Key traceLogGroup ) { state MonitorLeaderInfo info(connFile->get()); loop { choose { diff --git a/fdbclient/MonitorLeader.h b/fdbclient/MonitorLeader.h index 89a128ec4f..05935f935a 100644 --- a/fdbclient/MonitorLeader.h +++ b/fdbclient/MonitorLeader.h @@ -57,9 +57,13 @@ Future monitorLeader( Reference const& connFile, Re Future monitorLeaderForProxies( Value const& key, vector const& coordinators, ClientData* const& clientData ); -Future monitorProxies( Reference>> const& connFile, Reference> const& clientInfo, Standalone> const& supportedVersions, Key const& traceLogGroup ); +Future monitorProxies( Reference>> const& connFile, Reference> const& clientInfo, Reference>>> const& supportedVersions, Key const& traceLogGroup ); +void shrinkProxyList( ClientDBInfo& ni, std::vector& lastProxyUIDs, std::vector& lastProxies ); + +#ifndef __INTEL_COMPILER #pragma region Implementation +#endif Future monitorLeaderInternal( Reference const& connFile, Reference> const& outSerializedLeaderInfo ); @@ -67,7 +71,7 @@ template struct LeaderDeserializer { Future operator()(const Reference>& serializedInfo, const Reference>>& outKnownLeader) { - return asyncDeserialize(serializedInfo, outKnownLeader, FLOW_KNOBS->USE_OBJECT_SERIALIZER); + return asyncDeserialize(serializedInfo, outKnownLeader); } }; @@ -91,6 +95,8 @@ Future monitorLeader(Reference const& connFile, return m || deserializer( serializedInfo, outKnownLeader ); } +#ifndef __INTEL_COMPILER #pragma endregion +#endif #endif diff --git a/fdbclient/MultiVersionTransaction.actor.cpp b/fdbclient/MultiVersionTransaction.actor.cpp index 767805d785..bb1ef53260 100644 --- a/fdbclient/MultiVersionTransaction.actor.cpp +++ b/fdbclient/MultiVersionTransaction.actor.cpp @@ -145,6 +145,20 @@ ThreadFuture> DLTransaction::getVersionstamp() { }); } +ThreadFuture DLTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) { + if (!api->transactionGetEstimatedRangeSizeBytes) { + return unsupported_operation(); + } + FdbCApi::FDBFuture *f = api->transactionGetEstimatedRangeSizeBytes(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()); + + return toThreadFuture(api, f, [](FdbCApi::FDBFuture *f, FdbCApi *api) { + int64_t sampledSize; + FdbCApi::fdb_error_t error = api->futureGetInt64(f, &sampledSize); + ASSERT(!error); + return sampledSize; + }); +} + void DLTransaction::addReadConflictRange(const KeyRangeRef& keys) { throwIfError(api->transactionAddConflictRange(tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDBConflictRangeTypes::READ)); } @@ -307,6 +321,7 @@ void DLApi::init() { loadClientFunction(&api->transactionReset, lib, fdbCPath, "fdb_transaction_reset"); loadClientFunction(&api->transactionCancel, lib, fdbCPath, "fdb_transaction_cancel"); loadClientFunction(&api->transactionAddConflictRange, lib, fdbCPath, "fdb_transaction_add_conflict_range"); + loadClientFunction(&api->transactionGetEstimatedRangeSizeBytes, lib, fdbCPath, "fdb_transaction_get_estimated_range_size_bytes", headerVersion >= 630); loadClientFunction(&api->futureGetInt64, lib, fdbCPath, headerVersion >= 620 ? "fdb_future_get_int64" : "fdb_future_get_version"); loadClientFunction(&api->futureGetError, lib, fdbCPath, "fdb_future_get_error"); @@ -547,6 +562,12 @@ void MultiVersionTransaction::addReadConflictRange(const KeyRangeRef& keys) { } } +ThreadFuture MultiVersionTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) { + auto tr = getTransaction(); + auto f = tr.transaction ? tr.transaction->getEstimatedRangeSizeBytes(keys) : ThreadFuture(Never()); + return abortableFuture(f, tr.onChange); +} + void MultiVersionTransaction::atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) { auto tr = getTransaction(); if(tr.transaction) { diff --git a/fdbclient/MultiVersionTransaction.h b/fdbclient/MultiVersionTransaction.h index e71f6e895f..a657f49cbb 100644 --- a/fdbclient/MultiVersionTransaction.h +++ b/fdbclient/MultiVersionTransaction.h @@ -81,6 +81,9 @@ struct FdbCApi : public ThreadSafeReferenceCounted { void (*transactionClear)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength); void (*transactionClearRange)(FDBTransaction *tr, uint8_t const *beginKeyName, int beginKeyNameLength, uint8_t const *endKeyName, int endKeyNameLength); void (*transactionAtomicOp)(FDBTransaction *tr, uint8_t const *keyName, int keyNameLength, uint8_t const *param, int paramLength, FDBMutationTypes::Option operationType); + + FDBFuture* (*transactionGetEstimatedRangeSizeBytes)(FDBTransaction* tr, uint8_t const* begin_key_name, + int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length); FDBFuture* (*transactionCommit)(FDBTransaction *tr); fdb_error_t (*transactionGetCommittedVersion)(FDBTransaction *tr, int64_t *outVersion); @@ -129,6 +132,7 @@ public: ThreadFuture> getRange( const KeyRangeRef& keys, GetRangeLimits limits, bool snapshot=false, bool reverse=false) override; ThreadFuture>> getAddressesForKey(const KeyRef& key) override; ThreadFuture> getVersionstamp() override; + ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; void addReadConflictRange(const KeyRangeRef& keys) override; @@ -228,6 +232,7 @@ public: ThreadFuture> getVersionstamp() override; void addReadConflictRange(const KeyRangeRef& keys) override; + ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; void atomicOp(const KeyRef& key, const ValueRef& value, uint32_t operationType) override; void set(const KeyRef& key, const ValueRef& value) override; diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp index 78f40bf7d8..92828882ff 100644 --- a/fdbclient/NativeAPI.actor.cpp +++ b/fdbclient/NativeAPI.actor.cpp @@ -21,29 +21,32 @@ #include "fdbclient/NativeAPI.actor.h" #include +#include +#include #include "fdbclient/Atomic.h" #include "fdbclient/ClusterInterface.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/DatabaseContext.h" -#include "fdbclient/FailureMonitorClient.h" #include "fdbclient/KeyRangeMap.h" #include "fdbclient/Knobs.h" #include "fdbclient/ManagementAPI.actor.h" #include "fdbclient/MasterProxyInterface.h" #include "fdbclient/MonitorLeader.h" #include "fdbclient/MutationList.h" +#include "fdbclient/ReadYourWrites.h" +#include "fdbclient/SpecialKeySpace.actor.h" #include "fdbclient/StorageServerInterface.h" #include "fdbclient/SystemData.h" #include "fdbrpc/LoadBalance.h" #include "fdbrpc/Net2FileSystem.h" #include "fdbrpc/simulator.h" -#include "fdbrpc/TLSConnection.h" #include "flow/ActorCollection.h" #include "flow/DeterministicRandom.h" #include "flow/Knobs.h" #include "flow/Platform.h" #include "flow/SystemMonitor.h" +#include "flow/TLSConfig.actor.h" #include "flow/UnitTest.h" #if defined(CMAKE_BUILD) || !defined(WIN32) @@ -67,13 +70,13 @@ using std::min; using std::pair; NetworkOptions networkOptions; -Reference tlsOptions; +TLSConfig tlsConfig(TLSEndpointType::CLIENT); -static void initTLSOptions() { - if (!tlsOptions) { - tlsOptions = Reference(new TLSOptions()); - } -} +// The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h. +NetworkOptions::NetworkOptions() + : localAddress(""), clusterFile(""), traceDirectory(Optional()), + traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"), + traceFormat("xml"), traceClockSource("now"), runLoopProfilingEnabled(false), supportedVersions(new ReferencedObject>>()) {} static const Key CLIENT_LATENCY_INFO_PREFIX = LiteralStringRef("client_latency/"); static const Key CLIENT_LATENCY_INFO_CTR_PREFIX = LiteralStringRef("client_latency_counter/"); @@ -215,7 +218,7 @@ template <> void delref( DatabaseContext* ptr ) { ptr->delref(); } ACTOR Future databaseLogger( DatabaseContext *cx ) { state double lastLogged = 0; loop { - wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, cx->taskID)); + wait(delay(CLIENT_KNOBS->SYSTEM_MONITOR_INTERVAL, TaskPriority::FlushTrace)); TraceEvent ev("TransactionMetrics", cx->dbId); ev.detail("Elapsed", (lastLogged == 0) ? 0 : now() - lastLogged) @@ -256,24 +259,6 @@ ACTOR Future databaseLogger( DatabaseContext *cx ) { } } -ACTOR static Future > getSampleVersionStamp(Transaction *tr) { - loop{ - try { - tr->reset(); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - wait(success(tr->get(LiteralStringRef("\xff/StatusJsonTestKey62793")))); - state Future > vstamp = tr->getVersionstamp(); - tr->makeSelfConflicting(); - wait(tr->commit()); - Standalone val = wait(vstamp); - return val; - } - catch (Error& e) { - wait(tr->onError(e)); - } - } -} - struct TrInfoChunk { ValueRef value; Key key; @@ -510,26 +495,49 @@ ACTOR static Future getHealthMetricsActor(DatabaseContext *cx, bo Future DatabaseContext::getHealthMetrics(bool detailed = false) { return getHealthMetricsActor(this, detailed); } -DatabaseContext::DatabaseContext( - Reference>> connectionFile, Reference> clientInfo, Future clientInfoMonitor, - TaskPriority taskID, LocalityData const& clientLocality, bool enableLocalityLoadBalance, bool lockAware, bool internal, int apiVersion, bool switchable ) - : connectionFile(connectionFile),clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), taskID(taskID), clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), - lockAware(lockAware), apiVersion(apiVersion), switchable(switchable), provisional(false), cc("TransactionMetrics"), - transactionReadVersions("ReadVersions", cc), transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc), - transactionCommittedMutations("CommittedMutations", cc), transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionsCommitStarted("CommitStarted", cc), - transactionsCommitCompleted("CommitCompleted", cc), transactionsTooOld("TooOld", cc), transactionsFutureVersions("FutureVersions", cc), - transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc), - transactionsProcessBehind("ProcessBehind", cc), outstandingWatches(0), - latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), mvCacheInsertLocation(0), - healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0), internal(internal) -{ +DatabaseContext::DatabaseContext(Reference>> connectionFile, + Reference> clientInfo, Future clientInfoMonitor, + TaskPriority taskID, LocalityData const& clientLocality, + bool enableLocalityLoadBalance, bool lockAware, bool internal, int apiVersion, + bool switchable) + : connectionFile(connectionFile), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor), taskID(taskID), + clientLocality(clientLocality), enableLocalityLoadBalance(enableLocalityLoadBalance), lockAware(lockAware), + apiVersion(apiVersion), switchable(switchable), provisional(false), cc("TransactionMetrics"), + transactionReadVersions("ReadVersions", cc), transactionReadVersionsCompleted("ReadVersionsCompleted", cc), + transactionReadVersionBatches("ReadVersionBatches", cc), + transactionBatchReadVersions("BatchPriorityReadVersions", cc), + transactionDefaultReadVersions("DefaultPriorityReadVersions", cc), + transactionImmediateReadVersions("ImmediatePriorityReadVersions", cc), + transactionBatchReadVersionsCompleted("BatchPriorityReadVersionsCompleted", cc), + transactionDefaultReadVersionsCompleted("DefaultPriorityReadVersionsCompleted", cc), + transactionImmediateReadVersionsCompleted("ImmediatePriorityReadVersionsCompleted", cc), + transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc), + transactionPhysicalReadsCompleted("PhysicalReadRequestsCompleted", cc), + transactionGetKeyRequests("GetKeyRequests", cc), transactionGetValueRequests("GetValueRequests", cc), + transactionGetRangeRequests("GetRangeRequests", cc), transactionWatchRequests("WatchRequests", cc), + transactionGetAddressesForKeyRequests("GetAddressesForKeyRequests", cc), transactionBytesRead("BytesRead", cc), + transactionKeysRead("KeysRead", cc), transactionMetadataVersionReads("MetadataVersionReads", cc), + transactionCommittedMutations("CommittedMutations", cc), + transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionSetMutations("SetMutations", cc), + transactionClearMutations("ClearMutations", cc), transactionAtomicMutations("AtomicMutations", cc), + transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc), + transactionKeyServerLocationRequests("KeyServerLocationRequests", cc), + transactionKeyServerLocationRequestsCompleted("KeyServerLocationRequestsCompleted", cc), + transactionsTooOld("TooOld", cc), transactionsFutureVersions("FutureVersions", cc), + transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc), + transactionsResourceConstrained("ResourceConstrained", cc), transactionsThrottled("Throttled", cc), + transactionsProcessBehind("ProcessBehind", cc), outstandingWatches(0), latencies(1000), readLatencies(1000), + commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), mvCacheInsertLocation(0), + healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0), internal(internal), + specialKeySpace(std::make_shared(normalKeys.begin, specialKeys.end)), + cKImpl(std::make_shared(conflictingKeysRange)) { dbId = deterministicRandom()->randomUniqueID(); connected = clientInfo->get().proxies.size() ? Void() : clientInfo->onChange(); metadataVersionCache.resize(CLIENT_KNOBS->METADATA_VERSION_CACHE_SIZE); maxOutstandingWatches = CLIENT_KNOBS->DEFAULT_MAX_OUTSTANDING_WATCHES; - snapshotRywEnabled = apiVersionAtLeast(300) ? 1 : 0; + snapshotRywEnabled = apiVersionAtLeast(300) ? 1 : 0; logger = databaseLogger( this ); locationCacheSize = g_network->isSimulated() ? @@ -541,14 +549,22 @@ DatabaseContext::DatabaseContext( monitorMasterProxiesInfoChange = monitorMasterProxiesChange(clientInfo, &masterProxiesChangeTrigger); clientStatusUpdater.actor = clientStatusUpdateActor(this); + specialKeySpace->registerKeyRange(conflictingKeysRange, cKImpl.get()); } -DatabaseContext::DatabaseContext( const Error &err ) : deferredError(err), cc("TransactionMetrics"), - transactionReadVersions("ReadVersions", cc), transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc), - transactionCommittedMutations("CommittedMutations", cc), transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionsCommitStarted("CommitStarted", cc), - transactionsCommitCompleted("CommitCompleted", cc), transactionsTooOld("TooOld", cc), transactionsFutureVersions("FutureVersions", cc), - transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc), transactionsResourceConstrained("ResourceConstrained", cc), - transactionsProcessBehind("ProcessBehind", cc), latencies(1000), readLatencies(1000), commitLatencies(1000), +DatabaseContext::DatabaseContext( const Error &err ) : deferredError(err), cc("TransactionMetrics"), transactionReadVersions("ReadVersions", cc), + transactionReadVersionsCompleted("ReadVersionsCompleted", cc), transactionReadVersionBatches("ReadVersionBatches", cc), transactionBatchReadVersions("BatchPriorityReadVersions", cc), + transactionDefaultReadVersions("DefaultPriorityReadVersions", cc), transactionImmediateReadVersions("ImmediatePriorityReadVersions", cc), + transactionBatchReadVersionsCompleted("BatchPriorityReadVersionsCompleted", cc), transactionDefaultReadVersionsCompleted("DefaultPriorityReadVersionsCompleted", cc), + transactionImmediateReadVersionsCompleted("ImmediatePriorityReadVersionsCompleted", cc), transactionLogicalReads("LogicalUncachedReads", cc), transactionPhysicalReads("PhysicalReadRequests", cc), + transactionPhysicalReadsCompleted("PhysicalReadRequestsCompleted", cc), transactionGetKeyRequests("GetKeyRequests", cc), transactionGetValueRequests("GetValueRequests", cc), + transactionGetRangeRequests("GetRangeRequests", cc), transactionWatchRequests("WatchRequests", cc), transactionGetAddressesForKeyRequests("GetAddressesForKeyRequests", cc), + transactionBytesRead("BytesRead", cc), transactionKeysRead("KeysRead", cc), transactionMetadataVersionReads("MetadataVersionReads", cc), transactionCommittedMutations("CommittedMutations", cc), + transactionCommittedMutationBytes("CommittedMutationBytes", cc), transactionSetMutations("SetMutations", cc), transactionClearMutations("ClearMutations", cc), + transactionAtomicMutations("AtomicMutations", cc), transactionsCommitStarted("CommitStarted", cc), transactionsCommitCompleted("CommitCompleted", cc), + transactionKeyServerLocationRequests("KeyServerLocationRequests", cc), transactionKeyServerLocationRequestsCompleted("KeyServerLocationRequestsCompleted", cc), transactionsTooOld("TooOld", cc), + transactionsFutureVersions("FutureVersions", cc), transactionsNotCommitted("NotCommitted", cc), transactionsMaybeCommitted("MaybeCommitted", cc), + transactionsResourceConstrained("ResourceConstrained", cc), transactionsThrottled("Throttled", cc), transactionsProcessBehind("ProcessBehind", cc), latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), internal(false) {} @@ -737,7 +753,7 @@ ACTOR static Future switchConnectionFileImpl(ReferencerandomUniqueID(); self->clientInfo->set(clearedClientInfo); self->connectionFile->set(connFile); - + state Database db(Reference::addRef(self)); state Transaction tr(db); loop { @@ -788,7 +804,10 @@ Database Database::createDatabase( Reference connFile, in auto publicIP = determinePublicIPAutomatically( connFile->getConnectionString() ); selectTraceFormatter(networkOptions.traceFormat); - openTraceFile(NetworkAddress(publicIP, ::getpid()), networkOptions.traceRollSize, networkOptions.traceMaxLogsSize, networkOptions.traceDirectory.get(), "trace", networkOptions.traceLogGroup); + selectTraceClockSource(networkOptions.traceClockSource); + openTraceFile(NetworkAddress(publicIP, ::getpid()), networkOptions.traceRollSize, + networkOptions.traceMaxLogsSize, networkOptions.traceDirectory.get(), "trace", + networkOptions.traceLogGroup, networkOptions.traceFileIdentifier); TraceEvent("ClientStart") .detail("SourceVersion", getSourceVersion()) @@ -808,6 +827,8 @@ Database Database::createDatabase( Reference connFile, in } } + g_network->initTLS(); + Reference> clientInfo(new AsyncVar()); Reference>> connectionFile(new AsyncVar>()); connectionFile->set(connFile); @@ -835,8 +856,9 @@ const UniqueOrderedOptionList& Database::getTransactionDe } void setNetworkOption(FDBNetworkOptions::Option option, Optional value) { + std::regex identifierRegex("^[a-zA-Z0-9_]*$"); switch(option) { - // SOMEDAY: If the network is already started, should these three throw an error? + // SOMEDAY: If the network is already started, should these five throw an error? case FDBNetworkOptions::TRACE_ENABLE: networkOptions.traceDirectory = value.present() ? value.get().toString() : ""; break; @@ -848,10 +870,6 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu validateOptionValue(value, true); networkOptions.traceMaxLogsSize = extractIntOption(value, 0, std::numeric_limits::max()); break; - case FDBNetworkOptions::TRACE_LOG_GROUP: - if(value.present()) - networkOptions.traceLogGroup = value.get().toString(); - break; case FDBNetworkOptions::TRACE_FORMAT: validateOptionValue(value, true); networkOptions.traceFormat = value.get().toString(); @@ -860,6 +878,36 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu throw invalid_option_value(); } break; + case FDBNetworkOptions::TRACE_FILE_IDENTIFIER: + validateOptionValue(value, true); + networkOptions.traceFileIdentifier = value.get().toString(); + if (networkOptions.traceFileIdentifier.length() > CLIENT_KNOBS->TRACE_LOG_FILE_IDENTIFIER_MAX_LENGTH) { + fprintf(stderr, "Trace file identifier provided is too long.\n"); + throw invalid_option_value(); + } else if (!std::regex_match(networkOptions.traceFileIdentifier, identifierRegex)) { + fprintf(stderr, "Trace file identifier should only contain alphanumerics and underscores.\n"); + throw invalid_option_value(); + } + break; + + case FDBNetworkOptions::TRACE_LOG_GROUP: + if(value.present()) { + if (traceFileIsOpen()) { + setTraceLogGroup(value.get().toString()); + } + else { + networkOptions.traceLogGroup = value.get().toString(); + } + } + break; + case FDBNetworkOptions::TRACE_CLOCK_SOURCE: + validateOptionValue(value, true); + networkOptions.traceClockSource = value.get().toString(); + if (!validateTraceClockSource(networkOptions.traceClockSource)) { + fprintf(stderr, "Unrecognized trace clock source: `%s'\n", networkOptions.traceClockSource.c_str()); + throw invalid_option_value(); + } + break; case FDBNetworkOptions::KNOB: { validateOptionValue(value, true); @@ -874,8 +922,17 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu std::string knobName = optionValue.substr(0, eq); std::string knobValue = optionValue.substr(eq+1); - if (!const_cast(FLOW_KNOBS)->setKnob( knobName, knobValue ) && - !const_cast(CLIENT_KNOBS)->setKnob( knobName, knobValue )) + if (const_cast(FLOW_KNOBS)->setKnob(knobName, knobValue)) + { + // update dependent knobs + const_cast(FLOW_KNOBS)->initialize(); + } + else if (const_cast(CLIENT_KNOBS)->setKnob(knobName, knobValue)) + { + // update dependent knobs + const_cast(CLIENT_KNOBS)->initialize(); + } + else { TraceEvent(SevWarnAlways, "UnrecognizedKnob").detail("Knob", knobName.c_str()); fprintf(stderr, "FoundationDB client ignoring unrecognized knob option '%s'\n", knobName.c_str()); @@ -887,49 +944,40 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu break; case FDBNetworkOptions::TLS_CERT_PATH: validateOptionValue(value, true); - initTLSOptions(); - tlsOptions->set_cert_file( value.get().toString() ); + tlsConfig.setCertificatePath(value.get().toString()); break; - case FDBNetworkOptions::TLS_CERT_BYTES: - initTLSOptions(); - tlsOptions->set_cert_data( value.get().toString() ); - break; - case FDBNetworkOptions::TLS_CA_PATH: + case FDBNetworkOptions::TLS_CERT_BYTES: { validateOptionValue(value, true); - initTLSOptions(); - tlsOptions->set_ca_file( value.get().toString() ); + tlsConfig.setCertificateBytes(value.get().toString()); break; - case FDBNetworkOptions::TLS_CA_BYTES: + } + case FDBNetworkOptions::TLS_CA_PATH: { validateOptionValue(value, true); - initTLSOptions(); - tlsOptions->set_ca_data(value.get().toString()); + tlsConfig.setCAPath(value.get().toString()); break; + } + case FDBNetworkOptions::TLS_CA_BYTES: { + validateOptionValue(value, true); + tlsConfig.setCABytes(value.get().toString()); + break; + } case FDBNetworkOptions::TLS_PASSWORD: validateOptionValue(value, true); - initTLSOptions(); - tlsOptions->set_key_password(value.get().toString()); + tlsConfig.setPassword(value.get().toString()); break; case FDBNetworkOptions::TLS_KEY_PATH: validateOptionValue(value, true); - initTLSOptions(); - tlsOptions->set_key_file( value.get().toString() ); + tlsConfig.setKeyPath(value.get().toString()); break; - case FDBNetworkOptions::TLS_KEY_BYTES: + case FDBNetworkOptions::TLS_KEY_BYTES: { validateOptionValue(value, true); - initTLSOptions(); - tlsOptions->set_key_data( value.get().toString() ); + tlsConfig.setKeyBytes(value.get().toString()); break; + } case FDBNetworkOptions::TLS_VERIFY_PEERS: validateOptionValue(value, true); - initTLSOptions(); - try { - tlsOptions->set_verify_peers({ value.get().toString() }); - } catch( Error& e ) { - TraceEvent(SevWarnAlways, "TLSValidationSetError") - .error( e ) - .detail("Input", value.get().toString() ); - throw invalid_option_value(); - } + tlsConfig.clearVerifyPeers(); + tlsConfig.addVerifyPeers( value.get().toString() ); break; case FDBNetworkOptions::CLIENT_BUGGIFY_ENABLE: enableBuggify(true, BuggifyType::Client); @@ -956,24 +1004,25 @@ void setNetworkOption(FDBNetworkOptions::Option option, Optional valu ASSERT(g_network); ASSERT(value.present()); - networkOptions.supportedVersions.resize(networkOptions.supportedVersions.arena(), 0); + Standalone> supportedVersions; std::string versionString = value.get().toString(); size_t index = 0; size_t nextIndex = 0; while(nextIndex != versionString.npos) { nextIndex = versionString.find(';', index); - networkOptions.supportedVersions.push_back_deep(networkOptions.supportedVersions.arena(), ClientVersionRef(versionString.substr(index, nextIndex-index))); + supportedVersions.push_back_deep(supportedVersions.arena(), ClientVersionRef(versionString.substr(index, nextIndex-index))); index = nextIndex + 1; } - ASSERT(networkOptions.supportedVersions.size() > 0); + ASSERT(supportedVersions.size() > 0); + networkOptions.supportedVersions->set(supportedVersions); break; } - case FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING: + case FDBNetworkOptions::ENABLE_RUN_LOOP_PROFILING: // Same as ENABLE_SLOW_TASK_PROFILING validateOptionValue(value, false); - networkOptions.slowTaskProfilingEnabled = true; + networkOptions.runLoopProfilingEnabled = true; break; default: break; @@ -987,23 +1036,20 @@ void setupNetwork(uint64_t transportId, bool useMetrics) { if (!networkOptions.logClientInfo.present()) networkOptions.logClientInfo = true; - g_network = newNet2(false, useMetrics || networkOptions.traceDirectory.present()); + TLS::DisableOpenSSLAtExitHandler(); + g_network = newNet2(tlsConfig, false, useMetrics || networkOptions.traceDirectory.present()); + g_network->addStopCallback( Net2FileSystem::stop ); + g_network->addStopCallback( TLS::DestroyOpenSSLGlobalState ); FlowTransport::createInstance(true, transportId); Net2FileSystem::newFileSystem(); - - initTLSOptions(); - -#ifndef TLS_DISABLED - tlsOptions->register_network(); -#endif } void runNetwork() { if(!g_network) throw network_not_setup(); - if(networkOptions.traceDirectory.present() && networkOptions.slowTaskProfilingEnabled) { - setupSlowTaskProfiler(); + if(networkOptions.traceDirectory.present() && networkOptions.runLoopProfilingEnabled) { + setupRunLoopProfiler(); } g_network->run(); @@ -1168,9 +1214,11 @@ ACTOR Future< pair> > getKeyLocation_internal( g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.Before"); loop { + ++cx->transactionKeyServerLocationRequests; choose { when ( wait( cx->onMasterProxiesChanged() ) ) {} when ( GetKeyServerLocationsReply rep = wait( basicLoadBalance( cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::getKeyServersLocations, GetKeyServerLocationsRequest(key, Optional(), 100, isBackward, key.arena()), TaskPriority::DefaultPromiseEndpoint ) ) ) { + ++cx->transactionKeyServerLocationRequestsCompleted; if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocation.After"); ASSERT( rep.results.size() == 1 ); @@ -1205,9 +1253,11 @@ ACTOR Future< vector< pair> > > getKeyRangeLoca g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocations.Before"); loop { + ++cx->transactionKeyServerLocationRequests; choose { when ( wait( cx->onMasterProxiesChanged() ) ) {} when ( GetKeyServerLocationsReply _rep = wait( basicLoadBalance( cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::getKeyServersLocations, GetKeyServerLocationsRequest(keys.begin, keys.end, limit, reverse, keys.arena()), TaskPriority::DefaultPromiseEndpoint ) ) ) { + ++cx->transactionKeyServerLocationRequestsCompleted; state GetKeyServerLocationsReply rep = _rep; if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKeyLocations.After"); @@ -1305,8 +1355,6 @@ ACTOR Future> getValue( Future version, Key key, Databa state uint64_t startTime; state double startTimeD; try { - //GetValueReply r = wait( deterministicRandom()->randomChoice( ssi->get() ).getValue.getReply( GetValueRequest(key,ver) ) ); - //return r.value; if( info.debugID.present() ) { getValueID = nondeterministicRandom()->randomUniqueID(); @@ -1323,19 +1371,26 @@ ACTOR Future> getValue( Future version, Key key, Databa startTimeD = now(); ++cx->transactionPhysicalReads; - if (CLIENT_BUGGIFY) { - throw deterministicRandom()->randomChoice( - std::vector{ transaction_too_old(), future_version() }); - } state GetValueReply reply; - choose { - when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } - when(GetValueReply _reply = - wait(loadBalance(ssi.second, &StorageServerInterface::getValue, - GetValueRequest(key, ver, getValueID), TaskPriority::DefaultPromiseEndpoint, false, - cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { - reply = _reply; + try { + if (CLIENT_BUGGIFY) { + throw deterministicRandom()->randomChoice( + std::vector{ transaction_too_old(), future_version() }); } + choose { + when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } + when(GetValueReply _reply = + wait(loadBalance(ssi.second, &StorageServerInterface::getValue, + GetValueRequest(key, ver, getValueID), TaskPriority::DefaultPromiseEndpoint, false, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { + reply = _reply; + } + } + ++cx->transactionPhysicalReadsCompleted; + } + catch(Error&) { + ++cx->transactionPhysicalReadsCompleted; + throw; } double latency = now() - startTimeD; @@ -1354,6 +1409,9 @@ ACTOR Future> getValue( Future version, Key key, Databa .detail("ReqVersion", ver) .detail("ReplySize", reply.value.present() ? reply.value.get().size() : -1);*/ } + + cx->transactionBytesRead += reply.value.present() ? reply.value.get().size() : 0; + ++cx->transactionKeysRead; return reply.value; } catch (Error& e) { cx->getValueCompleted->latency = timer_int() - startTime; @@ -1401,14 +1459,20 @@ ACTOR Future getKey( Database cx, KeySelector k, Future version, T g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.Before"); //.detail("StartKey", k.getKey()).detail("Offset",k.offset).detail("OrEqual",k.orEqual); ++cx->transactionPhysicalReads; state GetKeyReply reply; - choose { - when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } - when(GetKeyReply _reply = - wait(loadBalance(ssi.second, &StorageServerInterface::getKey, GetKeyRequest(k, version.get()), - TaskPriority::DefaultPromiseEndpoint, false, - cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { - reply = _reply; + try { + choose { + when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } + when(GetKeyReply _reply = + wait(loadBalance(ssi.second, &StorageServerInterface::getKey, GetKeyRequest(k, version.get()), + TaskPriority::DefaultPromiseEndpoint, false, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { + reply = _reply; + } } + ++cx->transactionPhysicalReadsCompleted; + } catch(Error&) { + ++cx->transactionPhysicalReadsCompleted; + throw; } if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getKey.After"); //.detail("NextKey",reply.sel.key).detail("Offset", reply.sel.offset).detail("OrEqual", k.orEqual); @@ -1439,7 +1503,7 @@ ACTOR Future waitForCommittedVersion( Database cx, Version version ) { when ( wait( cx->onMasterProxiesChanged() ) ) {} when ( GetReadVersionReply v = wait( basicLoadBalance( cx->getMasterProxies(false), &MasterProxyInterface::getConsistentReadVersion, GetReadVersionRequest( 0, GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE ), cx->taskID ) ) ) { cx->minAcceptableReadVersion = std::min(cx->minAcceptableReadVersion, v.version); - + if (v.version >= version) return v.version; // SOMEDAY: Do the wait on the server side, possibly use less expensive source of committed version (causal consistency is not needed for this purpose) @@ -1453,6 +1517,17 @@ ACTOR Future waitForCommittedVersion( Database cx, Version version ) { } } +ACTOR Future getRawVersion( Database cx ) { + loop { + choose { + when ( wait( cx->onMasterProxiesChanged() ) ) {} + when ( GetReadVersionReply v = wait( loadBalance( cx->getMasterProxies(false), &MasterProxyInterface::getConsistentReadVersion, GetReadVersionRequest( 0, GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE ), cx->taskID ) ) ) { + return v.version; + } + } + } +} + ACTOR Future readVersionBatcher( DatabaseContext* cx, FutureStream, Optional>> versionStream, uint32_t flags); @@ -1576,14 +1651,20 @@ ACTOR Future> getExactRange( Database cx, Version ver } ++cx->transactionPhysicalReads; state GetKeyValuesReply rep; - choose { - when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } - when(GetKeyValuesReply _rep = - wait(loadBalance(locations[shard].second, &StorageServerInterface::getKeyValues, req, - TaskPriority::DefaultPromiseEndpoint, false, - cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { - rep = _rep; + try { + choose { + when(wait(cx->connectionFileChanged())) { throw transaction_too_old(); } + when(GetKeyValuesReply _rep = + wait(loadBalance(locations[shard].second, &StorageServerInterface::getKeyValues, req, + TaskPriority::DefaultPromiseEndpoint, false, + cx->enableLocalityLoadBalance ? &cx->queueModel : nullptr))) { + rep = _rep; + } } + ++cx->transactionPhysicalReadsCompleted; + } catch(Error&) { + ++cx->transactionPhysicalReadsCompleted; + throw; } if( info.debugID.present() ) g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getExactRange.After"); @@ -1732,14 +1813,19 @@ ACTOR Future> getRangeFallback( Database cx, Version return r; } -void getRangeFinished(Reference trLogInfo, double startTime, KeySelector begin, KeySelector end, bool snapshot, +void getRangeFinished(Database cx, Reference trLogInfo, double startTime, KeySelector begin, KeySelector end, bool snapshot, Promise> conflictRange, bool reverse, Standalone result) { + int64_t bytes = 0; + for(const KeyValueRef &kv : result) { + bytes += kv.key.size() + kv.value.size(); + } + + cx->transactionBytesRead += bytes; + cx->transactionKeysRead += result.size(); + if( trLogInfo ) { - int rangeSize = 0; - for (const KeyValueRef &kv : result.contents()) - rangeSize += kv.key.size() + kv.value.size(); - trLogInfo->addLog(FdbClientLogEvents::EventGetRange(startTime, now()-startTime, rangeSize, begin.getKey(), end.getKey())); + trLogInfo->addLog(FdbClientLogEvents::EventGetRange(startTime, now()-startTime, bytes, begin.getKey(), end.getKey())); } if( !snapshot ) { @@ -1805,7 +1891,7 @@ ACTOR Future> getRange( Database cx, Reference> getRange( Database cx, ReferencetransactionPhysicalReads; - if (CLIENT_BUGGIFY) { - throw deterministicRandom()->randomChoice(std::vector{ - transaction_too_old(), future_version() - }); + ++cx->transactionGetRangeRequests; + state GetKeyValuesReply rep; + try { + if (CLIENT_BUGGIFY) { + throw deterministicRandom()->randomChoice(std::vector{ + transaction_too_old(), future_version() + }); + } + GetKeyValuesReply _rep = wait( loadBalance(beginServer.second, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL ) ); + rep = _rep; + ++cx->transactionPhysicalReadsCompleted; + } catch(Error&) { + ++cx->transactionPhysicalReadsCompleted; + throw; } - GetKeyValuesReply rep = wait( loadBalance(beginServer.second, &StorageServerInterface::getKeyValues, req, TaskPriority::DefaultPromiseEndpoint, false, cx->enableLocalityLoadBalance ? &cx->queueModel : NULL ) ); if( info.debugID.present() ) { g_traceBatch.addEvent("TransactionDebug", info.debugID.get().first(), "NativeAPI.getRange.After");//.detail("SizeOf", rep.data.size()); @@ -1898,7 +1993,7 @@ ACTOR Future> getRange( Database cx, Reference std::max(1, originalLimits.minRows) ) { output.more = true; output.resize(output.arena(), deterministicRandom()->randomInt(std::max(1,originalLimits.minRows),output.size())); - getRangeFinished(trLogInfo, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, output); + getRangeFinished(cx, trLogInfo, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, output); return output; } @@ -1907,7 +2002,7 @@ ACTOR Future> getRange( Database cx, Reference> getRange( Database cx, Reference> getRange( Database cx, Reference result = wait( getRangeFallback(cx, version, originalBegin, originalEnd, originalLimits, reverse, info ) ); - getRangeFinished(trLogInfo, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, result); + getRangeFinished(cx, trLogInfo, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, result); return result; } @@ -1962,7 +2057,7 @@ ACTOR Future> getRange( Database cx, Reference result = wait( getRangeFallback(cx, version, originalBegin, originalEnd, originalLimits, reverse, info ) ); - getRangeFinished(trLogInfo, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, result); + getRangeFinished(cx, trLogInfo, startTime, originalBegin, originalEnd, snapshot, conflictRange, reverse, result); return result; } @@ -2040,6 +2135,7 @@ void Transaction::setVersion( Version v ) { Future> Transaction::get( const Key& key, bool snapshot ) { ++cx->transactionLogicalReads; + ++cx->transactionGetValueRequests; //ASSERT (key < allKeys.end); //There are no keys in the database with size greater than KEY_SIZE_LIMIT @@ -2055,6 +2151,7 @@ Future> Transaction::get( const Key& key, bool snapshot ) { tr.transaction.read_conflict_ranges.push_back(tr.arena, singleKeyRange(key, tr.arena)); if(key == metadataVersionKey) { + ++cx->transactionMetadataVersionReads; if(!ver.isReady() || metadataVersion.isSet()) { return metadataVersion.getFuture(); } else { @@ -2095,12 +2192,9 @@ void Watch::setWatch(Future watchFuture) { } //FIXME: This seems pretty horrible. Now a Database can't die until all of its watches do... -ACTOR Future watch( Reference watch, Database cx, Transaction *self ) { - state TransactionInfo info = self->info; +ACTOR Future watch(Reference watch, Database cx, TransactionInfo info) { cx->addWatch(); try { - self->watches.push_back(watch); - choose { // RYOW write to value that is being watched (if applicable) // Errors @@ -2133,8 +2227,15 @@ ACTOR Future watch( Reference watch, Database cx, Transaction *self return Void(); } +Future Transaction::getRawReadVersion() { + return ::getRawVersion(cx); +} + Future< Void > Transaction::watch( Reference watch ) { - return ::watch(watch, cx, this); + ++cx->transactionWatchRequests; + cx->addWatch(); + watches.push_back(watch); + return ::watch(watch, cx, info); } ACTOR Future>> getAddressesForKeyActor(Key key, Future ver, Database cx, @@ -2145,6 +2246,8 @@ ACTOR Future>> getAddressesForKeyActor(Key key // If key >= allKeys.end, then getRange will return a kv-pair with an empty value. This will result in our serverInterfaces vector being empty, which will cause us to return an empty addresses list. state Key ksKey = keyServersKey(key); + state Standalone serverTagResult = wait( getRange(cx, ver, lastLessOrEqual(serverTagKeys.begin), firstGreaterThan(serverTagKeys.end), GetRangeLimits(CLIENT_KNOBS->TOO_MANY), false, info ) ); + ASSERT( !serverTagResult.more && serverTagResult.size() < CLIENT_KNOBS->TOO_MANY ); Future> futureServerUids = getRange(cx, ver, lastLessOrEqual(ksKey), firstGreaterThan(ksKey), GetRangeLimits(1), false, info); Standalone serverUids = wait( futureServerUids ); @@ -2152,7 +2255,7 @@ ACTOR Future>> getAddressesForKeyActor(Key key vector src; vector ignore; // 'ignore' is so named because it is the vector into which we decode the 'dest' servers in the case where this key is being relocated. But 'src' is the canonical location until the move is finished, because it could be cancelled at any time. - decodeKeyServersValue(serverUids[0].value, src, ignore); + decodeKeyServersValue(serverTagResult, serverUids[0].value, src, ignore); Optional> serverInterfaces = wait( transactionalGetServerInterfaces(ver, cx, info, src) ); ASSERT( serverInterfaces.present() ); // since this is happening transactionally, /FF/keyServers and /FF/serverList need to be consistent with one another @@ -2170,6 +2273,7 @@ ACTOR Future>> getAddressesForKeyActor(Key key Future< Standalone< VectorRef< const char*>>> Transaction::getAddressesForKey( const Key& key ) { ++cx->transactionLogicalReads; + ++cx->transactionGetAddressesForKeyRequests; auto ver = getReadVersion(); return getAddressesForKeyActor(key, ver, cx, info, options); @@ -2193,6 +2297,7 @@ ACTOR Future< Key > getKeyAndConflictRange( Future< Key > Transaction::getKey( const KeySelector& key, bool snapshot ) { ++cx->transactionLogicalReads; + ++cx->transactionGetKeyRequests; if( snapshot ) return ::getKey(cx, key, getReadVersion(), info); @@ -2209,6 +2314,7 @@ Future< Standalone > Transaction::getRange( bool reverse ) { ++cx->transactionLogicalReads; + ++cx->transactionGetRangeRequests; if( limits.isReached() ) return Standalone(); @@ -2285,7 +2391,7 @@ void Transaction::makeSelfConflicting() { } void Transaction::set( const KeyRef& key, const ValueRef& value, bool addConflictRange ) { - + ++cx->transactionSetMutations; if(key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) throw key_too_large(); if(value.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) @@ -2303,6 +2409,7 @@ void Transaction::set( const KeyRef& key, const ValueRef& value, bool addConflic } void Transaction::atomicOp(const KeyRef& key, const ValueRef& operand, MutationRef::Type operationType, bool addConflictRange) { + ++cx->transactionAtomicMutations; if(key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) throw key_too_large(); if(operand.size() > CLIENT_KNOBS->VALUE_SIZE_LIMIT) @@ -2329,6 +2436,7 @@ void Transaction::atomicOp(const KeyRef& key, const ValueRef& operand, MutationR } void Transaction::clear( const KeyRangeRef& range, bool addConflictRange ) { + ++cx->transactionClearMutations; auto &req = tr; auto &t = req.transaction; @@ -2351,7 +2459,7 @@ void Transaction::clear( const KeyRangeRef& range, bool addConflictRange ) { t.write_conflict_ranges.push_back( req.arena, r ); } void Transaction::clear( const KeyRef& key, bool addConflictRange ) { - + ++cx->transactionClearMutations; //There aren't any keys in the database with size larger than KEY_SIZE_LIMIT if(key.size() > (key.startsWith(systemKeys.begin) ? CLIENT_KNOBS->SYSTEM_KEY_SIZE_LIMIT : CLIENT_KNOBS->KEY_SIZE_LIMIT)) return; @@ -2394,8 +2502,10 @@ void Transaction::addWriteConflictRange( const KeyRangeRef& keys ) { double Transaction::getBackoff(int errCode) { double b = backoff * deterministicRandom()->random01(); - backoff = errCode == error_code_proxy_memory_limit_exceeded ? std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF) : - std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, options.maxBackoff); + backoff = + errCode == error_code_proxy_memory_limit_exceeded + ? std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, CLIENT_KNOBS->RESOURCE_CONSTRAINED_MAX_BACKOFF) + : std::min(backoff * CLIENT_KNOBS->BACKOFF_GROWTH_RATE, options.maxBackoff); return b; } @@ -2417,6 +2527,9 @@ void TransactionOptions::reset(Database const& cx) { maxBackoff = CLIENT_KNOBS->DEFAULT_MAX_BACKOFF; sizeLimit = CLIENT_KNOBS->TRANSACTION_SIZE_LIMIT; lockAware = cx->lockAware; + if (cx->apiVersionAtLeast(630)) { + includePort = true; + } } void Transaction::reset() { @@ -2533,8 +2646,8 @@ ACTOR void checkWrites( Database cx, Future committed, Promise outCo } else { Optional val = wait( tr.get( it->range().begin ) ); if( !val.present() || val.get() != m.setValue ) { - TraceEvent evt = TraceEvent(SevError, "CheckWritesFailed") - .detail("Class", "Set") + TraceEvent evt(SevError, "CheckWritesFailed"); + evt.detail("Class", "Set") .detail("Key", it->range().begin) .detail("Expected", m.setValue); if( !val.present() ) @@ -2605,7 +2718,6 @@ ACTOR static Future tryCommit( Database cx, Reference state double startTime = now(); if (info.debugID.present()) TraceEvent(interval.begin()).detail( "Parent", info.debugID.get() ); - try { if(CLIENT_BUGGIFY) { throw deterministicRandom()->randomChoice(std::vector{ @@ -2629,8 +2741,12 @@ ACTOR static Future tryCommit( Database cx, Reference req.debugID = commitID; state Future reply; if (options.commitOnFirstProxy) { - const std::vector& proxies = cx->clientInfo->get().proxies; - reply = proxies.size() ? throwErrorOr ( brokenPromiseToMaybeDelivered ( proxies[0].commit.tryGetReply(req) ) ) : Never(); + if(cx->clientInfo->get().firstProxy.present()) { + reply = throwErrorOr ( brokenPromiseToMaybeDelivered ( cx->clientInfo->get().firstProxy.get().commit.tryGetReply(req) ) ); + } else { + const std::vector& proxies = cx->clientInfo->get().proxies; + reply = proxies.size() ? throwErrorOr ( brokenPromiseToMaybeDelivered ( proxies[0].commit.tryGetReply(req) ) ) : Never(); + } } else { reply = basicLoadBalance( cx->getMasterProxies(info.useProvisionalProxies), &MasterProxyInterface::commit, req, TaskPriority::DefaultPromiseEndpoint, true ); } @@ -2673,6 +2789,24 @@ ACTOR static Future tryCommit( Database cx, Reference trLogInfo->addLog(FdbClientLogEvents::EventCommit(startTime, latency, req.transaction.mutations.size(), req.transaction.mutations.expectedSize(), req)); return Void(); } else { + // clear the RYW transaction which contains previous conflicting keys + tr->info.conflictingKeys.reset(); + if (ci.conflictingKRIndices.present()) { + tr->info.conflictingKeys = + std::make_shared>(conflictingKeysFalse, specialKeys.end); + state Standalone> conflictingKRIndices = ci.conflictingKRIndices.get(); + // drop duplicate indices and merge overlapped ranges + // Note: addReadConflictRange in native transaction object does not merge overlapped ranges + state std::unordered_set mergedIds(conflictingKRIndices.begin(), + conflictingKRIndices.end()); + for (auto const& rCRIndex : mergedIds) { + const KeyRangeRef kr = req.transaction.read_conflict_ranges[rCRIndex]; + const KeyRange krWithPrefix = KeyRangeRef(kr.begin.withPrefix(conflictingKeysRange.begin), + kr.end.withPrefix(conflictingKeysRange.begin)); + tr->info.conflictingKeys->insert(krWithPrefix, conflictingKeysTrue); + } + } + if (info.debugID.present()) TraceEvent(interval.end()).detail("Conflict", 1); @@ -2708,7 +2842,8 @@ ACTOR static Future tryCommit( Database cx, Reference if (e.code() != error_code_transaction_too_old && e.code() != error_code_not_committed && e.code() != error_code_database_locked - && e.code() != error_code_proxy_memory_limit_exceeded) + && e.code() != error_code_proxy_memory_limit_exceeded + && e.code() != error_code_batch_transaction_throttled) TraceEvent(SevError, "TryCommitError").error(e); if (trLogInfo) trLogInfo->addLog(FdbClientLogEvents::EventCommitError(startTime, static_cast(e.code()), req)); @@ -2784,6 +2919,9 @@ Future Transaction::commitMutations() { if(options.firstInBatch) { tr.flags = tr.flags | CommitTransactionRequest::FLAG_FIRST_IN_BATCH; } + if (options.reportConflictingKeys) { + tr.transaction.report_conflicting_keys = true; + } Future commitResult = tryCommit( cx, trLogInfo, tr, readVersion, info, &this->committedVersion, this, options ); @@ -2908,6 +3046,12 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional(new TransactionLogInfo(value.get().printable(), TransactionLogInfo::DONT_LOG)); trLogInfo->maxFieldLength = options.maxTransactionLoggingFieldLength; } + if (info.debugID.present()) { + TraceEvent(SevInfo, "TransactionBeingTraced") + .detail("DebugTransactionID", trLogInfo->identifier) + .detail("ServerTraceID", info.debugID.get().toString()); + + } break; case FDBTransactionOptions::LOG_TRANSACTION: @@ -2935,6 +3079,16 @@ void Transaction::setOption( FDBTransactionOptions::Option option, OptionalrandomUniqueID()); + if (trLogInfo && !trLogInfo->identifier.empty()) { + TraceEvent(SevInfo, "TransactionBeingTraced") + .detail("DebugTransactionID", trLogInfo->identifier) + .detail("ServerTraceID", info.debugID.get().toString()); + } + break; + case FDBTransactionOptions::MAX_RETRY_DELAY: validateOptionValue(value, true); options.maxBackoff = extractIntOption(value, 0, std::numeric_limits::max()) / 1000.0; @@ -2975,13 +3129,19 @@ void Transaction::setOption( FDBTransactionOptions::Option option, Optional getConsistentReadVersion( DatabaseContext *cx, uint32_t transactionCount, uint32_t flags, Optional debugID ) { try { + ++cx->transactionReadVersionBatches; if( debugID.present() ) g_traceBatch.addEvent("TransactionDebug", debugID.get().first(), "NativeAPI.getConsistentReadVersion.Before"); loop { @@ -2998,7 +3158,7 @@ ACTOR Future getConsistentReadVersion( DatabaseContext *cx, } } } catch (Error& e) { - if( e.code() != error_code_broken_promise ) + if (e.code() != error_code_broken_promise && e.code() != error_code_batch_transaction_throttled) TraceEvent(SevError, "GetConsistentReadVersionError").error(e); throw; } @@ -3016,45 +3176,41 @@ ACTOR Future readVersionBatcher( DatabaseContext *cx, FutureStream< std::p state PromiseStream replyTimes; state PromiseStream _errorStream; state double batchTime = 0; - loop { send_batch = false; choose { - when(std::pair< Promise, Optional > req = waitNext(versionStream)) { + when(std::pair, Optional> req = waitNext(versionStream)) { if (req.second.present()) { - if (!debugID.present()) + if (!debugID.present()) { debugID = nondeterministicRandom()->randomUniqueID(); + } g_traceBatch.addAttach("TransactionAttachID", req.second.get().first(), debugID.get().first()); } requests.push_back(req.first); if (requests.size() == CLIENT_KNOBS->MAX_BATCH_SIZE) send_batch = true; else if (!timeout.isValid()) - timeout = delay(batchTime, TaskPriority::ProxyGetConsistentReadVersion); - } - when(wait(timeout.isValid() ? timeout : Never())) { - send_batch = true; + timeout = delay(batchTime, TaskPriority::GetConsistentReadVersion); } + when(wait(timeout.isValid() ? timeout : Never())) { send_batch = true; } // dynamic batching monitors reply latencies - when(double reply_latency = waitNext(replyTimes.getFuture())){ + when(double reply_latency = waitNext(replyTimes.getFuture())) { double target_latency = reply_latency * 0.5; batchTime = min(0.1 * target_latency + 0.9 * batchTime, CLIENT_KNOBS->GRV_BATCH_TIMEOUT); } - when(wait(collection)){} // for errors + when(wait(collection)) {} // for errors } if (send_batch) { int count = requests.size(); ASSERT(count); - // dynamic batching Promise GRVReply; requests.push_back(GRVReply); - addActor.send(timeReply(GRVReply.getFuture(), replyTimes)); + addActor.send(ready(timeReply(GRVReply.getFuture(), replyTimes))); - Future batch = - incrementalBroadcast( - getConsistentReadVersion(cx, count, flags, std::move(debugID)), - std::vector< Promise >(std::move(requests)), CLIENT_KNOBS->BROADCAST_BATCH_SIZE); + Future batch = incrementalBroadcastWithError( + getConsistentReadVersion(cx, count, flags, std::move(debugID)), + std::vector>(std::move(requests)), CLIENT_KNOBS->BROADCAST_BATCH_SIZE); debugID = Optional(); requests = std::vector< Promise >(); addActor.send(batch); @@ -3069,11 +3225,28 @@ ACTOR Future extractReadVersion(DatabaseContext* cx, uint32_t flags, Re cx->GRVLatencies.addSample(latency); if (trLogInfo) trLogInfo->addLog(FdbClientLogEvents::EventGetVersion_V2(startTime, latency, flags & GetReadVersionRequest::FLAG_PRIORITY_MASK)); + if (rep.version == 1 && rep.locked) { + throw proxy_memory_limit_exceeded(); + } if(rep.locked && !lockAware) throw database_locked(); + ++cx->transactionReadVersionsCompleted; + if((flags & GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE) == GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE) { + ++cx->transactionImmediateReadVersionsCompleted; + } + else if((flags & GetReadVersionRequest::PRIORITY_DEFAULT) == GetReadVersionRequest::PRIORITY_DEFAULT) { + ++cx->transactionDefaultReadVersionsCompleted; + } + else if((flags & GetReadVersionRequest::PRIORITY_BATCH) == GetReadVersionRequest::PRIORITY_BATCH) { + ++cx->transactionBatchReadVersionsCompleted; + } + else { + ASSERT(false); + } + if(rep.version > cx->metadataVersionCache[cx->mvCacheInsertLocation].first) { - cx->mvCacheInsertLocation = (cx->mvCacheInsertLocation + 1)%cx->metadataVersionCache.size(); + cx->mvCacheInsertLocation = (cx->mvCacheInsertLocation + 1) % cx->metadataVersionCache.size(); cx->metadataVersionCache[cx->mvCacheInsertLocation] = std::make_pair(rep.version, rep.metadataVersion); } @@ -3085,6 +3258,18 @@ Future Transaction::getReadVersion(uint32_t flags) { if (!readVersion.isValid()) { ++cx->transactionReadVersions; flags |= options.getReadVersionFlags; + if((flags & GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE) == GetReadVersionRequest::PRIORITY_SYSTEM_IMMEDIATE) { + ++cx->transactionImmediateReadVersions; + } + else if((flags & GetReadVersionRequest::PRIORITY_DEFAULT) == GetReadVersionRequest::PRIORITY_DEFAULT) { + ++cx->transactionDefaultReadVersions; + } + else if((flags & GetReadVersionRequest::PRIORITY_BATCH) == GetReadVersionRequest::PRIORITY_BATCH) { + ++cx->transactionBatchReadVersions; + } + else { + ASSERT(false); + } auto& batcher = cx->versionBatcher[ flags ]; if (!batcher.actor.isValid()) { @@ -3099,6 +3284,14 @@ Future Transaction::getReadVersion(uint32_t flags) { return readVersion; } +Optional Transaction::getCachedReadVersion() { + if (readVersion.isValid() && readVersion.isReady() && !readVersion.isError()) { + return readVersion.get(); + } else { + return Optional(); + } +} + Future> Transaction::getVersionstamp() { if(committing.isValid()) { return transaction_invalid_version(); @@ -3120,16 +3313,19 @@ Future Transaction::onError( Error const& e ) { e.code() == error_code_commit_unknown_result || e.code() == error_code_database_locked || e.code() == error_code_proxy_memory_limit_exceeded || - e.code() == error_code_process_behind) + e.code() == error_code_process_behind || + e.code() == error_code_batch_transaction_throttled) { if(e.code() == error_code_not_committed) ++cx->transactionsNotCommitted; - if(e.code() == error_code_commit_unknown_result) + else if (e.code() == error_code_commit_unknown_result) ++cx->transactionsMaybeCommitted; - if (e.code() == error_code_proxy_memory_limit_exceeded) + else if (e.code() == error_code_proxy_memory_limit_exceeded) ++cx->transactionsResourceConstrained; - if (e.code() == error_code_process_behind) + else if (e.code() == error_code_process_behind) ++cx->transactionsProcessBehind; + else if (e.code() == error_code_batch_transaction_throttled) + ++cx->transactionsThrottled; double backoff = getBackoff(e.code()); reset(); @@ -3153,6 +3349,46 @@ Future Transaction::onError( Error const& e ) { return e; } +ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRangeRef keys); + +ACTOR Future doGetStorageMetrics(Database cx, KeyRangeRef keys, Reference locationInfo) { + loop { + try { + WaitMetricsRequest req(keys, StorageMetrics(), StorageMetrics()); + req.min.bytes = 0; + req.max.bytes = -1; + StorageMetrics m = wait( + loadBalance(locationInfo, &StorageServerInterface::waitMetrics, req, TaskPriority::DataDistribution)); + return m; + } catch (Error& e) { + if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { + TraceEvent(SevError, "WaitStorageMetricsError").error(e); + throw; + } + wait(delay(CLIENT_KNOBS->WRONG_SHARD_SERVER_DELAY, TaskPriority::DataDistribution)); + cx->invalidateCache(keys); + StorageMetrics m = wait(getStorageMetricsLargeKeyRange(cx, keys)); + return m; + } + } +} + +ACTOR Future getStorageMetricsLargeKeyRange(Database cx, KeyRangeRef keys) { + + vector>> locations = wait(getKeyRangeLocations( + cx, keys, std::numeric_limits::max(), false, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution))); + state int nLocs = locations.size(); + state vector> fx(nLocs); + state StorageMetrics total; + for (int i = 0; i < nLocs; i++) { + fx[i] = doGetStorageMetrics(cx, locations[i].first, locations[i].second); + } + wait(waitForAll(fx)); + for (int i = 0; i < nLocs; i++) { + total += fx[i].get(); + } + return total; +} ACTOR Future trackBoundedStorageMetrics( KeyRange keys, @@ -3174,14 +3410,11 @@ ACTOR Future trackBoundedStorageMetrics( } } -ACTOR Future< StorageMetrics > waitStorageMetricsMultipleLocations( - vector< pair> > locations, - StorageMetrics min, - StorageMetrics max, - StorageMetrics permittedError) -{ +ACTOR Future waitStorageMetricsMultipleLocations( + vector>> locations, StorageMetrics min, StorageMetrics max, + StorageMetrics permittedError) { state int nLocs = locations.size(); - state vector> fx( nLocs ); + state vector> fx(nLocs); state StorageMetrics total; state PromiseStream deltas; state vector> wx( fx.size() ); @@ -3189,17 +3422,17 @@ ACTOR Future< StorageMetrics > waitStorageMetricsMultipleLocations( state StorageMetrics maxPlus = max + halfErrorPerMachine * (nLocs-1); state StorageMetrics minMinus = min - halfErrorPerMachine * (nLocs-1); - for(int i=0; i waitStorageMetricsMultipleLocations( } } -ACTOR Future< StorageMetrics > waitStorageMetrics( +ACTOR Future< StorageMetrics > extractMetrics( Future, int>> fMetrics ) { + std::pair, int> x = wait(fMetrics); + return x.first.get(); +} + +ACTOR Future< std::pair, int> > waitStorageMetrics( Database cx, KeyRange keys, StorageMetrics min, StorageMetrics max, StorageMetrics permittedError, - int shardLimit ) + int shardLimit, + int expectedShardCount ) { loop { vector< pair> > locations = wait( getKeyRangeLocations( cx, keys, shardLimit, false, &StorageServerInterface::waitMetrics, TransactionInfo(TaskPriority::DataDistribution) ) ); + if(expectedShardCount >= 0 && locations.size() != expectedShardCount) { + return std::make_pair(Optional(), locations.size()); + } //SOMEDAY: Right now, if there are too many shards we delay and check again later. There may be a better solution to this. if(locations.size() < shardLimit) { try { Future fx; if (locations.size() > 1) { - fx = waitStorageMetricsMultipleLocations( locations, min, max, permittedError ); + fx = waitStorageMetricsMultipleLocations(locations, min, max, permittedError); } else { WaitMetricsRequest req( keys, min, max ); fx = loadBalance( locations[0].second, &StorageServerInterface::waitMetrics, req, TaskPriority::DataDistribution ); } StorageMetrics x = wait(fx); - return x; + return std::make_pair(x,-1); } catch (Error& e) { if (e.code() != error_code_wrong_shard_server && e.code() != error_code_all_alternatives_failed) { TraceEvent(SevError, "WaitStorageMetricsError").error(e); @@ -3258,20 +3500,25 @@ ACTOR Future< StorageMetrics > waitStorageMetrics( } } -Future< StorageMetrics > Transaction::waitStorageMetrics( +Future< std::pair, int> > Transaction::waitStorageMetrics( KeyRange const& keys, StorageMetrics const& min, StorageMetrics const& max, StorageMetrics const& permittedError, - int shardLimit ) + int shardLimit, + int expectedShardCount ) { - return ::waitStorageMetrics( cx, keys, min, max, permittedError, shardLimit ); + return ::waitStorageMetrics( cx, keys, min, max, permittedError, shardLimit, expectedShardCount ); } Future< StorageMetrics > Transaction::getStorageMetrics( KeyRange const& keys, int shardLimit ) { - StorageMetrics m; - m.bytes = -1; - return ::waitStorageMetrics( cx, keys, StorageMetrics(), m, StorageMetrics(), shardLimit ); + if (shardLimit > 0) { + StorageMetrics m; + m.bytes = -1; + return extractMetrics(::waitStorageMetrics(cx, keys, StorageMetrics(), m, StorageMetrics(), shardLimit, -1)); + } else { + return ::getStorageMetricsLargeKeyRange(cx, keys); + } } ACTOR Future< Standalone> > splitStorageMetrics( Database cx, KeyRange keys, StorageMetrics limit, StorageMetrics estimated ) diff --git a/fdbclient/NativeAPI.actor.h b/fdbclient/NativeAPI.actor.h index eadca69fe0..5c9f76d9b2 100644 --- a/fdbclient/NativeAPI.actor.h +++ b/fdbclient/NativeAPI.actor.h @@ -25,7 +25,6 @@ #elif !defined(FDBCLIENT_NATIVEAPI_ACTOR_H) #define FDBCLIENT_NATIVEAPI_ACTOR_H - #include "flow/flow.h" #include "flow/TDMetric.actor.h" #include "fdbclient/FDBTypes.h" @@ -34,6 +33,7 @@ #include "fdbclient/CoordinationInterface.h" #include "fdbclient/ClusterInterface.h" #include "fdbclient/ClientLogEvents.h" +#include "fdbclient/KeyRangeMap.h" #include "flow/actorcompiler.h" // has to be last include // CLIENT_BUGGIFY should be used to randomly introduce failures at run time (like BUGGIFY but for client side testing) @@ -55,18 +55,16 @@ struct NetworkOptions { std::string clusterFile; Optional traceDirectory; uint64_t traceRollSize; - uint64_t traceMaxLogsSize; + uint64_t traceMaxLogsSize; std::string traceLogGroup; std::string traceFormat; + std::string traceClockSource; + std::string traceFileIdentifier; Optional logClientInfo; - Standalone> supportedVersions; - bool slowTaskProfilingEnabled; + Reference>>> supportedVersions; + bool runLoopProfilingEnabled; - // The default values, TRACE_DEFAULT_ROLL_SIZE and TRACE_DEFAULT_MAX_LOGS_SIZE are located in Trace.h. - NetworkOptions() - : localAddress(""), clusterFile(""), traceDirectory(Optional()), - traceRollSize(TRACE_DEFAULT_ROLL_SIZE), traceMaxLogsSize(TRACE_DEFAULT_MAX_LOGS_SIZE), traceLogGroup("default"), - traceFormat("xml"), slowTaskProfilingEnabled(false) {} + NetworkOptions(); }; class Database { @@ -132,6 +130,7 @@ struct TransactionOptions { bool readOnly : 1; bool firstInBatch : 1; bool includePort : 1; + bool reportConflictingKeys : 1; TransactionOptions(Database const& cx); TransactionOptions(); @@ -139,10 +138,15 @@ struct TransactionOptions { void reset(Database const& cx); }; +class ReadYourWritesTransaction; // workaround cyclic dependency struct TransactionInfo { Optional debugID; TaskPriority taskID; bool useProvisionalProxies; + // Used to save conflicting keys if FDBTransactionOptions::REPORT_CONFLICTING_KEYS is enabled + // prefix/ : '1' - any keys equal or larger than this key are (probably) conflicting keys + // prefix/ : '0' - any keys equal or larger than this key are (definitely) not conflicting keys + std::shared_ptr> conflictingKeys; explicit TransactionInfo( TaskPriority taskID ) : taskID(taskID), useProvisionalProxies(false) {} }; @@ -211,6 +215,8 @@ public: void setVersion( Version v ); Future getReadVersion() { return getReadVersion(0); } + Future getRawReadVersion(); + Optional getCachedReadVersion(); [[nodiscard]] Future> get(const Key& key, bool snapshot = false); [[nodiscard]] Future watch(Reference watch); @@ -241,7 +247,8 @@ public: Future< Void > warmRange( Database cx, KeyRange keys ); - Future< StorageMetrics > waitStorageMetrics( KeyRange const& keys, StorageMetrics const& min, StorageMetrics const& max, StorageMetrics const& permittedError, int shardLimit ); + Future< std::pair, int> > waitStorageMetrics( KeyRange const& keys, StorageMetrics const& min, StorageMetrics const& max, StorageMetrics const& permittedError, int shardLimit, int expectedShardCount ); + // Pass a negative value for `shardLimit` to indicate no limit on the shard number. Future< StorageMetrics > getStorageMetrics( KeyRange const& keys, int shardLimit ); Future< Standalone> > splitStorageMetrics( KeyRange const& keys, StorageMetrics const& limit, StorageMetrics const& estimated ); diff --git a/fdbclient/RYWIterator.cpp b/fdbclient/RYWIterator.cpp index 3f8decfaab..7e9960b40f 100644 --- a/fdbclient/RYWIterator.cpp +++ b/fdbclient/RYWIterator.cpp @@ -334,31 +334,31 @@ ACTOR Standalone getRange( Transaction* tr, KeySelector begin, K -static void printWriteMap(WriteMap *p) { - WriteMap::iterator it(p); - for (it.skip(allKeys.begin); it.beginKey() < allKeys.end; ++it) { - if (it.is_cleared_range()) { - printf("CLEARED "); - } - if (it.is_conflict_range()) { - printf("CONFLICT "); - } - if (it.is_operation()) { - printf("OPERATION "); - printf(it.is_independent() ? "INDEPENDENT " : "DEPENDENT "); - } - if (it.is_unmodified_range()) { - printf("UNMODIFIED "); - } - if (it.is_unreadable()) { - printf("UNREADABLE "); - } - printf(": \"%s\" -> \"%s\"\n", - printable(it.beginKey().toStandaloneStringRef()).c_str(), - printable(it.endKey().toStandaloneStringRef()).c_str()); - } - printf("\n"); -} +//static void printWriteMap(WriteMap *p) { +// WriteMap::iterator it(p); +// for (it.skip(allKeys.begin); it.beginKey() < allKeys.end; ++it) { +// if (it.is_cleared_range()) { +// printf("CLEARED "); +// } +// if (it.is_conflict_range()) { +// printf("CONFLICT "); +// } +// if (it.is_operation()) { +// printf("OPERATION "); +// printf(it.is_independent() ? "INDEPENDENT " : "DEPENDENT "); +// } +// if (it.is_unmodified_range()) { +// printf("UNMODIFIED "); +// } +// if (it.is_unreadable()) { +// printf("UNREADABLE "); +// } +// printf(": \"%s\" -> \"%s\"\n", +// printable(it.beginKey().toStandaloneStringRef()).c_str(), +// printable(it.endKey().toStandaloneStringRef()).c_str()); +// } +// printf("\n"); +//} static int getWriteMapCount(WriteMap *p) { // printWriteMap(p); diff --git a/fdbclient/ReadYourWrites.actor.cpp b/fdbclient/ReadYourWrites.actor.cpp index 1d9efbef2d..4ff52e426f 100644 --- a/fdbclient/ReadYourWrites.actor.cpp +++ b/fdbclient/ReadYourWrites.actor.cpp @@ -21,6 +21,7 @@ #include "fdbclient/ReadYourWrites.h" #include "fdbclient/Atomic.h" #include "fdbclient/DatabaseContext.h" +#include "fdbclient/SpecialKeySpace.actor.h" #include "fdbclient/StatusClient.h" #include "fdbclient/MonitorLeader.h" #include "flow/Util.h" @@ -1018,7 +1019,12 @@ public: return Void(); } - watchFuture = ryw->tr.watch(watch); // throws if there are too many outstanding watches + try { + watchFuture = ryw->tr.watch(watch); // throws if there are too many outstanding watches + } catch( Error &e ) { + done.send(Void()); + throw; + } done.send(Void()); wait(watchFuture); @@ -1156,7 +1162,7 @@ Future ReadYourWritesTransaction::getReadVersion() { Optional getValueFromJSON(StatusObject statusObj) { try { - Value output = StringRef(json_spirit::write_string(json_spirit::mValue(statusObj), json_spirit::Output_options::raw_utf8).c_str()); + Value output = StringRef(json_spirit::write_string(json_spirit::mValue(statusObj), json_spirit::Output_options::none)); return output; } catch (std::exception& e){ @@ -1165,8 +1171,8 @@ Optional getValueFromJSON(StatusObject statusObj) { } } -ACTOR Future> getJSON(Reference clusterFile) { - StatusObject statusObj = wait(StatusClient::statusFetcher(clusterFile)); +ACTOR Future> getJSON(Database db) { + StatusObject statusObj = wait(StatusClient::statusFetcher(db)); return getValueFromJSON(statusObj); } @@ -1194,7 +1200,7 @@ Future< Optional > ReadYourWritesTransaction::get( const Key& key, bool s if (key == LiteralStringRef("\xff\xff/status/json")){ if (tr.getDatabase().getPtr() && tr.getDatabase()->getConnectionFile()) { - return getJSON(tr.getDatabase()->getConnectionFile()); + return getJSON(tr.getDatabase()); } else { return Optional(); @@ -1228,6 +1234,10 @@ Future< Optional > ReadYourWritesTransaction::get( const Key& key, bool s return Optional(); } + // special key space are only allowed to query if both begin and end are in \xff\xff, \xff\xff\xff + if (specialKeys.contains(key)) + return getDatabase()->specialKeySpace->get(Reference::addRef(this), key); + if(checkUsedDuringCommit()) { return used_during_commit(); } @@ -1278,7 +1288,12 @@ Future< Standalone > ReadYourWritesTransaction::getRange( return Standalone(); } } - + + // special key space are only allowed to query if both begin and end are in \xff\xff, \xff\xff\xff + if (specialKeys.contains(begin.getKey()) && specialKeys.contains(end.getKey())) + return getDatabase()->specialKeySpace->getRange(Reference::addRef(this), begin, end, + limits, reverse); + if(checkUsedDuringCommit()) { return used_during_commit(); } @@ -1343,6 +1358,16 @@ Future< Standalone >> ReadYourWritesTransaction::getAddre return result; } +Future ReadYourWritesTransaction::getEstimatedRangeSizeBytes(const KeyRangeRef& keys) { + if(checkUsedDuringCommit()) { + throw used_during_commit(); + } + if( resetPromise.isSet() ) + return resetPromise.getFuture().getError(); + + return map(waitOrError(tr.getStorageMetrics(keys, -1), resetPromise.getFuture()), [](const StorageMetrics& m) { return m.bytes; }); +} + void ReadYourWritesTransaction::addReadConflictRange( KeyRangeRef const& keys ) { if(checkUsedDuringCommit()) { throw used_during_commit(); @@ -1565,11 +1590,16 @@ void ReadYourWritesTransaction::atomicOp( const KeyRef& key, const ValueRef& ope } if(operationType == MutationRef::SetVersionstampedKey) { - KeyRangeRef range = getVersionstampKeyRange(arena, k, getMaxReadKey()); // this does validation of the key and needs to be performed before the readYourWritesDisabled path + // this does validation of the key and needs to be performed before the readYourWritesDisabled path + KeyRangeRef range = getVersionstampKeyRange(arena, k, tr.getCachedReadVersion().orDefault(0), getMaxReadKey()); if(!options.readYourWritesDisabled) { writeRangeToNativeTransaction(range); writes.addUnmodifiedAndUnreadableRange(range); } + // k is the unversionstamped key provided by the user. If we've filled in a minimum bound + // for the versionstamp, we need to make sure that's reflected when we insert it into the + // WriteMap below. + transformVersionstampKey( k, tr.getCachedReadVersion().orDefault(0), 0 ); } if(operationType == MutationRef::SetVersionstampedValue) { diff --git a/fdbclient/ReadYourWrites.h b/fdbclient/ReadYourWrites.h index 363c9f4639..ea0b7dffa3 100644 --- a/fdbclient/ReadYourWrites.h +++ b/fdbclient/ReadYourWrites.h @@ -69,6 +69,7 @@ public: void setVersion( Version v ) { tr.setVersion(v); } Future getReadVersion(); + Optional getCachedReadVersion() { return tr.getCachedReadVersion(); } Future< Optional > get( const Key& key, bool snapshot = false ); Future< Key > getKey( const KeySelector& key, bool snapshot = false ); Future< Standalone > getRange( const KeySelector& begin, const KeySelector& end, int limit, bool snapshot = false, bool reverse = false ); @@ -83,6 +84,7 @@ public: } [[nodiscard]] Future>> getAddressesForKey(const Key& key); + Future getEstimatedRangeSizeBytes( const KeyRangeRef& keys ); void addReadConflictRange( KeyRangeRef const& keys ); void makeSelfConflicting() { tr.makeSelfConflicting(); } @@ -129,6 +131,10 @@ public: Database getDatabase() const { return tr.getDatabase(); } + + const TransactionInfo& getTransactionInfo() const { + return tr.info; + } private: friend class RYWImpl; diff --git a/fdbclient/RestoreWorkerInterface.actor.h b/fdbclient/RestoreWorkerInterface.actor.h index 5871d1483e..a1fc942b28 100644 --- a/fdbclient/RestoreWorkerInterface.actor.h +++ b/fdbclient/RestoreWorkerInterface.actor.h @@ -29,6 +29,7 @@ #define FDBCLIENT_RESTORE_WORKER_INTERFACE_ACTOR_H #include +#include #include "flow/Stats.h" #include "flow/flow.h" #include "fdbrpc/fdbrpc.h" @@ -51,6 +52,7 @@ struct RestoreSendMutationsToAppliersRequest; struct RestoreSendVersionedMutationsRequest; struct RestoreSysInfo; struct RestoreApplierInterface; +struct RestoreFinishRequest; // RestoreSysInfo includes information each (type of) restore roles should know. // At this moment, it only include appliers. We keep the name for future extension. @@ -128,8 +130,9 @@ struct RestoreLoaderInterface : RestoreRoleInterface { RequestStream loadFile; RequestStream sendMutations; RequestStream initVersionBatch; + RequestStream finishVersionBatch; RequestStream collectRestoreRoleInterfaces; - RequestStream finishRestore; + RequestStream finishRestore; bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); } bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); } @@ -147,6 +150,7 @@ struct RestoreLoaderInterface : RestoreRoleInterface { loadFile.getEndpoint(TaskPriority::LoadBalancedEndpoint); sendMutations.getEndpoint(TaskPriority::LoadBalancedEndpoint); initVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint); + finishVersionBatch.getEndpoint(TaskPriority::LoadBalancedEndpoint); collectRestoreRoleInterfaces.getEndpoint(TaskPriority::LoadBalancedEndpoint); finishRestore.getEndpoint(TaskPriority::LoadBalancedEndpoint); } @@ -154,7 +158,7 @@ struct RestoreLoaderInterface : RestoreRoleInterface { template void serialize(Ar& ar) { serializer(ar, *(RestoreRoleInterface*)this, heartbeat, updateRestoreSysInfo, loadFile, sendMutations, - initVersionBatch, collectRestoreRoleInterfaces, finishRestore); + initVersionBatch, finishVersionBatch, collectRestoreRoleInterfaces, finishRestore); } }; @@ -166,7 +170,7 @@ struct RestoreApplierInterface : RestoreRoleInterface { RequestStream applyToDB; RequestStream initVersionBatch; RequestStream collectRestoreRoleInterfaces; - RequestStream finishRestore; + RequestStream finishRestore; bool operator==(RestoreWorkerInterface const& r) const { return id() == r.id(); } bool operator!=(RestoreWorkerInterface const& r) const { return id() != r.id(); } @@ -205,19 +209,23 @@ struct RestoreAsset { KeyRange range; // Only use mutations in range int fileIndex; + // Partition ID for mutation log files, which is also encoded in the filename of mutation logs. + int partitionId = -1; std::string filename; int64_t offset; int64_t len; + UID uid; + RestoreAsset() = default; bool operator==(const RestoreAsset& r) const { - return fileIndex == r.fileIndex && filename == r.filename && offset == r.offset && len == r.len && - beginVersion == r.beginVersion && endVersion == r.endVersion && range == r.range; + return beginVersion == r.beginVersion && endVersion == r.endVersion && range == r.range && + fileIndex == r.fileIndex && partitionId == r.partitionId && filename == r.filename && + offset == r.offset && len == r.len; } bool operator!=(const RestoreAsset& r) const { - return fileIndex != r.fileIndex || filename != r.filename || offset != r.offset || len != r.len || - beginVersion != r.beginVersion || endVersion != r.endVersion || range != r.range; + return !(*this == r); } bool operator<(const RestoreAsset& r) const { return std::make_tuple(fileIndex, filename, offset, len, beginVersion, endVersion, range.begin, range.end) < @@ -227,30 +235,39 @@ struct RestoreAsset { template void serialize(Ar& ar) { - serializer(ar, beginVersion, endVersion, range, filename, fileIndex, offset, len); + serializer(ar, beginVersion, endVersion, range, filename, fileIndex, partitionId, offset, len, uid); } std::string toString() { std::stringstream ss; - ss << "begin:" << beginVersion << " end:" << endVersion << " range:" << range.toString() - << " filename:" << filename << " fileIndex:" << fileIndex << " offset:" << offset << " len:" << len; + ss << "UID:" << uid.toString() << " begin:" << beginVersion << " end:" << endVersion + << " range:" << range.toString() << " filename:" << filename << " fileIndex:" << fileIndex + << " partitionId:" << partitionId << " offset:" << offset << " len:" << len; return ss.str(); } + // RestoreAsset and VersionBatch both use endVersion as exclusive in version range bool isInVersionRange(Version commitVersion) const { return commitVersion >= beginVersion && commitVersion < endVersion; } + + // Is mutation's begin and end keys are in RestoreAsset's range + bool isInKeyRange(MutationRef mutation) const { + if (isRangeMutation(mutation)) { + // Range mutation's right side is exclusive + return mutation.param1 >= range.begin && mutation.param2 <= range.end; + } else { + return mutation.param1 >= range.begin && mutation.param1 < range.end; + } + } }; -// TODO: It is probably better to specify the (beginVersion, endVersion] for each loadingParam. -// beginVersion (endVersion) is the version the applier is before (after) it receives the request. struct LoadingParam { constexpr static FileIdentifier file_identifier = 17023837; bool isRangeFile; Key url; - Version prevVersion; - Version endVersion; // range file's mutations are all at the endVersion + Optional rangeVersion; // range file's version int64_t blockSize; RestoreAsset asset; @@ -264,15 +281,20 @@ struct LoadingParam { return (isRangeFile < r.isRangeFile) || (isRangeFile == r.isRangeFile && asset < r.asset); } + bool isPartitionedLog() const { + return !isRangeFile && asset.partitionId >= 0; + } + template void serialize(Ar& ar) { - serializer(ar, isRangeFile, url, prevVersion, endVersion, blockSize, asset); + serializer(ar, isRangeFile, url, rangeVersion, blockSize, asset); } std::string toString() { std::stringstream str; - str << "isRangeFile:" << isRangeFile << " url:" << url.toString() << " prevVersion:" << prevVersion - << " endVersion:" << endVersion << " blockSize:" << blockSize << " RestoreAsset:" << asset.toString(); + str << "isRangeFile:" << isRangeFile << " url:" << url.toString() + << " rangeVersion:" << (rangeVersion.present() ? rangeVersion.get() : -1) << " blockSize:" << blockSize + << " RestoreAsset:" << asset.toString(); return str.str(); } }; @@ -335,24 +357,29 @@ struct RestoreRecruitRoleRequest : TimedRequest { std::string toString() { return printable(); } }; +// Static info. across version batches struct RestoreSysInfoRequest : TimedRequest { constexpr static FileIdentifier file_identifier = 75960741; RestoreSysInfo sysInfo; + Standalone>> rangeVersions; ReplyPromise reply; RestoreSysInfoRequest() = default; - explicit RestoreSysInfoRequest(RestoreSysInfo sysInfo) : sysInfo(sysInfo) {} + explicit RestoreSysInfoRequest(RestoreSysInfo sysInfo, + Standalone>> rangeVersions) + : sysInfo(sysInfo), rangeVersions(rangeVersions) {} template void serialize(Ar& ar) { - serializer(ar, sysInfo, reply); + serializer(ar, sysInfo, rangeVersions, reply); } std::string toString() { std::stringstream ss; - ss << "RestoreSysInfoRequest"; + ss << "RestoreSysInfoRequest " + << "rangeVersions.size:" << rangeVersions.size(); return ss.str(); } }; @@ -362,18 +389,21 @@ struct RestoreLoadFileReply : TimedRequest { LoadingParam param; MutationsVec samples; // sampled mutations + bool isDuplicated; // true if loader thinks the request is a duplicated one RestoreLoadFileReply() = default; - explicit RestoreLoadFileReply(LoadingParam param, MutationsVec samples) : param(param), samples(samples) {} + explicit RestoreLoadFileReply(LoadingParam param, MutationsVec samples, bool isDuplicated) + : param(param), samples(samples), isDuplicated(isDuplicated) {} template void serialize(Ar& ar) { - serializer(ar, param, samples); + serializer(ar, param, samples, isDuplicated); } std::string toString() { std::stringstream ss; - ss << "LoadingParam:" << param.toString() << " samples.size:" << samples.size(); + ss << "LoadingParam:" << param.toString() << " samples.size:" << samples.size() + << " isDuplicated:" << isDuplicated; return ss.str(); } }; @@ -382,21 +412,22 @@ struct RestoreLoadFileReply : TimedRequest { struct RestoreLoadFileRequest : TimedRequest { constexpr static FileIdentifier file_identifier = 26557364; + int batchIndex; LoadingParam param; ReplyPromise reply; RestoreLoadFileRequest() = default; - explicit RestoreLoadFileRequest(LoadingParam& param) : param(param){}; + explicit RestoreLoadFileRequest(int batchIndex, LoadingParam& param) : batchIndex(batchIndex), param(param){}; template void serialize(Ar& ar) { - serializer(ar, param, reply); + serializer(ar, batchIndex, param, reply); } std::string toString() { std::stringstream ss; - ss << "RestoreLoadFileRequest param:" << param.toString(); + ss << "RestoreLoadFileRequest batchIndex:" << batchIndex << " param:" << param.toString(); return ss.str(); } }; @@ -404,24 +435,25 @@ struct RestoreLoadFileRequest : TimedRequest { struct RestoreSendMutationsToAppliersRequest : TimedRequest { constexpr static FileIdentifier file_identifier = 68827305; + int batchIndex; // version batch index std::map rangeToApplier; bool useRangeFile; // Send mutations parsed from range file? ReplyPromise reply; RestoreSendMutationsToAppliersRequest() = default; - explicit RestoreSendMutationsToAppliersRequest(std::map rangeToApplier, bool useRangeFile) - : rangeToApplier(rangeToApplier), useRangeFile(useRangeFile) {} + explicit RestoreSendMutationsToAppliersRequest(int batchIndex, std::map rangeToApplier, bool useRangeFile) + : batchIndex(batchIndex), rangeToApplier(rangeToApplier), useRangeFile(useRangeFile) {} template void serialize(Ar& ar) { - serializer(ar, rangeToApplier, useRangeFile, reply); + serializer(ar, batchIndex, rangeToApplier, useRangeFile, reply); } std::string toString() { std::stringstream ss; - ss << "RestoreSendMutationsToAppliersRequest keyToAppliers.size:" << rangeToApplier.size() - << " useRangeFile:" << useRangeFile; + ss << "RestoreSendMutationsToAppliersRequest batchIndex:" << batchIndex + << " keyToAppliers.size:" << rangeToApplier.size() << " useRangeFile:" << useRangeFile; return ss.str(); } }; @@ -429,50 +461,77 @@ struct RestoreSendMutationsToAppliersRequest : TimedRequest { struct RestoreSendVersionedMutationsRequest : TimedRequest { constexpr static FileIdentifier file_identifier = 69764565; + int batchIndex; // version batch index RestoreAsset asset; // Unique identifier for the current restore asset - Version prevVersion, version; // version is the commitVersion of the mutation vector. + Version msgIndex; // Monitonically increasing index of mutation messages bool isRangeFile; - MutationsVec mutations; // All mutations at the same version parsed by one loader + MutationsVec mutations; // Mutations that may be at different versions parsed by one loader + LogMessageVersionVec mVersions; // (version, subversion) of each mutation in mutations field ReplyPromise reply; RestoreSendVersionedMutationsRequest() = default; - explicit RestoreSendVersionedMutationsRequest(const RestoreAsset& asset, Version prevVersion, Version version, - bool isRangeFile, MutationsVec mutations) - : asset(asset), prevVersion(prevVersion), version(version), isRangeFile(isRangeFile), mutations(mutations) {} + explicit RestoreSendVersionedMutationsRequest(int batchIndex, const RestoreAsset& asset, Version msgIndex, + bool isRangeFile, MutationsVec mutations, + LogMessageVersionVec mVersions) + : batchIndex(batchIndex), asset(asset), msgIndex(msgIndex), isRangeFile(isRangeFile), mutations(mutations), + mVersions(mVersions) {} std::string toString() { std::stringstream ss; - ss << "RestoreAsset:" << asset.toString() << " prevVersion:" << prevVersion << " version:" << version - << " isRangeFile:" << isRangeFile << " mutations.size:" << mutations.size(); + ss << "VersionBatchIndex:" << batchIndex << "RestoreAsset:" << asset.toString() << " msgIndex:" << msgIndex + << " isRangeFile:" << isRangeFile << " mutations.size:" << mutations.size() + << " mVersions.size:" << mVersions.size(); return ss.str(); } template void serialize(Ar& ar) { - serializer(ar, asset, prevVersion, version, isRangeFile, mutations, reply); + serializer(ar, batchIndex, asset, msgIndex, isRangeFile, mutations, mVersions, reply); } }; struct RestoreVersionBatchRequest : TimedRequest { - constexpr static FileIdentifier file_identifier = 13018413; + constexpr static FileIdentifier file_identifier = 97223537; - int batchID; + int batchIndex; ReplyPromise reply; RestoreVersionBatchRequest() = default; - explicit RestoreVersionBatchRequest(int batchID) : batchID(batchID) {} + explicit RestoreVersionBatchRequest(int batchIndex) : batchIndex(batchIndex) {} template void serialize(Ar& ar) { - serializer(ar, batchID, reply); + serializer(ar, batchIndex, reply); } std::string toString() { std::stringstream ss; - ss << "RestoreVersionBatchRequest BatchID:" << batchID; + ss << "RestoreVersionBatchRequest batchIndex:" << batchIndex; + return ss.str(); + } +}; + +struct RestoreFinishRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 13018413; + + bool terminate; // role exits if terminate = true + + ReplyPromise reply; + + RestoreFinishRequest() = default; + explicit RestoreFinishRequest(bool terminate) : terminate(terminate) {} + + template + void serialize(Ar& ar) { + serializer(ar, terminate, reply); + } + + std::string toString() { + std::stringstream ss; + ss << "RestoreFinishRequest terminate:" << terminate; return ss.str(); } }; @@ -528,7 +587,7 @@ std::string getRoleStr(RestoreRole role); ////--- Interface functions ACTOR Future _restoreWorker(Database cx, LocalityData locality); -ACTOR Future restoreWorker(Reference ccf, LocalityData locality); +ACTOR Future restoreWorker(Reference ccf, LocalityData locality, std::string coordFolder); #include "flow/unactorcompiler.h" #endif diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 53fd7641c6..1d7099196a 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -68,6 +68,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "kvstore_available_bytes":12341234, "kvstore_free_bytes":12341234, "kvstore_total_bytes":12341234, + "kvstore_total_size":12341234, + "kvstore_total_nodes":12341234, + "kvstore_inline_keys":12341234, "durable_bytes":{ "hz":0.0, "counter":0, @@ -159,6 +162,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "$enum":[ "file_open_error", "incorrect_cluster_file_contents", + "trace_log_file_write_error", + "trace_log_could_not_create_file", + "trace_log_writer_thread_unresponsive", "process_error", "io_error", "io_timeout", @@ -210,6 +216,9 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( }, "megabits_received":{ "hz":0.0 + }, + "tls_policy_failures":{ + "hz":0.0 } }, "run_loop_busy":0.2 @@ -389,14 +398,19 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "consistencycheck_suspendkey_fetch_timeout", "consistencycheck_disabled", "duplicate_mutation_streams", - "duplicate_mutation_fetch_timeout" + "duplicate_mutation_fetch_timeout", + "primary_dc_missing", + "fetch_primary_dc_timeout" ] }, "issues":[ { "name":{ "$enum":[ - "incorrect_cluster_file_contents" + "incorrect_cluster_file_contents", + "trace_log_file_write_error", + "trace_log_could_not_create_file", + "trace_log_writer_thread_unresponsive" ] }, "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally." @@ -405,7 +419,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "description":"abc" } ], -)statusSchema" R"statusSchema( +)statusSchema" + R"statusSchema( "recovery_state":{ "required_resolvers":1, "required_proxies":1, @@ -430,6 +445,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( }, "required_logs":3, "missing_logs":"7f8d623d0cb9966e", + "active_generations":1, "description":"Recovery complete." }, "workload":{ @@ -448,6 +464,16 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "hz":0.0, "counter":0, "roughness":0.0 + }, + "location_requests":{ + "hz":0.0, + "counter":0, + "roughness":0.0 + }, + "memory_errors":{ + "hz":0.0, + "counter":0, + "roughness":0.0 } }, "bytes":{ @@ -511,6 +537,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "data_distribution_disabled_for_ss_failures":true, "data_distribution_disabled_for_rebalance":true, "data_distribution_disabled":true, + "active_primary_dc":"pv", "configuration":{ "log_anti_quorum":0, "log_replicas":2, @@ -574,7 +601,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "ssd-redwood-experimental", "memory", "memory-1", - "memory-2" + "memory-2", + "memory-radixtree-beta" ]}, "coordinators_count":1, "excluded_servers":[ @@ -585,7 +613,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "auto_proxies":3, "auto_resolvers":1, "auto_logs":3, - "proxies":5 + "proxies":5, + "backup_worker_enabled":1 }, "data":{ "least_operating_space_bytes_log_server":0, @@ -599,6 +628,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "missing_data", "healing", "optimizing_team_collections", + "healthy_populating_region", "healthy_repartitioning", "healthy_removing_server", "healthy_rebalancing", @@ -633,6 +663,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "missing_data", "healing", "optimizing_team_collections", + "healthy_populating_region", "healthy_repartitioning", "healthy_removing_server", "healthy_rebalancing", diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp new file mode 100644 index 0000000000..762d97281c --- /dev/null +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -0,0 +1,377 @@ +/* + * SpecialKeySpace.actor.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbclient/SpecialKeySpace.actor.h" +#include "flow/UnitTest.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +// This function will normalize the given KeySelector to a standard KeySelector: +// orEqual == false && offset == 1 (Standard form) +// If the corresponding key is not in this special key range, it will move as far as possible to adjust the offset to 1 +// It does have overhead here since we query all keys twice in the worst case. +// However, moving the KeySelector while handling other parameters like limits makes the code much more complex and hard +// to maintain Separate each part to make the code easy to understand and more compact +ACTOR Future SpecialKeyRangeBaseImpl::normalizeKeySelectorActor(const SpecialKeyRangeBaseImpl* pkrImpl, + Reference ryw, + KeySelector* ks) { + ASSERT(!ks->orEqual); // should be removed before calling + ASSERT(ks->offset != 1); // never being called if KeySelector is already normalized + + state Key startKey(pkrImpl->range.begin); + state Key endKey(pkrImpl->range.end); + + if (ks->offset < 1) { + // less than the given key + if (pkrImpl->range.contains(ks->getKey())) endKey = keyAfter(ks->getKey()); + } else { + // greater than the given key + if (pkrImpl->range.contains(ks->getKey())) startKey = ks->getKey(); + } + + TraceEvent(SevDebug, "NormalizeKeySelector") + .detail("OriginalKey", ks->getKey()) + .detail("OriginalOffset", ks->offset) + .detail("SpecialKeyRangeStart", pkrImpl->range.begin) + .detail("SpecialKeyRangeEnd", pkrImpl->range.end); + + Standalone result = wait(pkrImpl->getRange(ryw, KeyRangeRef(startKey, endKey))); + if (result.size() == 0) { + TraceEvent("ZeroElementsIntheRange").detail("Start", startKey).detail("End", endKey); + return Void(); + } + // Note : KeySelector::setKey has byte limit according to the knobs, customize it if needed + if (ks->offset < 1) { + if (result.size() >= 1 - ks->offset) { + ks->setKey(KeyRef(ks->arena(), result[result.size() - (1 - ks->offset)].key)); + ks->offset = 1; + } else { + ks->setKey(KeyRef(ks->arena(), result[0].key)); + ks->offset += result.size(); + } + } else { + if (result.size() >= ks->offset) { + ks->setKey(KeyRef(ks->arena(), result[ks->offset - 1].key)); + ks->offset = 1; + } else { + ks->setKey(KeyRef(ks->arena(), keyAfter(result[result.size() - 1].key))); + ks->offset -= result.size(); + } + } + TraceEvent(SevDebug, "NormalizeKeySelector") + .detail("NormalizedKey", ks->getKey()) + .detail("NormalizedOffset", ks->offset) + .detail("SpecialKeyRangeStart", pkrImpl->range.begin) + .detail("SpecialKeyRangeEnd", pkrImpl->range.end); + return Void(); +} + +ACTOR Future> SpecialKeySpace::getRangeAggregationActor( + SpecialKeySpace* pks, Reference ryw, KeySelector begin, KeySelector end, + GetRangeLimits limits, bool reverse) { + // This function handles ranges which cover more than one keyrange and aggregates all results + // KeySelector, GetRangeLimits and reverse are all handled here + state Standalone result; + state RangeMap::Iterator iter; + state int actualBeginOffset; + state int actualEndOffset; + + // make sure offset == 1 + state RangeMap::Iterator beginIter = + pks->impls.rangeContaining(begin.getKey()); + while ((begin.offset < 1 && beginIter != pks->impls.ranges().begin()) || + (begin.offset > 1 && beginIter != pks->impls.ranges().end())) { + if (beginIter->value() != nullptr) + wait(beginIter->value()->normalizeKeySelectorActor(beginIter->value(), ryw, &begin)); + begin.offset < 1 ? --beginIter : ++beginIter; + } + + actualBeginOffset = begin.offset; + if (beginIter == pks->impls.ranges().begin()) + begin.setKey(pks->range.begin); + else if (beginIter == pks->impls.ranges().end()) + begin.setKey(pks->range.end); + + if (!begin.isFirstGreaterOrEqual()) { + // The Key Selector points to key outside the whole special key space + TraceEvent(SevInfo, "BeginKeySelectorPointsOutside") + .detail("TerminateKey", begin.getKey()) + .detail("TerminateOffset", begin.offset); + if (begin.offset < 1 && beginIter == pks->impls.ranges().begin()) + result.readToBegin = true; + else + result.readThroughEnd = true; + begin.offset = 1; + } + state RangeMap::Iterator endIter = + pks->impls.rangeContaining(end.getKey()); + while ((end.offset < 1 && endIter != pks->impls.ranges().begin()) || + (end.offset > 1 && endIter != pks->impls.ranges().end())) { + if (endIter->value() != nullptr) wait(endIter->value()->normalizeKeySelectorActor(endIter->value(), ryw, &end)); + end.offset < 1 ? --endIter : ++endIter; + } + + actualEndOffset = end.offset; + if (endIter == pks->impls.ranges().begin()) + end.setKey(pks->range.begin); + else if (endIter == pks->impls.ranges().end()) + end.setKey(pks->range.end); + + if (!end.isFirstGreaterOrEqual()) { + // The Key Selector points to key outside the whole special key space + TraceEvent(SevInfo, "EndKeySelectorPointsOutside") + .detail("TerminateKey", end.getKey()) + .detail("TerminateOffset", end.offset); + if (end.offset < 1 && endIter == pks->impls.ranges().begin()) + result.readToBegin = true; + else + result.readThroughEnd = true; + end.offset = 1; + } + // Handle all corner cases like what RYW does + // return if range inverted + if (actualBeginOffset >= actualEndOffset && begin.getKey() >= end.getKey()) { + TEST(true); + return RangeResultRef(false, false); + } + // If touches begin or end, return with readToBegin and readThroughEnd flags + if (beginIter == pks->impls.ranges().end() || endIter == pks->impls.ranges().begin()) { + TEST(true); + return result; + } + state RangeMap::Ranges ranges = + pks->impls.intersectingRanges(KeyRangeRef(begin.getKey(), end.getKey())); + // TODO : workaround to write this two together to make the code compact + // The issue here is boost::iterator_range<> doest not provide rbegin(), rend() + iter = reverse ? ranges.end() : ranges.begin(); + if (reverse) { + while (iter != ranges.begin()) { + --iter; + if (iter->value() == nullptr) continue; + KeyRangeRef kr = iter->range(); + KeyRef keyStart = kr.contains(begin.getKey()) ? begin.getKey() : kr.begin; + KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end; + Standalone pairs = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd))); + result.arena().dependsOn(pairs.arena()); + // limits handler + for (int i = pairs.size() - 1; i >= 0; --i) { + result.push_back(result.arena(), pairs[i]); + // Note : behavior here is even the last k-v pair makes total bytes larger than specified, it's still + // returned. In other words, the total size of the returned value (less the last entry) will be less + // than byteLimit + limits.decrement(pairs[i]); + if (limits.isReached()) { + result.more = true; + result.readToBegin = false; + return result; + }; + } + } + } else { + for (iter = ranges.begin(); iter != ranges.end(); ++iter) { + if (iter->value() == nullptr) continue; + KeyRangeRef kr = iter->range(); + KeyRef keyStart = kr.contains(begin.getKey()) ? begin.getKey() : kr.begin; + KeyRef keyEnd = kr.contains(end.getKey()) ? end.getKey() : kr.end; + Standalone pairs = wait(iter->value()->getRange(ryw, KeyRangeRef(keyStart, keyEnd))); + result.arena().dependsOn(pairs.arena()); + // limits handler + for (int i = 0; i < pairs.size(); ++i) { + result.push_back(result.arena(), pairs[i]); + // Note : behavior here is even the last k-v pair makes total bytes larger than specified, it's still + // returned. In other words, the total size of the returned value (less the last entry) will be less + // than byteLimit + limits.decrement(pairs[i]); + if (limits.isReached()) { + result.more = true; + result.readThroughEnd = false; + return result; + }; + } + } + } + return result; +} + +Future> SpecialKeySpace::getRange(Reference ryw, + KeySelector begin, KeySelector end, GetRangeLimits limits, + bool reverse) { + // validate limits here + if (!limits.isValid()) return range_limits_invalid(); + if (limits.isReached()) { + TEST(true); // read limit 0 + return Standalone(); + } + // make sure orEqual == false + begin.removeOrEqual(begin.arena()); + end.removeOrEqual(end.arena()); + + return getRangeAggregationActor(this, ryw, begin, end, limits, reverse); +} + +ACTOR Future> SpecialKeySpace::getActor(SpecialKeySpace* pks, Reference ryw, + KeyRef key) { + // use getRange to workaround this + Standalone result = + wait(pks->getRange(ryw, KeySelector(firstGreaterOrEqual(key)), KeySelector(firstGreaterOrEqual(keyAfter(key))), + GetRangeLimits(CLIENT_KNOBS->TOO_MANY), false)); + ASSERT(result.size() <= 1); + if (result.size()) { + return Optional(result[0].value); + } else { + return Optional(); + } +} + +Future> SpecialKeySpace::get(Reference ryw, const Key& key) { + return getActor(this, ryw, key); +} + +ConflictingKeysImpl::ConflictingKeysImpl(KeyRangeRef kr) : SpecialKeyRangeBaseImpl(kr) {} + +Future> ConflictingKeysImpl::getRange(Reference ryw, + KeyRangeRef kr) const { + Standalone result; + if (ryw->getTransactionInfo().conflictingKeys) { + auto krMapPtr = ryw->getTransactionInfo().conflictingKeys.get(); + auto beginIter = krMapPtr->rangeContaining(kr.begin); + if (beginIter->begin() != kr.begin) ++beginIter; + auto endIter = krMapPtr->rangeContaining(kr.end); + for (auto it = beginIter; it != endIter; ++it) { + // it->begin() is stored in the CoalescedKeyRangeMap in TransactionInfo + // it->value() is always constants in SystemData.cpp + // Thus, push_back() can be used + result.push_back(result.arena(), KeyValueRef(it->begin(), it->value())); + } + if (endIter->begin() != kr.end) + result.push_back(result.arena(), KeyValueRef(endIter->begin(), endIter->value())); + } + return result; +} + +class SpecialKeyRangeTestImpl : public SpecialKeyRangeBaseImpl { +public: + explicit SpecialKeyRangeTestImpl(KeyRangeRef kr, const std::string& prefix, int size) + : SpecialKeyRangeBaseImpl(kr), prefix(prefix), size(size) { + ASSERT(size > 0); + for (int i = 0; i < size; ++i) { + kvs.push_back_deep(kvs.arena(), + KeyValueRef(getKeyForIndex(i), deterministicRandom()->randomAlphaNumeric(16))); + } + } + + KeyValueRef getKeyValueForIndex(int idx) { return kvs[idx]; } + + Key getKeyForIndex(int idx) { return Key(prefix + format("%010d", idx)).withPrefix(range.begin); } + int getSize() { return size; } + Future> getRange(Reference ryw, + KeyRangeRef kr) const override { + int startIndex = 0, endIndex = size; + while (startIndex < size && kvs[startIndex].key < kr.begin) ++startIndex; + while (endIndex > startIndex && kvs[endIndex - 1].key >= kr.end) --endIndex; + if (startIndex == endIndex) + return Standalone(); + else + return Standalone(RangeResultRef(kvs.slice(startIndex, endIndex), false)); + } + +private: + Standalone> kvs; + std::string prefix; + int size; +}; + +TEST_CASE("/fdbclient/SpecialKeySpace/Unittest") { + SpecialKeySpace pks(normalKeys.begin, normalKeys.end); + SpecialKeyRangeTestImpl pkr1(KeyRangeRef(LiteralStringRef("/cat/"), LiteralStringRef("/cat/\xff")), "small", 10); + SpecialKeyRangeTestImpl pkr2(KeyRangeRef(LiteralStringRef("/dog/"), LiteralStringRef("/dog/\xff")), "medium", 100); + SpecialKeyRangeTestImpl pkr3(KeyRangeRef(LiteralStringRef("/pig/"), LiteralStringRef("/pig/\xff")), "large", 1000); + pks.registerKeyRange(pkr1.getKeyRange(), &pkr1); + pks.registerKeyRange(pkr2.getKeyRange(), &pkr2); + pks.registerKeyRange(pkr3.getKeyRange(), &pkr3); + auto nullRef = Reference(); + // get + { + auto resultFuture = pks.get(nullRef, LiteralStringRef("/cat/small0000000009")); + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue().get(); + ASSERT(result == pkr1.getKeyValueForIndex(9).value); + auto emptyFuture = pks.get(nullRef, LiteralStringRef("/cat/small0000000010")); + ASSERT(emptyFuture.isReady()); + auto emptyResult = emptyFuture.getValue(); + ASSERT(!emptyResult.present()); + } + // general getRange + { + KeySelector start = KeySelectorRef(LiteralStringRef("/elepant"), false, -9); + KeySelector end = KeySelectorRef(LiteralStringRef("/frog"), false, +11); + auto resultFuture = pks.getRange(nullRef, start, end, GetRangeLimits()); + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + ASSERT(result.size() == 20); + ASSERT(result[0].key == pkr2.getKeyForIndex(90)); + ASSERT(result[result.size() - 1].key == pkr3.getKeyForIndex(9)); + } + // KeySelector points outside + { + KeySelector start = KeySelectorRef(pkr3.getKeyForIndex(999), true, -1110); + KeySelector end = KeySelectorRef(pkr1.getKeyForIndex(0), false, +1112); + auto resultFuture = pks.getRange(nullRef, start, end, GetRangeLimits()); + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + ASSERT(result.size() == 1110); + ASSERT(result[0].key == pkr1.getKeyForIndex(0)); + ASSERT(result[result.size() - 1].key == pkr3.getKeyForIndex(999)); + } + // GetRangeLimits with row limit + { + KeySelector start = KeySelectorRef(pkr2.getKeyForIndex(0), true, 0); + KeySelector end = KeySelectorRef(pkr3.getKeyForIndex(0), false, 0); + auto resultFuture = pks.getRange(nullRef, start, end, GetRangeLimits(2)); + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + ASSERT(result.size() == 2); + ASSERT(result[0].key == pkr2.getKeyForIndex(0)); + ASSERT(result[1].key == pkr2.getKeyForIndex(1)); + } + // GetRangeLimits with byte limit + { + KeySelector start = KeySelectorRef(pkr2.getKeyForIndex(0), true, 0); + KeySelector end = KeySelectorRef(pkr3.getKeyForIndex(0), false, 0); + auto resultFuture = pks.getRange(nullRef, start, end, GetRangeLimits(10, 100)); + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + int bytes = 0; + for (int i = 0; i < result.size() - 1; ++i) bytes += 8 + pkr2.getKeyValueForIndex(i).expectedSize(); + ASSERT(bytes < 100); + ASSERT(bytes + 8 + pkr2.getKeyValueForIndex(result.size()).expectedSize() >= 100); + } + // reverse test with overlapping key range + { + KeySelector start = KeySelectorRef(pkr2.getKeyForIndex(0), true, 0); + KeySelector end = KeySelectorRef(pkr3.getKeyForIndex(999), true, +1); + auto resultFuture = pks.getRange(nullRef, start, end, GetRangeLimits(1100), true); + ASSERT(resultFuture.isReady()); + auto result = resultFuture.getValue(); + for (int i = 0; i < pkr3.getSize(); ++i) ASSERT(result[i] == pkr3.getKeyValueForIndex(pkr3.getSize() - 1 - i)); + for (int i = 0; i < pkr2.getSize(); ++i) + ASSERT(result[i + pkr3.getSize()] == pkr2.getKeyValueForIndex(pkr2.getSize() - 1 - i)); + } + return Void(); +} diff --git a/fdbclient/SpecialKeySpace.actor.h b/fdbclient/SpecialKeySpace.actor.h new file mode 100644 index 0000000000..c0ae6dbdf6 --- /dev/null +++ b/fdbclient/SpecialKeySpace.actor.h @@ -0,0 +1,99 @@ +/* + * SpecialKeySpace.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_SPECIALKEYSPACE_ACTOR_G_H) +#define FDBCLIENT_SPECIALKEYSPACE_ACTOR_G_H +#include "fdbclient/SpecialKeySpace.actor.g.h" +#elif !defined(FDBCLIENT_SPECIALKEYSPACE_ACTOR_H) +#define FDBCLIENT_SPECIALKEYSPACE_ACTOR_H + +#include "flow/flow.h" +#include "flow/Arena.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/KeyRangeMap.h" +#include "fdbclient/ReadYourWrites.h" +#include "flow/actorcompiler.h" // This must be the last #include. + +class SpecialKeyRangeBaseImpl { +public: + // Each derived class only needs to implement this simple version of getRange + virtual Future> getRange(Reference ryw, + KeyRangeRef kr) const = 0; + + explicit SpecialKeyRangeBaseImpl(KeyRangeRef kr) : range(kr) {} + KeyRangeRef getKeyRange() const { return range; } + ACTOR Future normalizeKeySelectorActor(const SpecialKeyRangeBaseImpl* pkrImpl, + Reference ryw, KeySelector* ks); + +protected: + KeyRange range; // underlying key range for this function +}; + +class SpecialKeySpace { +public: + Future> get(Reference ryw, const Key& key); + + Future> getRange(Reference ryw, KeySelector begin, + KeySelector end, GetRangeLimits limits, bool reverse = false); + + SpecialKeySpace(KeyRef spaceStartKey = Key(), KeyRef spaceEndKey = normalKeys.end) { + // Default value is nullptr, begin of KeyRangeMap is Key() + impls = KeyRangeMap(nullptr, spaceEndKey); + range = KeyRangeRef(spaceStartKey, spaceEndKey); + } + void registerKeyRange(const KeyRangeRef& kr, SpecialKeyRangeBaseImpl* impl) { + // range check + // TODO: add range check not to be replaced by overlapped ones + ASSERT(kr.begin >= range.begin && kr.end <= range.end); + // make sure the registered range is not overlapping with existing ones + // Note: kr.end should not be the same as another range's begin, although it should work even they are the same + ASSERT(impls.rangeContaining(kr.begin) == impls.rangeContaining(kr.end) && impls[kr.begin] == nullptr); + impls.insert(kr, impl); + } + +private: + ACTOR Future> getActor(SpecialKeySpace* pks, Reference ryw, KeyRef key); + + ACTOR Future> getRangeAggregationActor(SpecialKeySpace* pks, + Reference ryw, + KeySelector begin, KeySelector end, + GetRangeLimits limits, bool reverse); + + KeyRangeMap impls; + KeyRange range; +}; + +// Use special key prefix "\xff\xff/transaction/conflicting_keys/", +// to retrieve keys which caused latest not_committed(conflicting with another transaction) error. +// The returned key value pairs are interpretted as : +// prefix/ : '1' - any keys equal or larger than this key are (probably) conflicting keys +// prefix/ : '0' - any keys equal or larger than this key are (definitely) not conflicting keys +// Currently, the conflicting keyranges returned are original read_conflict_ranges or union of them. +class ConflictingKeysImpl : public SpecialKeyRangeBaseImpl { +public: + explicit ConflictingKeysImpl(KeyRangeRef kr); + Future> getRange(Reference ryw, + KeyRangeRef kr) const override; +}; + +#include "flow/unactorcompiler.h" +#endif diff --git a/fdbclient/Status.h b/fdbclient/Status.h index 6d7384abfb..8a6e49ff25 100644 --- a/fdbclient/Status.h +++ b/fdbclient/Status.h @@ -68,7 +68,7 @@ struct StatusValue : json_spirit::mValue { StatusValue(json_spirit::mValue const& o) : json_spirit::mValue(o) {} }; -static StatusObject makeMessage(const char *name, const char *description) { +inline StatusObject makeMessage(const char *name, const char *description) { StatusObject out; out["name"] = name; out["description"] = description; @@ -88,7 +88,7 @@ template <> inline bool JSONDoc::get(const std::string path, StatusObje } // Takes an object by reference so make usage look clean and avoid the client doing object["messages"] which will create the key. -static bool findMessagesByName(StatusObjectReader object, std::set to_find) { +inline bool findMessagesByName(StatusObjectReader object, std::set to_find) { if (!object.has("messages") || object.last().type() != json_spirit::array_type) return false; diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp index 4798b217f4..ab5dce1aa9 100644 --- a/fdbclient/StatusClient.actor.cpp +++ b/fdbclient/StatusClient.actor.cpp @@ -21,7 +21,6 @@ #include "flow/flow.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/MonitorLeader.h" -#include "fdbclient/FailureMonitorClient.h" #include "fdbclient/ClusterInterface.h" #include "fdbclient/StatusClient.h" #include "fdbclient/Status.h" @@ -452,7 +451,7 @@ StatusObject getClientDatabaseStatus(StatusObjectReader client, StatusObjectRead return databaseStatus; } -ACTOR Future statusFetcherImpl( Reference f ) { +ACTOR Future statusFetcherImpl( Reference f, Reference>> clusterInterface) { if (!g_network) throw network_not_setup(); state StatusObject statusObj; @@ -462,13 +461,10 @@ ACTOR Future statusFetcherImpl( Reference f // This could be read from the JSON but doing so safely is ugly so using a real var. state bool quorum_reachable = false; state int coordinatorsFaultTolerance = 0; - state Reference>> clusterInterface(new AsyncVar>); try { state int64_t clientTime = time(0); - state Future leaderMon = monitorLeader(f, clusterInterface); - StatusObject _statusObjClient = wait(clientStatusFetcher(f, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance)); statusObjClient = _statusObjClient; @@ -548,6 +544,23 @@ ACTOR Future statusFetcherImpl( Reference f return statusObj; } -Future StatusClient::statusFetcher( Reference clusterFile ) { - return statusFetcherImpl(clusterFile); +ACTOR Future timeoutMonitorLeader(Database db) { + state Future leadMon = monitorLeader(db->getConnectionFile(), db->statusClusterInterface); + loop { + wait(delay(CLIENT_KNOBS->STATUS_IDLE_TIMEOUT + 0.00001 + db->lastStatusFetch - now())); + if(now() - db->lastStatusFetch > CLIENT_KNOBS->STATUS_IDLE_TIMEOUT) { + db->statusClusterInterface = Reference>>(); + return Void(); + } + } +} + +Future StatusClient::statusFetcher( Database db ) { + db->lastStatusFetch = now(); + if(!db->statusClusterInterface) { + db->statusClusterInterface = Reference>>(new AsyncVar>); + db->statusLeaderMon = timeoutMonitorLeader(db); + } + + return statusFetcherImpl(db->getConnectionFile(), db->statusClusterInterface); } diff --git a/fdbclient/StatusClient.h b/fdbclient/StatusClient.h index 5a78b9b20f..6b780163a4 100755 --- a/fdbclient/StatusClient.h +++ b/fdbclient/StatusClient.h @@ -23,11 +23,12 @@ #include "flow/flow.h" #include "fdbclient/Status.h" +#include "fdbclient/DatabaseContext.h" class StatusClient { public: enum StatusLevel { MINIMAL = 0, NORMAL = 1, DETAILED = 2, JSON = 3 }; - static Future statusFetcher(Reference clusterFile); + static Future statusFetcher(Database db); }; #endif \ No newline at end of file diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index 423b099018..69fe31c5ee 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -74,10 +74,12 @@ struct StorageServerInterface { explicit StorageServerInterface(UID uid) : uniqueID( uid ) {} StorageServerInterface() : uniqueID( deterministicRandom()->randomUniqueID() ) {} NetworkAddress address() const { return getValue.getEndpoint().getPrimaryAddress(); } + NetworkAddress stableAddress() const { return getValue.getEndpoint().getStableAddress(); } + Optional secondaryAddress() const { return getValue.getEndpoint().addresses.secondaryAddress; } UID id() const { return uniqueID; } std::string toString() const { return id().shortString(); } - template - void serialize( Ar& ar ) { + template + void serialize(Ar& ar) { // StorageServerInterface is persisted in the database and in the tLog's data structures, so changes here have to be // versioned carefully! @@ -267,7 +269,7 @@ struct GetShardStateRequest { FETCHING = 1, READABLE = 2 }; - + KeyRange keys; int32_t mode; ReplyPromise reply; @@ -282,15 +284,13 @@ struct GetShardStateRequest { struct StorageMetrics { constexpr static FileIdentifier file_identifier = 13622226; - int64_t bytes; // total storage - int64_t bytesPerKSecond; // network bandwidth (average over 10s) - int64_t iosPerKSecond; - int64_t bytesReadPerKSecond; + int64_t bytes = 0; // total storage + int64_t bytesPerKSecond = 0; // network bandwidth (average over 10s) + int64_t iosPerKSecond = 0; + int64_t bytesReadPerKSecond = 0; static const int64_t infinity = 1LL<<60; - StorageMetrics() : bytes(0), bytesPerKSecond(0), iosPerKSecond(0), bytesReadPerKSecond(0) {} - bool allLessOrEqual( const StorageMetrics& rhs ) const { return bytes <= rhs.bytes && bytesPerKSecond <= rhs.bytesPerKSecond && iosPerKSecond <= rhs.iosPerKSecond && bytesReadPerKSecond <= rhs.bytesReadPerKSecond; @@ -392,15 +392,17 @@ struct SplitMetricsRequest { struct GetStorageMetricsReply { constexpr static FileIdentifier file_identifier = 15491478; StorageMetrics load; - StorageMetrics free; + StorageMetrics available; StorageMetrics capacity; double bytesInputRate; + int64_t versionLag; + double lastUpdate; GetStorageMetricsReply() : bytesInputRate(0) {} template void serialize(Ar& ar) { - serializer(ar, load, free, capacity, bytesInputRate); + serializer(ar, load, available, capacity, bytesInputRate, versionLag, lastUpdate); } }; diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 99bbed5429..e1375b7e13 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -21,6 +21,8 @@ #include "fdbclient/SystemData.h" #include "fdbclient/StorageServerInterface.h" #include "flow/TDMetric.actor.h" +#include "fdbclient/NativeAPI.actor.h" + const KeyRef systemKeysPrefix = LiteralStringRef("\xff"); const KeyRangeRef normalKeys(KeyRef(), systemKeysPrefix); @@ -28,6 +30,7 @@ const KeyRangeRef systemKeys(systemKeysPrefix, LiteralStringRef("\xff\xff") ); const KeyRangeRef nonMetadataSystemKeys(LiteralStringRef("\xff\x02"), LiteralStringRef("\xff\x03")); const KeyRangeRef allKeys = KeyRangeRef(normalKeys.begin, systemKeys.end); const KeyRef afterAllKeys = LiteralStringRef("\xff\xff\x00"); +const KeyRangeRef specialKeys = KeyRangeRef(LiteralStringRef("\xff\xff"), LiteralStringRef("\xff\xff\xff")); // keyServersKeys.contains(k) iff k.startsWith(keyServersPrefix) const KeyRangeRef keyServersKeys( LiteralStringRef("\xff/keyServers/"), LiteralStringRef("\xff/keyServers0") ); @@ -42,22 +45,74 @@ const Key keyServersKey( const KeyRef& k ) { const KeyRef keyServersKey( const KeyRef& k, Arena& arena ) { return k.withPrefix( keyServersPrefix, arena ); } -const Value keyServersValue( const vector& src, const vector& dest ) { +const Value keyServersValue( Standalone result, const std::vector& src, const std::vector& dest ) { + std::vector srcTag; + std::vector destTag; + + for (const KeyValueRef kv : result) { + UID uid = decodeServerTagKey(kv.key); + if (std::find(src.begin(), src.end(), uid) != src.end()) { + srcTag.push_back( decodeServerTagValue(kv.value) ); + } + if (std::find(dest.begin(), dest.end(), uid) != dest.end()) { + destTag.push_back( decodeServerTagValue(kv.value) ); + } + } + + return keyServersValue(srcTag, destTag); +} +const Value keyServersValue( const std::vector& srcTag, const std::vector& destTag ) { // src and dest are expected to be sorted - ASSERT( std::is_sorted(src.begin(), src.end()) && std::is_sorted(dest.begin(), dest.end()) ); - BinaryWriter wr((IncludeVersion())); wr << src << dest; + BinaryWriter wr(IncludeVersion()); wr << srcTag << destTag; return wr.toValue(); } -void decodeKeyServersValue( const ValueRef& value, vector& src, vector& dest ) { - if (value.size()) { - BinaryReader rd(value, IncludeVersion()); - rd >> src >> dest; - } else { + +void decodeKeyServersValue( Standalone result, const ValueRef& value, + std::vector& src, std::vector& dest ) { + if (value.size() == 0) { src.clear(); dest.clear(); + return; } + + BinaryReader rd(value, IncludeVersion()); + rd.checkpoint(); + int srcLen, destLen; + rd >> srcLen; + rd.readBytes(srcLen * sizeof(Tag)); + rd >> destLen; + rd.rewind(); + + if (value.size() != sizeof(ProtocolVersion) + sizeof(int) + srcLen * sizeof(Tag) + sizeof(int) + destLen * sizeof(Tag)) { + rd >> src >> dest; + rd.assertEnd(); + return; + } + + std::vector srcTag, destTag; + rd >> srcTag >> destTag; + + src.clear(); + dest.clear(); + + for (const KeyValueRef kv : result) { + Tag tag = decodeServerTagValue(kv.value); + if (std::find(srcTag.begin(), srcTag.end(), tag) != srcTag.end()) { + src.push_back( decodeServerTagKey(kv.key) ); + } + if (std::find(destTag.begin(), destTag.end(), tag) != destTag.end()) { + dest.push_back( decodeServerTagKey(kv.key) ); + } + } + std::sort(src.begin(), src.end()); + std::sort(dest.begin(), dest.end()); } +const KeyRangeRef conflictingKeysRange = KeyRangeRef(LiteralStringRef("\xff\xff/transaction/conflicting_keys/"), + LiteralStringRef("\xff\xff/transaction/conflicting_keys/\xff")); +const ValueRef conflictingKeysTrue = LiteralStringRef("1"); +const ValueRef conflictingKeysFalse = LiteralStringRef("0"); + // "\xff/storageCache/[[begin]]" := "[[vector]]" const KeyRangeRef storageCacheKeys( LiteralStringRef("\xff/storageCache/"), LiteralStringRef("\xff/storageCache0") ); const KeyRef storageCachePrefix = storageCacheKeys.begin; @@ -177,7 +232,7 @@ const KeyRangeRef serverTagConflictKeys( const KeyRef serverTagConflictPrefix = serverTagConflictKeys.begin; // serverTagHistoryKeys is the old tag a storage server uses before it is migrated to a different location. // For example, we can copy a SS file to a remote DC and start the SS there; -// The new SS will need to cnosume the last bits of data from the old tag it is responsible for. +// The new SS will need to consume the last bits of data from the old tag it is responsible for. const KeyRangeRef serverTagHistoryKeys( LiteralStringRef("\xff/serverTagHistory/"), LiteralStringRef("\xff/serverTagHistory0") ); @@ -491,6 +546,52 @@ ProcessData decodeWorkerListValue( ValueRef const& value ) { return s; } +const KeyRangeRef backupProgressKeys(LiteralStringRef("\xff\x02/backupProgress/"), + LiteralStringRef("\xff\x02/backupProgress0")); +const KeyRef backupProgressPrefix = backupProgressKeys.begin; +const KeyRef backupStartedKey = LiteralStringRef("\xff\x02/backupStarted"); +extern const KeyRef backupPausedKey = LiteralStringRef("\xff\x02/backupPaused"); + +const Key backupProgressKeyFor(UID workerID) { + BinaryWriter wr(Unversioned()); + wr.serializeBytes(backupProgressPrefix); + wr << workerID; + return wr.toValue(); +} + +const Value backupProgressValue(const WorkerBackupStatus& status) { + BinaryWriter wr(IncludeVersion()); + wr << status; + return wr.toValue(); +} + +UID decodeBackupProgressKey(const KeyRef& key) { + UID serverID; + BinaryReader rd(key.removePrefix(backupProgressPrefix), Unversioned()); + rd >> serverID; + return serverID; +} + +WorkerBackupStatus decodeBackupProgressValue(const ValueRef& value) { + WorkerBackupStatus status; + BinaryReader reader(value, IncludeVersion()); + reader >> status; + return status; +} + +Value encodeBackupStartedValue(const std::vector>& ids) { + BinaryWriter wr(IncludeVersion()); + wr << ids; + return wr.toValue(); +} + +std::vector> decodeBackupStartedValue(const ValueRef& value) { + std::vector> ids; + BinaryReader reader(value, IncludeVersion()); + if (value.size() > 0) reader >> ids; + return ids; +} + const KeyRef coordinatorsKey = LiteralStringRef("\xff/coordinators"); const KeyRef logsKey = LiteralStringRef("\xff/logs"); const KeyRef minRequiredCommitVersionKey = LiteralStringRef("\xff/minRequiredCommitVersion"); @@ -705,20 +806,22 @@ const KeyRangeRef restoreApplierKeys(LiteralStringRef("\xff\x02/restoreApplier/" const KeyRef restoreApplierTxnValue = LiteralStringRef("1"); // restoreApplierKeys: track atomic transaction progress to ensure applying atomicOp exactly once -// Version is passed in as LittleEndian, it must be converted to BigEndian to maintain ordering in lexical order -const Key restoreApplierKeyFor(UID const& applierID, Version version) { +// Version and batchIndex are passed in as LittleEndian, +// they must be converted to BigEndian to maintain ordering in lexical order +const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version) { BinaryWriter wr(Unversioned()); wr.serializeBytes(restoreApplierKeys.begin); - wr << applierID << bigEndian64(version); + wr << applierID << bigEndian64(batchIndex) << bigEndian64(version); return wr.toValue(); } -std::pair decodeRestoreApplierKey(ValueRef const& key) { +std::tuple decodeRestoreApplierKey(ValueRef const& key) { BinaryReader rd(key, Unversioned()); UID applierID; + int64_t batchIndex; Version version; - rd >> applierID >> version; - return std::make_pair(applierID, bigEndian64(version)); + rd >> applierID >> batchIndex >> version; + return std::make_tuple(applierID, bigEndian64(batchIndex), bigEndian64(version)); } // Encode restore worker key for workerID diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index c3debfac3f..fe9407980c 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -28,6 +28,10 @@ #include "fdbclient/StorageServerInterface.h" #include "fdbclient/RestoreWorkerInterface.actor.h" +// Don't warn on constants being defined in this file. +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-variable" + struct RestoreLoaderInterface; struct RestoreApplierInterface; struct RestoreMasterInterface; @@ -36,25 +40,32 @@ extern const KeyRangeRef normalKeys; // '' to systemKeys.begin extern const KeyRangeRef systemKeys; // [FF] to [FF][FF] extern const KeyRangeRef nonMetadataSystemKeys; // [FF][00] to [FF][01] extern const KeyRangeRef allKeys; // '' to systemKeys.end +extern const KeyRangeRef specialKeys; // [FF][FF] to [FF][FF][FF], some client functions are exposed through FDB calls + // using these special keys, see pr#2662 extern const KeyRef afterAllKeys; -// "\xff/keyServers/[[begin]]" := "[[vector, vector]]" +// "\xff/keyServers/[[begin]]" := "[[vector, vector]|[vector, vector]]" extern const KeyRangeRef keyServersKeys, keyServersKeyServersKeys; extern const KeyRef keyServersPrefix, keyServersEnd, keyServersKeyServersKey; const Key keyServersKey( const KeyRef& k ); const KeyRef keyServersKey( const KeyRef& k, Arena& arena ); const Value keyServersValue( - const vector& src, - const vector& dest = vector() ); -void decodeKeyServersValue( const ValueRef& value, - vector& src, vector& dest ); + Standalone result, + const std::vector& src, + const std::vector& dest = std::vector() ); +const Value keyServersValue( + const std::vector& srcTag, + const std::vector& destTag = std::vector()); +// `result` must be the full result of getting serverTagKeys +void decodeKeyServersValue( Standalone result, const ValueRef& value, + std::vector& src, std::vector& dest ); // "\xff/storageCache/[[begin]]" := "[[vector]]" extern const KeyRangeRef storageCacheKeys; extern const KeyRef storageCachePrefix; const Key storageCacheKey( const KeyRef& k ); -const Value storageCacheValue( const vector& serverIndices ); -void decodeStorageCacheValue( const ValueRef& value, vector& serverIndices ); +const Value storageCacheValue( const std::vector& serverIndices ); +void decodeStorageCacheValue( const ValueRef& value, std::vector& serverIndices ); // "\xff/serverKeys/[[serverID]]/[[begin]]" := "" | "1" | "2" extern const KeyRef serverKeysPrefix; @@ -64,6 +75,9 @@ const Key serverKeysPrefixFor( UID serverID ); UID serverKeysDecodeServer( const KeyRef& key ); bool serverHasKey( ValueRef storedValue ); +extern const KeyRangeRef conflictingKeysRange; +extern const ValueRef conflictingKeysTrue, conflictingKeysFalse; + extern const KeyRef cacheKeysPrefix; const Key cacheKeysKey( uint16_t idx, const KeyRef& key ); @@ -77,6 +91,7 @@ extern const KeyRef cacheChangePrefix; const Key cacheChangeKeyFor( uint16_t idx ); uint16_t cacheChangeKeyDecodeIndex( const KeyRef& key ); +// "\xff/serverTag/[[serverID]]" = "[[Tag]]" extern const KeyRangeRef serverTagKeys; extern const KeyRef serverTagPrefix; extern const KeyRangeRef serverTagMaxKeys; @@ -174,6 +189,24 @@ const Value workerListValue( ProcessData const& ); Key decodeWorkerListKey( KeyRef const& ); ProcessData decodeWorkerListValue( ValueRef const& ); +// "\xff\x02/backupProgress/[[workerID]]" := "[[WorkerBackupStatus]]" +extern const KeyRangeRef backupProgressKeys; +extern const KeyRef backupProgressPrefix; +const Key backupProgressKeyFor(UID workerID); +const Value backupProgressValue(const WorkerBackupStatus& status); +UID decodeBackupProgressKey(const KeyRef& key); +WorkerBackupStatus decodeBackupProgressValue(const ValueRef& value); + +// The key to signal backup workers a new backup job is submitted. +// "\xff\x02/backupStarted" := "[[vector]]" +extern const KeyRef backupStartedKey; +Value encodeBackupStartedValue(const std::vector>& ids); +std::vector> decodeBackupStartedValue(const ValueRef& value); + +// The key to signal backup workers that they should pause or resume. +// "\xff\x02/backupPaused" := "[[0|1]]" +extern const KeyRef backupPausedKey; + extern const KeyRef coordinatorsKey; extern const KeyRef logsKey; extern const KeyRef minRequiredCommitVersionKey; @@ -317,8 +350,8 @@ extern const KeyRangeRef restoreRequestKeys; extern const KeyRangeRef restoreApplierKeys; extern const KeyRef restoreApplierTxnValue; -const Key restoreApplierKeyFor(UID const& applierID, Version version); -std::pair decodeRestoreApplierKey(ValueRef const& key); +const Key restoreApplierKeyFor(UID const& applierID, int64_t batchIndex, Version version); +std::tuple decodeRestoreApplierKey(ValueRef const& key); const Key restoreWorkerKeyFor(UID const& workerID); const Value restoreWorkerInterfaceValue(RestoreWorkerInterface const& server); RestoreWorkerInterface decodeRestoreWorkerInterfaceValue(ValueRef const& value); @@ -343,4 +376,6 @@ std::pair decodeHealthyZoneValue( ValueRef const& ); // Used to create artifically large txnStateStore instances in testing. extern const KeyRangeRef testOnlyTxnStateStorePrefixRange; +#pragma clang diagnostic pop + #endif diff --git a/fdbclient/TaskBucket.actor.cpp b/fdbclient/TaskBucket.actor.cpp index f40b0090a9..66d3b9571f 100644 --- a/fdbclient/TaskBucket.actor.cpp +++ b/fdbclient/TaskBucket.actor.cpp @@ -230,7 +230,7 @@ public: return task; } - // Verify that the user configured task verification key still has the user specificied value + // Verify that the user configured task verification key still has the user specified value ACTOR static Future taskVerify(Reference tb, Reference tr, Reference task) { if (task->params.find(Task::reservedTaskParamValidKey) == task->params.end()) { diff --git a/fdbclient/ThreadSafeTransaction.actor.cpp b/fdbclient/ThreadSafeTransaction.actor.cpp index c71482b3b9..afafa8747c 100644 --- a/fdbclient/ThreadSafeTransaction.actor.cpp +++ b/fdbclient/ThreadSafeTransaction.actor.cpp @@ -64,9 +64,9 @@ void ThreadSafeDatabase::setOption( FDBDatabaseOptions::Option option, Optional< Standalone> passValue = value; // ThreadSafeDatabase is not allowed to do anything with options except pass them through to RYW. - onMainThreadVoid( [db, option, passValue](){ + onMainThreadVoid( [db, option, passValue](){ db->checkDeferredError(); - db->setOption(option, passValue.contents()); + db->setOption(option, passValue.contents()); }, &db->deferredError ); } @@ -77,7 +77,7 @@ ThreadSafeDatabase::ThreadSafeDatabase(std::string connFilename, int apiVersion) // but run its constructor on the main thread DatabaseContext *db = this->db = DatabaseContext::allocateOnForeignThread(); - onMainThreadVoid([db, connFile, apiVersion](){ + onMainThreadVoid([db, connFile, apiVersion](){ try { Database::createDatabase(Reference(connFile), apiVersion, false, LocalityData(), db).extractPtr(); } @@ -157,6 +157,17 @@ ThreadFuture< Key > ThreadSafeTransaction::getKey( const KeySelectorRef& key, bo } ); } +ThreadFuture ThreadSafeTransaction::getEstimatedRangeSizeBytes( const KeyRangeRef& keys ) { + KeyRange r = keys; + + ReadYourWritesTransaction *tr = this->tr; + return onMainThread( [tr, r]() -> Future { + tr->checkDeferredError(); + return tr->getEstimatedRangeSizeBytes(r); + } ); +} + + ThreadFuture< Standalone > ThreadSafeTransaction::getRange( const KeySelectorRef& begin, const KeySelectorRef& end, int limit, bool snapshot, bool reverse ) { KeySelector b = begin; KeySelector e = end; @@ -292,7 +303,7 @@ void ThreadSafeTransaction::setOption( FDBTransactionOptions::Option option, Opt TraceEvent("UnknownTransactionOption").detail("Option", option); throw invalid_option(); } - + ReadYourWritesTransaction *tr = this->tr; Standalone> passValue = value; diff --git a/fdbclient/ThreadSafeTransaction.h b/fdbclient/ThreadSafeTransaction.h index c5832cec45..61b64aa4b4 100644 --- a/fdbclient/ThreadSafeTransaction.h +++ b/fdbclient/ThreadSafeTransaction.h @@ -71,6 +71,7 @@ public: } ThreadFuture>> getAddressesForKey(const KeyRef& key) override; ThreadFuture> getVersionstamp() override; + ThreadFuture getEstimatedRangeSizeBytes(const KeyRangeRef& keys) override; void addReadConflictRange( const KeyRangeRef& keys ) override; void makeSelfConflicting(); diff --git a/fdbclient/fdbclient.vcxproj b/fdbclient/fdbclient.vcxproj index 974aa896a8..613f5f0ac1 100644 --- a/fdbclient/fdbclient.vcxproj +++ b/fdbclient/fdbclient.vcxproj @@ -48,7 +48,6 @@ - @@ -84,6 +83,7 @@ + @@ -111,7 +111,6 @@ - @@ -131,6 +130,7 @@ + diff --git a/fdbclient/local.mk b/fdbclient/local.mk deleted file mode 100644 index f3631dbee9..0000000000 --- a/fdbclient/local.mk +++ /dev/null @@ -1,32 +0,0 @@ -# -# local.mk -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -*- mode: makefile; -*- - -fdbclient_CFLAGS := $(fdbrpc_CFLAGS) - -fdbclient_GENERATED_SOURCES += fdbclient/FDBOptions.g.h - -fdbclient/FDBOptions.g.cpp: fdbclient/FDBOptions.g.h -fdbclient/FDBOptions.g.h: bin/vexillographer.exe fdbclient/vexillographer/fdb.options fdbclient/FDBOptions.h - @echo "Building $@" - @$(MONO) bin/vexillographer.exe fdbclient/vexillographer/fdb.options cpp fdbclient/FDBOptions.g - -lib/libfdbclient.a: bin/coverage.fdbclient.xml diff --git a/fdbclient/md5/md5.c b/fdbclient/md5/md5.c index 52d96accd3..1032ccfdaf 100644 --- a/fdbclient/md5/md5.c +++ b/fdbclient/md5/md5.c @@ -35,7 +35,7 @@ * compile-time configuration. */ -#ifndef HAVE_OPENSSL +#if !defined(HAVE_OPENSSL) || defined(TLS_DISABLED) #include diff --git a/fdbclient/md5/md5.h b/fdbclient/md5/md5.h index e73fb29c35..5731872376 100644 --- a/fdbclient/md5/md5.h +++ b/fdbclient/md5/md5.h @@ -23,7 +23,7 @@ * See md5.c for more information. */ -#ifdef HAVE_OPENSSL +#if defined(HAVE_OPENSSL) && !defined(TLS_DISABLED) #include #elif !defined(_MD5_H) #define _MD5_H diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options index 890dea4864..2987f9572f 100644 --- a/fdbclient/vexillographer/fdb.options +++ b/fdbclient/vexillographer/fdb.options @@ -51,6 +51,12 @@ description is not currently required but encouraged.