From cdd64ebc15c4a84704999b263e6719016da567af Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Thu, 15 Jun 2017 13:30:15 -0700 Subject: [PATCH 1/3] fix: asyncFileNonDurable could never complete deleting a file in rare situations --- fdbrpc/AsyncFileNonDurable.actor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbrpc/AsyncFileNonDurable.actor.h b/fdbrpc/AsyncFileNonDurable.actor.h index 951147b162..9b26bbc3ae 100644 --- a/fdbrpc/AsyncFileNonDurable.actor.h +++ b/fdbrpc/AsyncFileNonDurable.actor.h @@ -425,7 +425,7 @@ private: std::vector> priorModifications = self->getModificationsAndInsert(offset, length, true, writeEnded); if(BUGGIFY_WITH_PROB(0.001)) - priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY)); + priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture()); else priorModifications.push_back(waitUntilDiskReady(self->diskParameters, length) || self->killed.getFuture()); @@ -547,7 +547,7 @@ private: std::vector> priorModifications = self->getModificationsAndInsert(size, -1, true, truncateEnded); if(BUGGIFY_WITH_PROB(0.001)) - priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY)); + priorModifications.push_back(delay(g_random->random01() * FLOW_KNOBS->MAX_PRIOR_MODIFICATION_DELAY) || self->killed.getFuture()); else priorModifications.push_back(waitUntilDiskReady(self->diskParameters, 0) || self->killed.getFuture()); From 5de6c703cffa34c73bfe5c3012b6d61e39be7dd9 Mon Sep 17 00:00:00 2001 From: Stephen Atherton Date: Thu, 15 Jun 2017 16:57:35 -0700 Subject: [PATCH 2/3] Fixed line endings which were changed to UNIX style during a recent merge. --- fdbserver/fdbserver.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index d4b40d4e03..5ca4bce27d 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -817,7 +817,7 @@ int main(int argc, char* argv[]) { //Enables profiling on this thread (but does not start it) registerThreadForProfiling(); - + std::string commandLine; for (int a = 0; a Date: Thu, 15 Jun 2017 17:40:19 -0700 Subject: [PATCH 3/3] Merge branch 'release-4.6' into release-5.0 # Conflicts: # fdbrpc/AsyncFileKAIO.actor.h # fdbrpc/sim2.actor.cpp # fdbserver/optimisttest.actor.cpp # versions.target --- fdbrpc/sim2.actor.cpp | 12 +++++++----- fdbrpc/simulator.h | 3 +-- fdbserver/KeyValueStoreSQLite.actor.cpp | 2 +- fdbserver/fdbserver.actor.cpp | 1 + fdbserver/worker.actor.cpp | 17 ++++++++++++----- packaging/msi/FDBInstaller.wxs | 2 +- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/fdbrpc/sim2.actor.cpp b/fdbrpc/sim2.actor.cpp index 89b5df40db..99eda434ad 100644 --- a/fdbrpc/sim2.actor.cpp +++ b/fdbrpc/sim2.actor.cpp @@ -49,14 +49,13 @@ bool simulator_should_inject_fault( const char* context, const char* file, int l h2 = p->fault_injection_r; if (h1 < p->fault_injection_p1*std::numeric_limits::max()) { - TEST(true); - TEST(error_code == error_code_io_timeout); - TEST(error_code == error_code_io_error); - TEST(error_code == error_code_platform_error); + TEST(true); // A fault was injected + TEST(error_code == error_code_io_timeout); // An io timeout was injected + TEST(error_code == error_code_io_error); // An io error was injected + TEST(error_code == error_code_platform_error); // A platform error was injected. TraceEvent(SevWarn, "FaultInjected").detail("Context", context).detail("File", file).detail("Line", line).detail("ErrorCode", error_code); if(error_code == error_code_io_timeout) { g_network->setGlobal(INetwork::enASIOTimedOut, (flowGlobalType)true); - g_pSimulator->getCurrentProcess()->io_timeout_injected = true; } return true; } @@ -949,6 +948,7 @@ public: m->setGlobal(enTDMetrics, (flowGlobalType) &m->tdmetrics); m->setGlobal(enNetworkConnections, (flowGlobalType) m->network); + m->setGlobal(enASIOTimedOut, (flowGlobalType) false); TraceEvent("NewMachine").detail("Name", name).detail("Address", m->address).detailext("zoneId", m->locality.zoneId()); @@ -1172,6 +1172,8 @@ public: TraceEvent("KillMachine", zoneId).detailext("ZoneId", zoneId).detail("Kt", kt).detail("KtOrig", ktOrig).detail("KilledMachines", killedMachines).detail("KillableMachines", processesOnMachine).detail("ProcessPerMachine", processesPerMachine).detail("KillChanged", kt!=ktOrig).detail("killIsSafe", killIsSafe); if (kt < RebootAndDelete ) { + if(kt == InjectFaults && machines[zoneId].machineProcess != nullptr) + killProcess_internal( machines[zoneId].machineProcess, kt ); for (auto& process : machines[zoneId].processes) { TraceEvent("KillMachineProcess", zoneId).detail("KillType", kt).detail("Process", process->toString()).detail("startingClass", process->startingClass.toString()); if (process->startingClass != ProcessClass::TesterClass) diff --git a/fdbrpc/simulator.h b/fdbrpc/simulator.h index 92e4594fbf..8aa9bda3ec 100644 --- a/fdbrpc/simulator.h +++ b/fdbrpc/simulator.h @@ -64,14 +64,13 @@ public: uint64_t fault_injection_r; double fault_injection_p1, fault_injection_p2; - bool io_timeout_injected; ProcessInfo(const char* name, LocalityData locality, ProcessClass startingClass, NetworkAddress address, INetworkConnections *net, const char* dataFolder, const char* coordinationFolder ) : name(name), locality(locality), startingClass(startingClass), address(address), dataFolder(dataFolder), network(net), coordinationFolder(coordinationFolder), failed(false), excluded(false), cpuTicks(0), rebooting(false), fault_injection_p1(0), fault_injection_p2(0), - fault_injection_r(0), machine(0), io_timeout_injected(false) + fault_injection_r(0), machine(0) {} Future onShutdown() { return shutdownSignal.getFuture(); } diff --git a/fdbserver/KeyValueStoreSQLite.actor.cpp b/fdbserver/KeyValueStoreSQLite.actor.cpp index 869c3fd344..7b41a6a0e2 100644 --- a/fdbserver/KeyValueStoreSQLite.actor.cpp +++ b/fdbserver/KeyValueStoreSQLite.actor.cpp @@ -227,7 +227,7 @@ struct SQLiteDB : NonCopyable { // Our exceptions don't propagate through sqlite, so we don't know for sure if the error that caused this was // an injected fault. Assume that if fault injection is happening, this is an injected fault. Error err = io_error(); - if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting)) + if (g_network->isSimulated() && (g_simulator.getCurrentProcess()->fault_injection_p1 || g_simulator.getCurrentProcess()->machine->machineProcess->fault_injection_p1 || g_simulator.getCurrentProcess()->rebooting)) err = err.asInjectedFault(); if (db) diff --git a/fdbserver/fdbserver.actor.cpp b/fdbserver/fdbserver.actor.cpp index 5ca4bce27d..32bb6d4cad 100644 --- a/fdbserver/fdbserver.actor.cpp +++ b/fdbserver/fdbserver.actor.cpp @@ -107,6 +107,7 @@ CSimpleOpt::SOption g_rgOptions[] = { { OPT_NOBOX, "-q", SO_NONE }, { OPT_NOBOX, "--no_dialog", SO_NONE }, #endif + { OPT_KVFILE, "--kvfile", SO_REQ_SEP }, { OPT_TESTFILE, "-f", SO_REQ_SEP }, { OPT_TESTFILE, "--testfile", SO_REQ_SEP }, { OPT_RESTARTING, "-R", SO_NONE }, diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp index bc908dc76b..3dd6f610d5 100644 --- a/fdbserver/worker.actor.cpp +++ b/fdbserver/worker.actor.cpp @@ -77,10 +77,16 @@ struct ErrorInfo { Error checkIOTimeout(Error const &e) { // Convert all_errors to io_timeout if global timeout bool was set - if((bool)g_network->global(INetwork::enASIOTimedOut)) { + bool timeoutOccurred = (bool)g_network->global(INetwork::enASIOTimedOut); + // In simulation, have to check global timed out flag for both this process and the machine process on which IO is done + if(g_network->isSimulated() && !timeoutOccurred) + timeoutOccurred = g_pSimulator->getCurrentProcess()->machine->machineProcess->global(INetwork::enASIOTimedOut); + + if(timeoutOccurred) { + TEST(true); // Timeout occurred Error timeout = io_timeout(); - // If this error was injected OR if the timeout was injected then make the resulting io_timeout injected - if(e.isInjectedFault() || (g_network->isSimulated() && g_pSimulator->getCurrentProcess()->io_timeout_injected) ) + // Preserve injectedness of error + if(e.isInjectedFault()) timeout = timeout.asInjectedFault(); return timeout; } @@ -124,14 +130,15 @@ ACTOR Future workerHandleErrors(FutureStream errors) { loop choose { when( ErrorInfo _err = waitNext(errors) ) { ErrorInfo err = _err; - err.error = checkIOTimeout(err.error); // Possibly convert error to io_timeout - bool ok = err.error.code() == error_code_success || err.error.code() == error_code_please_reboot || err.error.code() == error_code_actor_cancelled || err.error.code() == error_code_coordinators_changed; // The worker server was cancelled + if(!ok) + err.error = checkIOTimeout(err.error); // Possibly convert error to io_timeout + endRole(err.id, err.context, "Error", ok, err.error); if (err.error.code() == error_code_please_reboot || err.error.code() == error_code_io_timeout) throw err.error; diff --git a/packaging/msi/FDBInstaller.wxs b/packaging/msi/FDBInstaller.wxs index 2250f6a628..69b5f34e39 100644 --- a/packaging/msi/FDBInstaller.wxs +++ b/packaging/msi/FDBInstaller.wxs @@ -32,7 +32,7 @@