From 855f03a41fa980a97b4d8f8a9cc3be166320dbeb Mon Sep 17 00:00:00 2001 From: Evan Tschannen Date: Fri, 10 Jan 2020 15:58:36 -0800 Subject: [PATCH] ratekeeper needed to check remoteDC in another location the storage server scoped a transaction incorrectly --- fdbclient/Knobs.cpp | 2 +- fdbclient/Knobs.h | 2 +- fdbclient/MonitorLeader.actor.cpp | 6 +++--- fdbserver/Ratekeeper.actor.cpp | 2 +- fdbserver/storageserver.actor.cpp | 5 ++--- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp index 1cfc09c15e..d67b7fb62c 100644 --- a/fdbclient/Knobs.cpp +++ b/fdbclient/Knobs.cpp @@ -45,7 +45,7 @@ ClientKnobs::ClientKnobs(bool randomize) { init( COORDINATOR_RECONNECTION_DELAY, 1.0 ); init( CLIENT_EXAMPLE_AMOUNT, 20 ); init( MAX_CLIENT_STATUS_AGE, 1.0 ); - init( MAX_CLIENT_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_CLIENT_PROXY_CONNECTIONS = 1; + init( MAX_PROXY_CONNECTIONS, 5 ); if( randomize && BUGGIFY ) MAX_PROXY_CONNECTIONS = 1; // wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h index 6e1a50ed4a..b95746cd86 100644 --- a/fdbclient/Knobs.h +++ b/fdbclient/Knobs.h @@ -44,7 +44,7 @@ public: double COORDINATOR_RECONNECTION_DELAY; int CLIENT_EXAMPLE_AMOUNT; double MAX_CLIENT_STATUS_AGE; - int MAX_CLIENT_PROXY_CONNECTIONS; + int MAX_PROXY_CONNECTIONS; // wrong_shard_server sometimes comes from the only nonfailed server, so we need to avoid a fast spin double WRONG_SHARD_SERVER_DELAY; // SOMEDAY: This delay can limit performance of retrieving data when the cache is mostly wrong (e.g. dumping the database after a test) diff --git a/fdbclient/MonitorLeader.actor.cpp b/fdbclient/MonitorLeader.actor.cpp index 8bf1ba69f8..6a5c4195de 100644 --- a/fdbclient/MonitorLeader.actor.cpp +++ b/fdbclient/MonitorLeader.actor.cpp @@ -671,7 +671,7 @@ ACTOR Future monitorLeaderForProxies( Key clusterKey, vector& lastProxyUIDs, std::vector& lastProxies ) { - if(ni.proxies.size() > CLIENT_KNOBS->MAX_CLIENT_PROXY_CONNECTIONS) { + if(ni.proxies.size() > CLIENT_KNOBS->MAX_PROXY_CONNECTIONS) { std::vector proxyUIDs; for(auto& proxy : ni.proxies) { proxyUIDs.push_back(proxy.id()); @@ -680,9 +680,9 @@ void shrinkProxyList( ClientDBInfo& ni, std::vector& lastProxyUIDs, std::ve lastProxyUIDs = proxyUIDs; lastProxies = ni.proxies; deterministicRandom()->randomShuffle(lastProxies); - lastProxies.resize(CLIENT_KNOBS->MAX_CLIENT_PROXY_CONNECTIONS); + lastProxies.resize(CLIENT_KNOBS->MAX_PROXY_CONNECTIONS); for(int i = 0; i < lastProxies.size(); i++) { - TraceEvent("ServerConnectedProxy").detail("Proxy", lastProxies[i].id()); + TraceEvent("ConnectedProxy").detail("Proxy", lastProxies[i].id()); } } ni.proxies = lastProxies; diff --git a/fdbserver/Ratekeeper.actor.cpp b/fdbserver/Ratekeeper.actor.cpp index b5a094240f..360e196acd 100644 --- a/fdbserver/Ratekeeper.actor.cpp +++ b/fdbserver/Ratekeeper.actor.cpp @@ -538,7 +538,7 @@ void updateRate(RatekeeperData* self, RatekeeperLimits* limits) { Version minLimitingSSVer = std::numeric_limits::max(); for (const auto& it : self->storageQueueInfo) { auto& ss = it.value; - if (!ss.valid) continue; + if (!ss.valid || (self->remoteDC.present() && ss.locality.dcId() == self->remoteDC)) continue; minSSVer = std::min(minSSVer, ss.lastReply.version); diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index d162cc29a7..ebb0863062 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -3454,12 +3454,11 @@ ACTOR Future checkBehind( StorageServer* self ) { state int behindCount = 0; loop { wait( delay(SERVER_KNOBS->BEHIND_CHECK_DELAY) ); + state Transaction tr(self->cx); loop { try { - state Transaction tr(self->cx); Version readVersion = wait( tr.getRawReadVersion() ); - Version storageVersion = self->version.get(); - if( readVersion > storageVersion + SERVER_KNOBS->BEHIND_CHECK_VERSIONS ) { + if( readVersion > self->version.get() + SERVER_KNOBS->BEHIND_CHECK_VERSIONS ) { behindCount++; } else { behindCount = 0;