From c1b12058e71b2a06a20b284c2498979efcd63633 Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Wed, 29 Nov 2023 12:09:59 +0100 Subject: [PATCH] Do not set RPC_READY to false on transports shutdown in order to not fail counter updates for deployments with coordinator and storage nodes with transports turned off This is the follow-up commit of CASSANDRA-18935 where we set RPC_READY to false when transports were shut down in runtime. The problem is that the current logic in StorageProxy.findSuitableEndpoint method, used for the selection of a leader for counter mutations, is filtering out all endpoints which do not have RPC_READY set to true. Hence, if there is a deployment of a coordinator node (not joining a ring) and storage nodes which have transports turned off (e.g. for security reasons), then a coordinator node will select no endpoint as a counter mutation leader which renders counter mutations impossible. This change just reverts the original fix which was setting RPC_READY to false when transports were shut down in runtime (e.g. by nodetool disablebinary). For trunk (to be 5.1 at time of writing this), there is already TCM in place and the correct fix yet to be implemented is to decouple from checking RCP_READY state and base it e.g. on JOINED state from TCM perspective. Please follow CASSANDRA-19103 where this will be addressed. patch by Stefan Miklosovic; reviewed by Brandon Williams for CASSANDRA-18935 --- CHANGES.txt | 1 + src/java/org/apache/cassandra/service/CassandraDaemon.java | 1 - .../cassandra/distributed/test/NativeProtocolTest.java | 5 +++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 4906a982f3..10c771ae2d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.30 + * Do not set RPC_READY to false on transports shutdown in order to not fail counter updates for deployments with coordinator and storage nodes with transports turned off (CASSANDRA-18935) * Suppress CVE-2023-44487 (CASSANDRA-18943) * Fix nodetool enable/disablebinary to correctly set rpc readiness in gossip (CASSANDRA-18935) * Implement the logic in bin/stop-server (CASSANDRA-18838) diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 629b85a622..d2987f45e2 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -726,7 +726,6 @@ public class CassandraDaemon { if (nativeTransportService != null) { - StorageService.instance.setRpcReady(false); nativeTransportService.stop(); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java b/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java index 44e989c235..fd75e40538 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/NativeProtocolTest.java @@ -139,9 +139,10 @@ public class NativeProtocolTest extends TestBaseImpl i.runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> StorageService.instance.startNativeTransport()); assertTrue(i.callOnInstance((IIsolatedExecutor.SerializableCallable) () -> StorageService.instance.isRpcReady(FBUtilities.getBroadcastAddress()))); - // by calling e.g. nodetool disablebinary, rpc will be set to false again + // by calling e.g. nodetool disablebinary, rpc will NOT be set to false again + // please read CASSANDRA-18935 for in-depth explanation why it is so i.runOnInstance((IIsolatedExecutor.SerializableRunnable) () -> StorageService.instance.stopNativeTransport()); - assertFalse(i.callOnInstance((IIsolatedExecutor.SerializableCallable) () -> StorageService.instance.isRpcReady(FBUtilities.getBroadcastAddress()))); + assertTrue(i.callOnInstance((IIsolatedExecutor.SerializableCallable) () -> StorageService.instance.isRpcReady(FBUtilities.getBroadcastAddress()))); } } } \ No newline at end of file