diff --git a/CHANGES.txt b/CHANGES.txt index 8fd0200504..d29cdc1ef6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -66,6 +66,7 @@ * Fully utilise specified compaction threads (CASSANDRA-14210) * Pre-create deletion log records to finish compactions quicker (CASSANDRA-12763) Merged from 2.2: + * Don't enable client transports when bootstrap is pending (CASSANDRA-14525) * Fix bug that prevented compaction of SSTables after full repairs (CASSANDRA-14423) * Incorrect counting of pending messages in OutboundTcpConnection (CASSANDRA-11551) * Fix compaction failure caused by reading un-flushed data (CASSANDRA-12743) diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 177f8e3ba7..6869d2c4c7 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -472,6 +472,30 @@ public class CassandraDaemon */ public void start() { + // We only start transports if bootstrap has completed and we're not in survey mode, OR if we are in + // survey mode and streaming has completed but we're not using auth. + // OR if we have not joined the ring yet. + if (StorageService.instance.hasJoined()) + { + if (StorageService.instance.isSurveyMode()) + { + if (StorageService.instance.isBootstrapMode() || DatabaseDescriptor.getAuthenticator().requireAuthentication()) + { + logger.info("Not starting client transports in write_survey mode as it's bootstrapping or " + + "auth is enabled"); + return; + } + } + else + { + if (!SystemKeyspace.bootstrapComplete()) + { + logger.info("Not starting client transports as bootstrap has not completed"); + return; + } + } + } + String nativeFlag = System.getProperty("cassandra.start_native_transport"); if ((nativeFlag != null && Boolean.parseBoolean(nativeFlag)) || (nativeFlag == null && DatabaseDescriptor.startNativeTransport())) { @@ -611,6 +635,29 @@ public class CassandraDaemon public void startNativeTransport() { + // We only start transports if bootstrap has completed and we're not in survey mode, OR if we are in + // survey mode and streaming has completed but we're not using auth. + // OR if we have not joined the ring yet. + if (StorageService.instance.hasJoined()) + { + if (StorageService.instance.isSurveyMode()) + { + if (StorageService.instance.isBootstrapMode() || DatabaseDescriptor.getAuthenticator().requireAuthentication()) + { + throw new IllegalStateException("Not starting client transports in write_survey mode as it's bootstrapping or " + + "auth is enabled"); + } + } + else + { + if (!SystemKeyspace.bootstrapComplete()) + { + throw new IllegalStateException("Node is not yet bootstrapped completely. Use nodetool to check bootstrap" + + " state and resume. For more, see `nodetool help bootstrap`"); + } + } + } + if (nativeTransportService == null) throw new IllegalStateException("setup() must be called first for CassandraDaemon"); else diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 5f76f7dba9..55bc9ae23d 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -175,7 +175,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE private volatile boolean isBootstrapMode; /* we bootstrap but do NOT join the ring unless told to do so */ - private boolean isSurveyMode = Boolean.parseBoolean(System.getProperty("cassandra.write_survey", "false")); + private boolean isSurveyMode = Boolean.parseBoolean(System.getProperty + ("cassandra.write_survey", "false")); + /* true if node is rebuilding and receiving data */ private final AtomicBoolean isRebuilding = new AtomicBoolean(); @@ -211,6 +213,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE private final StreamStateStore streamStateStore = new StreamStateStore(); + public boolean isSurveyMode() + { + return isSurveyMode; + } + + public boolean hasJoined() + { + return joined; + } + /** This method updates the local token on disk */ public void setTokens(Collection tokens) { @@ -352,6 +364,29 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { throw new IllegalStateException("No configured daemon"); } + + // We only start transports if bootstrap has completed and we're not in survey mode, OR if we are in + // survey mode and streaming has completed but we're not using auth. + // OR if we have not joined the ring yet. + if (StorageService.instance.hasJoined()) + { + if (StorageService.instance.isSurveyMode()) + { + if (StorageService.instance.isBootstrapMode() || DatabaseDescriptor.getAuthenticator().requireAuthentication()) + { + throw new IllegalStateException("Not starting RPC server in write_survey mode as it's bootstrapping or " + + "auth is enabled"); + } + } + else + { + if (!SystemKeyspace.bootstrapComplete()) + { + throw new IllegalStateException("Node is not yet bootstrapped completely. Use nodetool to check bootstrap state and resume. For more, see `nodetool help bootstrap`"); + } + } + } + daemon.thriftServer.start(); } @@ -946,7 +981,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } else { - logger.info("Startup complete, but write survey mode is active, not becoming an active ring member. Use JMX (StorageService->joinRing()) to finalize ring joining."); + if (dataAvailable) + logger.info("Startup complete, but write survey mode is active, not becoming an active ring member. Use JMX (StorageService->joinRing()) to finalize ring joining."); + else + logger.warn("Some data streaming failed. Use nodetool to check bootstrap state and resume. For more, see `nodetool help bootstrap`. {}", SystemKeyspace.getBootstrapState()); } } @@ -980,9 +1018,24 @@ public class StorageService extends NotificationBroadcasterSupport implements IE } else if (isSurveyMode) { - isSurveyMode = false; - logger.info("Leaving write survey mode and joining ring at operator request"); - finishJoiningRing(SystemKeyspace.getSavedTokens()); + // if isSurveyMode is on then verify isBootstrapMode + // node can join the ring even if isBootstrapMode is true which should not happen + if (!isBootstrapMode()) + { + isSurveyMode = false; + logger.info("Leaving write survey mode and joining ring at operator request"); + finishJoiningRing(SystemKeyspace.getSavedTokens()); + daemon.start(); + } + else + { + logger.warn("Can't join the ring because in write_survey mode and bootstrap hasn't completed"); + } + } + else if (isBootstrapMode()) + { + // bootstrap is not complete hence node cannot join the ring + logger.warn("Can't join the ring because bootstrap hasn't completed."); } } @@ -1285,19 +1338,18 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public void onSuccess(StreamState streamState) { bootstrapFinished(); - // start participating in the ring. - // pretend we are in survey mode so we can use joinRing() here - isSurveyMode = true; - try + if (isSurveyMode) { - progressSupport.progress("bootstrap", ProgressEvent.createNotification("Joining ring...")); - joinRing(); + logger.info("Startup complete, but write survey mode is active, not becoming an active ring member. Use JMX (StorageService->joinRing()) to finalize ring joining."); } - catch (IOException ignore) + else { - // joinRing with survey mode does not throw IOException + isSurveyMode = false; + progressSupport.progress("bootstrap", ProgressEvent.createNotification("Joining ring...")); + finishJoiningRing(bootstrapTokens); } progressSupport.progress("bootstrap", new ProgressEvent(ProgressEventType.COMPLETE, 1, 1, "Resume bootstrap complete")); + daemon.start(); logger.info("Resume complete"); } diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 10d47f716f..835da17f80 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -495,6 +495,13 @@ public interface StorageServiceMBean extends NotificationEmitter public boolean isDrained(); public boolean isDraining(); + /** Check if currently bootstrapping. + * Note this becomes false before {@link org.apache.cassandra.db.SystemKeyspace#bootstrapComplete()} is called, + * as setting bootstrap to complete is called only when the node joins the ring. + * @return True prior to bootstrap streaming completing. False prior to start of bootstrap and post streaming. + */ + public boolean isBootstrapMode(); + public void setStreamThroughputMbPerSec(int value); public int getStreamThroughputMbPerSec(); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 0d3c0789d3..7ed40bbbcb 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -623,6 +623,11 @@ public class NodeProbe implements AutoCloseable return ssProxy.isDraining(); } + public boolean isBootstrapMode() + { + return ssProxy.isBootstrapMode(); + } + public void joinRing() throws IOException { ssProxy.joinRing(); diff --git a/src/java/org/apache/cassandra/tools/nodetool/Join.java b/src/java/org/apache/cassandra/tools/nodetool/Join.java index 58155911ea..a4a7cad5d0 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Join.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Join.java @@ -32,6 +32,7 @@ public class Join extends NodeToolCmd public void execute(NodeProbe probe) { checkState(!probe.isJoined(), "This node has already joined the ring."); + checkState(!probe.isBootstrapMode(), "Cannot join the ring until bootstrap completes"); try {