diff --git a/CHANGES.txt b/CHANGES.txt index 587ca0e973..c3dfe1b04b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ * Allow nodetool garbagecollect to take a user defined list of SSTables (CASSANDRA-16767) * Add a guardrail for misprepared statements (CASSANDRA-21139) Merged from 6.0: + * Fix operationMode reporting DECOMMISSION_FAILED instead of LEAVING when resuming a failed decommission (CASSANDRA-21493) * Avoid megamorphic calls when serializing and deserializing fixed-length values (CASSANDRA-21536) * Avoid megamorphic calls for Cell.timestamp/ttl/path/localDeletionTimeAsUnsignedInt methods (CASSANDRA-21526) * Reduce allocations in DefaultQueryOptions (CASSANDRA-21467) diff --git a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java index 313700aaaf..82a1059b6e 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java +++ b/src/java/org/apache/cassandra/tcm/sequences/SingleNodeSequences.java @@ -92,6 +92,7 @@ public interface SingleNodeSequences else if (InProgressSequences.isLeave(inProgress)) { logger.info("Resuming decommission @ {} (current epoch = {}): {}", inProgress.latestModification, metadata.epoch, inProgress.status()); + StorageService.instance.clearTransientMode(); } else { diff --git a/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java index ad0120513b..41f8274ae5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/DecommissionTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.distributed.test; import java.io.IOException; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import net.bytebuddy.ByteBuddy; import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; @@ -141,6 +142,7 @@ public class DecommissionTest extends TestBaseImpl public static class BB { + static final AtomicBoolean injected = new AtomicBoolean(false); public static void install(ClassLoader classLoader, Integer num) { if (num == 2) @@ -157,7 +159,7 @@ public class DecommissionTest extends TestBaseImpl public static void execute(NodeId leaving, PlacementDeltas startLeave, PlacementDeltas midLeave, PlacementDeltas finishLeave, @SuperCall Callable zuper) throws ExecutionException, InterruptedException { - if (!StorageService.instance.isDecommissionFailed()) + if (injected.compareAndSet(false, true)) throw new ExecutionException(new RuntimeException("simulated error in prepareUnbootstrapStreaming")); try diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java index 307956828e..f6ba5c33a1 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ring/DecommissionTest.java @@ -53,8 +53,10 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.transformations.PrepareLeave; import org.apache.cassandra.tcm.transformations.Startup; import org.apache.cassandra.utils.CassandraVersion; import org.apache.cassandra.utils.FBUtilities; @@ -62,6 +64,8 @@ import org.apache.cassandra.utils.FBUtilities; import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.distributed.shared.ClusterUtils.pauseBeforeCommit; +import static org.apache.cassandra.distributed.shared.ClusterUtils.unpauseCommits; import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack; import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology; import static org.apache.cassandra.distributed.test.ring.BootstrapTest.populate; @@ -84,6 +88,43 @@ public class DecommissionTest extends TestBaseImpl } } + @Test + public void testOperationModeOnDecomResume() throws Exception + { + // Node 2's first decomission attempt fails mid-stream (injected by BB), leaving it in + // DECOMISSION_FAILED. On resume, operationMode() must transition back to LEAVING before + // MID_LEAVE is committed. We pause the CMS just before that commit to assert the mode + // in the window where streaming has finished but the epoch has not yet advanced. + try (Cluster cluster = builder().withNodes(3) + .withConfig(config -> config.with(NETWORK, GOSSIP)) + .withInstanceInitializer(BB::install) + .start()) + { + populate(cluster, 0, 100, 1, 2, ConsistencyLevel.QUORUM); + + IInvokableInstance cmsNode = cluster.get(1); + IInvokableInstance leavingNode = cluster.get(2); + + // --force required: cie_internal keyspace has RF=3, decommission would fail replication check otherwise + leavingNode.nodetoolResult("decommission", "--force").asserts().failure(); + leavingNode.runOnInstance(() -> assertEquals(StorageService.Mode.DECOMMISSION_FAILED, StorageService.instance.operationMode())); + + Callable midLeavePaused = pauseBeforeCommit(cmsNode, e -> e instanceof PrepareLeave.MidLeave); + + Thread resumeThread = new Thread(() -> leavingNode.nodetoolResult("decommission").asserts().success()); + resumeThread.start(); + midLeavePaused.call(); + + leavingNode.runOnInstance(() -> + assertEquals("operationMode during resumed decommission streaming should be LEAVING, not DECOMMISSION_FAILED", + StorageService.Mode.LEAVING, + StorageService.instance.operationMode())); + + unpauseCommits(cmsNode); + resumeThread.join(TimeUnit.MINUTES.toMillis(2)); + } + } + @Test public void testAddressReuseAfterDecommission() throws IOException, ExecutionException, InterruptedException {