From e1e56e5d5dc3e47ea1a1d3f19161f930a93e8b3d Mon Sep 17 00:00:00 2001 From: Sam Tunnicliffe Date: Mon, 6 Jul 2026 11:52:18 +0100 Subject: [PATCH 1/6] Add CMS membership directly to ClusterMetadata Patch by Sam Tunnicliffe; reviewed by Marcus Eriksson for CASSANDRA-20736 --- CHANGES.txt | 1 + .../schema/CreateKeyspaceStatement.java | 2 +- .../db/AbstractMutationVerbHandler.java | 2 +- .../cassandra/db/ReadCommandVerbHandler.java | 4 +- .../db/compaction/CompactionManager.java | 2 +- .../apache/cassandra/db/view/ViewUtils.java | 6 +- .../org/apache/cassandra/gms/Gossiper.java | 9 +- .../cassandra/hints/HintsDispatcher.java | 2 +- .../locator/CMSPlacementStrategy.java | 3 +- .../cassandra/locator/EndpointsForToken.java | 2 +- .../cassandra/locator/MetaStrategy.java | 4 +- .../cassandra/locator/ReplicaLayout.java | 14 +- .../cassandra/locator/ReplicaPlans.java | 2 +- .../apache/cassandra/metrics/TCMMetrics.java | 3 +- .../cassandra/net/ResponseVerbHandler.java | 3 +- .../cassandra/repair/RepairCoordinator.java | 3 +- .../service/ActiveRepairService.java | 2 +- .../cassandra/service/CassandraDaemon.java | 2 +- .../org/apache/cassandra/service/Rebuild.java | 12 +- .../cassandra/service/StorageProxy.java | 3 +- .../cassandra/service/StorageService.java | 26 +- .../accord/topology/AccordTopology.java | 2 +- .../apache/cassandra/service/paxos/Paxos.java | 2 +- .../cassandra/service/paxos/PaxosRepair.java | 2 +- .../streaming/DataMovementVerbHandler.java | 2 +- .../cassandra/tcm/AbstractLocalProcessor.java | 4 +- .../tcm/AtomicLongBackedProcessor.java | 12 + .../apache/cassandra/tcm/CMSMembership.java | 257 +++++++++++++ .../apache/cassandra/tcm/CMSOperations.java | 3 +- .../apache/cassandra/tcm/ClusterMetadata.java | 354 ++++++++++++++---- .../cassandra/tcm/ClusterMetadataService.java | 63 ++-- .../apache/cassandra/tcm/MetadataKeys.java | 4 +- .../cassandra/tcm/PaxosBackedProcessor.java | 6 + .../org/apache/cassandra/tcm/Startup.java | 36 +- .../tcm/StubClusterMetadataService.java | 3 +- .../tcm/compatibility/GossipHelper.java | 10 +- .../tcm/listeners/LegacyStateListener.java | 4 +- .../listeners/PlacementsChangeListener.java | 24 +- .../listeners/UpgradeMigrationListener.java | 20 +- .../apache/cassandra/tcm/log/LogState.java | 2 +- .../cassandra/tcm/membership/Directory.java | 2 +- .../cassandra/tcm/membership/NodeId.java | 1 + .../cassandra/tcm/membership/NodeVersion.java | 2 +- .../cassandra/tcm/migration/Election.java | 5 - .../tcm/ownership/DataPlacement.java | 5 + .../tcm/ownership/DataPlacements.java | 29 +- .../tcm/ownership/UniformRangePlacement.java | 8 +- .../tcm/sequences/BootstrapAndJoin.java | 4 +- .../tcm/sequences/BootstrapAndReplace.java | 8 +- .../sequences/CancelCMSReconfiguration.java | 68 ++-- .../apache/cassandra/tcm/sequences/Move.java | 4 +- .../tcm/sequences/ProgressBarrier.java | 6 +- .../tcm/sequences/RemoveNodeStreams.java | 2 +- .../tcm/sequences/ReplaceSameAddress.java | 2 +- .../tcm/sequences/UnbootstrapAndLeave.java | 2 +- .../cassandra/tcm/serialization/Version.java | 14 +- .../tcm/transformations/AccordMarkStale.java | 2 +- .../tcm/transformations/AlterSchema.java | 2 +- .../tcm/transformations/AlterTopology.java | 4 +- .../transformations/ApplyPlacementDeltas.java | 2 +- .../tcm/transformations/PrepareJoin.java | 6 +- .../tcm/transformations/PrepareLeave.java | 2 +- .../tcm/transformations/PrepareMove.java | 4 +- .../tcm/transformations/PrepareReplace.java | 2 +- .../tcm/transformations/Register.java | 2 +- .../tcm/transformations/Startup.java | 27 +- .../cms/AdvanceCMSReconfiguration.java | 68 ++-- .../transformations/cms/FinishAddToCMS.java | 10 +- .../transformations/cms/PreInitialize.java | 40 +- .../cms/PrepareCMSReconfiguration.java | 13 +- .../transformations/cms/RemoveFromCMS.java | 25 +- .../transformations/cms/StartAddToCMS.java | 29 +- .../cassandra/tools/CMSOfflineTool.java | 32 +- .../tools/TransformClusterMetadataHelper.java | 27 +- .../cassandra/distributed/impl/Instance.java | 2 +- .../distributed/shared/ClusterUtils.java | 8 +- .../AssassinateCMSNodeTest.java | 5 +- .../test/log/BootWithMetadataTest.java | 16 +- .../test/log/ClusterMetadataDumpTest.java | 3 +- .../test/log/ClusterMetadataTestHelper.java | 36 +- .../log/MetadataChangeSimulationTest.java | 12 +- .../test/log/OperationalEquivalenceTest.java | 3 +- .../test/log/ReconfigureCMSTest.java | 18 +- .../distributed/test/log/RegisterTest.java | 156 +++++--- .../test/log/ResumableStartupTest.java | 4 +- .../test/log/SimulatedOperation.java | 4 +- .../distributed/test/log/SnapshotTest.java | 2 +- .../test/ring/RangeVersioningTest.java | 2 +- .../tcm/CMSPlacementAfterReplacementTest.java | 3 +- ...ClusterMetadataUpgradeAssassinateTest.java | 2 +- ...rMetadataUpgradeDelayedInitializeTest.java | 8 +- .../fuzz/topology/TopologyMixupTestBase.java | 2 +- .../simulator/cluster/OnClusterReplace.java | 4 +- .../cassandra/auth/GrantAndRevokeTest.java | 8 + .../cassandra/dht/BootStrapperTest.java | 3 +- .../cassandra/locator/MetaStrategyTest.java | 4 +- .../cassandra/locator/SimpleStrategyTest.java | 2 +- .../service/accord/AccordTopologyUtils.java | 2 +- .../service/accord/EpochSyncTest.java | 6 +- .../cassandra/tcm/BootWithMetadataTest.java | 8 +- .../ClusterMetadataTransformationTest.java | 43 ++- .../apache/cassandra/tcm/GetLogStateTest.java | 17 +- .../apache/cassandra/tcm/UnregisterTest.java | 2 +- .../tcm/compatibility/GossipHelperTest.java | 6 +- .../MetadataSnapshotListenerTest.java | 2 +- .../PlacementsChangeListenerTest.java | 179 +++++++++ .../tcm/log/DistributedLogStateTest.java | 5 +- .../tcm/membership/MembershipUtils.java | 6 +- .../ownership/LocalRangesAllSettledTest.java | 12 +- .../tcm/ownership/OwnershipUtils.java | 2 +- .../InProgressSequenceCancellationTest.java | 10 +- .../tcm/sequences/ProgressBarrierTest.java | 12 +- .../transformations/EventsMetadataTest.java | 14 +- .../tcm/transformations/PrepareLeaveTest.java | 15 +- .../cassandra/tools/CMSOfflineToolTest.java | 23 +- .../cassandra/utils/CassandraGenerators.java | 4 +- 116 files changed, 1398 insertions(+), 653 deletions(-) create mode 100644 src/java/org/apache/cassandra/tcm/CMSMembership.java create mode 100644 test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 1a62d6acf9..dcd7972231 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 6.0-alpha2 + * Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736) * Fix maven remote publishing of Accord artifacts (CASSANDRA-21261) * Move long running TCM operations to a longer timout (CASSANDRA-21453) * Offline nodetool commands should not print network options in help (CASSANDRA-20876) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java index a622b86192..eac58187b1 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java @@ -96,7 +96,7 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement // as we have as keys in metadata.placements to have a fast map lookup // ReplicationParams are immutable, so it is a safe optimization KeyspaceParams keyspaceParams = attrs.asNewKeyspaceParams(); - ReplicationParams replicationParams = metadata.placements.deduplicateReplicationParams(keyspaceParams.replication); + ReplicationParams replicationParams = metadata.placements().deduplicateReplicationParams(keyspaceParams.replication); keyspaceParams = keyspaceParams.withSwapped(replicationParams); KeyspaceMetadata keyspaceMetadata = KeyspaceMetadata.create(keyspaceName, keyspaceParams); diff --git a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java index 5d2e4e3cdc..18aa78552b 100644 --- a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java @@ -195,6 +195,6 @@ public abstract class AbstractMutationVerbHandler implement private static VersionedEndpoints.ForToken writePlacements(ClusterMetadata metadata, String keyspace, DecoratedKey key) { - return metadata.placements.get(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken()); + return metadata.placement(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken()); } } diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index b418ecdfeb..caac99cac1 100644 --- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -240,8 +240,8 @@ public class ReadCommandVerbHandler implements IVerbHandler private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace) { - return metadata.placements - .get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication) + return metadata + .placement(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication) .reads .forToken(token) .get() diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 84c1193517..e7ae548ffd 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -780,7 +780,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan // we only consider write placements during cleanup as range movements always ensure // overlap between new replicas accepting reads and old replicas accepting writes ClusterMetadata cm = ClusterMetadata.current(); - DataPlacement placement = cm.placements.get(keyspace.getMetadata().params.replication); + DataPlacement placement = cm.placement(keyspace.getMetadata().params.replication); InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); RangesAtEndpoint localWrites = placement.writes.byEndpoint().get(local); // TODO review: Hack to get local partitioner not to fail out because it's handled very poorly with data placements diff --git a/src/java/org/apache/cassandra/db/view/ViewUtils.java b/src/java/org/apache/cassandra/db/view/ViewUtils.java index 1bab97b312..ee27ace329 100644 --- a/src/java/org/apache/cassandra/db/view/ViewUtils.java +++ b/src/java/org/apache/cassandra/db/view/ViewUtils.java @@ -30,6 +30,7 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.ownership.DataPlacement; public final class ViewUtils { @@ -64,8 +65,9 @@ public final class ViewUtils Location local = metadata.locator.local(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); - EndpointsForToken naturalBaseReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(baseToken).get(); - EndpointsForToken naturalViewReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(viewToken).get(); + DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication); + EndpointsForToken naturalBaseReplicas = placement.reads.forToken(baseToken).get(); + EndpointsForToken naturalViewReplicas = placement.reads.forToken(viewToken).get(); Optional localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isSelf).toJavaUtil(); if (localReplica.isPresent()) diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index 3d6414913f..48ef04fa07 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -1764,7 +1764,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, register(DatabaseDescriptor.getLocalAddressReconnectionHelper()); ClusterMetadata metadata = ClusterMetadata.current(); - if (mergeLocalStates && metadata.myNodeId() != null) + if (mergeLocalStates && metadata.myNodeId() != NodeId.UNREGISTERED) mergeNodeToGossip(metadata.myNodeId(), metadata); shutdownAnnounced.set(false); @@ -2234,7 +2234,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, { checkProperThreadForStateMutation(); assert !endpoint.equals(getBroadcastAddressAndPort()) || epstate.getHeartBeatState().getGeneration() > 0 : - "We should not update epstates with generation = 0 for the local host"; + String.format("We should not update epstates with generation = 0 for the local host " + + "(endpoint: %s, broadcast: %s, generation: %s)", + endpoint, getBroadcastAddressAndPort(), epstate.getHeartBeatState().getGeneration()); EndpointState old = endpointStateMap.get(endpoint); if (old == null) endpointStateMap.put(endpoint, epstate); @@ -2278,6 +2280,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, taskLock.lock(); try { + if (nodeId == NodeId.UNREGISTERED) + return; + boolean isLocal = nodeId.equals(metadata.myNodeId()); IPartitioner partitioner = metadata.tokenMap.partitioner(); NodeAddresses addresses = metadata.directory.getNodeAddresses(nodeId); diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java index 3ba38bdeaf..025d52579b 100644 --- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java +++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java @@ -276,7 +276,7 @@ final class HintsDispatcher implements AutoCloseable // Also may need to apply locally because it's possible this is from the batchlog // and we never applied it locally // TODO (review): Additional error handling necessary? Hints are lossy - DataPlacement dataPlacement = cm.placements.get(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication); + DataPlacement dataPlacement = cm.placement(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication); VersionedEndpoints.ForToken forToken = dataPlacement.writes.forToken(mutation.key().getToken()); Replica self = forToken.get().selfIfPresent(); if (self != null) diff --git a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java index e00289aef6..cb4dfa82c6 100644 --- a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java +++ b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java @@ -90,7 +90,8 @@ public class CMSPlacementStrategy // Although MetaStrategy has its own entireRange, it uses a custom partitioner which isn't compatible with // regular, non-CMS placements. For that reason, we select replicas here using tokens provided by the - // globally configured partitioner. + // globally configured partitioner. This also has the benefit of making concurrent operations, such as + // bounces/upgrades/etc, safe for the CMS if they are replica aware. Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken(); EndpointsForRange endpoints = NetworkTopologyStrategy.calculateNaturalReplicas(minToken, new Range<>(minToken, minToken), diff --git a/src/java/org/apache/cassandra/locator/EndpointsForToken.java b/src/java/org/apache/cassandra/locator/EndpointsForToken.java index fc88fa761b..7342816015 100644 --- a/src/java/org/apache/cassandra/locator/EndpointsForToken.java +++ b/src/java/org/apache/cassandra/locator/EndpointsForToken.java @@ -164,7 +164,7 @@ public class EndpointsForToken extends Endpoints public static VersionedEndpoints.ForToken natural(Keyspace keyspace, Token token) { - return ClusterMetadata.current().placements.get(keyspace.getMetadata().params.replication).reads.forToken(token); + return ClusterMetadata.current().placement(keyspace.getMetadata().params.replication).reads.forToken(token); } } diff --git a/src/java/org/apache/cassandra/locator/MetaStrategy.java b/src/java/org/apache/cassandra/locator/MetaStrategy.java index a7afa4898c..fdebff72b1 100644 --- a/src/java/org/apache/cassandra/locator/MetaStrategy.java +++ b/src/java/org/apache/cassandra/locator/MetaStrategy.java @@ -82,13 +82,13 @@ public class MetaStrategy extends SystemStrategy @Override public EndpointsForRange calculateNaturalReplicas(Token token, ClusterMetadata metadata) { - return metadata.placements.get(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get(); + return metadata.placement(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get(); } @Override public DataPlacement calculateDataPlacement(Epoch epoch, List> ranges, ClusterMetadata metadata) { - return metadata.placements.get(ReplicationParams.meta(metadata)); + return metadata.placement(ReplicationParams.meta(metadata)); } @Override diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java index 440b2bed73..14915fc518 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaLayout.java +++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java @@ -239,7 +239,7 @@ public abstract class ReplicaLayout> { // todo deduplicate so that "pending" contains "read - write", // which is a hack until we revisit how consistency level handles pending - DataPlacement dataPlacement = metadata.placements.get(ks.params.replication); + DataPlacement dataPlacement = metadata.placement(ks.params.replication); natural = forNonLocalStrategyTokenRead(dataPlacement, token); // perf optimization to avoid double endpoints search and filtering for a typical case // DataPlacement constructor does a deduplication of reads/writes, so we can use cheap == comparision here @@ -394,12 +394,12 @@ public abstract class ReplicaLayout> static EndpointsForRange forNonLocalStategyRangeRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, AbstractBounds range) { - return metadata.placements.get(keyspace.params.replication).reads.forRange(range.right.getToken()).get(); + return metadata.placement(keyspace.params.replication).reads.forRange(range.right.getToken()).get(); } public static EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) { - return forNonLocalStrategyTokenRead(metadata.placements.get(keyspace.params.replication), token); + return forNonLocalStrategyTokenRead(metadata.placement(keyspace.params.replication), token); } public static EndpointsForToken forNonLocalStrategyTokenRead(DataPlacement dataPlacement, Token token) @@ -407,17 +407,11 @@ public abstract class ReplicaLayout> return dataPlacement.reads.forToken(token).get(); } - static EndpointsForToken forNonLocalStrategyTokenWrite(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) - { - return forNonLocalStrategyTokenWrite(metadata.placements.get(keyspace.params.replication), token); - } - - static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token) + private static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token) { return dataPlacement.writes.forToken(token).get(); } - static EndpointsForRange forLocalStrategyRange(ClusterMetadata metadata, AbstractReplicationStrategy replicationStrategy, AbstractBounds range) { return replicationStrategy.calculateNaturalReplicas(range.right.getToken(), metadata); diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java index 61def76d1a..366708248e 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java @@ -234,7 +234,7 @@ public class ReplicaPlans NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); - EndpointsForToken replicas = metadata.placements.get(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get(); + EndpointsForToken replicas = metadata.placement(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get(); // CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint())); diff --git a/src/java/org/apache/cassandra/metrics/TCMMetrics.java b/src/java/org/apache/cassandra/metrics/TCMMetrics.java index 9544c2215a..8b74217bbb 100644 --- a/src/java/org/apache/cassandra/metrics/TCMMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TCMMetrics.java @@ -29,7 +29,6 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.EpochAwareDebounce; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; import static org.apache.cassandra.tcm.transformations.cms.PrepareCMSReconfiguration.needsReconfiguration; @@ -94,7 +93,7 @@ public class TCMMetrics isCMSMember = Metrics.register(factory.createMetricName("IsCMSMember"), () -> { ClusterMetadata metadata = ClusterMetadata.currentNullable(); - return metadata != null && metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) ? 1 : 0; + return metadata != null && metadata.isCMSMember() ? 1 : 0; }); needsCMSReconfiguration = Metrics.register(factory.createMetricName("NeedsCMSReconfiguration"), () -> { diff --git a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java index fb36c67c66..215b575c10 100644 --- a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java +++ b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java @@ -30,7 +30,6 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.FBUtilities; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND; @@ -98,7 +97,7 @@ class ResponseVerbHandler implements IVerbHandler if (SKIP_CATCHUP_FOR.contains(message.verb())) return; - if (metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) && CMS_SKIP_CATCHUP_FOR.contains(message.verb())) + if (metadata.isCMSMember() && CMS_SKIP_CATCHUP_FOR.contains(message.verb())) return; // Gossip stage is single-threaded, so we may end up in a deadlock with after-commit hook diff --git a/src/java/org/apache/cassandra/repair/RepairCoordinator.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java index 3ba51e4df6..8aa3482b5e 100644 --- a/src/java/org/apache/cassandra/repair/RepairCoordinator.java +++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java @@ -81,7 +81,6 @@ import org.apache.cassandra.tracing.TraceState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.transport.messages.ResultMessage; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; @@ -402,7 +401,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai //calculation multiple times Iterable> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges(); boolean isMeta = Keyspace.open(state.keyspace).getMetadata().params.replication.isMeta(); - boolean isCMS = ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()); + boolean isCMS = ClusterMetadata.current().isCMSMember(); for (Range range : state.options.getRanges()) { EndpointsForRange allForRange = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range); diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index 286e8e86fc..bf187247e7 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -1213,7 +1213,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai // are based on the system partitioner EndpointsForRange endpoints = replication.isMeta() ? ClusterMetadata.current().fullCMSMembersAsReplicas() - : ClusterMetadata.current().placements.get(replication).reads.forRange(range).get(); + : ClusterMetadata.current().placement(replication).reads.forRange(range).get(); Set liveEndpoints = endpoints.filter(FailureDetector.isReplicaAlive).endpoints(); if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, liveEndpoints)) diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index a69124bb7d..7cf1494bb1 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -360,7 +360,7 @@ public class CassandraDaemon { CommitLog.instance.recoverSegmentsOnDisk(); NodeId self = ClusterMetadata.current().myNodeId(); - if (self != null) + if (self != NodeId.UNREGISTERED) AccordService.localStartup(self); } catch (IOException e) diff --git a/src/java/org/apache/cassandra/service/Rebuild.java b/src/java/org/apache/cassandra/service/Rebuild.java index 7c28960b96..27d814c177 100644 --- a/src/java/org/apache/cassandra/service/Rebuild.java +++ b/src/java/org/apache/cassandra/service/Rebuild.java @@ -52,12 +52,12 @@ import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.MovementMap; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; public class Rebuild @@ -232,21 +232,23 @@ public class Rebuild private static MovementMap movementMap(ClusterMetadata metadata, String keyspace, String tokens) { MovementMap.Builder movementMapBuilder = MovementMap.builder(); - DataPlacements placements = metadata.placements; if (keyspace == null) { - placements.forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null))); + metadata.placements().forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null))); + // Special case to include the system metadata keyspace + ReplicationParams metaParams = Keyspace.open(METADATA_KEYSPACE_NAME).getMetadata().params.replication; + movementMapBuilder.put(metaParams, addMovementsForParams(metadata.placement(metaParams), null)); } else if (tokens == null) { ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication; - movementMapBuilder.put(params, addMovementsForParams(placements.get(params), null)); + movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), null)); } else { ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication; RangesAtEndpoint ranges = rangesForRebuildWithTokens(tokens, keyspace); - movementMapBuilder.put(params, addMovementsForParams(placements.get(params), ranges)); + movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), ranges)); } return movementMapBuilder.build(); } diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 9e61567cb4..d10f8bb36b 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -175,6 +175,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.range.RangeCommands; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.ownership.VersionedEndpoints; import org.apache.cassandra.tracing.Tracing; @@ -2197,7 +2198,7 @@ public class StorageProxy implements StorageProxyMBean if (metadata == null) return false; - if (metadata.myNodeId() == null) + if (metadata.myNodeId() == NodeId.UNREGISTERED) return false; return metadata.myNodeState() == NodeState.JOINED; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 53f0986e0c..98f5d11ff8 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -2057,9 +2057,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE // creation and initialization of cluster metadata service. Metadata collector does accept // null localhost ID values, it's just that TokenMetadata was created earlier. ClusterMetadata metadata = ClusterMetadata.currentNullable(); - if (metadata == null || metadata.directory.peerId(getBroadcastAddressAndPort()) == null) - return null; - return metadata.directory.peerId(getBroadcastAddressAndPort()).toUUID(); + if (metadata == null || metadata.myNodeId() == NodeId.UNREGISTERED) + { + // this condition is to prevent accessing the tables when the node is not started yet, and in particular, + // when it is not going to be started at all (e.g. when running some unit tests or client tools). + if ((DatabaseDescriptor.isDaemonInitialized() || DatabaseDescriptor.isToolInitialized()) && CommitLog.instance.isStarted()) + return SystemKeyspace.getLocalHostId(); + else + return null; + } + return metadata.myNodeId().toUUID(); } public Map getHostIdMap() @@ -2067,7 +2074,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return getEndpointToHostId(); } - public Map getEndpointToHostId() { return getEndpointToHostId(false); @@ -2119,7 +2125,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { if (keyspaceMetadata.params.replication.isMeta()) { - DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication); + DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication); // May be empty if mid-upgrade and CMS is not yet initialized if (!placement.reads.isEmpty()) rangeToEndpointMap.put(MetaStrategy.entireRange, placement.reads.forRange(MetaStrategy.entireRange).get()); @@ -2129,8 +2135,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE TokenMap tokenMap = metadata.tokenMap; for (Range range : ranges) { - Token token = tokenMap.nextToken(tokenMap.tokens(), range.right.getToken()); - rangeToEndpointMap.put(range, metadata.placements.get(keyspaceMetadata.params.replication) + Token token = TokenMap.nextToken(tokenMap.tokens(), range.right.getToken()); + rangeToEndpointMap.put(range, metadata.placement(keyspaceMetadata.params.replication) .reads.forRange(token).get()); } } @@ -3447,14 +3453,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE token = MetaStrategy.partitioner.getToken(key); else token = metadata.partitioner.getToken(key); - return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(token).get(); + return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(token).get(); } public boolean isEndpointValidForWrite(String keyspace, Token token) { ClusterMetadata metadata = ClusterMetadata.current(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); - return keyspaceMetadata != null && metadata.placements.get(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf(); + return keyspaceMetadata != null && metadata.placement(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf(); } public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception @@ -4262,7 +4268,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (replicationParams.isMeta()) { LinkedHashMap ownership = Maps.newLinkedHashMap(); - metadata.placements.get(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> { + metadata.placement(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> { ownership.put(r.endpoint(), 1.0f); }); return ownership; diff --git a/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java b/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java index c5298c0164..9c713abcbb 100644 --- a/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java +++ b/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java @@ -312,7 +312,7 @@ public class AccordTopology public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup) { - return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas); + return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements(), metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas); } public static Topology createAccordTopology(ClusterMetadata metadata, Topology current) diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java index a40900f73c..da0d2b4e0d 100644 --- a/src/java/org/apache/cassandra/service/paxos/Paxos.java +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java @@ -277,7 +277,7 @@ public class Paxos final Token token = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : key.getToken(); ClusterMetadata metadata = ClusterMetadata.current(); Keyspace keyspace = Keyspace.open(table.keyspace); - DataPlacement placement = metadata.placements.get(keyspace.getMetadata().params.replication); + DataPlacement placement = metadata.placement(keyspace.getMetadata().params.replication); Epoch epoch = placement.writes.forToken(token).lastModified(); ForTokenWrite electorate = forTokenWriteLiveAndDown(metadata, keyspace, token); if (consistency == LOCAL_SERIAL) diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java index 5948086394..33189404e6 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java @@ -586,7 +586,7 @@ public class PaxosRepair extends AbstractPaxosRepair ClusterMetadata metadata = ClusterMetadata.current(); Collection allEndpoints = replication.isMeta() ? metadata.fullCMSMembers() - : metadata.placements.get(replication).reads.forRange(range).endpoints(); + : metadata.placement(replication).reads.forRange(range).endpoints(); return hasSufficientLiveNodesForTopologyChange(allEndpoints, liveEndpoints, ep -> metadata.locator.location(ep).datacenter, diff --git a/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java b/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java index 1bccf29044..0bf743fef5 100644 --- a/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java +++ b/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java @@ -50,7 +50,7 @@ public class DataMovementVerbHandler implements IVerbHandler StreamPlan streamPlan = new StreamPlan(StreamOperation.fromString(message.payload.streamOperation)); ClusterMetadata metadata = ClusterMetadata.current(); Schema.instance.getNonLocalStrategyKeyspaces().stream().forEach((ksm) -> { - if (metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1) + if (metadata.placement(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1) return; message.payload.movements.get(ksm.params.replication).asMap().forEach((local, endpoints) -> { diff --git a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java index e545e726ed..a9a75ac8fb 100644 --- a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java @@ -46,6 +46,7 @@ public abstract class AbstractLocalProcessor implements Processor this.log = log; } + /** * Epoch returned by processor in the Result is _not_ guaranteed to be visible by the Follower by * the time when this method returns. @@ -59,7 +60,7 @@ public abstract class AbstractLocalProcessor implements Processor while (!retryPolicy.hasExpired()) { ClusterMetadata previous = log.waitForHighestConsecutive(); - if (!previous.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort())) + if (!acceptCommit(previous)) { String msg = String.format("Node %s is not a CMS member in epoch %s; members=%s", FBUtilities.getBroadcastAddressAndPort(), @@ -228,4 +229,5 @@ public abstract class AbstractLocalProcessor implements Processor public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy); protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch); + protected abstract boolean acceptCommit(ClusterMetadata metadata); } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java index bbbe4fa806..fd0c92c181 100644 --- a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java @@ -59,6 +59,18 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor this.epochHolder = new AtomicLong(epoch.getEpoch()); } + @Override + public boolean acceptCommit(ClusterMetadata metadata) + { + // AtomicLongBackedProcessor is only for use in tests and offline tools and it should be safe to assume that it + // is always allowed to process commit requests. In a non-test setup, when the CMS is initialized the initiating + // node is also registered. This is not the case in tests (see initCMS/recreateCMS in ServerTestUtils for more + // detail), as this simplifies setting up/tearing down topologies in test cases. The processor intended for use + // in real clusters (PaxosBackedProcessor) performs some actual validation here to ensure that the node is + // really a CMS member. + return true; + } + @Override protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch) { diff --git a/src/java/org/apache/cassandra/tcm/CMSMembership.java b/src/java/org/apache/cassandra/tcm/CMSMembership.java new file mode 100644 index 0000000000..67c30dc860 --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/CMSMembership.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.MetaStrategy; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.tcm.membership.Directory; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.ownership.DataPlacement; +import org.apache.cassandra.tcm.ownership.VersionedEndpoints; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.utils.btree.BTreeSet; + +public class CMSMembership implements MetadataValue +{ + public static final Serializer serializer = new Serializer(); + public static final CMSMembership EMPTY = new CMSMembership(); + + private final Epoch lastModified; + private final BTreeSet fullMembers; + private final BTreeSet joiningMembers; + + /** + * Used to derive a CMSMembership when deserializing a ClusterMetadata instance written with a metadata version + * prior to V9. At that time, CMS membership was always inferred from the data placements of the distributed + * cluster metadata keyspace. Read replicas are full members of the CMS and write-only replicas are in the process + * of joining. Note: every read replica must also be a write replica, leaving the CMS is atomic in respect of the + * placements. + * @param placement + * @param directory + * @return + */ + public static CMSMembership reconstruct(DataPlacement placement, Directory directory) + { + BTreeSet.Builder fullMembersBuilder = BTreeSet.builder(NodeId::compareTo); + BTreeSet.Builder joiningMembersBuilder = BTreeSet.builder(NodeId::compareTo); + Epoch lm = Epoch.EMPTY; + for (VersionedEndpoints.ForRange endpoints : placement.reads.endpoints) + { + lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm; + endpoints.get().endpoints().forEach(e -> fullMembersBuilder.add(directory.peerId(e))); + } + BTreeSet full = fullMembersBuilder.build(); + + for (VersionedEndpoints.ForRange endpoints : placement.writes.endpoints) + { + lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm; + endpoints.get().endpoints().forEach(e -> { + NodeId id = directory.peerId(e); + if (!full.contains(id)) + joiningMembersBuilder.add(id); + }); + } + BTreeSet joining = joiningMembersBuilder.build(); + + return new CMSMembership(lm, full, joining); + } + + public DataPlacement toPlacement(Directory directory) + { + DataPlacement.Builder builder = DataPlacement.builder(); + for (NodeId id : fullMembers) + { + Replica replica = MetaStrategy.replica(directory.endpoint(id)); + builder.withReadReplica(lastModified, replica); + builder.withWriteReplica(lastModified, replica); + } + for(NodeId id : joiningMembers) + { + builder.withWriteReplica(lastModified, MetaStrategy.replica(directory.endpoint(id))); + } + return builder.build(); + } + + private CMSMembership() + { + this(Epoch.EMPTY, + BTreeSet.empty(NodeId::compareTo), + BTreeSet.empty(NodeId::compareTo)); + } + + private CMSMembership(Epoch lastModified, BTreeSet fullMembers, BTreeSet joiningMembers) + { + this.lastModified = lastModified; + this.fullMembers = fullMembers; + this.joiningMembers = joiningMembers; + } + + + @Override + public CMSMembership withLastModified(Epoch epoch) + { + return lastModified.is(epoch) ? this : new CMSMembership(epoch, fullMembers, joiningMembers); + } + + @Override + public Epoch lastModified() + { + return lastModified; + } + + public Set joiningMembers() + { + return joiningMembers; + } + + public Set fullMembers() + { + return fullMembers; + } + + public CMSMembership startJoining(NodeId id) + { + if (joiningMembers.contains(id)) + throw new IllegalStateException(id + " is already joining the CMS"); + if (fullMembers.contains(id)) + throw new IllegalStateException(id + " has already fully joined the CMS"); + + return new CMSMembership(lastModified, fullMembers, joiningMembers.with(id)); + } + + public CMSMembership cancelJoining(NodeId id) + { + if (!joiningMembers.contains(id)) + throw new IllegalStateException(id + " is not currently joining the CMS"); + if (fullMembers.contains(id)) + throw new IllegalStateException(id + " has already fully joined the CMS"); + + return new CMSMembership(lastModified, fullMembers, joiningMembers.without(id)); + } + + public CMSMembership finishJoining(NodeId id) + { + if (!joiningMembers.contains(id)) + throw new IllegalStateException(id + " is not currently joining the CMS"); + if (fullMembers.contains(id)) + throw new IllegalStateException(id + " has already fully joined the CMS"); + + return new CMSMembership(lastModified, fullMembers.with(id), joiningMembers.without(id)); + } + + public CMSMembership leave(NodeId id) + { + if (joiningMembers.contains(id)) + throw new IllegalStateException(id + " is currently joining the CMS, "); + if (!fullMembers.contains(id)) + throw new IllegalStateException(id + " is not a CMS member"); + + return new CMSMembership(lastModified, fullMembers.without(id), joiningMembers); + } + + @Override + public String toString() + { + return "CMSMembership{" + + "lastModified=" + lastModified + + ", fullMembers=" + fullMembers + + ", joiningMembers=" + joiningMembers + + '}'; + } + + @Override + public final boolean equals(Object o) + { + if (!(o instanceof CMSMembership)) return false; + + CMSMembership that = (CMSMembership) o; + return Objects.equals(lastModified, that.lastModified) && + Objects.equals(fullMembers, that.fullMembers) && + Objects.equals(joiningMembers, that.joiningMembers); + } + + @Override + public int hashCode() + { + int result = Objects.hashCode(lastModified); + result = 31 * result + Objects.hashCode(fullMembers); + result = 31 * result + Objects.hashCode(joiningMembers); + return result; + } + + public static class Serializer implements MetadataSerializer + { + @Override + public void serialize(CMSMembership t, DataOutputPlus out, Version version) throws IOException + { + Epoch.serializer.serialize(t.lastModified, out); + + out.writeUnsignedVInt32(t.fullMembers.size()); + for (NodeId id : t.fullMembers) + NodeId.serializer.serialize(id, out, version); + + out.writeUnsignedVInt32(t.joiningMembers.size()); + for (NodeId id : t.joiningMembers) + NodeId.serializer.serialize(id, out, version); + } + + @Override + public CMSMembership deserialize(DataInputPlus in, Version version) throws IOException + { + Epoch lastModified = Epoch.serializer.deserialize(in, version); + + int fullMemberCount = in.readUnsignedVInt32(); + BTreeSet.Builder fullMembers = BTreeSet.builder(NodeId::compareTo); + for (int i = 0; i < fullMemberCount; i++) + fullMembers.add(NodeId.serializer.deserialize(in, version)); + + int joiningMemberCount = in.readUnsignedVInt32(); + BTreeSet.Builder joiningMembers = BTreeSet.builder(NodeId::compareTo); + for (int i = 0; i < joiningMemberCount; i++) + joiningMembers.add(NodeId.serializer.deserialize(in, version)); + + return new CMSMembership(lastModified, fullMembers.build(), joiningMembers.build()) ; + } + + @Override + public long serializedSize(CMSMembership t, Version version) + { + long size = Epoch.serializer.serializedSize(t.lastModified); + + size += TypeSizes.sizeofUnsignedVInt(t.fullMembers.size()); + for (NodeId id : t.fullMembers) + size += NodeId.serializer.serializedSize(id, version); + + size += TypeSizes.sizeofUnsignedVInt(t.joiningMembers.size()); + for (NodeId id : t.joiningMembers) + size += NodeId.serializer.serializedSize(id, version); + + return size; + } + } + +} diff --git a/src/java/org/apache/cassandra/tcm/CMSOperations.java b/src/java/org/apache/cassandra/tcm/CMSOperations.java index 460513fbf5..60f4e67d1a 100644 --- a/src/java/org/apache/cassandra/tcm/CMSOperations.java +++ b/src/java/org/apache/cassandra/tcm/CMSOperations.java @@ -48,7 +48,6 @@ import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.tcm.transformations.Unregister; import org.apache.cassandra.tcm.transformations.cms.AdvanceCMSReconfiguration; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MBeanWrapper; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -217,7 +216,7 @@ public class CMSOperations implements CMSOperationsMBean String members = metadata.fullCMSMembers().stream().sorted().map(Object::toString).collect(Collectors.joining(",")); info.put(MEMBERS, members); info.put(NEEDS_RECONFIGURATION, Boolean.toString(metadata.epoch.isBefore(Epoch.FIRST) || needsReconfiguration(metadata))); - info.put(IS_MEMBER, Boolean.toString(cms.isCurrentMember(FBUtilities.getBroadcastAddressAndPort()))); + info.put(IS_MEMBER, Boolean.toString(metadata.isCMSMember())); info.put(SERVICE_STATE, ClusterMetadataService.state(metadata).toString()); info.put(IS_MIGRATING, Boolean.toString(cms.isMigrating())); info.put(EPOCH, Long.toString(metadata.epoch.getEpoch())); diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index be6d0bd212..0224ca5bfb 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -28,7 +28,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -77,6 +79,7 @@ import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator; import org.apache.cassandra.tcm.ownership.ReplicaGroups; @@ -89,7 +92,6 @@ import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -import static com.google.common.collect.ImmutableSet.toImmutableSet; import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR; import static org.apache.cassandra.db.TypeSizes.sizeof; import static org.apache.cassandra.tcm.serialization.Version.MIN_ACCORD_VERSION; @@ -107,13 +109,14 @@ public class ClusterMetadata public final DistributedSchema schema; public final Directory directory; public final TokenMap tokenMap; - public final DataPlacements placements; + private final DataPlacements placements; public final AccordFastPath accordFastPath; public final LockedRanges lockedRanges; public final InProgressSequences inProgressSequences; public final ConsensusMigrationState consensusMigrationState; public final ImmutableMap, ExtensionValue> extensions; public final AccordStaleReplicas accordStaleReplicas; + public final CMSMembership cmsMembership; // This isn't serialized as part of ClusterMetadata it's really just a view over the Directory. public final Locator locator; @@ -121,9 +124,10 @@ public class ClusterMetadata // These fields are lazy but only for the test purposes, since their computation requires initialization of the log ks private EndpointsForRange fullCMSReplicas; private Set fullCMSEndpoints; - private Set fullCMSIds; private volatile Map localRangesAllSettled = null; private static final RangesAtEndpoint EMPTY_LOCAL_RANGES = RangesAtEndpoint.empty(FBUtilities.getBroadcastAddressAndPort()); + private DataPlacement cmsDataPlacement; + private final NodeId localNodeId; public ClusterMetadata(IPartitioner partitioner) { @@ -151,7 +155,8 @@ public class ClusterMetadata InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public ClusterMetadata(Epoch epoch, @@ -165,7 +170,8 @@ public class ClusterMetadata InProgressSequences inProgressSequences, ConsensusMigrationState consensusMigrationState, Map, ExtensionValue> extensions, - AccordStaleReplicas accordStaleReplicas) + AccordStaleReplicas accordStaleReplicas, + CMSMembership cmsMembership) { this(EMPTY_METADATA_IDENTIFIER, epoch, @@ -179,22 +185,25 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas); + accordStaleReplicas, + cmsMembership); } + private ClusterMetadata(int metadataIdentifier, - Epoch epoch, - IPartitioner partitioner, - DistributedSchema schema, - Directory directory, - TokenMap tokenMap, - DataPlacements placements, - AccordFastPath accordFastPath, - LockedRanges lockedRanges, - InProgressSequences inProgressSequences, - ConsensusMigrationState consensusMigrationState, - Map, ExtensionValue> extensions, - AccordStaleReplicas accordStaleReplicas) + Epoch epoch, + IPartitioner partitioner, + DistributedSchema schema, + Directory directory, + TokenMap tokenMap, + DataPlacements placements, + AccordFastPath accordFastPath, + LockedRanges lockedRanges, + InProgressSequences inProgressSequences, + ConsensusMigrationState consensusMigrationState, + Map, ExtensionValue> extensions, + AccordStaleReplicas accordStaleReplicas, + CMSMembership cmsMembership) { // TODO: token map is a feature of the specific placement strategy, and so may not be a relevant component of // ClusterMetadata in the long term. We need to consider how the actual components of metadata can be evolved @@ -206,14 +215,42 @@ public class ClusterMetadata this.schema = schema; this.directory = directory; this.tokenMap = tokenMap; - this.placements = placements; this.accordFastPath = accordFastPath; + this.placements = placements; this.lockedRanges = lockedRanges; this.inProgressSequences = inProgressSequences; this.consensusMigrationState = consensusMigrationState; this.extensions = ImmutableMap.copyOf(extensions); this.locator = Locator.usingDirectory(directory); this.accordStaleReplicas = accordStaleReplicas; + this.cmsMembership = cmsMembership; + this.cmsDataPlacement = calculateCMSPlacement(placements, cmsMembership); + + InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); + this.localNodeId = directory.allAddresses().contains(broadcastAddress) + ? directory.peerId(broadcastAddress) + : NodeId.UNREGISTERED; + } + + public Set fullCMSMemberIds() + { + return cmsMembership.fullMembers(); + } + + public boolean isCMSMember() + { + if (epoch.isEqualOrBefore(Epoch.FIRST)) + return isCMSMember(FBUtilities.getBroadcastAddressAndPort()); + + return fullCMSMemberIds().contains(localNodeId); + } + + public boolean isCMSMember(InetAddressAndPort endpoint) + { + if (epoch.isEqualOrBefore(Epoch.FIRST)) + return cmsDataPlacement.reads.byEndpoint().keySet().contains(endpoint); + + return fullCMSMembers().contains(endpoint); } public Set fullCMSMembers() @@ -223,27 +260,14 @@ public class ClusterMetadata if (fullCMSEndpoints == null) { - if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) - return Collections.emptySet(); - this.fullCMSEndpoints = ImmutableSet.copyOf(placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet()); + fullCMSEndpoints = ImmutableSet.copyOf(cmsMembership.fullMembers() + .stream() + .map(directory::endpoint) + .collect(Collectors.toSet())); } return fullCMSEndpoints; } - public Set fullCMSMemberIds() - { - if (epoch.isBefore(Epoch.FIRST)) - return Collections.emptySet(); - - if (fullCMSIds == null) - { - if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) - return Collections.emptySet(); - this.fullCMSIds = placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet().stream().map(directory::peerId).collect(toImmutableSet()); - } - return fullCMSIds; - } - public EndpointsForRange fullCMSMembersAsReplicas() { if (epoch.isBefore(Epoch.FIRST)) @@ -251,16 +275,70 @@ public class ClusterMetadata if (fullCMSReplicas == null) { - if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) - return EndpointsForRange.empty(MetaStrategy.entireRange); - fullCMSReplicas = placements.get(ReplicationParams.meta(this)).reads.forRange(MetaStrategy.entireRange).get(); + EndpointsForRange.Builder builder = EndpointsForRange.builder(MetaStrategy.entireRange); + for (NodeId nodeId : fullCMSMemberIds()) + builder.add(MetaStrategy.replica(directory.endpoint(nodeId))); + fullCMSReplicas = builder.build(); } return fullCMSReplicas; } - public boolean isCMSMember(InetAddressAndPort endpoint) + private DataPlacement calculateCMSPlacement(DataPlacements placements, CMSMembership cms) { - return fullCMSMembers().contains(endpoint); + if (epoch.isBefore(Epoch.FIRST) || schema.getKeyspaces().get(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) + return DataPlacement.empty(); + + if (directory.isEmpty()) + { + if (epoch.is(Epoch.FIRST)) + { + // PRE_INITIALIZE_CMS: placements need to be hardcoded to the local address so that the subsequent + // INITIALIZE_CMS can be committed + Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort()); + return DataPlacement.builder() + .withReadReplica(Epoch.FIRST, localReplica) + .withWriteReplica(Epoch.FIRST, localReplica) + .build(); + } + else + { + // This cluster did not previously upgrade from a gossip based version (i.e. pre-6.0) but did at some point + // run a version prior to MetadataVersion.V9 where we started to encode CMS membership directly. This + // condition implies that we are reconstructing a serialized cluster metadata during replay or else the + // directory should not be empty after Epoch.FIRST as the base state in INITIALIZE_CMS now includes the + // first CMS node. Similarly, if the cluster had previously been running a gossip-based version, the + // directory would contain entries for each of the live nodes at the time of upgrade. + // Given this state, the very next transformation that is/was applied will be to register the node that + // committed the PRE_INITIALIZE_CMS and INTIALIZE_CMS transformations. So we just extract the placements + // associated with the MetaStrategy params as they will already contain that node as an endpoint. + for (ReplicationParams params : placements.keys()) + if (params.isMeta()) + return placements.get(params); + + // This point can only be reached in tests. + // TODO enforce that invariant somehow + Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort()); + return DataPlacement.builder() + .withReadReplica(epoch, localReplica) + .withWriteReplica(epoch, localReplica) + .build(); + } + } + else + { + // Build a placement based on the CMS membership + return cms.toPlacement(directory); + } + } + + public DataPlacement placement(ReplicationParams params) + { + return params.isMeta() ? cmsDataPlacement : placements.get(params); + } + + public DataPlacements placements() + { + return placements; } public Transformer transformer() @@ -291,10 +369,42 @@ public class ClusterMetadata capLastModified(inProgressSequences, epoch), capLastModified(consensusMigrationState, epoch), capLastModified(extensions, epoch), - capLastModified(accordStaleReplicas, epoch)); + capLastModified(accordStaleReplicas, epoch), + capLastModified(cmsMembership, epoch)); } - public ClusterMetadata initializeClusterIdentifier(int clusterIdentifier) + /** + * To be used only during the execute of PRE_INITIALIZE_CMS, this sets the DataPlacement for the metadata keyspace + * so that the global log can be initialized and the subsequent entry containing the INITIALIZE_CMS transformation + * can be committed. + * @param initialCMSPlacement Expected to be a singleton placement identifying the local node as the sole replica + * for the metadata keyspace + * @return ClusterMetadata instance in the correct state to constitute the result of the PRE_INITIALIZE_CMS + * transformation + */ + public ClusterMetadata forcePreInitializedState(DataPlacement initialCMSPlacement) + { + assert epoch.isEqualOrBefore(Epoch.FIRST); + // double check that all metadata elements have had the last modified epoch set correctly + ClusterMetadata initial = forceEpoch(Epoch.FIRST); + initial.cmsDataPlacement = initialCMSPlacement; + return initial; + } + + /** + * Produce a ClusterMetadata suitable for use as the base state in the INITIALIZE_CMS transformation. This should + * only be used on the first CMS node when bootstrapping the CMS after upgrade or in a brand new cluster. + * @param clusterIdentifier Unique identifier for split brain detection & protection + * @param addresses The NodeAddresses of the first CMS node. + * @param version Version info for the first CMS node. + * @param location The rack & DC of the first CMS node. + * @return ClusterMetadata instance in the correct state to constitute the base state of the INITIALIZE_CMS + * transformation + */ + public ClusterMetadata forceInitializedState(int clusterIdentifier, + NodeAddresses addresses, + NodeVersion version, + Location location) { if (this.metadataIdentifier != EMPTY_METADATA_IDENTIFIER) throw new IllegalStateException(String.format("Can only initialize cluster identifier once, but it was already set to %d", this.metadataIdentifier)); @@ -302,11 +412,21 @@ public class ClusterMetadata if (clusterIdentifier == EMPTY_METADATA_IDENTIFIER) throw new IllegalArgumentException("Can not initialize cluster with empty cluster identifier"); + if (this.epoch.isAfter(Epoch.FIRST)) + throw new IllegalStateException(String.format("Can only initialize cluster identifier during epoch %d, but current epoch is %d", Epoch.FIRST.getEpoch(), epoch.getEpoch())); + + // Maybe register the first CMS node. If upgrading from gossip, this should be a no-op + Directory withRegistered = directory.with(addresses, location, version); + NodeId firstNode = withRegistered.peerId(addresses.broadcastAddress); + if (firstNode == null) + throw new IllegalStateException("Failed to find first CMS node in directory"); + + CMSMembership initialCMS = cmsMembership.startJoining(firstNode).finishJoining(firstNode); return new ClusterMetadata(clusterIdentifier, epoch, partitioner, schema, - directory, + withRegistered, tokenMap, placements, accordFastPath, @@ -314,7 +434,8 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas); + accordStaleReplicas, + initialCMS); } private static Map, ExtensionValue> capLastModified(Map, ExtensionValue> original, Epoch maxEpoch) @@ -413,8 +534,8 @@ public class ClusterMetadata // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges public boolean hasPendingRangesFor(KeyspaceMetadata ksm, Token token) { - ReplicaGroups writes = placements.get(ksm.params.replication).writes; - ReplicaGroups reads = placements.get(ksm.params.replication).reads; + ReplicaGroups writes = placement(ksm.params.replication).writes; + ReplicaGroups reads = placement(ksm.params.replication).reads; if (ksm.params.replication.isMeta()) return !reads.equals(writes); return !reads.forToken(token).equals(writes.forToken(token)); @@ -423,8 +544,8 @@ public class ClusterMetadata // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges public boolean hasPendingRangesFor(KeyspaceMetadata ksm, InetAddressAndPort endpoint) { - ReplicaGroups writes = placements.get(ksm.params.replication).writes; - ReplicaGroups reads = placements.get(ksm.params.replication).reads; + ReplicaGroups writes = placement(ksm.params.replication).writes; + ReplicaGroups reads = placement(ksm.params.replication).reads; return !writes.byEndpoint().get(endpoint).equals(reads.byEndpoint().get(endpoint)); } @@ -435,22 +556,22 @@ public class ClusterMetadata public RangesAtEndpoint writeRanges(KeyspaceMetadata metadata, InetAddressAndPort peer) { - return placements.get(metadata.params.replication).writes.byEndpoint().get(peer); + return placement(metadata.params.replication).writes.byEndpoint().get(peer); } // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges public Map, VersionedEndpoints.ForRange> pendingRanges(KeyspaceMetadata metadata) { Map, VersionedEndpoints.ForRange> map = new HashMap<>(); - ReplicaGroups writes = placements.get(metadata.params.replication).writes; - ReplicaGroups reads = placements.get(metadata.params.replication).reads; + ReplicaGroups writes = placement(metadata.params.replication).writes; + ReplicaGroups reads = placement(metadata.params.replication).reads; // first, pending ranges as the result of range splitting or merging // i.e. new ranges being created through join/leave List> pending = new ArrayList<>(writes.ranges()); pending.removeAll(reads.ranges()); for (Range p : pending) - map.put(p, placements.get(metadata.params.replication).writes.forRange(p)); + map.put(p, placement(metadata.params.replication).writes.forRange(p)); // next, ranges where the ranges themselves are not changing, but the replicas are // i.e. replacement or RF increase @@ -467,8 +588,8 @@ public class ClusterMetadata // TODO Remove this as it isn't really an equivalent to the previous concept of pending endpoints public VersionedEndpoints.ForToken pendingEndpointsFor(KeyspaceMetadata metadata, Token t) { - VersionedEndpoints.ForToken writeEndpoints = placements.get(metadata.params.replication).writes.forToken(t); - VersionedEndpoints.ForToken readEndpoints = placements.get(metadata.params.replication).reads.forToken(t); + VersionedEndpoints.ForToken writeEndpoints = placement(metadata.params.replication).writes.forToken(t); + VersionedEndpoints.ForToken readEndpoints = placement(metadata.params.replication).reads.forToken(t); EndpointsForToken.Builder endpointsForToken = writeEndpoints.get().newBuilder(writeEndpoints.size() - readEndpoints.size()); for (Replica writeReplica : writeEndpoints.get()) @@ -495,6 +616,7 @@ public class ClusterMetadata private final Map, ExtensionValue> extensions; private final Set modifiedKeys; private AccordStaleReplicas accordStaleReplicas; + private CMSMembership cmsMembership; private Transformer(ClusterMetadata metadata, Epoch epoch) { @@ -512,6 +634,7 @@ public class ClusterMetadata extensions = new HashMap<>(metadata.extensions); modifiedKeys = new HashSet<>(); accordStaleReplicas = metadata.accordStaleReplicas; + cmsMembership = metadata.cmsMembership; } public Epoch epoch() @@ -637,6 +760,30 @@ public class ClusterMetadata return this; } + public Transformer startJoiningCMS(NodeId id) + { + cmsMembership = cmsMembership.startJoining(id); + return this; + } + + public Transformer finishJoiningCMS(NodeId id) + { + cmsMembership = cmsMembership.finishJoining(id); + return this; + } + + public Transformer cancelJoiningCMS(NodeId id) + { + cmsMembership = cmsMembership.cancelJoining(id); + return this; + } + + public Transformer leaveCMS(NodeId id) + { + cmsMembership = cmsMembership.leave(id); + return this; + } + public Transformer with(DataPlacements placements) { this.placements = placements; @@ -822,6 +969,12 @@ public class ClusterMetadata consensusMigrationState.validateAgainstSchema(schema); } + if (cmsMembership != base.cmsMembership) + { + modifiedKeys.add(MetadataKeys.CMS_MEMBERSHIP); + cmsMembership = cmsMembership.withLastModified(epoch); + } + return new Transformed(new ClusterMetadata(base.metadataIdentifier, epoch, partitioner, @@ -834,7 +987,8 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas), + accordStaleReplicas, + cmsMembership), ImmutableSet.copyOf(modifiedKeys)); } @@ -852,7 +1006,8 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas); + accordStaleReplicas, + cmsMembership); } @Override @@ -871,6 +1026,7 @@ public class ClusterMetadata ", inProgressSequences=" + inProgressSequences + ", consensusMigrationState=" + consensusMigrationState + ", extensions=" + extensions + + ", cmsMembership=" + cmsMembership + ", modifiedKeys=" + modifiedKeys + '}'; } @@ -966,6 +1122,9 @@ public class ClusterMetadata ", placements=" + placements + ", lockedRanges=" + lockedRanges + ", consensusMigrationState=" + lockedRanges + + ", inProgressSequences=" + inProgressSequences + + ", extensions=" + extensions + + ", cmsMembership=" + cmsMembership + '}'; } @@ -994,7 +1153,8 @@ public class ClusterMetadata inProgressSequences.equals(that.inProgressSequences) && consensusMigrationState.equals(that.consensusMigrationState) && accordStaleReplicas.equals(that.accordStaleReplicas) && - extensions.equals(that.extensions); + extensions.equals(that.extensions) && + cmsMembership.equals(that.cmsMembership); } private static final Logger logger = LoggerFactory.getLogger(ClusterMetadata.class); @@ -1037,12 +1197,16 @@ public class ClusterMetadata { logger.warn("Extensions differ: {} != {}", extensions, other.extensions); } + if (!cmsMembership.equals(other.cmsMembership)) + { + logger.warn("CMS Membership differ: {} != {}", cmsMembership, other.cmsMembership); + } } @Override public int hashCode() { - return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions); + return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions, cmsMembership); } public static ClusterMetadata current() @@ -1083,13 +1247,13 @@ public class ClusterMetadata public NodeId myNodeId() { - return directory.peerId(FBUtilities.getBroadcastAddressAndPort()); + return localNodeId; } public NodeState myNodeState() { NodeId nodeId = myNodeId(); - if (myNodeId() != null) + if (nodeId != NodeId.UNREGISTERED) return directory.peerState(nodeId); return null; } @@ -1113,7 +1277,10 @@ public class ClusterMetadata DistributedSchema.serializer.serialize(metadata.schema, out, version); Directory.serializer.serialize(metadata.directory, out, version); TokenMap.serializer.serialize(metadata.tokenMap, out, version); - DataPlacements.serializer.serialize(metadata.placements, out, version); + // Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements + // so when targetting such a version, emulate that. + DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements; + DataPlacements.serializer.serialize(placements, out, version); if (version.isAtLeast(MIN_ACCORD_VERSION)) { AccordFastPath.serializer.serialize(metadata.accordFastPath, out, version); @@ -1132,6 +1299,9 @@ public class ClusterMetadata assert key.valueType.isInstance(value); value.serialize(out, version); } + // From V9 CMS membership is directly encoded in ClusterMetadata + if (version.isAtLeast(Version.V9)) + CMSMembership.serializer.serialize(metadata.cmsMembership, out, version); } @Override @@ -1188,6 +1358,42 @@ public class ClusterMetadata value.deserialize(in, version); extensions.put(key, value); } + + CMSMembership cmsMembership = CMSMembership.EMPTY; + if (version.isAtLeast(Version.V9)) + cmsMembership = CMSMembership.serializer.deserialize(in, version); + else + { + Optional metadataKs = schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); + if (metadataKs.isPresent()) + { + // Pre-V9 the membership of the CMS was always inferred from the placement of the distributed + // metadata keyspace. + // If the directory is not empty the endpoints in the placement must belong to registered nodes, + // so we can derive the CMSMembership using the data placement and directory. + + // If the directory is empty, then the cluster metadata must be the payload of an INITIALIZE_CMS + // transformation of a cluster that began on a post-6.0, pre-MetadataVersion.V9 version. + // In this case, we can and must assume that the initial CMS membership will consist of a single + // node, with the node_id 1. + // Note: the only route to arrive at this scenario is if a cluster is initialized on a post-6.0, + // pre-V9 version and then upgraded to a post-V9 version without any metadata snapshots being taken. + // If there is a snapshot available locally, when the upgraded node starts up it will replay its + // local log from that point. The INITIALIZE_CMS transform will not be replayed. + if (!dir.isEmpty()) + { + DataPlacement placement = placements.get(metadataKs.get().params.replication); + cmsMembership = CMSMembership.reconstruct(placement, dir); + } + else + { + NodeId id = new NodeId(1); + cmsMembership = CMSMembership.EMPTY.startJoining(id).finishJoining(id); + } + placements = placements.unbuild().without(metadataKs.get().params.replication).build(); + } + } + return new ClusterMetadata(clusterIdentifier, epoch, partitioner, @@ -1200,7 +1406,8 @@ public class ClusterMetadata ips, consensusMigrationState, extensions, - staleReplicas); + staleReplicas, + cmsMembership); } private DistributedSchema deduplicateReplicationParams(DistributedSchema schema, DataPlacements placements) @@ -1235,8 +1442,7 @@ public class ClusterMetadata sizeof(metadata.partitioner.getClass().getCanonicalName()) + DistributedSchema.serializer.serializedSize(metadata.schema, version) + Directory.serializer.serializedSize(metadata.directory, version) + - TokenMap.serializer.serializedSize(metadata.tokenMap, version) + - DataPlacements.serializer.serializedSize(metadata.placements, version); + TokenMap.serializer.serializedSize(metadata.tokenMap, version); if (version.isAtLeast(MIN_ACCORD_VERSION)) { @@ -1248,9 +1454,27 @@ public class ClusterMetadata size += LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) + InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version); + // Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements + // so when targetting such a version, emulate that. + DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements; + size += DataPlacements.serializer.serializedSize(placements, version); + // From V9 CMS membership is directly encoded in ClusterMetadata + if (version.isAtLeast(Version.V9)) + size += CMSMembership.serializer.serializedSize(metadata.cmsMembership, version); + return size; } + private DataPlacements preV9Placements(ClusterMetadata metadata) + { + if (metadata.cmsDataPlacement.isEmpty()) + return metadata.placements; + + return metadata.placements.unbuild() + .with(ReplicationParams.meta(metadata), metadata.cmsDataPlacement) + .build(); + } + public static IPartitioner getPartitioner(DataInputPlus in, Version version) throws IOException { if (version.isAtLeast(Version.V1)) diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java index 6d1b054bec..08fadb2aa8 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java @@ -117,7 +117,7 @@ public class ClusterMetadataService trace); instance = newInstance; RegistrationStatus.instance.onInitialized(); - if (newInstance.metadata().myNodeId() != null) + if (newInstance.metadata().myNodeId() != NodeId.UNREGISTERED) RegistrationStatus.instance.onRegistration(); trace = new RuntimeException("Previously initialized trace"); DatabaseDescriptor.applyLocator(); @@ -176,8 +176,9 @@ public class ClusterMetadataService // The node is a full member of the CMS if it has started participating in reads for distributed metadata table (which // implies it is a write replica as well). In other words, it's a fully joined member of the replica set responsible for // the distributed metadata table. - if (ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())) + if (metadata.epoch.isEqualOrBefore(Epoch.FIRST) || metadata.isCMSMember()) return LOCAL; + return REMOTE; } @@ -327,14 +328,15 @@ public class ClusterMetadataService DatabaseDescriptor.getPartitioner(), new DistributedSchema(keyspaces), Directory.EMPTY, - new TokenMap(DatabaseDescriptor.getPartitioner()), - DataPlacements.empty(), - AccordFastPath.EMPTY, - LockedRanges.EMPTY, - InProgressSequences.EMPTY, - ConsensusMigrationState.EMPTY, - Collections.emptyMap(), - AccordStaleReplicas.EMPTY); + new TokenMap(DatabaseDescriptor.getPartitioner()), + DataPlacements.empty(), + AccordFastPath.EMPTY, + LockedRanges.EMPTY, + InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, + Collections.emptyMap(), + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); LocalLog.LogSpec logSpec = LocalLog.logSpec() @@ -360,6 +362,24 @@ public class ClusterMetadataService ClusterMetadataService.setInstance(cms); } + @SuppressWarnings("resource") + public static void initializeForClients() + { + if (instance != null) + return; + + ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools()); + } + + public static void initializeForClients(DistributedSchema initialSchema) + { + if (instance != null) + return; + + + ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema)); + } + /* * Hook to be executed when the LocalLog is bootstrapped with the PreInitialize transformation. This is done on * the first CMS member to set up the initial replication and data placements for the metadata keyspace. @@ -391,25 +411,8 @@ public class ClusterMetadataService }; } // otherwise, this is a noop. - return preInit -> {}; - } - - @SuppressWarnings("resource") - public static void initializeForClients() - { - if (instance != null) - return; - - ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools()); - } - - public static void initializeForClients(DistributedSchema initialSchema) - { - if (instance != null) - return; - - - ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema)); + return preInit -> { + }; } public boolean isCurrentMember(InetAddressAndPort peer) @@ -1100,6 +1103,6 @@ public class ClusterMetadataService public enum State { - LOCAL, REMOTE, GOSSIP, RESET + LOCAL, REMOTE, GOSSIP, RESET, OFFLINE_TOOL } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/MetadataKeys.java b/src/java/org/apache/cassandra/tcm/MetadataKeys.java index 0be621b201..f788f1a449 100644 --- a/src/java/org/apache/cassandra/tcm/MetadataKeys.java +++ b/src/java/org/apache/cassandra/tcm/MetadataKeys.java @@ -45,18 +45,20 @@ public class MetadataKeys public static final MetadataKey LOCKED_RANGES = make(CORE_NS, "sequences", "locked_ranges"); public static final MetadataKey IN_PROGRESS_SEQUENCES = make(CORE_NS, "sequences", "in_progress"); public static final MetadataKey CONSENSUS_MIGRATION_STATE = make(CORE_NS, "consensus", "migration_state"); + public static final MetadataKey CMS_MEMBERSHIP = make(CORE_NS, "cms_membership", "cms_membership"); public static final ImmutableMap>> CORE_METADATA = ImmutableMap.>>builder() .put(SCHEMA, cm -> cm.schema) .put(NODE_DIRECTORY, cm -> cm.directory) .put(TOKEN_MAP, cm -> cm.tokenMap) - .put(DATA_PLACEMENTS, cm -> cm.placements) + .put(DATA_PLACEMENTS, ClusterMetadata::placements) .put(LOCKED_RANGES, cm -> cm.lockedRanges) .put(IN_PROGRESS_SEQUENCES, cm -> cm.inProgressSequences) .put(ACCORD_FAST_PATH, cm -> cm.accordFastPath) .put(ACCORD_STALE_REPLICAS, cm -> cm.accordStaleReplicas) .put(CONSENSUS_MIGRATION_STATE, cm -> cm.consensusMigrationState) + .put(CMS_MEMBERSHIP, cm -> cm.cmsMembership) .build(); public static MetadataKey make(String...parts) diff --git a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java index 4e18964e4d..4e36cf1066 100644 --- a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java @@ -63,6 +63,12 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor super(log); } + @Override + protected boolean acceptCommit(ClusterMetadata metadata) + { + return metadata.isCMSMember(); + } + @Override protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch) { diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index 57169d9876..f9a4313afa 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -63,8 +63,11 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; +import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.migration.CMSInitializationException; import org.apache.cassandra.tcm.migration.CMSInitializationRequest; import org.apache.cassandra.tcm.migration.Election; @@ -148,19 +151,24 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; */ public static void initializeAsFirstCMSNode() { - InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort(); - String datacenter = DatabaseDescriptor.getLocator().local().datacenter; + NodeAddresses addresses = NodeAddresses.current(); + Location location = DatabaseDescriptor.getLocator().local(); ClusterMetadataService cms = ClusterMetadataService.instance(); - cms.log().bootstrap(addr, datacenter, cms.logBootstrapCallback()); - ClusterMetadata metadata = ClusterMetadata.current(); - assert ClusterMetadataService.state() == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata); - Initialize initialize = new Initialize(metadata.initializeClusterIdentifier(addr.hashCode())); - ClusterMetadataService.instance().commit(initialize, - m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;}, - (code, message) -> { - logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message); - throw new CMSInitializationException(); - }); + cms.log().bootstrap(addresses.broadcastAddress, location.datacenter, cms.logBootstrapCallback()); + ClusterMetadata metadata = ClusterMetadata.current().forceInitializedState(addresses.broadcastAddress.hashCode(), + addresses, + NodeVersion.CURRENT, + location); + assert ClusterMetadataService.state(metadata) == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata); + Initialize initialize = new Initialize(metadata); + ClusterMetadata initialized = ClusterMetadataService.instance().commit(initialize, + m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;}, + (code, message) -> { + logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message); + throw new CMSInitializationException(); + }); + NodeId id = initialized.myNodeId(); + SystemKeyspace.setLocalHostId(id.toUUID()); } public static void initializeAsNonCmsNode(Function wrapProcessor) throws StartupException @@ -178,7 +186,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; NodeId nodeId = ClusterMetadata.current().myNodeId(); UUID currentHostId = SystemKeyspace.getLocalHostId(); - if (nodeId != null && !Objects.equals(nodeId.toUUID(), currentHostId)) + if (nodeId != NodeId.UNREGISTERED && !Objects.equals(nodeId.toUUID(), currentHostId)) { if (currentHostId == null) { @@ -394,7 +402,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; DatabaseDescriptor.getPartitioner().getClass().getCanonicalName(), metadata.partitioner.getClass().getCanonicalName())); - if (!metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort())) + if (!metadata.isCMSMember()) throw new IllegalStateException("When reinitializing with cluster metadata, we must be in the CMS"); metadata = metadata.forceEpoch(metadata.epoch.nextEpoch()); diff --git a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java index a7956cd21a..6e4a201613 100644 --- a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java @@ -184,7 +184,8 @@ public class StubClusterMetadataService extends ClusterMetadataService InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } return new StubClusterMetadataService(new UniformRangePlacement(), snapshots != null ? snapshots : MetadataSnapshots.NO_OP, diff --git a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java index 965751c554..c603737620 100644 --- a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java +++ b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java @@ -58,6 +58,7 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; @@ -306,7 +307,8 @@ public class GossipHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, Collections.emptyMap(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public static ClusterMetadata fromEndpointStates(DistributedSchema schema, Map epStates) @@ -396,7 +398,8 @@ public class GossipHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, extensions, - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); DataPlacements placements = new UniformRangePlacement().calculatePlacements(Epoch.UPGRADE_GOSSIP, forPlacementCalculation, schema.getKeyspaces()); @@ -411,7 +414,8 @@ public class GossipHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, extensions, - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public static boolean isValidForClusterMetadata(Map epstates) diff --git a/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java b/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java index d8b9dd746e..db1c250d01 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java @@ -82,9 +82,9 @@ public class LegacyStateListener implements ChangeListener } } - // next.myNodeId() can be null during replay (before we have registered) but if it is present and + // next.myNodeId() can be UNREGISTERED during replay (before we have registered) but if not and // there is a relevant change to the state of the local node, process that synchronously. - if (next.myNodeId() != null && changed.contains(next.myNodeId())) + if (next.myNodeId() != NodeId.UNREGISTERED && changed.contains(next.myNodeId())) { // Default is to process updates for the local node synchronously, overridable via config/hotprop if (DatabaseDescriptor.getLegacyStateListenerSyncLocalUpdates()) diff --git a/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java b/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java index 605b526378..8fd4bf87c8 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java @@ -18,23 +18,41 @@ package org.apache.cassandra.tcm.listeners; +import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; public class PlacementsChangeListener implements ChangeListener { + private final Runnable onChange; + + @VisibleForTesting + public PlacementsChangeListener(Runnable onChange) + { + this.onChange = onChange; + } + + public PlacementsChangeListener() + { + this(StorageService.instance::invalidateLocalRanges); + } + @Override public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot) { if (shouldInvalidate(prev, next)) - StorageService.instance.invalidateLocalRanges(); + onChange.run(); } private boolean shouldInvalidate(ClusterMetadata prev, ClusterMetadata next) { - if (!prev.placements.lastModified().equals(next.placements.lastModified()) && - !prev.placements.equivalentTo(next.placements)) // <- todo should we update lastModified if the result is the same? + if (!prev.placements().lastModified().equals(next.placements().lastModified()) && + !prev.placements().equivalentTo(next.placements())) // <- todo should we update lastModified if the result is the same? + return true; + + if (!prev.cmsMembership.equals(next.cmsMembership)) return true; if (prev.schema.getKeyspaces().size() != next.schema.getKeyspaces().size()) diff --git a/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java b/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java index feb191d32a..0be6243e53 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java @@ -47,17 +47,23 @@ public class UpgradeMigrationListener implements ChangeListener logger.info("Detected upgrade from gossip mode"); return; } - else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty()) // directory is non-empty after initialization during gossip upgrade + else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty()) { NodeId localId = next.myNodeId(); - if (localId != null) + if (localId != NodeId.UNREGISTERED) { - logger.info("Initialized CMS, updating local host id to {}", next.myNodeId()); - SystemKeyspace.setLocalHostId(next.myNodeId().toUUID()); - Gossiper.instance.mergeNodeToGossip(next.myNodeId(), next); + // assigning the local node id is done in Epoch.FIRST in one of two scenarios: + // * during an upgrade from gossip, as all pre-existing nodes will have an id assigned + // * during CMS initialization for a brand new cluster iff the local node is the first CMS member + logger.info("Initialized CMS, updating local host id to {}", localId); + SystemKeyspace.setLocalHostId(localId.toUUID()); + if (Gossiper.instance.isEnabled()) + { + Gossiper.instance.mergeNodeToGossip(localId, next); + if (Gossiper.instance.getQuarantineDisabled()) + Gossiper.instance.clearQuarantinedEndpoints(); + } } - if (Gossiper.instance.getQuarantineDisabled()) - Gossiper.instance.clearQuarantinedEndpoints(); } CassandraVersion prevMinVersion = prev.directory.clusterMinVersion.cassandraVersion; diff --git a/src/java/org/apache/cassandra/tcm/log/LogState.java b/src/java/org/apache/cassandra/tcm/log/LogState.java index 62e20ceddf..e18854f161 100644 --- a/src/java/org/apache/cassandra/tcm/log/LogState.java +++ b/src/java/org/apache/cassandra/tcm/log/LogState.java @@ -306,7 +306,7 @@ public class LogState if (metadata != null) { NodeId mynodeId = metadata.myNodeId(); - if (mynodeId != null) + if (mynodeId != NodeId.UNREGISTERED) SystemKeyspace.setLocalHostId(mynodeId.toUUID()); } } diff --git a/src/java/org/apache/cassandra/tcm/membership/Directory.java b/src/java/org/apache/cassandra/tcm/membership/Directory.java index f86481f141..8d444c6443 100644 --- a/src/java/org/apache/cassandra/tcm/membership/Directory.java +++ b/src/java/org/apache/cassandra/tcm/membership/Directory.java @@ -467,7 +467,7 @@ public class Directory implements MetadataValue public NodeState peerState(NodeId peer) { - return states.get(peer); + return peer == NodeId.UNREGISTERED ? null : states.get(peer); } public NodeVersion version(NodeId peer) diff --git a/src/java/org/apache/cassandra/tcm/membership/NodeId.java b/src/java/org/apache/cassandra/tcm/membership/NodeId.java index b73c4f00c7..1d5d3c9cd4 100644 --- a/src/java/org/apache/cassandra/tcm/membership/NodeId.java +++ b/src/java/org/apache/cassandra/tcm/membership/NodeId.java @@ -32,6 +32,7 @@ import org.apache.cassandra.tcm.serialization.Version; public class NodeId implements Comparable, MultiStepOperation.SequenceKey { + public static final NodeId UNREGISTERED = new NodeId(-1); private final static long NODE_ID_UUID_MAGIC = 7861390860069061072L; public static final Serializer serializer = new Serializer(); diff --git a/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java b/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java index de1bc71250..f3b78a1ee9 100644 --- a/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java +++ b/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java @@ -36,7 +36,7 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public class NodeVersion implements Comparable { public static final Serializer serializer = new Serializer(); - public static final Version CURRENT_METADATA_VERSION = Version.V8; + public static final Version CURRENT_METADATA_VERSION = Version.V9; public static final NodeVersion CURRENT = new NodeVersion(new CassandraVersion(FBUtilities.getReleaseVersionString()), CURRENT_METADATA_VERSION); private static final CassandraVersion SINCE_VERSION = CassandraVersion.CASSANDRA_5_1; diff --git a/src/java/org/apache/cassandra/tcm/migration/Election.java b/src/java/org/apache/cassandra/tcm/migration/Election.java index 906b647488..4e1cb5d4df 100644 --- a/src/java/org/apache/cassandra/tcm/migration/Election.java +++ b/src/java/org/apache/cassandra/tcm/migration/Election.java @@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; -import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; @@ -54,7 +53,6 @@ import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.ownership.TokenMap; -import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -157,9 +155,6 @@ public class Election initiator.compareAndSet(currentInitiator, MIGRATING)) { Startup.initializeAsFirstCMSNode(); - Register.maybeRegister(); - SystemKeyspace.setLocalHostId(ClusterMetadata.current().myNodeId().toUUID()); - updateInitiator(MIGRATING, MIGRATED); MessageDelivery.fanoutAndWait(messaging, sendTo, Verb.TCM_NOTIFY_REQ, DistributedMetadataLogKeyspace.getLogState(Epoch.EMPTY, false)); } diff --git a/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java b/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java index fd1a2618db..6c0ff2012c 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java +++ b/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java @@ -106,6 +106,11 @@ public class DataPlacement return EMPTY; } + public boolean isEmpty() + { + return reads.isEmpty() && writes.isEmpty(); + } + public static Builder builder() { return new Builder(ReplicaGroups.builder(), diff --git a/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java b/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java index d813e8c287..6bc836c770 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java +++ b/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java @@ -46,6 +46,7 @@ import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.db.TypeSizes.sizeof; +import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public class DataPlacements extends ReplicationMap implements MetadataValue { @@ -62,21 +63,6 @@ public class DataPlacements extends ReplicationMap implements Met this.lastModified = lastModified; } - public DataPlacements replaceParams(Epoch lastModified, ReplicationParams oldParams, ReplicationParams newParams) - { - Map newMap = Maps.newHashMapWithExpectedSize(map.size()); - assert map.containsKey(oldParams) : String.format("Can't replace key %s, since map doesn't contain it: %s", oldParams, map); - for (Map.Entry e : map.entrySet()) - { - if (e.getKey().equals(oldParams)) - newMap.put(newParams, e.getValue()); - else - newMap.put(e.getKey(), e.getValue()); - } - - return new DataPlacements(lastModified, newMap); - } - protected DataPlacement defaultValue() { return DataPlacement.empty(); @@ -256,7 +242,11 @@ public class DataPlacements extends ReplicationMap implements Met public void serialize(DataPlacements t, DataOutputPlus out, Version version) throws IOException { Map map = t.asMap(); - out.writeInt(map.size()); + if (version.isBefore(Version.V9)) + out.writeInt(map.size()); + else + out.writeUnsignedVInt32(map.size()); + for (Map.Entry entry : map.entrySet()) { ReplicationParams.serializer.serialize(entry.getKey(), out, version); @@ -267,7 +257,7 @@ public class DataPlacements extends ReplicationMap implements Met public DataPlacements deserialize(DataInputPlus in, Version version) throws IOException { - int size = in.readInt(); + int size = version.isBefore(Version.V9) ? in.readInt() : in.readUnsignedVInt32(); Map map = Maps.newHashMapWithExpectedSize(size); for (int i = 0; i < size; i++) { @@ -280,8 +270,9 @@ public class DataPlacements extends ReplicationMap implements Met public long serializedSize(DataPlacements t, Version version) { - long size = sizeof(t.size()); - for (Map.Entry entry : t.asMap().entrySet()) + Map map = t.asMap(); + long size = version.isBefore(Version.V9) ? sizeof(map.size()) : sizeofUnsignedVInt(map.size()); + for (Map.Entry entry : map.entrySet()) { size += ReplicationParams.serializer.serializedSize(entry.getKey(), version); size += DataPlacement.serializerFor(entry.getKey()).serializedSize(entry.getValue(), version); diff --git a/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java b/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java index 3d3b4da374..6fa832f8d4 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java +++ b/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java @@ -329,9 +329,13 @@ public class UniformRangePlacement implements PlacementProvider logger.trace("Calculating data placements for {}", ksMetadata.name); AbstractReplicationStrategy replication = ksMetadata.replicationStrategy; ReplicationParams params = ksMetadata.params.replication; - if (params.isMeta() || params.isLocal()) + if (params.isMeta()) { - placements.put(params, metadata.placements.get(params)); + // don't calculate meta strategy placements, these are derived from ClusterMetadata.cmsMembership + } + else if (params.isLocal()) + { + placements.put(params, metadata.placement(params)); } else { diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index 43891d5abc..07147fac17 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -326,7 +326,7 @@ public class BootstrapAndJoin extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { // need to undo MID_JOIN and START_JOIN, then merge the ranges split by PrepareJoin @@ -357,7 +357,7 @@ public class BootstrapAndJoin extends MultiStepOperation @VisibleForTesting public Pair getMovementMaps(ClusterMetadata metadata) { - MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements, startJoin.delta()); + MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements(), startJoin.delta()); MovementMap strictMovementMap = toStrict(movementMap, finishJoin.delta()); return Pair.create(movementMap, strictMovementMap); } diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java index 552818a774..bd6d3672f9 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java @@ -318,7 +318,7 @@ public class BootstrapAndReplace extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { // need to undo MID_REPLACE and START_REPLACE, but PREPARE_REPLACE doesn't affect placements @@ -355,7 +355,7 @@ public class BootstrapAndReplace extends MultiStepOperation private static MovementMap movementMap(InetAddressAndPort beingReplaced, PlacementDeltas startDelta) { MovementMap.Builder movementMapBuilder = MovementMap.builder(); - DataPlacements placements = ClusterMetadata.current().placements; + DataPlacements placements = ClusterMetadata.current().placements(); startDelta.forEach((params, delta) -> { EndpointsByReplica.Builder movements = new EndpointsByReplica.Builder(); DataPlacement originalPlacements = placements.get(params); @@ -451,6 +451,8 @@ public class BootstrapAndReplace extends MultiStepOperation public static void gossipStateToHibernate(ClusterMetadata metadata, NodeId nodeId) { + if (nodeId == NodeId.UNREGISTERED) + return; // order is important here, the gossiper can fire in between adding these two states. It's ok to send TOKENS without STATUS, but *not* vice versa. List> states = new ArrayList<>(); VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory; @@ -462,6 +464,8 @@ public class BootstrapAndReplace extends MultiStepOperation public static void gossipStateToNormal(ClusterMetadata metadata, NodeId nodeId) { + if (nodeId == NodeId.UNREGISTERED) + return; List> states = new ArrayList<>(); VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory; Collection tokens = metadata.tokenMap.tokens(nodeId); diff --git a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java index 4139110236..3dbd9311bd 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java @@ -25,25 +25,20 @@ import java.util.Map; import org.apache.cassandra.exceptions.ExceptionCode; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.accord.topology.FastPathStrategy; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.Directory; -import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; - public class CancelCMSReconfiguration implements Transformation { public static final Serializer serializer = new Serializer(); @@ -66,53 +61,46 @@ public class CancelCMSReconfiguration implements Transformation if (reconfigureCMS == null) return new Rejected(ExceptionCode.INVALID, "Can not cancel reconfiguration since there does not seem to be any in-flight"); - ReplicationParams metaParams = ReplicationParams.meta(prev); ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement placement = prev.placements.get(metaParams); // Reset any partially completed transition by removing the pending replica from the write group if (reconfigureCMS.next.activeTransition != null) { - InetAddressAndPort pendingEndpoint = prev.directory.endpoint(reconfigureCMS.next.activeTransition.nodeId); - Replica pendingReplica = new Replica(pendingEndpoint, entireRange, true); - placement = placement.unbuild() - .withoutWriteReplica(prev.nextEpoch(), pendingReplica) - .build(); + // see what the placements for the meta keyspace will be after cancelling the active join + CMSMembership cms = prev.cmsMembership.cancelJoining(reconfigureCMS.next.activeTransition.nodeId); + if (!cms.joiningMembers().isEmpty()) + return new Rejected(ExceptionCode.INVALID, + String.format("Placements will be inconsistent if this transformation is applied:" + + "\nFull members %s\nJoining members: %s", + cms.fullMembers(), + cms.joiningMembers())); + + // if all is good, actually cancel the joining member + transformer = transformer.cancelJoiningCMS(reconfigureCMS.next.activeTransition.nodeId); + // Recalculate the replication params for the meta keyspace based on the actual placement as it will be + ReplicationParams recalculated = getAccurateReplication(prev.directory, cms); + + // If they no longer match the replication params in schema, i.e. the transitions completed so far did not + // bring the membership/placements into line with configuration, update schema to match what we actually have + if (!recalculated.equals(ReplicationParams.meta(prev))) + { + KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); + KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, + recalculated, + FastPathStrategy.simple())); + transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))); + } } - if (!placement.reads.equivalentTo(placement.writes)) - return new Rejected(ExceptionCode.INVALID, String.format("Placements will be inconsistent if this transformation is applied:\nReads %s\nWrites: %s", - placement.reads, - placement.writes)); - - // Reset the replication params for the meta keyspace based on the actual placement in case they no longer match - ReplicationParams fromPlacement = getAccurateReplication(prev.directory, placement); - - // If they no longer match, i.e. the transitions completed so far did not bring the placements into line with - // the configuration, remove the entry keyed by the existing configured params. - DataPlacements.Builder builder = prev.placements.unbuild(); - if (!metaParams.equals(fromPlacement)) - { - builder = builder.without(metaParams); - - // Also update schema with the corrected params - KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); - KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple())); - transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))); - } - - // finally, add the possibly corrected placement keyed by the possibly corrected params - builder = builder.with(fromPlacement, placement); - transformer = transformer.with(builder.build()); return Transformation.success(transformer.with(prev.inProgressSequences.without(ReconfigureCMS.SequenceKey.instance)) .with(prev.lockedRanges.unlock(reconfigureCMS.next.lockKey)), MetaStrategy.affectedRanges(prev)); } - private ReplicationParams getAccurateReplication(Directory directory, DataPlacement placement) + private ReplicationParams getAccurateReplication(Directory directory, CMSMembership membership) { Map replicasPerDc = new HashMap<>(); - placement.writes.byEndpoint().keySet().forEach(i -> { - String dc = directory.location(directory.peerId(i)).datacenter; + membership.fullMembers().forEach(id -> { + String dc = directory.location(id).datacenter; int count = replicasPerDc.getOrDefault(dc, 0); replicasPerDc.put(dc, ++count); }); diff --git a/src/java/org/apache/cassandra/tcm/sequences/Move.java b/src/java/org/apache/cassandra/tcm/sequences/Move.java index cf81e5a3ce..a9f68812f2 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/Move.java +++ b/src/java/org/apache/cassandra/tcm/sequences/Move.java @@ -321,7 +321,7 @@ public class Move extends MultiStepOperation StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION); Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces(); Map movementMap = movementMap(FailureDetector.instance, - metadata.placements, + metadata.placements(), toSplitRanges, startMove.delta(), midMove.delta(), @@ -430,7 +430,7 @@ public class Move extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { diff --git a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java index db728ef67e..09950edfe8 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java @@ -58,6 +58,7 @@ import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Retry; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.AsyncPromise; @@ -170,8 +171,9 @@ public class ProgressBarrier Set> ranges = e.getValue(); for (Range range : ranges) { - EndpointsForRange writes = metadata.placements.get(params).writes.matchRange(range).get().filter(r -> filter.test(r.endpoint())); - EndpointsForRange reads = metadata.placements.get(params).reads.matchRange(range).get().filter(r -> filter.test(r.endpoint())); + DataPlacement placement = metadata.placement(params); + EndpointsForRange writes = placement.writes.matchRange(range).get().filter(r -> filter.test(r.endpoint())); + EndpointsForRange reads = placement.reads.matchRange(range).get().filter(r -> filter.test(r.endpoint())); // Affected ranges can contain ranges which are the results of merging or splitting and may not exist // as keys in the existing ReplicaGroups. As such, no replicas will be found for these ranges and so no // WaitFor is necessary. diff --git a/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java b/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java index d6e8ca3127..bd09f8aaca 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java +++ b/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java @@ -121,7 +121,7 @@ public class RemoveNodeStreams implements LeaveStreams RangesByEndpoint startWriteAdditions = startDelta.get(params).writes.additions; RangesByEndpoint startWriteRemovals = startDelta.get(params).writes.removals; // find current placements from the metadata, we need to stream from replicas that are not changed and are therefore not in the deltas - ReplicaGroups currentPlacements = metadata.placements.get(params).reads; + ReplicaGroups currentPlacements = metadata.placement(params).reads; startWriteAdditions.flattenValues() .forEach(newReplica -> { EndpointsForRange.Builder candidateBuilder = new EndpointsForRange.Builder(newReplica.range()); diff --git a/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java b/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java index 4580d716a1..ab819c5f96 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java @@ -44,7 +44,7 @@ public class ReplaceSameAddress { MovementMap.Builder builder = MovementMap.builder(); InetAddressAndPort addr = metadata.directory.endpoint(nodeId); - metadata.placements.forEach((params, placement) -> { + metadata.placements().forEach((params, placement) -> { EndpointsByReplica.Builder sources = new EndpointsByReplica.Builder(); placement.reads.byEndpoint().get(addr).forEach(destination -> { placement.reads.forRange(destination.range()).forEach(potentialSource -> { diff --git a/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java b/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java index 876caa3d96..011d35f866 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java +++ b/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java @@ -253,7 +253,7 @@ public class UnbootstrapAndLeave extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { // need to undo MID_LEAVE and START_LEAVE, but PrepareLeave doesn't affect placement diff --git a/src/java/org/apache/cassandra/tcm/serialization/Version.java b/src/java/org/apache/cassandra/tcm/serialization/Version.java index 9cbcb8587a..fb6e59cfd6 100644 --- a/src/java/org/apache/cassandra/tcm/serialization/Version.java +++ b/src/java/org/apache/cassandra/tcm/serialization/Version.java @@ -38,13 +38,13 @@ public enum Version */ V0(0), /** - * - Moved Partitioner in ClusterMetadata serializer to be the first field - * - Added a counter to Directory serializer to keep track of NodeIds + * - Moved Partitioner in ClusterMetadata serializer to be the first field + * - Added a counter to Directory serializer to keep track of NodeIds */ V1(1), /** - * - Added version to PlacementForRange serializer - * - Serialize MemtableParams when serializing TableParams + * - Added version to PlacementForRange serializer + * - Serialize MemtableParams when serializing TableParams */ V2(2), /** @@ -78,6 +78,11 @@ public enum Version * - Comments and security labels for schema elements (keyspaces, tables, columns, UDTs, and UDT fields) */ V8(8), + /** + * - DataPlacements don't include MetaStrategy, replaced by ClusterMetadata.CMSMembership + * - Size of DataPlacements is encoded as vint + */ + V9(9), UNKNOWN(Integer.MAX_VALUE); @@ -95,6 +100,7 @@ public enum Version } private final int version; + Version(int version) { this.version = version; diff --git a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java index 1e3aef5817..491a3a2118 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java @@ -78,7 +78,7 @@ public class AccordMarkStale implements Transformation for (KeyspaceMetadata keyspace : prev.schema.getKeyspaces().without(SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES)) { - List shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements, prev.directory); + List shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements(), prev.directory); for (AccordTopology.KeyspaceShard shard : shards) { diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java index 087ebfcc61..807fed2b17 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java @@ -244,7 +244,7 @@ public class AlterSchema implements Transformation DataPlacements.Builder newPlacementsBuilder = DataPlacements.builder(calculatedPlacements.size()); calculatedPlacements.forEach((params, newPlacement) -> { - DataPlacement previousPlacement = prev.placements.get(params); + DataPlacement previousPlacement = prev.placement(params); // Preserve placement versioning that has resulted from natural application where possible if (previousPlacement.equivalentTo(newPlacement)) newPlacementsBuilder.with(params, previousPlacement); diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java b/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java index e247f66a88..4eb3d7b364 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java @@ -130,11 +130,11 @@ public class AlterTopology implements Transformation for (Map.Entry update : updates.entrySet()) updated = updated.withUpdatedRackAndDc(update.getKey(), update.getValue()); ClusterMetadata proposed = prev.transformer().with(updated).build().metadata; - DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements.lastModified(), + DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements().lastModified(), proposed.tokenMap.toRanges(), proposed, proposed.schema.getKeyspaces()); - if (!proposedPlacements.equivalentTo(prev.placements)) + if (!proposedPlacements.equivalentTo(prev.placements())) { logger.info("Rejecting topology modifications which would materially change data placements: {}", updates); return new Rejected(INVALID, "Proposed updates modify data placements, violating consistency guarantees"); diff --git a/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java b/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java index 10f48609d4..384e04dc83 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java +++ b/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java @@ -82,7 +82,7 @@ public abstract class ApplyPlacementDeltas implements Transformation ClusterMetadata.Transformer next = prev.transformer(); if (!delta.isEmpty()) - next = next.with(delta.apply(prev.nextEpoch(), prev.placements)); + next = next.with(delta.apply(prev.nextEpoch(), prev.placements())); next = transform(prev, next); diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java index d8986e6829..08f9f10233 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java @@ -89,7 +89,7 @@ import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; */ public class PrepareJoin implements Transformation { - public static final Serializer serializer = new Serializer() + public static final Serializer serializer = new Serializer<>() { public PrepareJoin construct(NodeId nodeId, Set tokens, PlacementProvider placementProvider, boolean joinTokenRing, boolean streamData) { @@ -168,10 +168,10 @@ public class PrepareJoin implements Transformation startJoin, midJoin, finishJoin, joinTokenRing, streamData); if (!prev.tokenMap.isEmpty()) - assertPreExistingWriteReplica(prev.placements, transitionPlan); + assertPreExistingWriteReplica(prev.placements(), transitionPlan); LockedRanges newLockedRanges = prev.lockedRanges.lock(lockKey, rangesToLock); - DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements); + DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements()); ClusterMetadata.Transformer proposed = prev.transformer() .with(newLockedRanges) .with(startingPlacements) diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java index 0ad37f8bcd..74d2c2c810 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java @@ -115,7 +115,7 @@ public class PrepareLeave implements Transformation PlacementDeltas startDelta = transitionPlan.addToWrites(); PlacementDeltas midDelta = transitionPlan.moveReads(); PlacementDeltas finishDelta = transitionPlan.removeFromWrites(); - transitionPlan.assertPreExistingWriteReplica(prev.placements); + transitionPlan.assertPreExistingWriteReplica(prev.placements()); LockedRanges.Key unlockKey = LockedRanges.keyFor(proposed.epoch); diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java index e7e278d0d2..16dcdfca08 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java @@ -109,7 +109,7 @@ public class PrepareMove implements Transformation StartMove startMove = new StartMove(nodeId, transitionPlan.addToWrites(), lockKey); MidMove midMove = new MidMove(nodeId, transitionPlan.moveReads(), lockKey); FinishMove finishMove = new FinishMove(nodeId, tokens, transitionPlan.removeFromWrites(), lockKey); - transitionPlan.assertPreExistingWriteReplica(prev.placements); + transitionPlan.assertPreExistingWriteReplica(prev.placements()); Move sequence = Move.newSequence(prev.nextEpoch(), lockKey, @@ -123,7 +123,7 @@ public class PrepareMove implements Transformation return Transformation.success(prev.transformer() .withNodeState(nodeId, NodeState.MOVING) .with(prev.lockedRanges.lock(lockKey, rangesToLock)) - .with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements)) + .with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements())) .with(prev.inProgressSequences.with(nodeId, sequence)), rangesToLock); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java index 8b1577822b..94bf3508c0 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java @@ -109,7 +109,7 @@ public class PrepareReplace implements Transformation StartReplace start = new StartReplace(replaced, replacement, transitionPlan.addToWrites(), unlockKey); MidReplace mid = new MidReplace(replaced, replacement, transitionPlan.moveReads(), unlockKey); FinishReplace finish = new FinishReplace(replaced, replacement, transitionPlan.removeFromWrites(), unlockKey); - transitionPlan.assertPreExistingWriteReplica(prev.placements); + transitionPlan.assertPreExistingWriteReplica(prev.placements()); Set tokens = new HashSet<>(prev.tokenMap.tokens(replaced)); BootstrapAndReplace plan = BootstrapAndReplace.newSequence(prev.nextEpoch(), diff --git a/src/java/org/apache/cassandra/tcm/transformations/Register.java b/src/java/org/apache/cassandra/tcm/transformations/Register.java index a7551c7069..b2fce47011 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/Register.java +++ b/src/java/org/apache/cassandra/tcm/transformations/Register.java @@ -109,7 +109,7 @@ public class Register implements Transformation if (isReplacingSameAddress()) { NodeId self = ClusterMetadata.current().myNodeId(); - if (self == null) + if (self == NodeId.UNREGISTERED) throw new IllegalStateException("Tried to replace same address, but node does not seem to be registered"); return self; diff --git a/src/java/org/apache/cassandra/tcm/transformations/Startup.java b/src/java/org/apache/cassandra/tcm/transformations/Startup.java index 7b8c4cff7e..00b4023525 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/Startup.java +++ b/src/java/org/apache/cassandra/tcm/transformations/Startup.java @@ -24,10 +24,6 @@ import java.util.Objects; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.Keyspaces; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Transformation; @@ -35,14 +31,12 @@ import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; public class Startup implements Transformation { @@ -96,32 +90,13 @@ public class Startup implements Transformation if (!nodeId.equals(existingNodeId) && addresses.conflictsWith(existingAddresses)) return new Rejected(INVALID, String.format("New addresses %s conflicts with existing node %s with addresses %s", addresses, entry.getKey(), existingAddresses)); } - next = next.withNewAddresses(nodeId, addresses); - Keyspaces allKeyspaces = prev.schema.getKeyspaces().withAddedOrReplaced(prev.schema.getKeyspaces()); - DataPlacements newPlacement = ClusterMetadataService.instance() .placementProvider() .calculatePlacements(prev.nextEpoch(), prev.tokenMap.toRanges(), next.build().metadata, - allKeyspaces); - - if (prev.isCMSMember(prev.directory.endpoint(nodeId))) - { - ReplicationParams metaParams = ReplicationParams.meta(prev); - InetAddressAndPort endpoint = prev.directory.endpoint(nodeId); - Replica leavingReplica = new Replica(endpoint, entireRange, true); - Replica joiningReplica = new Replica(addresses.broadcastAddress, entireRange, true); - - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild(); - builder.reads.withoutReplica(prev.nextEpoch(), leavingReplica); - builder.writes.withoutReplica(prev.nextEpoch(), leavingReplica); - builder.reads.withReplica(prev.nextEpoch(), joiningReplica); - builder.writes.withReplica(prev.nextEpoch(), joiningReplica); - newPlacement = newPlacement.unbuild().with(metaParams, builder.build()).build(); - } - + prev.schema.getKeyspaces()); next = next.with(newPlacement); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java index 614dd8a4ed..dd4b49127b 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java @@ -29,15 +29,12 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.RangesByEndpoint; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; @@ -45,7 +42,6 @@ import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; import static org.apache.cassandra.tcm.MultiStepOperation.Kind.RECONFIGURE_CMS; /** @@ -154,38 +150,26 @@ public class AdvanceCMSReconfiguration implements Transformation { // Pop the next node to be added from the list diff.additions NodeId addition = diff.additions.get(0); - InetAddressAndPort endpoint = prev.directory.endpoint(addition); - Replica replica = new Replica(endpoint, entireRange, true); - List newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size())); // Check that the candidate is not already a CMS member - ReplicationParams metaParams = ReplicationParams.meta(prev); - RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint(); - RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint(); - if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint)) - return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS"); + CMSMembership cms = prev.cmsMembership; + if (cms.joiningMembers().contains(addition) || cms.fullMembers().contains(addition)) + return new Transformation.Rejected(INVALID, "Endpoint is already a full or joining member of the CMS: " + prev.directory.endpoint(addition)); - - ClusterMetadata.Transformer transformer = prev.transformer(); - // Add the candidate as a write replica - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild() - .withWriteReplica(prev.nextEpoch(), replica); - transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()); + // Add the candidate as a joining member + ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(addition); // Construct a set of sources for the new member to stream log tables from (essentially this is the existing members) - Set streamCandidates = new HashSet<>(); - for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues()) - { - if (!replica.equals(r)) - streamCandidates.add(r.endpoint()); - } + Set streamCandidates = prev.fullCMSMembers(); // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence, // including the remaining add/remove operations and the streaming that needs to be done by the joining node + List newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size())); AdvanceCMSReconfiguration next = next(prev.nextEpoch(), newAdditions, diff.removals, new ReconfigureCMS.ActiveTransition(addition, streamCandidates)); + // Create a new sequence instance with the next step to reflect that the state has progressed. ReconfigureCMS advanced = sequence.advance(next); // Finally, replace the existing reconfiguration sequence with this updated one. @@ -206,21 +190,21 @@ public class AdvanceCMSReconfiguration implements Transformation */ private Transformation.Result finishAdd(ClusterMetadata prev, ReconfigureCMS sequence, NodeId addition) { + // Check that the candidate is already a joining CMS member + CMSMembership cms = prev.cmsMembership; + if (!cms.joiningMembers().contains(addition)) + return new Transformation.Rejected(INVALID, "Endpoint is not a in the process of joining the CMS: " + prev.directory.endpoint(addition)); + // Add the new member as a full read replica, able to participate in quorums for log updates - ReplicationParams metaParams = ReplicationParams.meta(prev); - InetAddressAndPort endpoint = prev.directory.endpoint(addition); - Replica replica = new Replica(endpoint, entireRange, true); - ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement.Builder builder = prev.placements.get(metaParams) - .unbuild() - .withReadReplica(prev.nextEpoch(), replica); - transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()); + ClusterMetadata.Transformer transformer = prev.transformer().finishJoiningCMS(addition); // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence, // which includes the remaining add/remove operations AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, diff.removals, null); + // Create a new sequence instance with the next step to reflect that the state has progressed. ReconfigureCMS advanced = sequence.advance(next); + // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); @@ -238,29 +222,25 @@ public class AdvanceCMSReconfiguration implements Transformation List newRemovals = new ArrayList<>(diff.removals.subList(1, diff.removals.size())); // Check that the candidate is actually a CMS member - ClusterMetadata.Transformer transformer = prev.transformer(); + Set cms = prev.fullCMSMemberIds(); InetAddressAndPort endpoint = prev.directory.endpoint(removal); - Replica replica = new Replica(endpoint, entireRange, true); - ReplicationParams metaParams = ReplicationParams.meta(prev); - if (!prev.fullCMSMembers().contains(endpoint)) + if (!cms.contains(removal)) return new Transformation.Rejected(INVALID, String.format("%s is not currently a CMS member, cannot remove it", endpoint)); // Check that the candidate is not the only CMS member - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild(); - builder.reads.withoutReplica(prev.nextEpoch(), replica); - builder.writes.withoutReplica(prev.nextEpoch(), replica); - DataPlacement proposed = builder.build(); - if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty()) + if (cms.size() == 1) return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint)); - // Actually remove the candidate - transformer = transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()); + // Remove the CMS member + ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(removal); // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence, // which includes the remaining add/remove operations AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, newRemovals, null); + // Create a new sequence instance with the next step to reflect that the state has progressed. ReconfigureCMS advanced = sequence.advance(next); + // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java index a1bed8ed2a..9cef32aef5 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java @@ -26,7 +26,6 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.AddToCMS; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; @@ -84,12 +83,9 @@ public class FinishAddToCMS extends BaseMembershipTransformation InetAddressAndPort endpoint = prev.directory.endpoint(targetNode); Replica replica = new Replica(endpoint, entireRange, true); - ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement.Builder builder = prev.placements.get(metaParams) - .unbuild() - .withReadReplica(prev.nextEpoch(), replica); - transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()) - .with(prev.inProgressSequences.without(targetNode)); + ClusterMetadata.Transformer transformer = prev.transformer() + .finishJoiningCMS(targetNode) + .with(prev.inProgressSequences.without(targetNode)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java index 7286743f91..2725fffebf 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java @@ -20,6 +20,8 @@ package org.apache.cassandra.tcm.transformations.cms; import java.io.IOException; +import com.google.common.collect.ImmutableSet; + import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -29,12 +31,10 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.Keyspaces; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -72,7 +72,6 @@ public class PreInitialize implements Transformation assert metadata.epoch.isBefore(Epoch.FIRST); ClusterMetadata.Transformer transformer = metadata.transformer(); - // This null check is a leftover from previous implementations. In earlier versions the address and datacenter // were not be included in the serialized form of this transform and so were not written to the local or // distributed logs nor included in the log entries sent over the wire between peers. @@ -84,30 +83,33 @@ public class PreInitialize implements Transformation // PRE_INITIALIZE_CMS becomes irrelevant. if (addr != null) { - DataPlacement.Builder dataPlacementBuilder = DataPlacement.builder(); - Replica replica = new Replica(addr, - MetaStrategy.partitioner.getMinimumToken(), - MetaStrategy.partitioner.getMinimumToken(), - true); - dataPlacementBuilder.reads.withReplica(Epoch.FIRST, replica); - dataPlacementBuilder.writes.withReplica(Epoch.FIRST, replica); - DataPlacements initialPlacement = metadata.placements.unbuild() - .with(ReplicationParams.simpleMeta(1, datacenter), - dataPlacementBuilder.build()).build(); - - transformer.with(initialPlacement); // create the distributed metadata keyspace in schema with replication settings based on the DC of the // initial CMS node Keyspaces updated = metadata.schema.getKeyspaces() .withAddedOrReplaced(DistributedMetadataLogKeyspace.initialMetadata(datacenter)); transformer.with(new DistributedSchema(updated, Epoch.FIRST)); + ClusterMetadata.Transformer.Transformed transformed = transformer.build(); + + // This is required to bootstrap the initialization process. Because committing to the metadata log uses the + // previous ClusterMetadata to identify CMS members who form the consensus group, the first CMS member must + // be routable at that point. After the INITIALIZE_CMS is committed, and on instances other than the first + // CMS member, this placement is derived from the CMS membership list of node ids. + DataPlacement.Builder initialPlacement = DataPlacement.builder(); + Replica replica = new Replica(addr, + MetaStrategy.partitioner.getMinimumToken(), + MetaStrategy.partitioner.getMinimumToken(), + true); + initialPlacement.reads.withReplica(Epoch.FIRST, replica); + initialPlacement.writes.withReplica(Epoch.FIRST, replica); + metadata = transformed.metadata.forcePreInitializedState(initialPlacement.build()); + } + else + { + metadata = metadata.forceEpoch(Epoch.FIRST); } - ClusterMetadata.Transformer.Transformed transformed = transformer.build(); - metadata = transformed.metadata.forceEpoch(Epoch.FIRST); assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch; - - return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, transformed.modifiedKeys); + return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, ImmutableSet.of()); } @Override diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java index cfb7db66dd..5eec85e833 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +82,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation logger.info("Proposed CMS reconfiguration resulted in {}", diff); LockedRanges.Key lockKey = LockedRanges.keyFor(prev.nextEpoch()); - Set cms = prev.fullCMSMembers().stream().map(prev.directory::peerId).collect(Collectors.toSet()); + Set cms = prev.fullCMSMemberIds(); Set tmp = new HashSet<>(cms); tmp.addAll(diff.additions); tmp.removeAll(diff.removals); @@ -267,9 +266,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple())); - return executeInternal(prev, - transformer -> transformer.with(prev.placements.replaceParams(prev.nextEpoch(), ReplicationParams.meta(prev), replicationParams)) - .with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)))); + return executeInternal(prev, transformer -> transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)))); } public String toString() @@ -312,7 +309,6 @@ public abstract class PrepareCMSReconfiguration implements Transformation public static Diff diff(Set currentCms, Set newCms) { - assert !currentCms.contains(null) : "Current CMS contains a null value " + currentCms; assert !newCms.contains(null) : "New CMS contains a null value " + newCms; List additions = new ArrayList<>(); @@ -355,10 +351,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation public static boolean needsReconfiguration(ClusterMetadata metadata) { Map dcRf = extractRf(ReplicationParams.meta(metadata)); - Set currentCms = metadata.fullCMSMembers() - .stream() - .map(metadata.directory::peerId) - .collect(Collectors.toSet()); + Set currentCms = metadata.fullCMSMemberIds(); int expectedSize = dcRf.values().stream().mapToInt(Integer::intValue).sum(); if (currentCms.size() != expectedSize) return true; diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java index 8e37875e94..d958309ed8 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java @@ -29,20 +29,16 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, StartAddToCMS & FinishAddToCMS, contain a high degree of duplication with their intended @@ -87,11 +83,7 @@ public class RemoveFromCMS extends BaseMembershipTransformation if (sequence != null) return new Transformation.Rejected(INVALID, String.format("Can't remove %s from CMS as there are ongoing range movements on it", endpoint)); - ReplicationParams metaParams = ReplicationParams.meta(prev); - DataPlacement placements = prev.placements.get(metaParams); - - int minProposedSize = (int) Math.min(placements.reads.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count(), - placements.writes.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count()); + int minProposedSize = prev.fullCMSMemberIds().size() - 1; if (minProposedSize < MIN_SAFE_CMS_SIZE) { logger.warn("Removing {} from CMS members would reduce the service size to {} which is below the " + @@ -109,19 +101,8 @@ public class RemoveFromCMS extends BaseMembershipTransformation if (minProposedSize == 0) return new Transformation.Rejected(INVALID, String.format("Removing %s from the CMS would leave no members in CMS.", endpoint)); - ClusterMetadata.Transformer transformer = prev.transformer(); - Replica replica = new Replica(endpoint, entireRange, true); - - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild(); - builder.reads.withoutReplica(prev.nextEpoch(), replica); - builder.writes.withoutReplica(prev.nextEpoch(), replica); - DataPlacement proposed = builder.build(); - - if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty()) - return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint)); - - return Transformation.success(transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()), - MetaStrategy.affectedRanges(prev)); + ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(nodeId); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } @Override diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java index 90f3056f47..efa92c042a 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java @@ -18,25 +18,20 @@ package org.apache.cassandra.tcm.transformations.cms; -import java.util.HashSet; import java.util.Set; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.RangesByEndpoint; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.AddToCMS; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, FinishAddToCMS & RemoveFromCMS, contain a high degree of duplication with their intended @@ -76,27 +71,13 @@ public class StartAddToCMS extends BaseMembershipTransformation if (prev.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance) != null) return new Rejected(INVALID, String.format("Cannot add node to CMS as a CMS reconfiguration is currently active")); - Replica replica = new Replica(endpoint, entireRange, true); - ReplicationParams metaParams = ReplicationParams.meta(prev); - RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint(); - RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint(); - - if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint)) + CMSMembership cms = prev.cmsMembership; + if (cms.joiningMembers().contains(nodeId) || cms.fullMembers().contains(nodeId)) return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS"); - ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild() - .withWriteReplica(prev.nextEpoch(), replica); - - transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()); - - Set streamCandidates = new HashSet<>(); - for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues()) - { - if (!replica.equals(r)) - streamCandidates.add(r.endpoint()); - } + ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(nodeId); + Set streamCandidates = prev.fullCMSMembers(); AddToCMS joinSequence = new AddToCMS(prev.nextEpoch(), nodeId, streamCandidates, new FinishAddToCMS(endpoint)); transformer = transformer.with(prev.inProgressSequences.with(nodeId, joinSequence)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); diff --git a/src/java/org/apache/cassandra/tools/CMSOfflineTool.java b/src/java/org/apache/cassandra/tools/CMSOfflineTool.java index 90d44e3fc0..d42c91955b 100644 --- a/src/java/org/apache/cassandra/tools/CMSOfflineTool.java +++ b/src/java/org/apache/cassandra/tools/CMSOfflineTool.java @@ -37,10 +37,9 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.MultiStepOperation; @@ -441,24 +440,15 @@ public class CMSOfflineTool implements Runnable nodeState + " state. Only a JOINED node can be set as CMS member."); } InetAddressAndPort endpoint = metadata.directory.getNodeAddresses(nodeId).broadcastAddress; - ReplicationParams metaParams = ReplicationParams.meta(metadata); - Iterable currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues(); - DataPlacement.Builder placementBuilder = metadata.placements.get(metaParams).unbuild(); - for (Replica replica : currentReplicas) - { - placementBuilder.withoutReadReplica(metadata.epoch, replica) - .withoutWriteReplica(metadata.epoch, replica); - } + CMSMembership cms = metadata.cmsMembership; + ClusterMetadata.Transformer transformer = metadata.transformer(); + for (NodeId id : cms.fullMembers()) + transformer = transformer.leaveCMS(id); + for (NodeId id : cms.joiningMembers()) + transformer = transformer.cancelJoiningCMS(id); - Replica newCMS = MetaStrategy.replica(endpoint); - placementBuilder.withReadReplica(metadata.epoch, newCMS) - .withWriteReplica(metadata.epoch, newCMS); - - return metadata.transformer() - .with(metadata.placements.unbuild() - .with(metaParams, placementBuilder.build()) - .build()) - .build().metadata; + transformer = transformer.startJoiningCMS(nodeId).finishJoiningCMS(nodeId); + return transformer.build().metadata; } } @@ -619,7 +609,7 @@ public class CMSOfflineTool implements Runnable output.out.printf("Cluster Metadata Service:%n"); output.out.printf("Members: %s%n", members); output.out.printf("Needs reconfiguration: %s%n", needsReconfiguration(metadata)); - output.out.printf("Service State: %s%n", ClusterMetadataService.state(metadata)); + output.out.printf("Service State: %s%n", ClusterMetadataService.State.OFFLINE_TOOL); output.out.printf("Epoch: %s%n", metadata.epoch.getEpoch()); output.out.printf("Replication factor: %s%n", ReplicationParams.meta(metadata).toString()); } @@ -782,7 +772,7 @@ public class CMSOfflineTool implements Runnable throw new IllegalArgumentException("Keyspace " + keyspace + " not found in cluster metadata."); } - DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication); + DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication); List rows = new ArrayList<>(); rows.addAll(replicaGroupsToRows(placement.reads, "read")); rows.addAll(replicaGroupsToRows(placement.writes, "write")); diff --git a/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java b/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java index 59102ff27d..3c9d5a1ef9 100644 --- a/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java +++ b/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java @@ -27,13 +27,11 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -60,9 +58,9 @@ public class TransformClusterMetadataHelper DatabaseDescriptor.setPartitionerUnsafe(partitioner); ClusterMetadataService.initializeForTools(false); ClusterMetadata metadata = ClusterMetadataService.deserializeClusterMetadata(sourceFile); - System.out.println("Old CMS: " + metadata.placements.get(ReplicationParams.meta(metadata))); + System.out.println("Old CMS: " + metadata.placement(ReplicationParams.meta(metadata))); metadata = makeCMS(metadata, InetAddressAndPort.getByNameUnchecked(args[1])); - System.out.println("New CMS: " + metadata.placements.get(ReplicationParams.meta(metadata))); + System.out.println("New CMS: " + metadata.placement(ReplicationParams.meta(metadata))); Path p = Files.createTempFile("clustermetadata", "dump"); try (FileOutputStreamPlus out = new FileOutputStreamPlus(p)) { @@ -73,20 +71,9 @@ public class TransformClusterMetadataHelper public static ClusterMetadata makeCMS(ClusterMetadata metadata, InetAddressAndPort endpoint) { - ReplicationParams metaParams = ReplicationParams.meta(metadata); - Iterable currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues(); - DataPlacement.Builder builder = metadata.placements.get(metaParams).unbuild(); - for (Replica replica : currentReplicas) - { - builder.withoutReadReplica(metadata.epoch, replica) - .withoutWriteReplica(metadata.epoch, replica); - } - Replica newCMS = MetaStrategy.replica(endpoint); - builder.withReadReplica(metadata.epoch, newCMS) - .withWriteReplica(metadata.epoch, newCMS); - return metadata.transformer().with(metadata.placements.unbuild().with(metaParams, - builder.build()) - .build()) - .build().metadata; + NodeId id = metadata.directory.peerId(endpoint); + if (id == null) + throw new IllegalStateException("No node id found for endpoint: " + endpoint); + return metadata.transformer().startJoiningCMS(id).finishJoiningCMS(id).build().metadata; } } diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index ba47bda366..c3b0e3ebd4 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -839,7 +839,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { CommitLog.instance.recoverSegmentsOnDisk(); NodeId self = ClusterMetadata.current().myNodeId(); - if (self != null) + if (self != NodeId.UNREGISTERED) AccordService.localStartup(self); } catch (IOException e) diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index 19986671ec..006379ae3b 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -444,8 +444,8 @@ public class ClusterUtils for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces()) { List[] placements = new List[2]; - placements[0] = metadata.placements.get(keyspace.params.replication).reads.toReplicaStringList(); - placements[1] = metadata.placements.get(keyspace.params.replication).writes.toReplicaStringList(); + placements[0] = metadata.placement(keyspace.params.replication).reads.toReplicaStringList(); + placements[1] = metadata.placement(keyspace.params.replication).writes.toReplicaStringList(); byKeyspace.put(keyspace.name, placements); } return byKeyspace; @@ -465,10 +465,10 @@ public class ClusterUtils StringBuilder builder = new StringBuilder(); builder.append("'keyspace' { 'name':").append(keyspace.name).append("', "); builder.append("'reads':['"); - ReplicaGroups placement = metadata.placements.get(keyspace.params.replication).reads; + ReplicaGroups placement = metadata.placement(keyspace.params.replication).reads; builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString()); builder.append("'], 'writes':['"); - placement = metadata.placements.get(keyspace.params.replication).writes; + placement = metadata.placement(keyspace.params.replication).writes; builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString()); builder.append("']}"); keyspaces.add(builder.toString()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java index e84b51ff80..f39c16ecd7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java @@ -27,7 +27,6 @@ import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; @@ -46,8 +45,8 @@ public class AssassinateCMSNodeTest extends TestBaseImpl InetSocketAddress toAssassinate = cluster.get(2).broadcastAddress(); cluster.get(2).shutdown().get(); cluster.get(1).nodetoolResult("assassinate", toAssassinate.getHostString()).asserts().success(); - cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()))); - cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()))); + cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember())); + cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember())); cluster.get(1).nodetoolResult("cms").asserts().success(); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java index bcc02b89a6..a76d0d8d42 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java @@ -32,14 +32,11 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.CMSOperations; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import static org.apache.cassandra.distributed.shared.ClusterUtils.start; @@ -108,15 +105,10 @@ public class BootWithMetadataTest extends TestBaseImpl try { ClusterMetadata metadata = ClusterMetadata.current(); - Replica oldCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.1")); - Replica newCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.2")); + NodeId oldCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.1")); + NodeId newCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.2")); ClusterMetadata.Transformer transformer = metadata.transformer(); - DataPlacement.Builder builder = metadata.placements.get(ReplicationParams.meta(metadata)).unbuild() - .withoutReadReplica(metadata.nextEpoch(), oldCMS) - .withoutWriteReplica(metadata.nextEpoch(), oldCMS) - .withWriteReplica(metadata.nextEpoch(), newCMS) - .withReadReplica(metadata.nextEpoch(), newCMS); - transformer = transformer.with(metadata.placements.unbuild().with(ReplicationParams.meta(metadata), builder.build()).build()); + transformer.leaveCMS(oldCMS).startJoiningCMS(newCMS).finishJoiningCMS(newCMS); ClusterMetadata toDump = transformer.build().metadata.forceEpoch(Epoch.create(1000)); Path p = Files.createTempFile("clustermetadata", "dump"); try (FileOutputStreamPlus out = new FileOutputStreamPlus(p)) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java index 7731bde594..c0a4466e78 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java @@ -66,7 +66,8 @@ public class ClusterMetadataDumpTest extends TestBaseImpl epochsSeen++; } assertEquals(3, unsafeJoinSeen); - assertEquals(3, registerSeen); + // Only 2 REGISTER transforms are expected as the first CMS node is registered implicitly by INITIALIZE_CMS + assertEquals(2, registerSeen); assertTrue(epochsSeen > 15); res = cluster.get(1).nodetoolResult("cms", "dumplog", "--start", "10", "--end", "15"); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java index f1c669efa7..d0ea424f24 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java @@ -64,6 +64,7 @@ import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.AtomicLongBackedProcessor; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Commit; @@ -159,7 +160,8 @@ public class ClusterMetadataTestHelper public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner) { - return new ClusterMetadata(epoch, Murmur3Partitioner.instance, + return new ClusterMetadata(epoch, + Murmur3Partitioner.instance, DistributedSchema.empty(), Directory.EMPTY, new TokenMap(partitioner), @@ -169,7 +171,8 @@ public class ClusterMetadataTestHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public static ClusterMetadata minimalForTesting(IPartitioner partitioner) @@ -177,6 +180,11 @@ public class ClusterMetadataTestHelper return minimalForTesting(Epoch.EMPTY, partitioner); } + public static ClusterMetadata minimalForTesting(Epoch e, Keyspaces keyspaces, CMSMembership cms) + { + return minimalForTesting(e, Murmur3Partitioner.instance, new DistributedSchema(keyspaces), cms); + } + public static ClusterMetadata minimalForTesting(Keyspaces keyspaces) { return minimalForTesting(Murmur3Partitioner.instance).transformer() @@ -185,6 +193,28 @@ public class ClusterMetadataTestHelper .metadata.forceEpoch(Epoch.EMPTY); } + public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema) + { + return minimalForTesting(epoch, partitioner, schema, CMSMembership.EMPTY); + } + + public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema, CMSMembership cms) + { + return new ClusterMetadata(epoch, + Murmur3Partitioner.instance, + schema, + Directory.EMPTY, + new TokenMap(partitioner), + DataPlacements.empty(), + AccordFastPath.EMPTY, + LockedRanges.EMPTY, + InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, + ImmutableMap.of(), + AccordStaleReplicas.EMPTY, + cms); + } + public static ClusterMetadataService syncInstanceForTest() { LocalLog log = LocalLog.logSpec() @@ -1057,7 +1087,7 @@ public class ClusterMetadataTestHelper public static VersionedEndpoints.ForToken getNaturalReplicasForToken(ClusterMetadata metadata, String keyspace, Token searchPosition) { KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); - return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(searchPosition); + return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(searchPosition); } public static BootstrapAndJoin getBootstrapPlan(int idx) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java index b13cab2660..67331f0cb3 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java @@ -508,7 +508,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase Set bouncing = new HashSet<>(); Set replicasFromBouncedReplicaSets = new HashSet<>(); outer: - for (VersionedEndpoints.ForRange placements : sut.service.metadata().placements.get(rf.asKeyspaceParams().replication).writes.endpoints) + for (VersionedEndpoints.ForRange placements : sut.service.metadata().placement(rf.asKeyspaceParams().replication).writes.endpoints) { List replicas = new ArrayList<>(metadata.directory.toNodeIds(placements.get().endpoints())); List bounceCandidates = new ArrayList<>(); @@ -545,8 +545,8 @@ public class MetadataChangeSimulationTest extends CMSTestBase ClusterMetadata actualMetadata = sut.service.metadata(); ReplicationParams replication = actualMetadata.schema.getKeyspaces().get("test").get().params.replication; Assert.assertEquals(replication, sut.rf.asKeyspaceParams().replication); - match(actualMetadata.placements.get(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); - match(actualMetadata.placements.get(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); + match(actualMetadata.placement(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); + match(actualMetadata.placement(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); } public static void validatePlacements(CMSTestBase.CMSSut sut, ModelState modelState) throws Throwable @@ -559,7 +559,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase Assert.assertEquals(modelState.simulatedPlacements.nodes.stream().map(Node::token).collect(Collectors.toSet()), actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).getLongValue()).collect(Collectors.toSet())); - for (Map.Entry e : actualMetadata.placements.asMap().entrySet()) + for (Map.Entry e : actualMetadata.placements().asMap().entrySet()) { if (!e.getKey().equals(replication)) continue; @@ -569,7 +569,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase match(placement.reads, modelState.simulatedPlacements.readPlacements); } - validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements); + validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements()); } public static ModelChecker.Pair registerNewNode(ModelState state, CMSSut sut, int dcIdx, int rackIdx) @@ -953,7 +953,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase validatePlacements(sut, state); } // Finally verify that the predicted placements match the actual ones - Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placements.get(ksm.params.replication))); + Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placement(ksm.params.replication))); } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java index 2f3e9f81ef..af4354a133 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java @@ -116,8 +116,7 @@ public class OperationalEquivalenceTest extends CMSTestBase withMove = ClusterMetadata.current(); } - assertPlacements(simulateAndCompare(rf, equivalentNodes).placements, - withMove.placements); + assertPlacements(simulateAndCompare(rf, equivalentNodes).placements(), withMove.placements()); } private static ClusterMetadata simulateAndCompare(ReplicationFactor rf, List nodes) throws Exception diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java index ff1b7d50ac..3b99de22b3 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java @@ -41,7 +41,6 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.ReplicationParams; -import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.PaxosRepairHistory; import org.apache.cassandra.tcm.ClusterMetadata; @@ -57,6 +56,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart; import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack; import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology; +import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.psjava.util.AssertStatus.assertTrue; @@ -85,10 +85,10 @@ public class ReconfigureCMSTest extends FuzzTestBase ClusterMetadata metadata = ClusterMetadata.current(); assertEquals(5, metadata.fullCMSMembers().size()); assertEquals(ReplicationParams.simpleMeta(5, metadata.directory.knownDatacenters()), - metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get()); + metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication); }); cluster.stream().forEach(i -> { - Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0); + Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0); }); cluster.get(nodeSelector.get()).nodetoolResult("cms", "reconfigure", "1").asserts().success(); @@ -96,7 +96,7 @@ public class ReconfigureCMSTest extends FuzzTestBase ClusterMetadata metadata = ClusterMetadata.current(); assertEquals(1, metadata.fullCMSMembers().size()); assertEquals(ReplicationParams.simpleMeta(1, metadata.directory.knownDatacenters()), - metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get()); + metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication); }); } } @@ -136,7 +136,7 @@ public class ReconfigureCMSTest extends FuzzTestBase Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance)); assertEquals(2, metadata.fullCMSMembers().size()); ReplicationParams params = ReplicationParams.meta(metadata); - DataPlacement placements = metadata.placements.get(params); + DataPlacement placements = metadata.placements().get(params); assertTrue(placements.reads.equivalentTo(placements.writes)); assertEquals(metadata.fullCMSMembers().size(), Integer.parseInt(params.asMap().get("dc0"))); }); @@ -161,7 +161,7 @@ public class ReconfigureCMSTest extends FuzzTestBase Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance)); Assert.assertTrue(metadata.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort())); assertEquals(3, metadata.fullCMSMembers().size()); - DataPlacement placements = metadata.placements.get(ReplicationParams.meta(metadata)); + DataPlacement placements = metadata.placements().get(ReplicationParams.meta(metadata)); Assert.assertTrue(placements.reads.equivalentTo(placements.writes)); }); } @@ -216,7 +216,7 @@ public class ReconfigureCMSTest extends FuzzTestBase awaitRingJoin(replacingNode, cluster.get(1)); replacingNode.runOnInstance(() -> { ClusterMetadata metadata = ClusterMetadata.current(); - assertTrue(metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort())); + assertTrue(metadata.isCMSMember()); assertEquals(3, metadata.fullCMSMembers().size()); }); } @@ -326,11 +326,11 @@ public class ReconfigureCMSTest extends FuzzTestBase Object[][] rows = instance.executeInternal("select points from system.paxos_repair_history " + "where keyspace_name = ? " + "and table_name = ?", - SchemaConstants.METADATA_KEYSPACE_NAME, + METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME); if (rows.length == 0) - return PaxosRepairHistory.empty(SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME); + return PaxosRepairHistory.empty(METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME); assertEquals(1, rows.length); //noinspection unchecked List points = (List)rows[0][0]; diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java index 81006bbfc0..d12dddeef5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java @@ -19,12 +19,13 @@ package org.apache.cassandra.distributed.test.log; import java.io.IOException; -import java.net.UnknownHostException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.EnumSet; import org.junit.Test; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.IInstanceConfig; @@ -33,9 +34,11 @@ import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.io.util.DataInputBuffer; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.Location; @@ -45,15 +48,21 @@ import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.PlacementProvider; import org.apache.cassandra.tcm.sequences.LeaveStreams; +import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave; +import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.tcm.transformations.CustomTransformation; import org.apache.cassandra.tcm.transformations.PrepareLeave; import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.tcm.transformations.Unregister; -import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.vint.VIntCoding; +import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.addr; +import static org.apache.cassandra.tcm.membership.NodeVersion.CURRENT_METADATA_VERSION; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class RegisterTest extends TestBaseImpl { @@ -106,57 +115,42 @@ public class RegisterTest extends TestBaseImpl { cluster.get(1).startup(); cluster.get(1).runOnInstance(() -> { + + // Run a custom transformation to inject a fake node into the directory with a known id and an + // artificially lowered max supported serialization version + CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer); + CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME, + new RegisterNodeWithOldVersion()); + ClusterMetadataService.instance().commit(injectOldNode); + + // Doesn't matter which specific Transformation we use here, we're testing that the serializer uses + // the correct lower bound + Transformation t = new Register(NodeAddresses.current(), TEST_LOCATION, NodeVersion.CURRENT); try { - // Unregister to make directory empty - ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(), - EnumSet.allOf(NodeState.class), - ClusterMetadataService.instance().placementProvider())); - - // Register a ghost node with V0 (bypasses version check because directory is now empty). - // In a real world cluster we will always be upgrading from a smaller version. - ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")), - TEST_LOCATION, - new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0))); - NodeId oldNode = ClusterMetadata.current().directory.peerId(InetAddressAndPort.getByName("127.0.0.100")); - - // Register a node with upgraded version - CassandraVersion currentVersion = NodeVersion.CURRENT.cassandraVersion; - NodeVersion upgraded = new NodeVersion(new CassandraVersion(String.format("%d.%d.%d", currentVersion.major + 1, 0, 0)), - NodeVersion.CURRENT_METADATA_VERSION); - ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.200")), TEST_LOCATION, upgraded)); - - // Doesn't matter which specific Transformation we use here, we're testing that the serializer uses - // the correct lower bound - Transformation t = new Register(NodeAddresses.current(), new Location("DC", "RACK"), NodeVersion.CURRENT); - try + assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, RegisterNodeWithOldVersion.METADATA_VERSION); + ByteBuffer bytes = t.kind().toVersionedBytes(t); + try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) { - assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion, - Version.V0.asInt()); - ByteBuffer bytes = t.kind().toVersionedBytes(t); - try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) - { - // Because ClusterMetadata.current().directory still contains oldNode we must serialize at - // the version it supports - assertEquals(Version.V0, Version.fromInt(buf.readUnsignedVInt32())); - } - - // If we unregister oldNode, then the ceiling for serialization version will rise - ClusterMetadataService.instance().commit(new Unregister(oldNode, EnumSet.allOf(NodeState.class), ClusterMetadataService.instance().placementProvider())); - assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion, - NodeVersion.CURRENT_METADATA_VERSION.asInt()); - bytes = t.kind().toVersionedBytes(t); - try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) - { - assertEquals(NodeVersion.CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32())); - } + // Because ClusterMetadata.current().directory still contains the fake old node we must + // serialize at the version _it_ supports + assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32())); } - catch (IOException e) + + // If we unregister the fake node, then the ceiling for serialization version will rise + Unregister unregisterOldNode = new Unregister(RegisterNodeWithOldVersion.NODE_ID, + EnumSet.allOf(NodeState.class), + ClusterMetadataService.instance().placementProvider()); + ClusterMetadataService.instance().commit(unregisterOldNode); + + assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, CURRENT_METADATA_VERSION); + bytes = t.kind().toVersionedBytes(t); + try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) { - throw new RuntimeException(e); + assertEquals(CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32())); } } - catch (UnknownHostException e) + catch (IOException e) { throw new RuntimeException(e); } @@ -172,28 +166,70 @@ public class RegisterTest extends TestBaseImpl { cluster.get(1).startup(); cluster.get(1).runOnInstance(() -> { + // Run a custom transformation to inject a fake node into the directory with a known id and an + // artificially lowered max supported serialization version + CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer); + CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME, + new RegisterNodeWithOldVersion()); + ClusterMetadataService.instance().commit(injectOldNode); + // Now trigger a snapshot which must be written to the snapshot using the old serialization version + Epoch epoch = ClusterMetadataService.instance().commit(TriggerSnapshot.instance).epoch; + // fetch the raw bytes of the snapshot we just serialized + ByteBuffer bytes = SystemKeyspace.getSnapshot(epoch); + assertNotNull(bytes); + // assert the prepended version matches + Version writtenVersion = null; try { - // Unregister to make directory empty - ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(), - EnumSet.allOf(NodeState.class), - ClusterMetadataService.instance().placementProvider())); - - // Register a ghost node with V0 (bypasses version check because directory is now empty). - // In a real world cluster we will always be upgrading from a smaller version. - ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")), - TEST_LOCATION, - new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0))); + writtenVersion = Version.fromInt(VIntCoding.readUnsignedVInt32(new DataInputBuffer(bytes, false))); } - catch (UnknownHostException e) + catch (IOException e) { throw new RuntimeException(e); } - ClusterMetadataService.instance().commit(TriggerSnapshot.instance); - + assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, writtenVersion); + // load the snapshot using the standard mechanism and assert it matches current cluster metadata ClusterMetadata cm = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots().getSnapshot(ClusterMetadata.current().epoch); cm.equals(ClusterMetadata.current()); }); } } + + // Custom transforms to lock/unlock an arbitrary set of ranges to + // avoid having to actually initiate some range movement + public static class RegisterNodeWithOldVersion implements Transformation, Serializable + { + public static final AsymmetricMetadataSerializer serializer = new AsymmetricMetadataSerializer() + { + @Override + public void serialize(Transformation t, DataOutputPlus out, Version version){} + @Override + public RegisterNodeWithOldVersion deserialize(DataInputPlus in, Version version) {return new RegisterNodeWithOldVersion();} + @Override + public long serializedSize(Transformation t, Version version) {return 0;} + }; + + public static final String NAME = "TestRegisterNodeWithOldVersion"; + public static final NodeId NODE_ID = new NodeId(99); + public static final Version METADATA_VERSION = Version.V0; + + @Override + public Kind kind() + { + return Kind.CUSTOM; + } + + @Override + public Result execute(ClusterMetadata metadata) + { + ClusterMetadata.Transformer transformer = metadata.transformer() + .unsafeRegisterForTesting(NODE_ID, + new NodeAddresses(addr(99)), + TEST_LOCATION, + new NodeVersion(NodeVersion.CURRENT.cassandraVersion, + METADATA_VERSION)); + return Transformation.success(transformer, LockedRanges.AffectedRanges.EMPTY); + } + } + } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java index 07ec054d54..51ccb4d7ea 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java @@ -146,12 +146,12 @@ public class ResumableStartupTest extends FuzzTestBase KeyspaceMetadata ksm = metadata.schema.getKeyspaceMetadata(keyspace); boolean isWriteReplica = false; boolean isReadReplica = false; - for (InetAddressAndPort readReplica : metadata.placements.get(ksm.params.replication).reads.byEndpoint().keySet()) + for (InetAddressAndPort readReplica : metadata.placement(ksm.params.replication).reads.byEndpoint().keySet()) { if (readReplica.getHostAddressAndPort().equals(newAddress)) isReadReplica = true; } - for (InetAddressAndPort writeReplica : metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet()) + for (InetAddressAndPort writeReplica : metadata.placement(ksm.params.replication).writes.byEndpoint().keySet()) { if (writeReplica.getHostAddressAndPort().equals(newAddress)) isWriteReplica = true; diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java b/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java index 5275471476..d4131f060d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java @@ -177,8 +177,8 @@ public abstract class SimulatedOperation sutActions.next(); ClusterMetadata m2 = ClusterMetadata.current(); - Map, VersionedEndpoints.ForRange> after = m2.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.asMap(); - m1.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> { + Map, VersionedEndpoints.ForRange> after = m2.placement(simulatedState.rf.asKeyspaceParams().replication).reads.asMap(); + m1.placement(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> { if (after.containsKey(k)) { VersionedEndpoints.ForRange afterPlacements = after.get(k); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java index e355fdccc8..62416825bb 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java @@ -69,7 +69,7 @@ public class SnapshotTest extends TestBaseImpl ClusterMetadata before = ClusterMetadata.current(); ClusterMetadata after = ClusterMetadataService.instance().triggerSnapshot(); ClusterMetadata serialized = ClusterMetadataService.instance().snapshotManager().getSnapshot(after.epoch); - assertEquals(before.placements, serialized.placements); + assertEquals(before.placements(), serialized.placements()); assertEquals(before.tokenMap, serialized.tokenMap); assertEquals(before.directory, serialized.directory); assertEquals(before.schema, serialized.schema); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java index 35c81a53bb..1b7eda5ace 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java @@ -54,7 +54,7 @@ public class RangeVersioningTest extends FuzzTestBase for (int i = 1; i <= 4; i++) { Epoch smallestSeen = null; - for (VersionedEndpoints.ForRange fr : metadata.placements.get(ReplicationParams.simple(i)).writes.endpoints) + for (VersionedEndpoints.ForRange fr : metadata.placement(ReplicationParams.simple(i)).writes.endpoints) { if (smallestSeen == null || fr.lastModified().isBefore(smallestSeen)) smallestSeen = fr.lastModified(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java b/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java index 4a78f8805a..7722969a6a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java @@ -34,7 +34,6 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.distributed.shared.ClusterUtils.addInstance; import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin; @@ -80,7 +79,7 @@ public class CMSPlacementAfterReplacementTest extends TestBaseImpl IInvokableInstance nodeToRemove = cluster.get(2); cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success(); cluster.get(2).runOnInstance(() -> { - assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())); + assertTrue(ClusterMetadata.current().isCMSMember()); }); nodeToRemove.shutdown().get(); IInvokableInstance replacingNode = addInstance(cluster, nodeToRemove.config(), diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java index c020fdcc66..d5875e70fe 100644 --- a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java @@ -60,7 +60,7 @@ public class ClusterMetadataUpgradeAssassinateTest extends UpgradeTestBase ((IInvokableInstance) i).runOnInstance(() -> { ClusterMetadata metadata = ClusterMetadata.current(); InetAddressAndPort ep = InetAddressAndPort.getByNameUnchecked(host); - metadata.placements.asMap().forEach((key, value) -> { + metadata.placements().forEach((key, value) -> { if (key.isMeta()) return; boolean existsInPlacements = Streams.concat(value.reads.endpoints.stream(), diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java index e1fffc74b7..b406ae914c 100644 --- a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java @@ -181,7 +181,7 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase try { new ByteBuddy().rebase(ClusterMetadata.class) - .method(named("initializeClusterIdentifier")) + .method(named("forceInitializedState")) .intercept(MethodDelegation.to(ClusterMetadataUpgradeDelayedInitializeTest.BBInterceptor.class)) .make() .load(classLoader, ClassLoadingStrategy.Default.INJECTION); @@ -201,13 +201,13 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase public static class BBInterceptor { @SuppressWarnings("unused") - public static ClusterMetadata initializeClusterIdentifier(@SuperCall Callable zuper) + public static ClusterMetadata forceInitializedState(@SuperCall Callable zuper) { try { - logger.info("initializeClusterIdentifier waiting..."); + logger.info("forceInitializedState waiting..."); BBState.latch.await(60, TimeUnit.SECONDS); - logger.info("initializeClusterIdentifier continuing..."); + logger.info("forceInitializedState continuing..."); return zuper.call(); } catch (Throwable e) diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java b/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java index 481392149d..8cccb595e1 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java @@ -945,7 +945,7 @@ public abstract class TopologyMixupTestBase { ClusterMetadata current = ClusterMetadata.current(); - Set members = current.placements.get(ReplicationParams.meta(current)).writes.byEndpoint().keySet(); + Set members = current.placement(ReplicationParams.meta(current)).writes.byEndpoint().keySet(); // Why not just use 'current.fullCMSMembers()'? That uses the "read" replicas, so "could" have less endpoints // It would be more consistent to use fullCMSMembers but thought process is knowing the full set is better // than the coordination set. diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java index 140da9f18e..9718f86ab8 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java @@ -78,7 +78,7 @@ class OnClusterReplace extends OnClusterChangeTopology List> repairRanges = actions.cluster.get(leaving).unsafeApplyOnThisThread( (String keyspaceName) -> { ClusterMetadata metadata = ClusterMetadata.current(); - return metadata.placements.get(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication) + return metadata.placement(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication) .writes.ranges() .stream() .map(OnClusterReplace::toStringEntry) @@ -93,7 +93,7 @@ class OnClusterReplace extends OnClusterChangeTopology (String keyspaceName, String tk) -> { ClusterMetadata metadata = ClusterMetadata.current(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspaceName); - return metadata.placements.get(keyspaceMetadata.params.replication).reads + return metadata.placement(keyspaceMetadata.params.replication).reads .forToken(Utils.parseToken(tk)) .get() .stream().map(Replica::endpoint) diff --git a/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java b/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java index f0060dae93..58c3038229 100644 --- a/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java +++ b/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.auth; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -34,6 +35,8 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; @@ -59,6 +62,11 @@ public class GrantAndRevokeTest extends CQLTester ServerTestUtils.daemonInitialization(); DatabaseDescriptor.setPermissionsValidity(0); DatabaseDescriptor.setRolesValidity(0); + // for the tables in the distributed metadata keyspace to be queryable, there needs to be a valid placement + // which is derived from the CMS membership. Most unit tests don't actually need to query those dist tables + // which is why this isn't done as a matter of routine + ClusterMetadataTestHelper.reconfigureCms(ReplicationParams.ntsMeta(Collections.singletonMap(DatabaseDescriptor.getLocalDataCenter(), 1))); + ServerTestUtils.markCMS(); requireAuthentication(); requireNetwork(); CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces(); diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index 32e040f965..b1d4827253 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -204,7 +204,7 @@ public class BootStrapperTest ClusterMetadata metadata = ClusterMetadata.current(); Pair movements = Pair.create(MovementMap.empty(), MovementMap.empty()); - if (metadata.myNodeId() == null) + if (metadata.myNodeId() == NodeId.UNREGISTERED) { Token myToken = metadata.partitioner.getRandomToken(); InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1"); @@ -265,5 +265,4 @@ public class BootStrapperTest ClusterMetadataTestHelper.addEndpoint(addr, tokens); } } - } diff --git a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java index 5e3cdd4c42..ba30b10971 100644 --- a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java @@ -36,6 +36,7 @@ import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.Directory; @@ -96,7 +97,8 @@ public class MetaStrategyTest InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } @Test diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java index d2a59bdc71..25854eca3e 100644 --- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java @@ -372,6 +372,6 @@ public class SimpleStrategyTest extends CassandraTestBase ReplicationParams replicationParams, Token token) { - return metadata.placements.get(replicationParams).writes.forToken(token).get(); + return metadata.placement(replicationParams).writes.forToken(token).get(); } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java index 8b6b9ddb3c..22a62ac84c 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java @@ -105,7 +105,7 @@ public class AccordTopologyUtils { ReplicationParams replication = keyspace.params.replication; AbstractReplicationStrategy strategy = AbstractReplicationStrategy.createReplicationStrategy(keyspace.name, replication); - DataPlacements.Builder placements = metadata.placements.unbuild(); + DataPlacements.Builder placements = metadata.placements().unbuild(); DataPlacement placement = strategy.calculateDataPlacement(Epoch.EMPTY, metadata.tokenMap.toRanges(), metadata); placements.with(replication, placement); metadata = transformer.with(placements.build()).build().metadata; diff --git a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java index 5ca0272c6e..0c43c74ea8 100644 --- a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java @@ -344,7 +344,7 @@ public class EpochSyncTest private static boolean joined(ClusterMetadata metadata, Node.Id id) { NodeAddresses address = metadata.directory.getNodeAddresses(new NodeId(id.id)); - return metadata.placements.get(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress); + return metadata.placement(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress); } public enum EpochTracker { topologyManager, accordSyncPropagator } @@ -615,7 +615,7 @@ public class EpochSyncTest { Topology t = AccordTopology.createAccordTopology(current); Ranges ranges = t.ranges().mergeTouching(); - if (!current.placements.get(replication_params).reads.isEmpty()) + if (!current.placement(replication_params).reads.isEmpty()) Assertions.assertThat(ranges).hasSize(1); cms.setMetadata(current); for (Node.Id id : status(s -> s != Status.Removed)) @@ -721,7 +721,7 @@ public class EpochSyncTest case Registered: Invariants.require(!t.nodes().contains(id), "Node was in Init state but present in the Topology!"); Invariants.require(current.directory.peerId(address(id)) != null, "Node exists but not in TCM"); - if (current.placements.get(replication_params).writes.byEndpoint().keySet().contains(address(id))) + if (current.placement(replication_params).writes.byEndpoint().keySet().contains(address(id))) status = Status.Joining; break; case Joining: diff --git a/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java b/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java index 6b0d119448..fe9c6f0264 100644 --- a/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java +++ b/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java @@ -41,6 +41,7 @@ import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.MembershipUtils; +import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.DataPlacements; @@ -110,7 +111,10 @@ public class BootWithMetadataTest Directory directory = first.directory; int nodeCount = 10; int tokensPerNode = 5; - for (int i = 0; i < nodeCount; i++) + // Ensure that the "local" node is registered as it being member of the CMS is a precondition of booting from + // a ClusterMetadata. + directory = directory.with(NodeAddresses.current(), new Location("DC1", "RACK1")); + for (int i = 0; i < nodeCount-1; i++) directory = directory.with(nodeAddresses(random), new Location("DC1", "RACK1")); t = t.with(directory); @@ -156,7 +160,7 @@ public class BootWithMetadataTest assertEquals(toWrite.schema, fromRead.schema); assertEquals(toWrite.directory, fromRead.directory); assertEquals(toWrite.tokenMap, fromRead.tokenMap); - assertEquals(toWrite.placements, fromRead.placements); + assertEquals(toWrite.placements(), fromRead.placements()); assertEquals(toWrite.lockedRanges, fromRead.lockedRanges); assertEquals(toWrite.inProgressSequences, fromRead.inProgressSequences); assertEquals(toWrite.extensions, fromRead.extensions); diff --git a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java index 528dbae477..235a9735e8 100644 --- a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java +++ b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.tcm; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; @@ -33,6 +34,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata.Transformer.Transformed; @@ -55,6 +57,7 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import static org.apache.cassandra.tcm.MetadataKeys.ACCORD_FAST_PATH; import static org.apache.cassandra.tcm.MetadataKeys.ACCORD_STALE_REPLICAS; +import static org.apache.cassandra.tcm.MetadataKeys.CMS_MEMBERSHIP; import static org.apache.cassandra.tcm.MetadataKeys.CONSENSUS_MIGRATION_STATE; import static org.apache.cassandra.tcm.MetadataKeys.DATA_PLACEMENTS; import static org.apache.cassandra.tcm.MetadataKeys.IN_PROGRESS_SEQUENCES; @@ -137,6 +140,42 @@ public class ClusterMetadataTransformationTest assertModifications(transformed, NODE_DIRECTORY, TOKEN_MAP); } + @Test + public void testModifyCMSMembership() + { + ClusterMetadata metadata = new ClusterMetadata(Murmur3Partitioner.instance, Directory.EMPTY, DistributedSchema.empty()); + Transformed transformed = metadata.transformer().build(); + assertTrue(transformed.modifiedKeys.isEmpty()); + + List endpoints = MembershipUtils.uniqueEndpoints(random, 2); + NodeAddresses a1 = new NodeAddresses(endpoints.get(0)); + NodeAddresses a2 = new NodeAddresses(endpoints.get(1)); + + transformed = metadata.transformer() + .register(a1, new Location("dc1", "rack1"), NodeVersion.CURRENT) + .register(a2, new Location("dc1", "rack1"), NodeVersion.CURRENT) + .build(); + assertModifications(transformed, NODE_DIRECTORY); + NodeId n1 = transformed.metadata.directory.peerId(a1.broadcastAddress); + NodeId n2 = transformed.metadata.directory.peerId(a2.broadcastAddress); + + transformed = metadata.transformer().startJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + transformed = transformed.metadata.transformer().cancelJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + + transformed = transformed.metadata.transformer().startJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + transformed = transformed.metadata.transformer().finishJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + + transformed = transformed.metadata.transformer().startJoiningCMS(n2).finishJoiningCMS(n2).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + + transformed = transformed.metadata.transformer().leaveCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + } + @Test public void testModifySchema() { @@ -309,7 +348,7 @@ public class ClusterMetadataTransformationTest else if (key == TOKEN_MAP) return metadata.tokenMap; else if (key == DATA_PLACEMENTS) - return metadata.placements; + return metadata.placements(); else if (key == LOCKED_RANGES) return metadata.lockedRanges; else if (key == IN_PROGRESS_SEQUENCES) @@ -320,6 +359,8 @@ public class ClusterMetadataTransformationTest return metadata.consensusMigrationState; else if (key == ACCORD_STALE_REPLICAS) return metadata.accordStaleReplicas; + else if (key == CMS_MEMBERSHIP) + return metadata.cmsMembership; throw new IllegalArgumentException("Unknown metadata key " + key); } diff --git a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java index 167aa3992b..93ae4f1249 100644 --- a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java @@ -41,9 +41,12 @@ import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; +import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; import org.apache.cassandra.tcm.transformations.CustomTransformation; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.tcm.transformations.cms.Initialize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -79,7 +82,17 @@ public class GetLogStateTest ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(cms); log.readyUnchecked(); - log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); + + NodeAddresses addresses = NodeAddresses.current(); + Location location = DatabaseDescriptor.getInitialLocationProvider().initialLocation(); + log.unsafeBootstrapForTesting(addresses.broadcastAddress); + // register first node & make it CMS as this affects how the meta strategy placements are calculated + ClusterMetadata metadata = ClusterMetadata.current().forceInitializedState(addresses.broadcastAddress.hashCode(), + addresses, + NodeVersion.CURRENT, + location); + Initialize initialize = new Initialize(metadata); + cms.commit(initialize); } @Test diff --git a/test/unit/org/apache/cassandra/tcm/UnregisterTest.java b/test/unit/org/apache/cassandra/tcm/UnregisterTest.java index 62a6b160a2..3861eefd6b 100644 --- a/test/unit/org/apache/cassandra/tcm/UnregisterTest.java +++ b/test/unit/org/apache/cassandra/tcm/UnregisterTest.java @@ -110,7 +110,7 @@ public class UnregisterTest assertFalse(metadata.directory.allJoinedEndpoints().contains(ep)); assertFalse(metadata.directory.allDatacenterRacks().containsKey("dc2")); assertFalse(metadata.directory.knownDatacenters().contains("dc2")); - metadata.placements.asMap().forEach((params, placement) -> { + metadata.placements().forEach((params, placement) -> { assertFalse(Streams.concat(placement.writes.endpoints.stream(), placement.reads.endpoints.stream()).anyMatch((fr) -> fr.endpoints().contains(ep))); }); } diff --git a/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java b/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java index cac53d553f..17d515e1ff 100644 --- a/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java +++ b/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java @@ -104,7 +104,7 @@ public class GossipHelperTest assertEquals(internal, metadata.directory.addresses.get(nodeId).localAddress); assertEquals(nativeAddress, metadata.directory.addresses.get(nodeId).nativeAddress); - DataPlacements dp = metadata.placements; + DataPlacements dp = metadata.placements(); assertEquals(1, dp.get(KSM.params.replication).reads.forToken(token).get().size()); assertTrue(dp.get(KSM.params.replication).reads.forToken(token).get().contains(endpoint)); assertEquals(1, dp.get(KSM.params.replication).writes.forToken(token).get().size()); @@ -196,8 +196,8 @@ public class GossipHelperTest assertEquals(entry.getValue(), metadata.tokenMap.tokens(nodeId).iterator().next()); } - ReplicaGroups reads = metadata.placements.get(KSM_NTS.params.replication).reads; - ReplicaGroups writes = metadata.placements.get(KSM_NTS.params.replication).writes; + ReplicaGroups reads = metadata.placement(KSM_NTS.params.replication).reads; + ReplicaGroups writes = metadata.placement(KSM_NTS.params.replication).writes; assertEquals(reads, writes); // tokens are // dc1: 1: 1000, 3: 3000, 5: 5000, 6: 7000, 7: 9000 diff --git a/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java b/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java index 46adc7c67a..05bc00cadb 100644 --- a/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java +++ b/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java @@ -109,7 +109,7 @@ public class MetadataSnapshotListenerTest listener.notify(entry, result); ClusterMetadata snapshot = snapshots.getSnapshot(nextEpoch); assertEquals(nextEpoch, snapshot.epoch); - assertEquals(toSnapshot.placements, snapshot.placements); + assertEquals(toSnapshot.placements(), snapshot.placements()); } private MetadataSnapshots init() diff --git a/test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java b/test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java new file mode 100644 index 0000000000..70705ff4f9 --- /dev/null +++ b/test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm.listeners; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.tcm.CMSMembership; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.MembershipUtils; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.ownership.DataPlacement; +import org.apache.cassandra.tcm.ownership.DataPlacements; +import org.apache.cassandra.tcm.ownership.OwnershipUtils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.psjava.util.AssertStatus.assertTrue; + +public class PlacementsChangeListenerTest +{ + private static final Logger logger = LoggerFactory.getLogger(PlacementsChangeListenerTest.class); + static Random random; + static Epoch e; + static NodeId node1; + static KeyspaceMetadata ks1; + + @BeforeClass + public static void setupClass() + { + long seed = System.nanoTime(); + logger.info("Seed: {}", seed); + random = new Random(seed); + e = Epoch.create(10); + node1 = new NodeId(1); + ks1 = KeyspaceMetadata.create("ks1", KeyspaceParams.simple(1)); + CassandraRelevantProperties.TCM_SORT_REPLICA_GROUPS.setBoolean(false); + } + + @Test + public void testPlacementChange() + { + DataPlacements before = OwnershipUtils.randomPlacements(random).withLastModified(e); + DataPlacements.Builder builder = before.unbuild(); + before.forEach((params, placement) -> { + Replica remove = placement.writes.byEndpoint().flattenValues().iterator().next(); + Replica add = Replica.fullReplica(MembershipUtils.endpoint(99), remove.range()); + DataPlacement newPlacement = placement.unbuild() + .withoutWriteReplica(e.nextEpoch(), remove) + .withWriteReplica(e.nextEpoch(), add).build(); + builder.with(params, newPlacement); + }); + DataPlacements after = builder.build().withLastModified(e.nextEpoch()); + + // only placements are different + ClusterMetadata prev = metadata(e, before, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e.nextEpoch(), after, Keyspaces.of(ks1), node1); + assertNotEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertFalse(prev.placements().equivalentTo(next.placements())); + assertEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + @Test + public void testKeyspaceCountChange() + { + DataPlacements placements = OwnershipUtils.randomPlacements(random).withLastModified(e); + KeyspaceMetadata ks2 = KeyspaceMetadata.create("ks2", KeyspaceParams.simple(1)); + + // only keyspace counts are different + ClusterMetadata prev = metadata(e, placements, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e, placements, Keyspaces.of(ks1, ks2), node1); + assertEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertTrue(prev.placements().equivalentTo(next.placements())); + assertNotEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + @Test + public void testKeyspaceParamsChange() + { + DataPlacements placements = OwnershipUtils.randomPlacements(random).withLastModified(e); + KeyspaceMetadata ks1a = KeyspaceMetadata.create("ks1", KeyspaceParams.simple(2)); + + // only keyspace params are different + ClusterMetadata prev = metadata(e, placements, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e, placements, Keyspaces.of(ks1a), node1); + assertEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertTrue(prev.placements().equivalentTo(next.placements())); + assertEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertNotEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + @Test + public void testCMSMembershipChange() + { + DataPlacements placements = OwnershipUtils.randomPlacements(random).withLastModified(e); + NodeId node2 = new NodeId(2); + + // only cms memberships are different + ClusterMetadata prev = metadata(e, placements, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e, placements, Keyspaces.of(ks1), node1, node2); + assertEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertTrue(prev.placements().equivalentTo(next.placements())); + assertEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertNotEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + private static ClusterMetadata metadata(Epoch epoch, + DataPlacements placements, + Keyspaces keyspaces, + NodeId...cmsNode) + { + CMSMembership cms = CMSMembership.EMPTY; + for (NodeId n : cmsNode) + cms = cms.startJoining(n).finishJoining(n); + + ClusterMetadata.Transformer t = ClusterMetadataTestHelper.minimalForTesting(epoch, + Murmur3Partitioner.instance, + new DistributedSchema(keyspaces, epoch), + cms) + .forceEpoch(epoch) + .transformer() + .with(placements); + + return t.build().metadata; + } + + private static void assertOnChangeEvent(ClusterMetadata prev, ClusterMetadata next) + { + AtomicInteger cnt = new AtomicInteger(0); + PlacementsChangeListener listener = new PlacementsChangeListener(cnt::incrementAndGet); + listener.notifyPostCommit(prev, next, false); + assertEquals(1, cnt.get()); + } +} diff --git a/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java b/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java index a4c884406c..fc8c0559e1 100644 --- a/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java @@ -62,8 +62,9 @@ public class DistributedLogStateTest extends LogStateTestBase { return new LogStateSUT() { - - // start test entries at FIRST + 1 as the pre-init transform is automatically inserted with Epoch.FIRST + // we start test entries at FIRST, but in a real log the PRE_INITIALIZE_CMS transform is automatically + // inserted with Epoch.FIRST, followed by INITIALIZE_CMS so the next entry to be committed would be at + // epoch 3 Epoch currentEpoch = Epoch.FIRST; Epoch nextEpoch; boolean applied; diff --git a/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java b/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java index 91e1f1e1c3..3820c8f8d4 100644 --- a/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java +++ b/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java @@ -19,8 +19,8 @@ package org.apache.cassandra.tcm.membership; import java.net.UnknownHostException; +import java.util.List; import java.util.Random; -import java.util.Set; import java.util.stream.Collectors; import org.apache.cassandra.locator.InetAddressAndPort; @@ -38,13 +38,13 @@ public class MembershipUtils return endpoint(random.nextInt(254) + 1); } - public static Set uniqueEndpoints(Random random, int count) + public static List uniqueEndpoints(Random random, int count) { return random.ints(1, 255) .distinct() .limit(count) .mapToObj(MembershipUtils::endpoint) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); } public static InetAddressAndPort endpoint(int i) diff --git a/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java b/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java index f9e8104ca9..07a3dc4b03 100644 --- a/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java +++ b/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java @@ -97,7 +97,7 @@ public class LocalRangesAllSettledTest AllLocalRanges proposed = snapshotAllLocalRanges(LocalRangeStatus.SETTLED, INITIAL_NODES); assertEquals(initial, proposed); // Check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, initial, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), initial, INITIAL_NODES); // Initiate an operation which affects ownership. This will add the MultiStepOperation which encodes any // necessary range movements so subsequent calls to ClusterMetadata::localRangesAllSettled @@ -123,7 +123,7 @@ public class LocalRangesAllSettledTest assertEquals(proposed, finalized); // Finally, check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, finalized, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), finalized, INITIAL_NODES); } @Test @@ -134,7 +134,7 @@ public class LocalRangesAllSettledTest AllLocalRanges proposed = snapshotAllLocalRanges(LocalRangeStatus.SETTLED, INITIAL_NODES); assertEquals(initial, proposed); // Check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, initial, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), initial, INITIAL_NODES); // Initiate an operation which affects ownership. This will add the MultiStepOperation which encodes any // necessary range movements so subsequent calls to ClusterMetadata::localRangesAllSettled @@ -161,7 +161,7 @@ public class LocalRangesAllSettledTest assertEquals(proposed, finalized); // Finally, check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, finalized, expandedNodes); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), finalized, expandedNodes); } @Test @@ -172,7 +172,7 @@ public class LocalRangesAllSettledTest AllLocalRanges proposed = snapshotAllLocalRanges(LocalRangeStatus.SETTLED, INITIAL_NODES); assertEquals(initial, proposed); // Check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, initial, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), initial, INITIAL_NODES); // Initiate an operation which affects ownership. This will add the MultiStepOperation which encodes any // necessary range movements so subsequent calls to ClusterMetadata::localRangesAllSettled @@ -201,7 +201,7 @@ public class LocalRangesAllSettledTest assertEquals(proposed, finalized); // Finally, check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, finalized, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), finalized, INITIAL_NODES); } private void assertLocalRangesMatchPlacements(DataPlacements placements, diff --git a/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java b/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java index 897dea5e5e..4b8b4e4164 100644 --- a/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java +++ b/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java @@ -254,6 +254,6 @@ public class OwnershipUtils assert result.isSuccess(); workingMetadata = result.success().metadata; } - return workingMetadata.placements; + return workingMetadata.placements(); } } diff --git a/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java b/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java index d4e1b4df29..50946d7ac4 100644 --- a/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java +++ b/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java @@ -105,11 +105,13 @@ public class InProgressSequenceCancellationTest LockedRanges locked = lockedRanges(placements, random); // state of metadata before starting the sequence + // note: don't allow epoch to be Epoch.FIRST as this is a + // special case for calculating meta strategy placements. ClusterMetadata before = metadata(directory).transformer() .with(placements) .withNodeState(nodeId, NodeState.REGISTERED) .with(locked) - .build().metadata; + .build().metadata.forceEpoch(epoch(random)); // Placements after PREPARE_JOIN DataPlacements afterPrepare = placements(ranges(random), replication, random); @@ -179,11 +181,13 @@ public class InProgressSequenceCancellationTest // Ranges locked by other operations LockedRanges locked = lockedRanges(placements, random); // state of metadata before starting the sequence + // note: don't allow epoch to be Epoch.FIRST as this is a + // special case for calculating meta strategy placements. ClusterMetadata before = metadata(directory).transformer() .with(placements) .withNodeState(nodeId, NodeState.JOINED) .with(locked) - .build().metadata; + .build().metadata.forceEpoch(epoch(random)); // PREPARE_LEAVE does not modify placements, so first transformation is START_LEAVE @@ -301,7 +305,7 @@ public class InProgressSequenceCancellationTest private void assertRelevantMetadata(ClusterMetadata first, ClusterMetadata second) { - assertTrue(first.placements.equivalentTo(second.placements)); + assertTrue(first.placements().equivalentTo(second.placements())); assertTrue(first.directory.equivalentTo(second.directory)); assertTrue(first.tokenMap.equivalentTo(second.tokenMap)); assertEquals(first.lockedRanges.locked.keySet(), second.lockedRanges.locked.keySet()); diff --git a/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java b/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java index cb4f15e5ef..2f3a3a0fd0 100644 --- a/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java +++ b/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java @@ -113,7 +113,7 @@ public class ProgressBarrierTest extends CMSTestBase // Internally affectedRanges::toPeers uses the same logic as // the progress barrier does to identify the consensus group Set consensusGroup = leave.barrier().affectedRanges.toPeers(ReplicationParams.simple(1), - sut.service.metadata().placements, + sut.service.metadata().placements(), sut.service.metadata().directory); assertEquals(Set.of(node2.nodeId(), node3.nodeId()), consensusGroup); } @@ -217,7 +217,7 @@ public class ProgressBarrierTest extends CMSTestBase case ALL: { Set replicas = metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) .collect(Collectors.toSet()); @@ -235,7 +235,7 @@ public class ProgressBarrierTest extends CMSTestBase case QUORUM: { Set replicas = metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) .collect(Collectors.toSet()); @@ -253,7 +253,7 @@ public class ProgressBarrierTest extends CMSTestBase case LOCAL_QUORUM: { List replicas = new ArrayList<>(metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .filter((n) -> metadata.directory.location(n).datacenter.equals(dc)) .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) @@ -277,7 +277,7 @@ public class ProgressBarrierTest extends CMSTestBase { Map byDc = new HashMap<>(); metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .forEach(n -> byDc.compute(metadata.directory.location(n).datacenter, (k, v) -> v == null ? 1 : v + 1)); @@ -306,7 +306,7 @@ public class ProgressBarrierTest extends CMSTestBase } case ONE: Set replicas = metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) .collect(Collectors.toSet()); diff --git a/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java b/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java index 11fd03c696..0f9115805f 100644 --- a/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java +++ b/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java @@ -95,8 +95,8 @@ public class EventsMetadataTest // should not be in tokenMap (no tokens yet) assertTrue(metadata.tokenMap.tokens(nodeId).isEmpty()); - assertTrue(metadata.placements.get(KSM.params.replication).writes.byEndpoint().isEmpty()); - assertTrue(metadata.placements.get(KSM.params.replication).reads.byEndpoint().isEmpty()); + assertTrue(metadata.placement(KSM.params.replication).writes.byEndpoint().isEmpty()); + assertTrue(metadata.placement(KSM.params.replication).reads.byEndpoint().isEmpty()); assertTrue(metadata.lockedRanges.locked.isEmpty()); } @@ -130,9 +130,9 @@ public class EventsMetadataTest assertTrue(ClusterMetadata.current().tokenMap.tokens(nodeId).isEmpty()); assertEquals(NodeState.BOOTSTRAPPING, ClusterMetadata.current().directory.peerState(nodeId)); - assertTrue(ClusterMetadata.current().placements.get(KSM.params.replication).writes.byEndpoint().containsKey(node1)); + assertTrue(ClusterMetadata.current().placement(KSM.params.replication).writes.byEndpoint().containsKey(node1)); // the first joined node gets added to the read endpoints immediately - assertTrue(ClusterMetadata.current().placements.get(KSM.params.replication).reads.byEndpoint().containsKey(node1)); + assertTrue(ClusterMetadata.current().placement(KSM.params.replication).reads.byEndpoint().containsKey(node1)); ClusterMetadataService.instance().commit(plan.midJoin); ClusterMetadataService.instance().commit(plan.finishJoin); @@ -152,8 +152,8 @@ public class EventsMetadataTest assertTrue(ClusterMetadata.current().tokenMap.tokens(nodeId).isEmpty()); assertEquals(NodeState.BOOTSTRAPPING, ClusterMetadata.current().directory.peerState(nodeId)); - assertTrue(ClusterMetadata.current().placements.get(KSM.params.replication).writes.byEndpoint().containsKey(node2)); - assertFalse(ClusterMetadata.current().placements.get(KSM.params.replication).reads.byEndpoint().containsKey(node2)); + assertTrue(ClusterMetadata.current().placement(KSM.params.replication).writes.byEndpoint().containsKey(node2)); + assertFalse(ClusterMetadata.current().placement(KSM.params.replication).reads.byEndpoint().containsKey(node2)); } @Test @@ -178,7 +178,7 @@ public class EventsMetadataTest // no change in metadata after prepareLeave; assertEquals(before.directory, after.directory); assertEquals(before.tokenMap, after.tokenMap); - assertEquals(before.placements, after.placements); + assertEquals(before.placements(), after.placements()); assertEquals(before.schema, after.schema); ClusterMetadataService.instance().commit(leave.startLeave); diff --git a/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java b/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java index f5044a00fd..bbf58eda54 100644 --- a/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java +++ b/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java @@ -84,10 +84,11 @@ public class PrepareLeaveTest public void testCheckRF_Simple() throws Throwable { Keyspaces kss = Keyspaces.of(DistributedMetadataLogKeyspace.initialMetadata(Sets.newHashSet(hostDc.values())), KSM); + // should be accepted (2 nodes in dc1 where we remove the host): ClusterMetadata metadata = prepMetadata(kss, 2, 2); assertTrue(executeLeave(metadata)); - // should be rejected: - metadata = prepMetadata(kss, 1, 2); + // should be rejected because only 1 node in dc2: + metadata = prepMetadata(kss, 2, 1); assertFalse(executeLeave(metadata)); } @@ -97,17 +98,17 @@ public class PrepareLeaveTest Keyspaces kss = Keyspaces.of(DistributedMetadataLogKeyspace.initialMetadata(Sets.newHashSet(hostDc.values())), KSM_NTS); ClusterMetadata metadata = prepMetadata(kss, 4, 4); assertTrue(executeLeave(metadata)); - // should be accepted (4 nodes in dc1 where we remove the host): - metadata = prepMetadata(kss, 4, 2); + // should be accepted (4 nodes in dc2 where we remove the host): + metadata = prepMetadata(kss, 2, 4); assertTrue(executeLeave(metadata)); - // should be rejected - metadata = prepMetadata(kss, 3, 4); + // should be rejected because there are already only 3 replicas in dc2 + metadata = prepMetadata(kss, 4, 3); assertFalse(executeLeave(metadata)); } private boolean executeLeave(ClusterMetadata metadata) throws Throwable { - PrepareLeave prepareLeave = new PrepareLeave(metadata.directory.peerId(InetAddressAndPort.getByName("127.0.0.1")), + PrepareLeave prepareLeave = new PrepareLeave(metadata.directory.peerId(InetAddressAndPort.getByName("127.0.0.11")), false, dummyPlacementProvider, LeaveStreams.Kind.UNBOOTSTRAP); diff --git a/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java b/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java index 57d87d22e7..04209514bb 100644 --- a/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java +++ b/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -55,6 +54,7 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; @@ -110,7 +110,8 @@ public class CMSOfflineToolTest extends OfflineToolUtils InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } @Before @@ -669,7 +670,7 @@ public class CMSOfflineToolTest extends OfflineToolUtils "Cluster Metadata Service:\n" + "Members: /127.0.0.1:" + storagePort + ",/127.0.0.2:" + storagePort + ",/127.0.0.3:" + storagePort + '\n' + "Needs reconfiguration: false\n" + - "Service State: LOCAL\n" + + "Service State: " + ClusterMetadataService.State.OFFLINE_TOOL + '\n' + "Epoch: 2\n" + "Replication factor: ReplicationParams{class=org.apache.cassandra.locator.MetaStrategy, datacenter1=3}\n"; assertThat(result.getStdout()).isEqualTo(expectedOutput); @@ -1668,10 +1669,7 @@ public class CMSOfflineToolTest extends OfflineToolUtils KeyspaceMetadata normalKeyspace = KeyspaceMetadata.create("ks", KeyspaceParams.simple(3)); Keyspaces keyspaces = Keyspaces.none().with(metaKeyspace).with(normalKeyspace); - ClusterMetadata clusterMetadata = getClusterMetadata(keyspaces, partitioner, directory); - - - ClusterMetadata metadata = clusterMetadata + ClusterMetadata metadata = getClusterMetadata(keyspaces, partitioner, directory) .transformer() .with(directory) .join(nodeId1) @@ -1680,12 +1678,12 @@ public class CMSOfflineToolTest extends OfflineToolUtils .proposeToken(nodeId2, getRandomTokens(partitioner, tokenSize)) .join(nodeId3) .proposeToken(nodeId3, getRandomTokens(partitioner, tokenSize)) + .startJoiningCMS(nodeId1).finishJoiningCMS(nodeId1) + .startJoiningCMS(nodeId2).finishJoiningCMS(nodeId2) + .startJoiningCMS(nodeId3).finishJoiningCMS(nodeId3) .build().metadata; - // Create replicas for the metadata keyspace on all three nodes - ReplicationParams metaParams = ReplicationParams.ntsMeta(Collections.singletonMap(DC, 3)); DataPlacements placements = DataPlacements.empty().unbuild() - .with(metaParams, getCMSMemberPlacement(metadata, List.of(addr1, addr2, addr3))) .with(ReplicationParams.simple(3), getKeyspacePlacement(metadata, normalKeyspace)) .build(); @@ -1799,6 +1797,11 @@ public class CMSOfflineToolTest extends OfflineToolUtils ClusterMetadata startReplacing(NodeId oldNodeId, NodeId newNodeId, ClusterMetadata clusterMetadata) { + // In a real cluster, a node being replaced is removed from the CMS prior to the replacement starting + // i.e. before the PrepareReplace is committed + if (clusterMetadata.fullCMSMemberIds().contains(oldNodeId)) + clusterMetadata = clusterMetadata.transformer().leaveCMS(oldNodeId).build().metadata; + Register register = new Register(getNodeAddresses(newNodeId.id()), new Location(DC, "rack" + newNodeId.id()), NodeVersion.CURRENT); diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index 4c0a6758ec..1438fc8b62 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -140,6 +140,7 @@ import org.apache.cassandra.service.accord.topology.SimpleFastPathStrategy; import org.apache.cassandra.service.accord.topology.UpFastPathStrategy; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.extensions.ExtensionKey; @@ -1955,7 +1956,8 @@ public final class CassandraGenerators ConsensusMigrationState consensusMigrationState = ConsensusMigrationState.EMPTY; Map, ExtensionValue> extensions = ImmutableMap.of(); AccordStaleReplicas accordStaleReplicas = accordStaleReplicasGen.generate(rnd); - return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions, accordStaleReplicas); + CMSMembership cms = CMSMembership.EMPTY; + return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions, accordStaleReplicas, cms); }; } } From 1df3a8cef04d1bf2a92b4ca48f99b5b5be6bd57d Mon Sep 17 00:00:00 2001 From: Sam Tunnicliffe Date: Wed, 8 Jul 2026 15:35:09 +0100 Subject: [PATCH 2/6] Setup async transformation before making internode request Patch by Sam Tunnicliffe and Dmitry Konstantinov; reviewed by Sam Tunnicliffe and Dmitry Konstantinov for CASSANDRA-21384 Co-authored-by: Dmitry Konstantinov --- CHANGES.txt | 1 + .../apache/cassandra/tcm/PeerLogFetcher.java | 37 +- .../cassandra/tcm/PeerLogFetcherTest.java | 354 ++++++++++++++++++ 3 files changed, 374 insertions(+), 18 deletions(-) create mode 100644 test/unit/org/apache/cassandra/tcm/PeerLogFetcherTest.java diff --git a/CHANGES.txt b/CHANGES.txt index dcd7972231..51b576d242 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 6.0-alpha2 + * Avoid potential deadlock between GlobalLogFollower and GossipStage (CASSANDRA-21384) * Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736) * Fix maven remote publishing of Accord artifacts (CASSANDRA-21261) * Move long running TCM operations to a longer timout (CASSANDRA-21453) diff --git a/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java b/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java index 12cdc18e5d..4a131954cb 100644 --- a/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java +++ b/src/java/org/apache/cassandra/tcm/PeerLogFetcher.java @@ -88,34 +88,35 @@ public class PeerLogFetcher return res; } - Promise fetchRes = new AsyncPromise<>(); + Promise fetchFromRemote = new AsyncPromise<>(); + Future appendToLog = fetchFromRemote.map(logState -> { + log.append(logState); + ClusterMetadata fetched = log.waitForHighestConsecutive(); + if (fetched.epoch.isEqualOrAfter(awaitAtleast)) + { + TCMMetrics.instance.peerLogEntriesFetched(before, logState.latestEpoch()); + return fetched; + } + else + { + throw new IllegalStateException(String.format("Queried for epoch %s, but could not catch up. Current epoch: %s", awaitAtleast, fetched.epoch)); + } + }); + logger.info("Fetching log from {}, at least {}", remote, awaitAtleast); try (Timer.Context ctx = TCMMetrics.instance.fetchPeerLogLatency.time()) { RemoteProcessor.sendWithRetries(Verb.TCM_FETCH_PEER_LOG_REQ, new FetchPeerLog(before), - fetchRes, + fetchFromRemote, new RemoteProcessor.CandidateIterator(Collections.singletonList(remote), false), Retry.untilElapsed(DatabaseDescriptor.getCmsAwaitTimeout().to(TimeUnit.NANOSECONDS), TCMMetrics.instance.fetchLogRetries)); - - return fetchRes.map((logState) -> { - log.append(logState); - ClusterMetadata fetched = log.waitForHighestConsecutive(); - if (fetched.epoch.isEqualOrAfter(awaitAtleast)) - { - TCMMetrics.instance.peerLogEntriesFetched(before, logState.latestEpoch()); - return fetched; - } - else - { - throw new IllegalStateException(String.format("Queried for epoch %s, but could not catch up. Current epoch: %s", awaitAtleast, fetched.epoch)); - } - }); - + return appendToLog; } catch (Throwable t) { - fetchRes.cancel(true); + fetchFromRemote.cancel(true); + appendToLog.cancel(true); JVMStabilityInspector.inspectThrowable(t); logger.warn("Unable to fetch log entries from " + remote, t); diff --git a/test/unit/org/apache/cassandra/tcm/PeerLogFetcherTest.java b/test/unit/org/apache/cassandra/tcm/PeerLogFetcherTest.java new file mode 100644 index 0000000000..b3ee9ef9a8 --- /dev/null +++ b/test/unit/org/apache/cassandra/tcm/PeerLogFetcherTest.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm; + +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; + +import com.google.common.util.concurrent.Uninterruptibles; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.OutboundSink; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.tcm.listeners.ChangeListener; +import org.apache.cassandra.tcm.log.Entry; +import org.apache.cassandra.tcm.log.LocalLog; +import org.apache.cassandra.tcm.log.LogState; +import org.apache.cassandra.tcm.ownership.UniformRangePlacement; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PeerLogFetcherTest +{ + private static final InetAddressAndPort REMOTE = InetAddressAndPort.getByNameUnchecked("127.0.0.2"); + + @BeforeClass + public static void beforeClass() + { + DatabaseDescriptor.daemonInitialization(); + } + + /** + * Regression test for CASSANDRA-21384 (deadlock between GlobalLogFollower and GossipStage). + * + * When a node discovers a peer at a higher epoch (e.g. while processing gossip), it calls + * {@link PeerLogFetcher#asyncFetchLog}. Through {@link EpochAwareDebounce} that runs + * synchronously on the calling thread (GossipStage). The fetch attaches a continuation via + * {@code Future.map} that does {@code log.append(logState)} followed by the blocking + * {@code log.waitForHighestConsecutive()}. + * + * The bug: the original code attached that continuation *after* dispatching the internode + * request. If the response completed the request promise before the {@code map()} call (a + * small but real race, easily won over loopback), {@code map()} on an already-completed + * promise runs the mapper inline on the caller thread. That means the blocking + * {@code waitForHighestConsecutive()} ran on GossipStage, which then could not service the + * {@code Gossiper.runInGossipStageBlocking()} call made by the GlobalLogFollower's post-commit + * listeners -> circular wait. + * + * This test forces exactly that "promise already completed before the continuation is + * attached" ordering: the request promise is completed from a dedicated responder thread and + * the caller is blocked until that has happened. It then asserts that the append/wait + * continuation ran on the responder (messaging) thread and never on the caller thread. + * Against the pre-fix code the continuation runs on the caller thread and the assertion fails. + */ + @Test + public void continuationNeverRunsOnCallerThread() throws Exception + { + LocalLog log = LocalLog.logSpec() + .async() + .withInitialState(new ClusterMetadata(Murmur3Partitioner.instance)) + .createLog(); + ClusterMetadataService cms = new ClusterMetadataService(new UniformRangePlacement(), + MetadataSnapshots.NO_OP, + log, + new AtomicLongBackedProcessor(log), + Commit.Replicator.NO_OP, + false); + ClusterMetadataService.setInstance(cms); + log.readyUnchecked(); + + // Records the thread that executes the append/wait continuation. The filter fires inside + // log.append(...) - the first thing the continuation does - and drops the entry so no real + // processing is required. + AtomicReference continuationThread = new AtomicReference<>(); + Predicate captureThenDrop = entry -> { + continuationThread.compareAndSet(null, Thread.currentThread()); + return true; + }; + + ExecutorService responder = Executors.newSingleThreadExecutor(r -> new Thread(r, "test-fetch-responder")); + AtomicReference responderThread = new AtomicReference<>(); + CountDownLatch delivered = new CountDownLatch(1); + AtomicBoolean intercepted = new AtomicBoolean(false); + + // Intercept the outbound TCM_FETCH_PEER_LOG_REQ. Complete the request promise from the + // responder thread (mimicking the messaging response callback) and block the caller here + // until that has happened, so the promise is already done when the caller returns from the + // send. The message is always dropped (return false) so nothing is put on the wire. + OutboundSink.Filter interceptor = (message, to, type) -> { + if (message.verb() != Verb.TCM_FETCH_PEER_LOG_REQ) + return true; + + if (intercepted.compareAndSet(false, true)) + { + long id = message.id(); + Message request = message; + responder.execute(() -> { + responderThread.set(Thread.currentThread()); + LogState logState = LogState.make(new ClusterMetadata(Murmur3Partitioner.instance)); + Message response = request.responseWith(logState); + MessagingService.instance().callbacks.removeAndRespond(id, to, response); + delivered.countDown(); + }); + assertTrue("responder did not deliver in time", + Uninterruptibles.awaitUninterruptibly(delivered, 30, TimeUnit.SECONDS)); + } + return false; + }; + MessagingService.instance().outboundSink.add(interceptor); + + try + { + log.addFilter(captureThenDrop); + Thread callerThread = Thread.currentThread(); + + PeerLogFetcher fetcher = new PeerLogFetcher(log); + Future result = fetcher.asyncFetchLog(REMOTE, Epoch.FIRST); + // The continuation has run by the time asyncFetchLog returns (inline on the caller for + // the buggy code, on the responder thread otherwise). Wait for the future to settle - + // it completes exceptionally here since the fetched epoch never reaches FIRST, which is + // irrelevant: we only care which thread ran the continuation. + result.awaitUninterruptibly(30, TimeUnit.SECONDS); + + Thread ran = continuationThread.get(); + assertNotNull("The append/wait continuation never ran", ran); + assertNotSame("CASSANDRA-21384: the peer-log append/wait continuation must not run on the " + + "caller (e.g. GossipStage) thread, otherwise it can deadlock with GlobalLogFollower", + callerThread, ran); + assertSame("The continuation should run on the messaging response thread", + responderThread.get(), ran); + } + finally + { + MessagingService.instance().outboundSink.remove(interceptor); + responder.shutdownNow(); + log.close(); + ClusterMetadataService.unsetInstance(); + } + } + + /** + * Regression test for CASSANDRA-21384 that reproduces the *actual* circular wait from the + * ticket end-to-end (not just the thread on which the continuation runs), and verifies it with + * a thread dump. + * + * It runs the peer-log fetch on the real, single-threaded {@link Stage#GOSSIP} and registers a + * post-commit {@link ChangeListener} that - like the production listeners - calls + * {@link Gossiper#runInGossipStageBlocking} while an epoch is enacted. On the pre-fix code the + * fetch's append/wait continuation runs inline on the GossipStage thread and blocks it, so: + * + * 
+     *   GossipStage       : blocked in PeerLogFetcher -> LocalLog.waitForHighestConsecutive()
+     *                       (waiting for the log to advance)
+     *   GlobalLogFollower : enacting the epoch -> ChangeListener -> Gossiper.runInGossipStageBlocking()
+     *                       (waiting for the single GossipStage thread, which is stuck above)
+     *
+ * + * The test asserts the fetch returns within a timeout (i.e. no deadlock). If it does deadlock + * it captures a full thread dump and asserts the two stacks are exactly that cycle before + * failing. On the fixed code the continuation runs on the messaging thread, GossipStage returns + * from the fetch immediately, and the follower's task is serviced - the test passes. + */ + @Test + public void doesNotDeadlockBetweenGossipStageAndLogFollower() throws Exception + { + LocalLog log = LocalLog.logSpec() + .async() + .withInitialState(new ClusterMetadata(Murmur3Partitioner.instance)) + .createLog(); + ClusterMetadataService cms = new ClusterMetadataService(new UniformRangePlacement(), + MetadataSnapshots.NO_OP, + log, + new AtomicLongBackedProcessor(log), + Commit.Replicator.NO_OP, + false); + ClusterMetadataService.setInstance(cms); + log.readyUnchecked(); + + // Models the production post-commit listeners (e.g. LegacyStateListener) which block on the + // gossip stage while the follower enacts an epoch. Only armed for the fetch under test. + AtomicBoolean armed = new AtomicBoolean(false); + log.addListener(new ChangeListener() + { + @Override + public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot) + { + if (armed.get()) + Gossiper.runInGossipStageBlocking(() -> {}); + } + }); + + // Releases the caller from the intercepted send once the request promise has been completed + // (buggy path) or the continuation has begun appending on the responder thread (fixed path). + // The latter frees the GossipStage thread before the follower needs it, so the fixed code + // does not deadlock. + CountDownLatch releaseSend = new CountDownLatch(1); + log.addFilter(entry -> { + releaseSend.countDown(); + return false; // keep the entry so the follower enacts it and fires the listener + }); + + ExecutorService responder = Executors.newSingleThreadExecutor(r -> new Thread(r, "test-fetch-responder")); + + // Complete the request promise from the responder thread (mimicking the messaging response + // callback), holding the caller inside the send until that has happened - so the buggy + // code's map() sees an already-completed promise and runs the continuation inline on the + // GossipStage thread. The message is always dropped so nothing hits the network. + OutboundSink.Filter interceptor = (message, to, type) -> { + if (message.verb() != Verb.TCM_FETCH_PEER_LOG_REQ) + return true; + + long id = message.id(); + Message request = message; + responder.execute(() -> { + // A snapshot at FIRST that the follower will enact, advancing the epoch and firing + // the post-commit listener above. + LogState logState = LogState.make(new ClusterMetadata(Murmur3Partitioner.instance).forceEpoch(Epoch.FIRST)); + Message response = request.responseWith(logState); + MessagingService.instance().callbacks.removeAndRespond(id, to, response); + releaseSend.countDown(); + }); + Uninterruptibles.awaitUninterruptibly(releaseSend, 30, TimeUnit.SECONDS); + return false; + }; + MessagingService.instance().outboundSink.add(interceptor); + + AtomicReference> fetchFuture = new AtomicReference<>(); + CountDownLatch fetchReturned = new CountDownLatch(1); + try + { + PeerLogFetcher fetcher = new PeerLogFetcher(log); + armed.set(true); + // Run the fetch on the real, single-threaded GossipStage, exactly as the gossip path does. + Stage.GOSSIP.execute(() -> { + try + { + fetchFuture.set(fetcher.asyncFetchLog(REMOTE, Epoch.FIRST)); + } + finally + { + fetchReturned.countDown(); + } + }); + + if (!fetchReturned.await(20, TimeUnit.SECONDS)) + { + // GossipStage never returned from the fetch: the deadlock. Prove it is exactly the + // GlobalLogFollower <-> GossipStage cycle, then fail with the dump attached. + String dump = fullThreadDump(); + StackTraceElement[] gossip = stackOf("GossipStage"); + StackTraceElement[] follower = stackOf("GlobalLogFollower"); + + // break the deadlock so we don't leak the shared GossipStage thread + interruptThreads("GossipStage"); + + assertTrue("Expected GossipStage to be blocked advancing the log inside the peer-log " + + "fetch continuation, but was:\n" + dump, + stackContains(gossip, "PeerLogFetcher") && stackContains(gossip, "LocalLog")); + assertTrue("Expected GlobalLogFollower to be blocked handing off to the gossip stage, " + + "but was:\n" + dump, + stackContains(follower, "runInGossipStageBlocking")); + fail("CASSANDRA-21384: reproduced deadlock between GlobalLogFollower and GossipStage:\n" + dump); + } + + // No deadlock: the fetch returned on the gossip stage. Let it settle to be sure the + // whole chain (responder -> follower -> gossip stage) completed. + Future f = fetchFuture.get(); + assertNotNull("peer-log fetch future was not captured", f); + f.awaitUninterruptibly(20, TimeUnit.SECONDS); + } + finally + { + armed.set(false); + MessagingService.instance().outboundSink.remove(interceptor); + responder.shutdownNow(); + log.close(); + ClusterMetadataService.unsetInstance(); + } + } + + private static StackTraceElement[] stackOf(String namePrefix) + { + for (Map.Entry e : Thread.getAllStackTraces().entrySet()) + if (e.getKey().getName().startsWith(namePrefix)) + return e.getValue(); + return null; + } + + private static boolean stackContains(StackTraceElement[] stack, String needle) + { + if (stack == null) + return false; + for (StackTraceElement frame : stack) + if (frame.toString().contains(needle)) + return true; + return false; + } + + private static void interruptThreads(String namePrefix) + { + for (Thread t : Thread.getAllStackTraces().keySet()) + if (t.getName().startsWith(namePrefix)) + t.interrupt(); + } + + private static String fullThreadDump() + { + StringBuilder sb = new StringBuilder(); + for (Map.Entry e : Thread.getAllStackTraces().entrySet()) + { + Thread t = e.getKey(); + sb.append('"').append(t.getName()).append("\" ").append(t.getState()).append('\n'); + for (StackTraceElement frame : e.getValue()) + sb.append("\tat ").append(frame).append('\n'); + sb.append('\n'); + } + return sb.toString(); + } +} From 480320f3cfb54e30de5e3505ae730c71fbc48815 Mon Sep 17 00:00:00 2001 From: Dmitry Konstantinov Date: Thu, 25 Jun 2026 01:04:12 +0100 Subject: [PATCH 3/6] Expose immediately-executed tasks in the queries virtual table SEPExecutor.maybeExecuteImmediately() runs a task synchronously on the calling worker thread, nested within the task the worker is already running. Such immediate tasks were invisible in system_views.queries, which only exposed each worker's primary running task. This is common on the coordinator path, where a local read or mutation is executed immediately within the enclosing QUERY task. Each SEPWorker now also tracks an immediate current task, set around maybeExecuteImmediately(), and exposes it as an additional DebuggableTaskRunner, so the queries table reports both the enclosing task and the immediate one as separate rows. patch by Dmitry Konstantinov; reviewed by Caleb Rackliffe for CASSANDRA-21471 --- CHANGES.txt | 1 + .../cassandra/concurrent/CassandraThread.java | 17 ++++++- .../concurrent/ImmediateTaskHolder.java | 41 +++++++++++++++++ .../cassandra/concurrent/SEPExecutor.java | 13 ++++++ .../cassandra/concurrent/SEPWorker.java | 44 +++++++++++++++++-- .../concurrent/SharedExecutorPool.java | 2 +- .../distributed/test/QueriesTableTest.java | 14 +++--- 7 files changed, 121 insertions(+), 11 deletions(-) create mode 100644 src/java/org/apache/cassandra/concurrent/ImmediateTaskHolder.java diff --git a/CHANGES.txt b/CHANGES.txt index 51b576d242..4f69b61115 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 6.0-alpha2 + * Expose immediately-executed tasks in the queries virtual table (CASSANDRA-21471) * Avoid potential deadlock between GlobalLogFollower and GossipStage (CASSANDRA-21384) * Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736) * Fix maven remote publishing of Accord artifacts (CASSANDRA-21261) diff --git a/src/java/org/apache/cassandra/concurrent/CassandraThread.java b/src/java/org/apache/cassandra/concurrent/CassandraThread.java index 5afec8815b..62cbcdb6a4 100644 --- a/src/java/org/apache/cassandra/concurrent/CassandraThread.java +++ b/src/java/org/apache/cassandra/concurrent/CassandraThread.java @@ -27,19 +27,34 @@ public class CassandraThread extends FastThreadLocalThread private ThreadLocalMetrics threadLocalMetrics; private ExecutorLocals executorLocals; - public CassandraThread(ThreadGroup group, Runnable target, String name) + private final ImmediateTaskHolder immediateTaskHolder; + + public CassandraThread(ThreadGroup group, Runnable target, String name, ImmediateTaskHolder immediateTaskHolder) { super(group, target, name); + assert immediateTaskHolder != null; + this.immediateTaskHolder = immediateTaskHolder; + } + public CassandraThread(ThreadGroup group, Runnable target, String name) + { + this(group, target, name, ImmediateTaskHolder.NO_OP); } public CassandraThread() { super(); + this.immediateTaskHolder = ImmediateTaskHolder.NO_OP; } public CassandraThread(Runnable target) { super(target); + this.immediateTaskHolder = ImmediateTaskHolder.NO_OP; + } + + public ImmediateTaskHolder getImmediateTaskHolder() + { + return immediateTaskHolder; } public ThreadLocalMetrics getThreadLocalMetrics() diff --git a/src/java/org/apache/cassandra/concurrent/ImmediateTaskHolder.java b/src/java/org/apache/cassandra/concurrent/ImmediateTaskHolder.java new file mode 100644 index 0000000000..99c1afa397 --- /dev/null +++ b/src/java/org/apache/cassandra/concurrent/ImmediateTaskHolder.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.concurrent; + +public interface ImmediateTaskHolder +{ + /** + * @param currentTask to set as a current immediate task + * @return the previously set immediate task, can be null + */ + Runnable setImmediateTask(Runnable currentTask); + + class NoOp implements ImmediateTaskHolder + { + + @Override + public Runnable setImmediateTask(Runnable currentTask) + { + // nothing to do + return null; + } + } + + NoOp NO_OP = new NoOp(); +} diff --git a/src/java/org/apache/cassandra/concurrent/SEPExecutor.java b/src/java/org/apache/cassandra/concurrent/SEPExecutor.java index ccce797be7..c467b1da22 100644 --- a/src/java/org/apache/cassandra/concurrent/SEPExecutor.java +++ b/src/java/org/apache/cassandra/concurrent/SEPExecutor.java @@ -211,12 +211,15 @@ public class SEPExecutor implements LocalAwareExecutorPlus, SEPExecutorMBean } else { + ImmediateTaskHolder taskHolder = getNestedCurrentTaskHolder(); + Runnable previousTask = taskHolder.setImmediateTask(task); try { task.run(); } finally { + taskHolder.setImmediateTask(previousTask); returnWorkPermit(); // we have to maintain our invariant of always scheduling after any work is performed // in this case in particular we are not processing the rest of the queue anyway, and so @@ -226,6 +229,16 @@ public class SEPExecutor implements LocalAwareExecutorPlus, SEPExecutorMBean } } + private static ImmediateTaskHolder getNestedCurrentTaskHolder() + { + Thread currentThread = Thread.currentThread(); + if (currentThread instanceof CassandraThread) + { + return ((CassandraThread) currentThread).getImmediateTaskHolder(); + } + return ImmediateTaskHolder.NO_OP; + } + @Override public void execute(Runnable run) { diff --git a/src/java/org/apache/cassandra/concurrent/SEPWorker.java b/src/java/org/apache/cassandra/concurrent/SEPWorker.java index 8eb34303e2..f3cdbc345b 100644 --- a/src/java/org/apache/cassandra/concurrent/SEPWorker.java +++ b/src/java/org/apache/cassandra/concurrent/SEPWorker.java @@ -51,6 +51,35 @@ final class SEPWorker extends AtomicReference implements Runnabl private final AtomicReference currentTask = new AtomicReference<>(); + private class ImmediateDebuggableTaskRunner implements DebuggableTask.DebuggableTaskRunner, ImmediateTaskHolder + { + private final AtomicReference immediateCurrentTask = new AtomicReference<>(); + + @Override + public DebuggableTask running() + { + return getDebuggableTask(immediateCurrentTask.get()); + } + + @Override + public String id() + { + // derive from the current thread name so the nested row tracks renames and correlates with the worker's main row + return thread.getName() + "(immediate)"; + } + + @Override + public Runnable setImmediateTask(Runnable currentTask) + { + // plain is used to reduce overheads, the method is expected to be invoked only by a single thread + Runnable previousTask = immediateCurrentTask.getPlain(); + immediateCurrentTask.lazySet(currentTask); + return previousTask; + } + } + + private final ImmediateDebuggableTaskRunner immediateDebuggableTaskRunner; + private String lastUsedExecutorName; SEPWorker(ThreadGroup threadGroup, Long workerId, Work initialState, SharedExecutorPool pool) @@ -58,18 +87,27 @@ final class SEPWorker extends AtomicReference implements Runnabl this.pool = pool; this.workerId = workerId; this.workerIdThreadSuffix = '-' + workerId.toString(); - thread = new CassandraThread(threadGroup, this, threadGroup.getName() + "-Worker-" + workerId); + String threadName = threadGroup.getName() + "-Worker-" + workerId; + this.immediateDebuggableTaskRunner = new ImmediateDebuggableTaskRunner(); + thread = new CassandraThread(threadGroup, this, threadName, immediateDebuggableTaskRunner); thread.setDaemon(true); set(initialState); thread.start(); } + public DebuggableTask.DebuggableTaskRunner immediateRunner() + { + return immediateDebuggableTaskRunner; + } + @Override public DebuggableTask running() { - // can change after null check so go off local reference - Runnable task = currentTask.get(); + return getDebuggableTask(currentTask.get()); + } + private static DebuggableTask getDebuggableTask(Runnable task) + { // Local read and mutation Runnables are themselves debuggable if (task instanceof DebuggableTask) return (DebuggableTask) task; diff --git a/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java b/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java index 3ad848d325..8dbe563deb 100644 --- a/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java +++ b/src/java/org/apache/cassandra/concurrent/SharedExecutorPool.java @@ -123,7 +123,7 @@ public class SharedExecutorPool public Stream workers() { - return allWorkers.stream(); + return allWorkers.stream().flatMap(worker -> Stream.of(worker, worker.immediateRunner())); } void maybeStartSpinningWorker() diff --git a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java index 934987076a..627fe5e409 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/QueriesTableTest.java @@ -121,13 +121,14 @@ public class QueriesTableTest extends TestBaseImpl String task = row.get("task").toString(); boolean localReaderThread = threadId.contains("Read") || threadId.contains("SharedPool-Worker"); - readVisible |= localReaderThread && task.contains("SELECT"); + // QUERY in a task name is used as a way to differentiate a local replica read vs a coordinator read + readVisible |= localReaderThread && task.contains("SELECT") && !task.contains("QUERY"); boolean coordReaderThread = threadId.contains("Native-Transport-Requests") || threadId.contains("SharedPool-Worker"); - coordinatorReadVisible |= coordReaderThread && task.contains("SELECT"); + coordinatorReadVisible |= coordReaderThread && task.contains("SELECT") && task.contains("QUERY"); boolean localWriterThread = threadId.contains("Mutation") || threadId.contains("SharedPool-Worker"); - writeVisible |= localWriterThread && task.contains("Mutation"); + writeVisible |= localWriterThread && task.contains("Mutation") && !task.contains("QUERY"); boolean coordWriterThread = threadId.contains("Native-Transport-Requests") || threadId.contains("SharedPool-Worker"); - coordinatorWriteVisible |= coordWriterThread && task.contains("INSERT"); + coordinatorWriteVisible |= coordWriterThread && task.contains("INSERT") && task.contains("QUERY"); } assertTrue(readVisible); @@ -169,9 +170,10 @@ public class QueriesTableTest extends TestBaseImpl String task = row.get("task").toString(); boolean localReaderThread = threadId.contains("Read") || threadId.contains("SharedPool-Worker"); - readVisible |= localReaderThread && task.contains("SELECT"); + // QUERY in a task name is used as a way to differentiate a local replica read vs a coordinator read + readVisible |= localReaderThread && task.contains("SELECT") && !task.contains("QUERY"); boolean coordUpdateThread = threadId.contains("Native-Transport-Requests") || threadId.contains("SharedPool-Worker"); - coordinatorUpdateVisible |= coordUpdateThread && task.contains("UPDATE"); + coordinatorUpdateVisible |= coordUpdateThread && task.contains("UPDATE") && task.contains("QUERY"); } assertTrue(readVisible); From 208d87513f658f6fbf82cabcbb04142e7319fa55 Mon Sep 17 00:00:00 2001 From: mck Date: Thu, 9 Jul 2026 14:14:29 +0200 Subject: [PATCH 4/6] Fix how ubuntu-test.docker curls to find the latest patch version of each release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Mick Semb Wever; reviewed Štefan Miklošovič for CASSANDRA-21489 --- .build/docker/ubuntu-test.docker | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.build/docker/ubuntu-test.docker b/.build/docker/ubuntu-test.docker index 822f40364f..c5c98b5528 100644 --- a/.build/docker/ubuntu-test.docker +++ b/.build/docker/ubuntu-test.docker @@ -183,9 +183,9 @@ RUN /bin/bash -c "source ${BUILD_HOME}/env3.8/bin/activate && \ # Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra RUN bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \ - latest_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \ + latest_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.0\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \ for i in $(seq 1 $latest_4_0); do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \ - latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \ + latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \ for i in $(seq 1 $latest_4_1); do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done' # 5+ requires java11, sudo doesn't work on cross-platform builds @@ -200,10 +200,10 @@ RUN /bin/bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \ ccm create --quiet -n 1 -v git:cassandra-5.0 test && ccm remove test && \ ccm create --quiet -n 1 -v git:cassandra-6.0 test && ccm remove test && \ ccm create --quiet -n 1 -v git:trunk test && ccm remove test && \ - latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \ + latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \ for i in $(seq 1 $latest_5_0); do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done' # TODO uncomment when 6.0.0 is released - #latest_6_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")6\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \ + #latest_6_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")6\.0\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \ #for i in $(seq 1 $latest_6_0); do echo $i ; ccm create --quiet -n 1 -v binary:6.0.$i test && ccm remove test ; done' # the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them From 2397d52df7ea9fff8f55f1d2a497f5630bcbc0b9 Mon Sep 17 00:00:00 2001 From: Stefan Miklosovic Date: Wed, 1 Jul 2026 14:16:20 +0200 Subject: [PATCH 5/6] Depend only on platform-specific Zstd JNI native libraries patch by Stefan Miklosovic; reviewed by Yifan Cai for CASSANDRA-21483 --- .build/cassandra-deps-maven-pom.xml | 16 ++++++++++++++++ .build/parent-maven-pom.xml | 19 +++++++++++++++++++ CHANGES.txt | 1 + ide/nbproject/project.xml | 2 +- 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.build/cassandra-deps-maven-pom.xml b/.build/cassandra-deps-maven-pom.xml index 474c1037d0..4df4cf4e5d 100644 --- a/.build/cassandra-deps-maven-pom.xml +++ b/.build/cassandra-deps-maven-pom.xml @@ -311,6 +311,22 @@ com.github.luben zstd-jni + linux_amd64 + + + com.github.luben + zstd-jni + linux_aarch64 + + + com.github.luben + zstd-jni + darwin_x86_64 + + + com.github.luben + zstd-jni + darwin_aarch64 org.psjava diff --git a/.build/parent-maven-pom.xml b/.build/parent-maven-pom.xml index e8d82bc4ed..34d9e1d4ff 100644 --- a/.build/parent-maven-pom.xml +++ b/.build/parent-maven-pom.xml @@ -304,6 +304,25 @@ com.github.luben zstd-jni 1.5.7-2 + linux_aarch64 + + + com.github.luben + zstd-jni + 1.5.7-2 + linux_amd64 + + + com.github.luben + zstd-jni + 1.5.7-2 + darwin_aarch64 + + + com.github.luben + zstd-jni + 1.5.7-2 + darwin_x86_64 com.google.guava diff --git a/CHANGES.txt b/CHANGES.txt index 4f69b61115..55f62c138b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 6.0-alpha2 + * Depend only on platform-specific Zstd JNI native libraries (CASSANDRA-21483) * Expose immediately-executed tasks in the queries virtual table (CASSANDRA-21471) * Avoid potential deadlock between GlobalLogFollower and GossipStage (CASSANDRA-21384) * Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736) diff --git a/ide/nbproject/project.xml b/ide/nbproject/project.xml index 59e19ed585..0f4b3839ba 100644 --- a/ide/nbproject/project.xml +++ b/ide/nbproject/project.xml @@ -8,7 +8,7 @@ .. - ${project.dir}/build/lib/jars/affinity-3.23.3.jar:${project.dir}/build/lib/jars/agrona-1.17.1.jar:${project.dir}/build/lib/jars/AmazonCorrettoCryptoProvider-2.2.0-linux-aarch_64.jar:${project.dir}/build/lib/jars/antlr-3.5.2.jar:${project.dir}/build/lib/jars/antlr-runtime-3.5.2.jar:${project.dir}/build/lib/jars/asm-9.5.jar:${project.dir}/build/lib/jars/async-profiler-4.2.jar:${project.dir}/build/lib/jars/big-math-2.3.0.jar:${project.dir}/build/lib/jars/byteman-4.0.26.jar:${project.dir}/build/lib/jars/byteman-bmunit-4.0.26.jar:${project.dir}/build/lib/jars/byteman-install-4.0.26.jar:${project.dir}/build/lib/jars/byteman-submit-4.0.26.jar:${project.dir}/build/lib/jars/caffeine-3.1.8.jar:${project.dir}/build/lib/jars/cassandra-accord-6.0-SNAPSHOT.jar:${project.dir}/build/lib/jars/cassandra-driver-core-3.12.1-shaded.jar:${project.dir}/build/lib/jars/chronicle-bytes-2.25ea10.jar:${project.dir}/build/lib/jars/chronicle-core-2.25ea14.jar:${project.dir}/build/lib/jars/chronicle-queue-5.25ea16.jar:${project.dir}/build/lib/jars/chronicle-threads-2.25ea7.jar:${project.dir}/build/lib/jars/chronicle-wire-2.25ea15.jar:${project.dir}/build/lib/jars/commons-cli-1.5.0.jar:${project.dir}/build/lib/jars/commons-lang3-3.18.0.jar:${project.dir}/build/lib/jars/commons-math3-3.2.jar:${project.dir}/build/lib/jars/compile-command-annotations-1.2.0.jar:${project.dir}/build/lib/jars/concurrent-trees-2.4.0.jar:${project.dir}/build/lib/jars/ecj-3.33.0.jar:${project.dir}/build/lib/jars/failureaccess-1.0.1.jar:${project.dir}/build/lib/jars/guava-32.0.1-jre.jar:${project.dir}/build/lib/jars/HdrHistogram-2.1.12.jar:${project.dir}/build/lib/jars/high-scale-lib-1.0.6.jar:${project.dir}/build/lib/jars/hppc-0.8.1.jar:${project.dir}/build/lib/jars/ipaddress-5.3.3.jar:${project.dir}/build/lib/jars/j2objc-annotations-1.3.jar:${project.dir}/build/lib/jars/jackson-annotations-2.19.2.jar:${project.dir}/build/lib/jars/jackson-core-2.19.2.jar:${project.dir}/build/lib/jars/jackson-databind-2.19.2.jar:${project.dir}/build/lib/jars/jackson-datatype-jsr310-2.19.2.jar:${project.dir}/build/lib/jars/jacocoagent.jar:${project.dir}/build/lib/jars/jamm-0.4.0.jar:${project.dir}/build/lib/jars/javax.inject-1.jar:${project.dir}/build/lib/jars/jbcrypt-0.4.jar:${project.dir}/build/lib/jars/jcl-over-slf4j-2.0.17.jar:${project.dir}/build/lib/jars/jcommander-1.30.jar:${project.dir}/build/lib/jars/jctools-core-3.1.0.jar:${project.dir}/build/lib/jars/jffi-1.3.11-native.jar:${project.dir}/build/lib/jars/jffi-1.3.11.jar:${project.dir}/build/lib/jars/jna-5.13.0.jar:${project.dir}/build/lib/jars/jna-platform-5.13.0.jar:${project.dir}/build/lib/jars/jnr-a64asm-1.0.0.jar:${project.dir}/build/lib/jars/jnr-constants-0.10.4.jar:${project.dir}/build/lib/jars/jnr-ffi-2.2.13.jar:${project.dir}/build/lib/jars/jnr-x86asm-1.0.2.jar:${project.dir}/build/lib/jars/jsr305-2.0.2.jar:${project.dir}/build/lib/jars/jvector-1.0.2.jar:${project.dir}/build/lib/jars/jvm-attach-api-1.5.jar:${project.dir}/build/lib/jars/log4j-over-slf4j-2.0.17.jar:${project.dir}/build/lib/jars/logback-classic-1.5.18.jar:${project.dir}/build/lib/jars/logback-core-1.5.18.jar:${project.dir}/build/lib/jars/lucene-analysis-common-9.12.0.jar:${project.dir}/build/lib/jars/lucene-core-9.12.0.jar:${project.dir}/build/lib/jars/lz4-java-1.10.1.jar:${project.dir}/build/lib/jars/metrics-core-4.2.28.jar:${project.dir}/build/lib/jars/metrics-jvm-4.2.19.jar:${project.dir}/build/lib/jars/metrics-logback-4.2.19.jar:${project.dir}/build/lib/jars/mxdump-0.14.jar:${project.dir}/build/lib/jars/netty-all-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-buffer-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-codec-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-common-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-handler-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-handler-proxy-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-handler-ssl-ocsp-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-resolver-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final.jar:${project.dir}/build/lib/jars/netty-tcnative-classes-2.0.70.Final.jar:${project.dir}/build/lib/jars/netty-transport-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-classes-epoll-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-classes-kqueue-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-native-epoll-4.1.130.Final-linux-aarch_64.jar:${project.dir}/build/lib/jars/netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar:${project.dir}/build/lib/jars/netty-transport-native-epoll-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-native-unix-common-4.1.130.Final.jar:${project.dir}/build/lib/jars/ohc-core-0.5.1.jar:${project.dir}/build/lib/jars/ohc-core-j8-0.5.1.jar:${project.dir}/build/lib/jars/oshi-core-6.4.8.jar:${project.dir}/build/lib/jars/passay-1.6.4.jar:${project.dir}/build/lib/jars/picocli-4.7.7.jar:${project.dir}/build/lib/jars/posix-2.24ea4.jar:${project.dir}/build/lib/jars/psjava-0.1.19.jar:${project.dir}/build/lib/jars/semver4j-3.1.0.jar:${project.dir}/build/lib/jars/sjk-cli-0.14.jar:${project.dir}/build/lib/jars/sjk-core-0.14.jar:${project.dir}/build/lib/jars/sjk-json-0.14.jar:${project.dir}/build/lib/jars/sjk-stacktrace-0.14.jar:${project.dir}/build/lib/jars/slf4j-api-2.0.17.jar:${project.dir}/build/lib/jars/snakeyaml-2.4.jar:${project.dir}/build/lib/jars/snappy-java-1.1.10.4.jar:${project.dir}/build/lib/jars/ST4-4.0.8.jar:${project.dir}/build/lib/jars/stream-2.5.2.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.7-2.jar:${project.dir}/build/test/lib/jars/accessors-smart-2.4.7.jar:${project.dir}/build/test/lib/jars/agrona-1.17.1.jar:${project.dir}/build/test/lib/jars/ant-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-junit-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-launcher-1.10.12.jar:${project.dir}/build/test/lib/jars/antlr4-runtime-4.11.1.jar:${project.dir}/build/test/lib/jars/asm-9.5.jar:${project.dir}/build/test/lib/jars/asm-analysis-9.5.jar:${project.dir}/build/test/lib/jars/asm-commons-9.5.jar:${project.dir}/build/test/lib/jars/asm-tree-9.5.jar:${project.dir}/build/test/lib/jars/asm-util-9.5.jar:${project.dir}/build/test/lib/jars/asm-xml-6.0.jar:${project.dir}/build/test/lib/jars/assertj-core-3.24.2.jar:${project.dir}/build/test/lib/jars/awaitility-4.0.3.jar:${project.dir}/build/test/lib/jars/bcpkix-jdk18on-1.78.jar:${project.dir}/build/test/lib/jars/bcprov-jdk18on-1.78.jar:${project.dir}/build/test/lib/jars/bcutil-jdk18on-1.78.jar:${project.dir}/build/test/lib/jars/byte-buddy-1.12.21.jar:${project.dir}/build/test/lib/jars/byte-buddy-agent-1.14.15.jar:${project.dir}/build/test/lib/jars/cassandra-accord-6.0-SNAPSHOT-tests.jar:${project.dir}/build/test/lib/jars/checker-qual-3.27.0.jar:${project.dir}/build/test/lib/jars/checkstyle-10.26.1.jar:${project.dir}/build/test/lib/jars/commons-beanutils-1.9.4.jar:${project.dir}/build/test/lib/jars/commons-codec-1.15.jar:${project.dir}/build/test/lib/jars/commons-collections-3.2.2.jar:${project.dir}/build/test/lib/jars/commons-fileupload-1.4.jar:${project.dir}/build/test/lib/jars/commons-io-2.11.0.jar:${project.dir}/build/test/lib/jars/commons-lang3-3.12.0.jar:${project.dir}/build/test/lib/jars/commons-logging-1.2.jar:${project.dir}/build/test/lib/jars/commons-math3-3.6.1.jar:${project.dir}/build/test/lib/jars/dtest-api-0.0.18.jar:${project.dir}/build/test/lib/jars/error_prone_annotations-2.2.0.jar:${project.dir}/build/test/lib/jars/failureaccess-1.0.1.jar:${project.dir}/build/test/lib/jars/guava-32.0.1-jre.jar:${project.dir}/build/test/lib/jars/guava-testlib-27.0-jre.jar:${project.dir}/build/test/lib/jars/hamcrest-2.2.jar:${project.dir}/build/test/lib/jars/hamcrest-core-2.2.jar:${project.dir}/build/test/lib/jars/handlebars-4.3.1.jar:${project.dir}/build/test/lib/jars/handlebars-helpers-4.3.1.jar:${project.dir}/build/test/lib/jars/http2-common-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/http2-hpack-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/http2-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/httpclient5-5.1.3.jar:${project.dir}/build/test/lib/jars/httpcore5-5.1.3.jar:${project.dir}/build/test/lib/jars/httpcore5-h2-5.1.3.jar:${project.dir}/build/test/lib/jars/j2objc-annotations-1.1.jar:${project.dir}/build/test/lib/jars/jackson-annotations-2.19.2.jar:${project.dir}/build/test/lib/jars/jackson-core-2.19.2.jar:${project.dir}/build/test/lib/jars/jackson-databind-2.19.2.jar:${project.dir}/build/test/lib/jars/jackson-dataformat-yaml-2.19.2.jar:${project.dir}/build/test/lib/jars/jakarta.activation-api-1.2.2.jar:${project.dir}/build/test/lib/jars/jakarta.xml.bind-api-2.3.3.jar:${project.dir}/build/test/lib/jars/java-allocation-instrumenter-3.1.0.jar:${project.dir}/build/test/lib/jars/java-cup-runtime-11b-20160615.jar:${project.dir}/build/test/lib/jars/java-diff-utils-4.12.jar:${project.dir}/build/test/lib/jars/javassist-3.28.0-GA.jar:${project.dir}/build/test/lib/jars/javax.servlet-api-3.1.0.jar:${project.dir}/build/test/lib/jars/jetty-alpn-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-java-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-java-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-openjdk8-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-openjdk8-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-continuation-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-http-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-io-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-proxy-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-security-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-servlet-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-servlets-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-util-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-util-ajax-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-webapp-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-xml-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jflex-1.8.2.jar:${project.dir}/build/test/lib/jars/jimfs-1.1.jar:${project.dir}/build/test/lib/jars/jmh-core-1.37.jar:${project.dir}/build/test/lib/jars/jmh-generator-annprocess-1.37.jar:${project.dir}/build/test/lib/jars/jopt-simple-5.0.4.jar:${project.dir}/build/test/lib/jars/json-path-2.7.0.jar:${project.dir}/build/test/lib/jars/json-smart-2.4.7.jar:${project.dir}/build/test/lib/jars/json-unit-core-2.36.0.jar:${project.dir}/build/test/lib/jars/jsr305-3.0.2.jar:${project.dir}/build/test/lib/jars/junit-4.12.jar:${project.dir}/build/test/lib/jars/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:${project.dir}/build/test/lib/jars/mockito-core-5.12.0.jar:${project.dir}/build/test/lib/jars/mockito-inline-5.2.0.jar:${project.dir}/build/test/lib/jars/objenesis-3.3.jar:${project.dir}/build/test/lib/jars/org.jacoco.agent-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.ant-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.core-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.report-0.8.8.jar:${project.dir}/build/test/lib/jars/picocli-4.7.4.jar:${project.dir}/build/test/lib/jars/quicktheories-0.26.jar:${project.dir}/build/test/lib/jars/randomizedtesting-runner-2.1.2.jar:${project.dir}/build/test/lib/jars/reflections-0.10.2.jar:${project.dir}/build/test/lib/jars/Saxon-HE-12.2.jar:${project.dir}/build/test/lib/jars/semver4j-3.1.0.jar:${project.dir}/build/test/lib/jars/simulator-asm.jar:${project.dir}/build/test/lib/jars/simulator-bootstrap.jar:${project.dir}/build/test/lib/jars/slf4j-api-1.7.32.jar:${project.dir}/build/test/lib/jars/snakeyaml-2.4.jar:${project.dir}/build/test/lib/jars/wiremock-jre8-2.35.0.jar:${project.dir}/build/test/lib/jars/xmlresolver-5.1.2-data.jar:${project.dir}/build/test/lib/jars/xmlresolver-5.1.2.jar:${project.dir}/build/test/lib/jars/xmlunit-core-2.9.0.jar:${project.dir}/build/test/lib/jars/xmlunit-legacy-2.9.0.jar:${project.dir}/build/test/lib/jars/xmlunit-placeholders-2.9.0.jar: + ${project.dir}/build/lib/jars/affinity-3.23.3.jar:${project.dir}/build/lib/jars/agrona-1.17.1.jar:${project.dir}/build/lib/jars/AmazonCorrettoCryptoProvider-2.2.0-linux-aarch_64.jar:${project.dir}/build/lib/jars/antlr-3.5.2.jar:${project.dir}/build/lib/jars/antlr-runtime-3.5.2.jar:${project.dir}/build/lib/jars/asm-9.5.jar:${project.dir}/build/lib/jars/async-profiler-4.2.jar:${project.dir}/build/lib/jars/big-math-2.3.0.jar:${project.dir}/build/lib/jars/byteman-4.0.26.jar:${project.dir}/build/lib/jars/byteman-bmunit-4.0.26.jar:${project.dir}/build/lib/jars/byteman-install-4.0.26.jar:${project.dir}/build/lib/jars/byteman-submit-4.0.26.jar:${project.dir}/build/lib/jars/caffeine-3.1.8.jar:${project.dir}/build/lib/jars/cassandra-accord-6.0-SNAPSHOT.jar:${project.dir}/build/lib/jars/cassandra-driver-core-3.12.1-shaded.jar:${project.dir}/build/lib/jars/chronicle-bytes-2.25ea10.jar:${project.dir}/build/lib/jars/chronicle-core-2.25ea14.jar:${project.dir}/build/lib/jars/chronicle-queue-5.25ea16.jar:${project.dir}/build/lib/jars/chronicle-threads-2.25ea7.jar:${project.dir}/build/lib/jars/chronicle-wire-2.25ea15.jar:${project.dir}/build/lib/jars/commons-cli-1.5.0.jar:${project.dir}/build/lib/jars/commons-lang3-3.18.0.jar:${project.dir}/build/lib/jars/commons-math3-3.2.jar:${project.dir}/build/lib/jars/compile-command-annotations-1.2.0.jar:${project.dir}/build/lib/jars/concurrent-trees-2.4.0.jar:${project.dir}/build/lib/jars/ecj-3.33.0.jar:${project.dir}/build/lib/jars/failureaccess-1.0.1.jar:${project.dir}/build/lib/jars/guava-32.0.1-jre.jar:${project.dir}/build/lib/jars/HdrHistogram-2.1.12.jar:${project.dir}/build/lib/jars/high-scale-lib-1.0.6.jar:${project.dir}/build/lib/jars/hppc-0.8.1.jar:${project.dir}/build/lib/jars/ipaddress-5.3.3.jar:${project.dir}/build/lib/jars/j2objc-annotations-1.3.jar:${project.dir}/build/lib/jars/jackson-annotations-2.19.2.jar:${project.dir}/build/lib/jars/jackson-core-2.19.2.jar:${project.dir}/build/lib/jars/jackson-databind-2.19.2.jar:${project.dir}/build/lib/jars/jackson-datatype-jsr310-2.19.2.jar:${project.dir}/build/lib/jars/jacocoagent.jar:${project.dir}/build/lib/jars/jamm-0.4.0.jar:${project.dir}/build/lib/jars/javax.inject-1.jar:${project.dir}/build/lib/jars/jbcrypt-0.4.jar:${project.dir}/build/lib/jars/jcl-over-slf4j-2.0.17.jar:${project.dir}/build/lib/jars/jcommander-1.30.jar:${project.dir}/build/lib/jars/jctools-core-3.1.0.jar:${project.dir}/build/lib/jars/jffi-1.3.11-native.jar:${project.dir}/build/lib/jars/jffi-1.3.11.jar:${project.dir}/build/lib/jars/jna-5.13.0.jar:${project.dir}/build/lib/jars/jna-platform-5.13.0.jar:${project.dir}/build/lib/jars/jnr-a64asm-1.0.0.jar:${project.dir}/build/lib/jars/jnr-constants-0.10.4.jar:${project.dir}/build/lib/jars/jnr-ffi-2.2.13.jar:${project.dir}/build/lib/jars/jnr-x86asm-1.0.2.jar:${project.dir}/build/lib/jars/jsr305-2.0.2.jar:${project.dir}/build/lib/jars/jvector-1.0.2.jar:${project.dir}/build/lib/jars/jvm-attach-api-1.5.jar:${project.dir}/build/lib/jars/log4j-over-slf4j-2.0.17.jar:${project.dir}/build/lib/jars/logback-classic-1.5.18.jar:${project.dir}/build/lib/jars/logback-core-1.5.18.jar:${project.dir}/build/lib/jars/lucene-analysis-common-9.12.0.jar:${project.dir}/build/lib/jars/lucene-core-9.12.0.jar:${project.dir}/build/lib/jars/lz4-java-1.10.1.jar:${project.dir}/build/lib/jars/metrics-core-4.2.28.jar:${project.dir}/build/lib/jars/metrics-jvm-4.2.19.jar:${project.dir}/build/lib/jars/metrics-logback-4.2.19.jar:${project.dir}/build/lib/jars/mxdump-0.14.jar:${project.dir}/build/lib/jars/netty-all-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-buffer-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-codec-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-common-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-handler-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-handler-proxy-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-handler-ssl-ocsp-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-resolver-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-linux-aarch_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-linux-x86_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-osx-aarch_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final-osx-x86_64.jar:${project.dir}/build/lib/jars/netty-tcnative-boringssl-static-2.0.70.Final.jar:${project.dir}/build/lib/jars/netty-tcnative-classes-2.0.70.Final.jar:${project.dir}/build/lib/jars/netty-transport-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-classes-epoll-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-classes-kqueue-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-native-epoll-4.1.130.Final-linux-aarch_64.jar:${project.dir}/build/lib/jars/netty-transport-native-epoll-4.1.130.Final-linux-x86_64.jar:${project.dir}/build/lib/jars/netty-transport-native-epoll-4.1.130.Final.jar:${project.dir}/build/lib/jars/netty-transport-native-unix-common-4.1.130.Final.jar:${project.dir}/build/lib/jars/ohc-core-0.5.1.jar:${project.dir}/build/lib/jars/ohc-core-j8-0.5.1.jar:${project.dir}/build/lib/jars/oshi-core-6.4.8.jar:${project.dir}/build/lib/jars/passay-1.6.4.jar:${project.dir}/build/lib/jars/picocli-4.7.7.jar:${project.dir}/build/lib/jars/posix-2.24ea4.jar:${project.dir}/build/lib/jars/psjava-0.1.19.jar:${project.dir}/build/lib/jars/semver4j-3.1.0.jar:${project.dir}/build/lib/jars/sjk-cli-0.14.jar:${project.dir}/build/lib/jars/sjk-core-0.14.jar:${project.dir}/build/lib/jars/sjk-json-0.14.jar:${project.dir}/build/lib/jars/sjk-stacktrace-0.14.jar:${project.dir}/build/lib/jars/slf4j-api-2.0.17.jar:${project.dir}/build/lib/jars/snakeyaml-2.4.jar:${project.dir}/build/lib/jars/snappy-java-1.1.10.4.jar:${project.dir}/build/lib/jars/ST4-4.0.8.jar:${project.dir}/build/lib/jars/stream-2.5.2.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.7-2-darwin_aarch64.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.7-2-darwin_x86_64.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.7-2-linux_aarch64.jar:${project.dir}/build/lib/jars/zstd-jni-1.5.7-2-linux_amd64.jar:${project.dir}/build/test/lib/jars/accessors-smart-2.4.7.jar:${project.dir}/build/test/lib/jars/agrona-1.17.1.jar:${project.dir}/build/test/lib/jars/ant-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-junit-1.10.12.jar:${project.dir}/build/test/lib/jars/ant-launcher-1.10.12.jar:${project.dir}/build/test/lib/jars/antlr4-runtime-4.11.1.jar:${project.dir}/build/test/lib/jars/asm-9.5.jar:${project.dir}/build/test/lib/jars/asm-analysis-9.5.jar:${project.dir}/build/test/lib/jars/asm-commons-9.5.jar:${project.dir}/build/test/lib/jars/asm-tree-9.5.jar:${project.dir}/build/test/lib/jars/asm-util-9.5.jar:${project.dir}/build/test/lib/jars/asm-xml-6.0.jar:${project.dir}/build/test/lib/jars/assertj-core-3.24.2.jar:${project.dir}/build/test/lib/jars/awaitility-4.0.3.jar:${project.dir}/build/test/lib/jars/bcpkix-jdk18on-1.78.jar:${project.dir}/build/test/lib/jars/bcprov-jdk18on-1.78.jar:${project.dir}/build/test/lib/jars/bcutil-jdk18on-1.78.jar:${project.dir}/build/test/lib/jars/byte-buddy-1.12.21.jar:${project.dir}/build/test/lib/jars/byte-buddy-agent-1.14.15.jar:${project.dir}/build/test/lib/jars/cassandra-accord-6.0-SNAPSHOT-tests.jar:${project.dir}/build/test/lib/jars/checker-qual-3.27.0.jar:${project.dir}/build/test/lib/jars/checkstyle-10.26.1.jar:${project.dir}/build/test/lib/jars/commons-beanutils-1.9.4.jar:${project.dir}/build/test/lib/jars/commons-codec-1.15.jar:${project.dir}/build/test/lib/jars/commons-collections-3.2.2.jar:${project.dir}/build/test/lib/jars/commons-fileupload-1.4.jar:${project.dir}/build/test/lib/jars/commons-io-2.11.0.jar:${project.dir}/build/test/lib/jars/commons-lang3-3.12.0.jar:${project.dir}/build/test/lib/jars/commons-logging-1.2.jar:${project.dir}/build/test/lib/jars/commons-math3-3.6.1.jar:${project.dir}/build/test/lib/jars/dtest-api-0.0.18.jar:${project.dir}/build/test/lib/jars/error_prone_annotations-2.2.0.jar:${project.dir}/build/test/lib/jars/failureaccess-1.0.1.jar:${project.dir}/build/test/lib/jars/guava-32.0.1-jre.jar:${project.dir}/build/test/lib/jars/guava-testlib-27.0-jre.jar:${project.dir}/build/test/lib/jars/hamcrest-2.2.jar:${project.dir}/build/test/lib/jars/hamcrest-core-2.2.jar:${project.dir}/build/test/lib/jars/handlebars-4.3.1.jar:${project.dir}/build/test/lib/jars/handlebars-helpers-4.3.1.jar:${project.dir}/build/test/lib/jars/http2-common-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/http2-hpack-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/http2-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/httpclient5-5.1.3.jar:${project.dir}/build/test/lib/jars/httpcore5-5.1.3.jar:${project.dir}/build/test/lib/jars/httpcore5-h2-5.1.3.jar:${project.dir}/build/test/lib/jars/j2objc-annotations-1.1.jar:${project.dir}/build/test/lib/jars/jackson-annotations-2.19.2.jar:${project.dir}/build/test/lib/jars/jackson-core-2.19.2.jar:${project.dir}/build/test/lib/jars/jackson-databind-2.19.2.jar:${project.dir}/build/test/lib/jars/jackson-dataformat-yaml-2.19.2.jar:${project.dir}/build/test/lib/jars/jakarta.activation-api-1.2.2.jar:${project.dir}/build/test/lib/jars/jakarta.xml.bind-api-2.3.3.jar:${project.dir}/build/test/lib/jars/java-allocation-instrumenter-3.1.0.jar:${project.dir}/build/test/lib/jars/java-cup-runtime-11b-20160615.jar:${project.dir}/build/test/lib/jars/java-diff-utils-4.12.jar:${project.dir}/build/test/lib/jars/javassist-3.28.0-GA.jar:${project.dir}/build/test/lib/jars/javax.servlet-api-3.1.0.jar:${project.dir}/build/test/lib/jars/jetty-alpn-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-java-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-java-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-openjdk8-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-openjdk8-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-alpn-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-client-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-continuation-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-http-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-io-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-proxy-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-security-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-server-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-servlet-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-servlets-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-util-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-util-ajax-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-webapp-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jetty-xml-9.4.49.v20220914.jar:${project.dir}/build/test/lib/jars/jflex-1.8.2.jar:${project.dir}/build/test/lib/jars/jimfs-1.1.jar:${project.dir}/build/test/lib/jars/jmh-core-1.37.jar:${project.dir}/build/test/lib/jars/jmh-generator-annprocess-1.37.jar:${project.dir}/build/test/lib/jars/jopt-simple-5.0.4.jar:${project.dir}/build/test/lib/jars/json-path-2.7.0.jar:${project.dir}/build/test/lib/jars/json-smart-2.4.7.jar:${project.dir}/build/test/lib/jars/json-unit-core-2.36.0.jar:${project.dir}/build/test/lib/jars/jsr305-3.0.2.jar:${project.dir}/build/test/lib/jars/junit-4.12.jar:${project.dir}/build/test/lib/jars/listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar:${project.dir}/build/test/lib/jars/mockito-core-5.12.0.jar:${project.dir}/build/test/lib/jars/mockito-inline-5.2.0.jar:${project.dir}/build/test/lib/jars/objenesis-3.3.jar:${project.dir}/build/test/lib/jars/org.jacoco.agent-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.ant-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.core-0.8.8.jar:${project.dir}/build/test/lib/jars/org.jacoco.report-0.8.8.jar:${project.dir}/build/test/lib/jars/picocli-4.7.4.jar:${project.dir}/build/test/lib/jars/quicktheories-0.26.jar:${project.dir}/build/test/lib/jars/randomizedtesting-runner-2.1.2.jar:${project.dir}/build/test/lib/jars/reflections-0.10.2.jar:${project.dir}/build/test/lib/jars/Saxon-HE-12.2.jar:${project.dir}/build/test/lib/jars/semver4j-3.1.0.jar:${project.dir}/build/test/lib/jars/simulator-asm.jar:${project.dir}/build/test/lib/jars/simulator-bootstrap.jar:${project.dir}/build/test/lib/jars/slf4j-api-1.7.32.jar:${project.dir}/build/test/lib/jars/snakeyaml-2.4.jar:${project.dir}/build/test/lib/jars/wiremock-jre8-2.35.0.jar:${project.dir}/build/test/lib/jars/xmlresolver-5.1.2-data.jar:${project.dir}/build/test/lib/jars/xmlresolver-5.1.2.jar:${project.dir}/build/test/lib/jars/xmlunit-core-2.9.0.jar:${project.dir}/build/test/lib/jars/xmlunit-legacy-2.9.0.jar:${project.dir}/build/test/lib/jars/xmlunit-placeholders-2.9.0.jar: From 03b9b71d52aa119ff7f4cb1cbd2cc01cee9b16f5 Mon Sep 17 00:00:00 2001 From: Mick Semb Wever Date: Fri, 10 Jul 2026 16:53:07 +0200 Subject: [PATCH 6/6] Prepare debian changelog for 6.0-alpha2 --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index ea6e1278c4..d642a65289 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +cassandra (6.0~alpha2) unstable; urgency=medium + + * New release + + -- Mick Semb Wever Fri, 10 Jul 2026 16:50:29 +0200 + cassandra (6.0~alpha1) unstable; urgency=medium * New release