From e1e56e5d5dc3e47ea1a1d3f19161f930a93e8b3d Mon Sep 17 00:00:00 2001 From: Sam Tunnicliffe Date: Mon, 6 Jul 2026 11:52:18 +0100 Subject: [PATCH] Add CMS membership directly to ClusterMetadata Patch by Sam Tunnicliffe; reviewed by Marcus Eriksson for CASSANDRA-20736 --- CHANGES.txt | 1 + .../schema/CreateKeyspaceStatement.java | 2 +- .../db/AbstractMutationVerbHandler.java | 2 +- .../cassandra/db/ReadCommandVerbHandler.java | 4 +- .../db/compaction/CompactionManager.java | 2 +- .../apache/cassandra/db/view/ViewUtils.java | 6 +- .../org/apache/cassandra/gms/Gossiper.java | 9 +- .../cassandra/hints/HintsDispatcher.java | 2 +- .../locator/CMSPlacementStrategy.java | 3 +- .../cassandra/locator/EndpointsForToken.java | 2 +- .../cassandra/locator/MetaStrategy.java | 4 +- .../cassandra/locator/ReplicaLayout.java | 14 +- .../cassandra/locator/ReplicaPlans.java | 2 +- .../apache/cassandra/metrics/TCMMetrics.java | 3 +- .../cassandra/net/ResponseVerbHandler.java | 3 +- .../cassandra/repair/RepairCoordinator.java | 3 +- .../service/ActiveRepairService.java | 2 +- .../cassandra/service/CassandraDaemon.java | 2 +- .../org/apache/cassandra/service/Rebuild.java | 12 +- .../cassandra/service/StorageProxy.java | 3 +- .../cassandra/service/StorageService.java | 26 +- .../accord/topology/AccordTopology.java | 2 +- .../apache/cassandra/service/paxos/Paxos.java | 2 +- .../cassandra/service/paxos/PaxosRepair.java | 2 +- .../streaming/DataMovementVerbHandler.java | 2 +- .../cassandra/tcm/AbstractLocalProcessor.java | 4 +- .../tcm/AtomicLongBackedProcessor.java | 12 + .../apache/cassandra/tcm/CMSMembership.java | 257 +++++++++++++ .../apache/cassandra/tcm/CMSOperations.java | 3 +- .../apache/cassandra/tcm/ClusterMetadata.java | 354 ++++++++++++++---- .../cassandra/tcm/ClusterMetadataService.java | 63 ++-- .../apache/cassandra/tcm/MetadataKeys.java | 4 +- .../cassandra/tcm/PaxosBackedProcessor.java | 6 + .../org/apache/cassandra/tcm/Startup.java | 36 +- .../tcm/StubClusterMetadataService.java | 3 +- .../tcm/compatibility/GossipHelper.java | 10 +- .../tcm/listeners/LegacyStateListener.java | 4 +- .../listeners/PlacementsChangeListener.java | 24 +- .../listeners/UpgradeMigrationListener.java | 20 +- .../apache/cassandra/tcm/log/LogState.java | 2 +- .../cassandra/tcm/membership/Directory.java | 2 +- .../cassandra/tcm/membership/NodeId.java | 1 + .../cassandra/tcm/membership/NodeVersion.java | 2 +- .../cassandra/tcm/migration/Election.java | 5 - .../tcm/ownership/DataPlacement.java | 5 + .../tcm/ownership/DataPlacements.java | 29 +- .../tcm/ownership/UniformRangePlacement.java | 8 +- .../tcm/sequences/BootstrapAndJoin.java | 4 +- .../tcm/sequences/BootstrapAndReplace.java | 8 +- .../sequences/CancelCMSReconfiguration.java | 68 ++-- .../apache/cassandra/tcm/sequences/Move.java | 4 +- .../tcm/sequences/ProgressBarrier.java | 6 +- .../tcm/sequences/RemoveNodeStreams.java | 2 +- .../tcm/sequences/ReplaceSameAddress.java | 2 +- .../tcm/sequences/UnbootstrapAndLeave.java | 2 +- .../cassandra/tcm/serialization/Version.java | 14 +- .../tcm/transformations/AccordMarkStale.java | 2 +- .../tcm/transformations/AlterSchema.java | 2 +- .../tcm/transformations/AlterTopology.java | 4 +- .../transformations/ApplyPlacementDeltas.java | 2 +- .../tcm/transformations/PrepareJoin.java | 6 +- .../tcm/transformations/PrepareLeave.java | 2 +- .../tcm/transformations/PrepareMove.java | 4 +- .../tcm/transformations/PrepareReplace.java | 2 +- .../tcm/transformations/Register.java | 2 +- .../tcm/transformations/Startup.java | 27 +- .../cms/AdvanceCMSReconfiguration.java | 68 ++-- .../transformations/cms/FinishAddToCMS.java | 10 +- .../transformations/cms/PreInitialize.java | 40 +- .../cms/PrepareCMSReconfiguration.java | 13 +- .../transformations/cms/RemoveFromCMS.java | 25 +- .../transformations/cms/StartAddToCMS.java | 29 +- .../cassandra/tools/CMSOfflineTool.java | 32 +- .../tools/TransformClusterMetadataHelper.java | 27 +- .../cassandra/distributed/impl/Instance.java | 2 +- .../distributed/shared/ClusterUtils.java | 8 +- .../AssassinateCMSNodeTest.java | 5 +- .../test/log/BootWithMetadataTest.java | 16 +- .../test/log/ClusterMetadataDumpTest.java | 3 +- .../test/log/ClusterMetadataTestHelper.java | 36 +- .../log/MetadataChangeSimulationTest.java | 12 +- .../test/log/OperationalEquivalenceTest.java | 3 +- .../test/log/ReconfigureCMSTest.java | 18 +- .../distributed/test/log/RegisterTest.java | 156 +++++--- .../test/log/ResumableStartupTest.java | 4 +- .../test/log/SimulatedOperation.java | 4 +- .../distributed/test/log/SnapshotTest.java | 2 +- .../test/ring/RangeVersioningTest.java | 2 +- .../tcm/CMSPlacementAfterReplacementTest.java | 3 +- ...ClusterMetadataUpgradeAssassinateTest.java | 2 +- ...rMetadataUpgradeDelayedInitializeTest.java | 8 +- .../fuzz/topology/TopologyMixupTestBase.java | 2 +- .../simulator/cluster/OnClusterReplace.java | 4 +- .../cassandra/auth/GrantAndRevokeTest.java | 8 + .../cassandra/dht/BootStrapperTest.java | 3 +- .../cassandra/locator/MetaStrategyTest.java | 4 +- .../cassandra/locator/SimpleStrategyTest.java | 2 +- .../service/accord/AccordTopologyUtils.java | 2 +- .../service/accord/EpochSyncTest.java | 6 +- .../cassandra/tcm/BootWithMetadataTest.java | 8 +- .../ClusterMetadataTransformationTest.java | 43 ++- .../apache/cassandra/tcm/GetLogStateTest.java | 17 +- .../apache/cassandra/tcm/UnregisterTest.java | 2 +- .../tcm/compatibility/GossipHelperTest.java | 6 +- .../MetadataSnapshotListenerTest.java | 2 +- .../PlacementsChangeListenerTest.java | 179 +++++++++ .../tcm/log/DistributedLogStateTest.java | 5 +- .../tcm/membership/MembershipUtils.java | 6 +- .../ownership/LocalRangesAllSettledTest.java | 12 +- .../tcm/ownership/OwnershipUtils.java | 2 +- .../InProgressSequenceCancellationTest.java | 10 +- .../tcm/sequences/ProgressBarrierTest.java | 12 +- .../transformations/EventsMetadataTest.java | 14 +- .../tcm/transformations/PrepareLeaveTest.java | 15 +- .../cassandra/tools/CMSOfflineToolTest.java | 23 +- .../cassandra/utils/CassandraGenerators.java | 4 +- 116 files changed, 1398 insertions(+), 653 deletions(-) create mode 100644 src/java/org/apache/cassandra/tcm/CMSMembership.java create mode 100644 test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java diff --git a/CHANGES.txt b/CHANGES.txt index 1a62d6acf9..dcd7972231 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 6.0-alpha2 + * Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736) * Fix maven remote publishing of Accord artifacts (CASSANDRA-21261) * Move long running TCM operations to a longer timout (CASSANDRA-21453) * Offline nodetool commands should not print network options in help (CASSANDRA-20876) diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java index a622b86192..eac58187b1 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java @@ -96,7 +96,7 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement // as we have as keys in metadata.placements to have a fast map lookup // ReplicationParams are immutable, so it is a safe optimization KeyspaceParams keyspaceParams = attrs.asNewKeyspaceParams(); - ReplicationParams replicationParams = metadata.placements.deduplicateReplicationParams(keyspaceParams.replication); + ReplicationParams replicationParams = metadata.placements().deduplicateReplicationParams(keyspaceParams.replication); keyspaceParams = keyspaceParams.withSwapped(replicationParams); KeyspaceMetadata keyspaceMetadata = KeyspaceMetadata.create(keyspaceName, keyspaceParams); diff --git a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java index 5d2e4e3cdc..18aa78552b 100644 --- a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java @@ -195,6 +195,6 @@ public abstract class AbstractMutationVerbHandler implement private static VersionedEndpoints.ForToken writePlacements(ClusterMetadata metadata, String keyspace, DecoratedKey key) { - return metadata.placements.get(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken()); + return metadata.placement(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken()); } } diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index b418ecdfeb..caac99cac1 100644 --- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -240,8 +240,8 @@ public class ReadCommandVerbHandler implements IVerbHandler private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace) { - return metadata.placements - .get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication) + return metadata + .placement(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication) .reads .forToken(token) .get() diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 84c1193517..e7ae548ffd 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@ -780,7 +780,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan // we only consider write placements during cleanup as range movements always ensure // overlap between new replicas accepting reads and old replicas accepting writes ClusterMetadata cm = ClusterMetadata.current(); - DataPlacement placement = cm.placements.get(keyspace.getMetadata().params.replication); + DataPlacement placement = cm.placement(keyspace.getMetadata().params.replication); InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort(); RangesAtEndpoint localWrites = placement.writes.byEndpoint().get(local); // TODO review: Hack to get local partitioner not to fail out because it's handled very poorly with data placements diff --git a/src/java/org/apache/cassandra/db/view/ViewUtils.java b/src/java/org/apache/cassandra/db/view/ViewUtils.java index 1bab97b312..ee27ace329 100644 --- a/src/java/org/apache/cassandra/db/view/ViewUtils.java +++ b/src/java/org/apache/cassandra/db/view/ViewUtils.java @@ -30,6 +30,7 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.ownership.DataPlacement; public final class ViewUtils { @@ -64,8 +65,9 @@ public final class ViewUtils Location local = metadata.locator.local(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); - EndpointsForToken naturalBaseReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(baseToken).get(); - EndpointsForToken naturalViewReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(viewToken).get(); + DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication); + EndpointsForToken naturalBaseReplicas = placement.reads.forToken(baseToken).get(); + EndpointsForToken naturalViewReplicas = placement.reads.forToken(viewToken).get(); Optional localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isSelf).toJavaUtil(); if (localReplica.isPresent()) diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index 3d6414913f..48ef04fa07 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -1764,7 +1764,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, register(DatabaseDescriptor.getLocalAddressReconnectionHelper()); ClusterMetadata metadata = ClusterMetadata.current(); - if (mergeLocalStates && metadata.myNodeId() != null) + if (mergeLocalStates && metadata.myNodeId() != NodeId.UNREGISTERED) mergeNodeToGossip(metadata.myNodeId(), metadata); shutdownAnnounced.set(false); @@ -2234,7 +2234,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, { checkProperThreadForStateMutation(); assert !endpoint.equals(getBroadcastAddressAndPort()) || epstate.getHeartBeatState().getGeneration() > 0 : - "We should not update epstates with generation = 0 for the local host"; + String.format("We should not update epstates with generation = 0 for the local host " + + "(endpoint: %s, broadcast: %s, generation: %s)", + endpoint, getBroadcastAddressAndPort(), epstate.getHeartBeatState().getGeneration()); EndpointState old = endpointStateMap.get(endpoint); if (old == null) endpointStateMap.put(endpoint, epstate); @@ -2278,6 +2280,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean, taskLock.lock(); try { + if (nodeId == NodeId.UNREGISTERED) + return; + boolean isLocal = nodeId.equals(metadata.myNodeId()); IPartitioner partitioner = metadata.tokenMap.partitioner(); NodeAddresses addresses = metadata.directory.getNodeAddresses(nodeId); diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java index 3ba38bdeaf..025d52579b 100644 --- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java +++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java @@ -276,7 +276,7 @@ final class HintsDispatcher implements AutoCloseable // Also may need to apply locally because it's possible this is from the batchlog // and we never applied it locally // TODO (review): Additional error handling necessary? Hints are lossy - DataPlacement dataPlacement = cm.placements.get(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication); + DataPlacement dataPlacement = cm.placement(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication); VersionedEndpoints.ForToken forToken = dataPlacement.writes.forToken(mutation.key().getToken()); Replica self = forToken.get().selfIfPresent(); if (self != null) diff --git a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java index e00289aef6..cb4dfa82c6 100644 --- a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java +++ b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java @@ -90,7 +90,8 @@ public class CMSPlacementStrategy // Although MetaStrategy has its own entireRange, it uses a custom partitioner which isn't compatible with // regular, non-CMS placements. For that reason, we select replicas here using tokens provided by the - // globally configured partitioner. + // globally configured partitioner. This also has the benefit of making concurrent operations, such as + // bounces/upgrades/etc, safe for the CMS if they are replica aware. Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken(); EndpointsForRange endpoints = NetworkTopologyStrategy.calculateNaturalReplicas(minToken, new Range<>(minToken, minToken), diff --git a/src/java/org/apache/cassandra/locator/EndpointsForToken.java b/src/java/org/apache/cassandra/locator/EndpointsForToken.java index fc88fa761b..7342816015 100644 --- a/src/java/org/apache/cassandra/locator/EndpointsForToken.java +++ b/src/java/org/apache/cassandra/locator/EndpointsForToken.java @@ -164,7 +164,7 @@ public class EndpointsForToken extends Endpoints public static VersionedEndpoints.ForToken natural(Keyspace keyspace, Token token) { - return ClusterMetadata.current().placements.get(keyspace.getMetadata().params.replication).reads.forToken(token); + return ClusterMetadata.current().placement(keyspace.getMetadata().params.replication).reads.forToken(token); } } diff --git a/src/java/org/apache/cassandra/locator/MetaStrategy.java b/src/java/org/apache/cassandra/locator/MetaStrategy.java index a7afa4898c..fdebff72b1 100644 --- a/src/java/org/apache/cassandra/locator/MetaStrategy.java +++ b/src/java/org/apache/cassandra/locator/MetaStrategy.java @@ -82,13 +82,13 @@ public class MetaStrategy extends SystemStrategy @Override public EndpointsForRange calculateNaturalReplicas(Token token, ClusterMetadata metadata) { - return metadata.placements.get(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get(); + return metadata.placement(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get(); } @Override public DataPlacement calculateDataPlacement(Epoch epoch, List> ranges, ClusterMetadata metadata) { - return metadata.placements.get(ReplicationParams.meta(metadata)); + return metadata.placement(ReplicationParams.meta(metadata)); } @Override diff --git a/src/java/org/apache/cassandra/locator/ReplicaLayout.java b/src/java/org/apache/cassandra/locator/ReplicaLayout.java index 440b2bed73..14915fc518 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaLayout.java +++ b/src/java/org/apache/cassandra/locator/ReplicaLayout.java @@ -239,7 +239,7 @@ public abstract class ReplicaLayout> { // todo deduplicate so that "pending" contains "read - write", // which is a hack until we revisit how consistency level handles pending - DataPlacement dataPlacement = metadata.placements.get(ks.params.replication); + DataPlacement dataPlacement = metadata.placement(ks.params.replication); natural = forNonLocalStrategyTokenRead(dataPlacement, token); // perf optimization to avoid double endpoints search and filtering for a typical case // DataPlacement constructor does a deduplication of reads/writes, so we can use cheap == comparision here @@ -394,12 +394,12 @@ public abstract class ReplicaLayout> static EndpointsForRange forNonLocalStategyRangeRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, AbstractBounds range) { - return metadata.placements.get(keyspace.params.replication).reads.forRange(range.right.getToken()).get(); + return metadata.placement(keyspace.params.replication).reads.forRange(range.right.getToken()).get(); } public static EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) { - return forNonLocalStrategyTokenRead(metadata.placements.get(keyspace.params.replication), token); + return forNonLocalStrategyTokenRead(metadata.placement(keyspace.params.replication), token); } public static EndpointsForToken forNonLocalStrategyTokenRead(DataPlacement dataPlacement, Token token) @@ -407,17 +407,11 @@ public abstract class ReplicaLayout> return dataPlacement.reads.forToken(token).get(); } - static EndpointsForToken forNonLocalStrategyTokenWrite(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token) - { - return forNonLocalStrategyTokenWrite(metadata.placements.get(keyspace.params.replication), token); - } - - static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token) + private static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token) { return dataPlacement.writes.forToken(token).get(); } - static EndpointsForRange forLocalStrategyRange(ClusterMetadata metadata, AbstractReplicationStrategy replicationStrategy, AbstractBounds range) { return replicationStrategy.calculateNaturalReplicas(range.right.getToken(), metadata); diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java index 61def76d1a..366708248e 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java @@ -234,7 +234,7 @@ public class ReplicaPlans NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy(); - EndpointsForToken replicas = metadata.placements.get(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get(); + EndpointsForToken replicas = metadata.placement(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get(); // CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint())); diff --git a/src/java/org/apache/cassandra/metrics/TCMMetrics.java b/src/java/org/apache/cassandra/metrics/TCMMetrics.java index 9544c2215a..8b74217bbb 100644 --- a/src/java/org/apache/cassandra/metrics/TCMMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TCMMetrics.java @@ -29,7 +29,6 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.EpochAwareDebounce; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; import static org.apache.cassandra.tcm.transformations.cms.PrepareCMSReconfiguration.needsReconfiguration; @@ -94,7 +93,7 @@ public class TCMMetrics isCMSMember = Metrics.register(factory.createMetricName("IsCMSMember"), () -> { ClusterMetadata metadata = ClusterMetadata.currentNullable(); - return metadata != null && metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) ? 1 : 0; + return metadata != null && metadata.isCMSMember() ? 1 : 0; }); needsCMSReconfiguration = Metrics.register(factory.createMetricName("NeedsCMSReconfiguration"), () -> { diff --git a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java index fb36c67c66..215b575c10 100644 --- a/src/java/org/apache/cassandra/net/ResponseVerbHandler.java +++ b/src/java/org/apache/cassandra/net/ResponseVerbHandler.java @@ -30,7 +30,6 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.FBUtilities; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND; @@ -98,7 +97,7 @@ class ResponseVerbHandler implements IVerbHandler if (SKIP_CATCHUP_FOR.contains(message.verb())) return; - if (metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) && CMS_SKIP_CATCHUP_FOR.contains(message.verb())) + if (metadata.isCMSMember() && CMS_SKIP_CATCHUP_FOR.contains(message.verb())) return; // Gossip stage is single-threaded, so we may end up in a deadlock with after-commit hook diff --git a/src/java/org/apache/cassandra/repair/RepairCoordinator.java b/src/java/org/apache/cassandra/repair/RepairCoordinator.java index 3ba51e4df6..8aa3482b5e 100644 --- a/src/java/org/apache/cassandra/repair/RepairCoordinator.java +++ b/src/java/org/apache/cassandra/repair/RepairCoordinator.java @@ -81,7 +81,6 @@ import org.apache.cassandra.tracing.TraceState; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.transport.messages.ResultMessage; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; @@ -402,7 +401,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai //calculation multiple times Iterable> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges(); boolean isMeta = Keyspace.open(state.keyspace).getMetadata().params.replication.isMeta(); - boolean isCMS = ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()); + boolean isCMS = ClusterMetadata.current().isCMSMember(); for (Range range : state.options.getRanges()) { EndpointsForRange allForRange = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range); diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index 286e8e86fc..bf187247e7 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -1213,7 +1213,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai // are based on the system partitioner EndpointsForRange endpoints = replication.isMeta() ? ClusterMetadata.current().fullCMSMembersAsReplicas() - : ClusterMetadata.current().placements.get(replication).reads.forRange(range).get(); + : ClusterMetadata.current().placement(replication).reads.forRange(range).get(); Set liveEndpoints = endpoints.filter(FailureDetector.isReplicaAlive).endpoints(); if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, liveEndpoints)) diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index a69124bb7d..7cf1494bb1 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -360,7 +360,7 @@ public class CassandraDaemon { CommitLog.instance.recoverSegmentsOnDisk(); NodeId self = ClusterMetadata.current().myNodeId(); - if (self != null) + if (self != NodeId.UNREGISTERED) AccordService.localStartup(self); } catch (IOException e) diff --git a/src/java/org/apache/cassandra/service/Rebuild.java b/src/java/org/apache/cassandra/service/Rebuild.java index 7c28960b96..27d814c177 100644 --- a/src/java/org/apache/cassandra/service/Rebuild.java +++ b/src/java/org/apache/cassandra/service/Rebuild.java @@ -52,12 +52,12 @@ import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamResultFuture; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.MovementMap; import org.apache.cassandra.utils.concurrent.Future; import org.apache.cassandra.utils.concurrent.FutureCombiner; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; public class Rebuild @@ -232,21 +232,23 @@ public class Rebuild private static MovementMap movementMap(ClusterMetadata metadata, String keyspace, String tokens) { MovementMap.Builder movementMapBuilder = MovementMap.builder(); - DataPlacements placements = metadata.placements; if (keyspace == null) { - placements.forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null))); + metadata.placements().forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null))); + // Special case to include the system metadata keyspace + ReplicationParams metaParams = Keyspace.open(METADATA_KEYSPACE_NAME).getMetadata().params.replication; + movementMapBuilder.put(metaParams, addMovementsForParams(metadata.placement(metaParams), null)); } else if (tokens == null) { ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication; - movementMapBuilder.put(params, addMovementsForParams(placements.get(params), null)); + movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), null)); } else { ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication; RangesAtEndpoint ranges = rangesForRebuildWithTokens(tokens, keyspace); - movementMapBuilder.put(params, addMovementsForParams(placements.get(params), ranges)); + movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), ranges)); } return movementMapBuilder.build(); } diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index 9e61567cb4..d10f8bb36b 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -175,6 +175,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.range.RangeCommands; import org.apache.cassandra.service.reads.repair.ReadRepair; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.ownership.VersionedEndpoints; import org.apache.cassandra.tracing.Tracing; @@ -2197,7 +2198,7 @@ public class StorageProxy implements StorageProxyMBean if (metadata == null) return false; - if (metadata.myNodeId() == null) + if (metadata.myNodeId() == NodeId.UNREGISTERED) return false; return metadata.myNodeState() == NodeState.JOINED; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 53f0986e0c..98f5d11ff8 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -2057,9 +2057,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE // creation and initialization of cluster metadata service. Metadata collector does accept // null localhost ID values, it's just that TokenMetadata was created earlier. ClusterMetadata metadata = ClusterMetadata.currentNullable(); - if (metadata == null || metadata.directory.peerId(getBroadcastAddressAndPort()) == null) - return null; - return metadata.directory.peerId(getBroadcastAddressAndPort()).toUUID(); + if (metadata == null || metadata.myNodeId() == NodeId.UNREGISTERED) + { + // this condition is to prevent accessing the tables when the node is not started yet, and in particular, + // when it is not going to be started at all (e.g. when running some unit tests or client tools). + if ((DatabaseDescriptor.isDaemonInitialized() || DatabaseDescriptor.isToolInitialized()) && CommitLog.instance.isStarted()) + return SystemKeyspace.getLocalHostId(); + else + return null; + } + return metadata.myNodeId().toUUID(); } public Map getHostIdMap() @@ -2067,7 +2074,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return getEndpointToHostId(); } - public Map getEndpointToHostId() { return getEndpointToHostId(false); @@ -2119,7 +2125,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE { if (keyspaceMetadata.params.replication.isMeta()) { - DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication); + DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication); // May be empty if mid-upgrade and CMS is not yet initialized if (!placement.reads.isEmpty()) rangeToEndpointMap.put(MetaStrategy.entireRange, placement.reads.forRange(MetaStrategy.entireRange).get()); @@ -2129,8 +2135,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE TokenMap tokenMap = metadata.tokenMap; for (Range range : ranges) { - Token token = tokenMap.nextToken(tokenMap.tokens(), range.right.getToken()); - rangeToEndpointMap.put(range, metadata.placements.get(keyspaceMetadata.params.replication) + Token token = TokenMap.nextToken(tokenMap.tokens(), range.right.getToken()); + rangeToEndpointMap.put(range, metadata.placement(keyspaceMetadata.params.replication) .reads.forRange(token).get()); } } @@ -3447,14 +3453,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE token = MetaStrategy.partitioner.getToken(key); else token = metadata.partitioner.getToken(key); - return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(token).get(); + return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(token).get(); } public boolean isEndpointValidForWrite(String keyspace, Token token) { ClusterMetadata metadata = ClusterMetadata.current(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); - return keyspaceMetadata != null && metadata.placements.get(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf(); + return keyspaceMetadata != null && metadata.placement(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf(); } public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception @@ -4262,7 +4268,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE if (replicationParams.isMeta()) { LinkedHashMap ownership = Maps.newLinkedHashMap(); - metadata.placements.get(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> { + metadata.placement(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> { ownership.put(r.endpoint(), 1.0f); }); return ownership; diff --git a/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java b/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java index c5298c0164..9c713abcbb 100644 --- a/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java +++ b/src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java @@ -312,7 +312,7 @@ public class AccordTopology public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup) { - return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas); + return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements(), metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas); } public static Topology createAccordTopology(ClusterMetadata metadata, Topology current) diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java index a40900f73c..da0d2b4e0d 100644 --- a/src/java/org/apache/cassandra/service/paxos/Paxos.java +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java @@ -277,7 +277,7 @@ public class Paxos final Token token = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : key.getToken(); ClusterMetadata metadata = ClusterMetadata.current(); Keyspace keyspace = Keyspace.open(table.keyspace); - DataPlacement placement = metadata.placements.get(keyspace.getMetadata().params.replication); + DataPlacement placement = metadata.placement(keyspace.getMetadata().params.replication); Epoch epoch = placement.writes.forToken(token).lastModified(); ForTokenWrite electorate = forTokenWriteLiveAndDown(metadata, keyspace, token); if (consistency == LOCAL_SERIAL) diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java index 5948086394..33189404e6 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java @@ -586,7 +586,7 @@ public class PaxosRepair extends AbstractPaxosRepair ClusterMetadata metadata = ClusterMetadata.current(); Collection allEndpoints = replication.isMeta() ? metadata.fullCMSMembers() - : metadata.placements.get(replication).reads.forRange(range).endpoints(); + : metadata.placement(replication).reads.forRange(range).endpoints(); return hasSufficientLiveNodesForTopologyChange(allEndpoints, liveEndpoints, ep -> metadata.locator.location(ep).datacenter, diff --git a/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java b/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java index 1bccf29044..0bf743fef5 100644 --- a/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java +++ b/src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java @@ -50,7 +50,7 @@ public class DataMovementVerbHandler implements IVerbHandler StreamPlan streamPlan = new StreamPlan(StreamOperation.fromString(message.payload.streamOperation)); ClusterMetadata metadata = ClusterMetadata.current(); Schema.instance.getNonLocalStrategyKeyspaces().stream().forEach((ksm) -> { - if (metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1) + if (metadata.placement(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1) return; message.payload.movements.get(ksm.params.replication).asMap().forEach((local, endpoints) -> { diff --git a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java index e545e726ed..a9a75ac8fb 100644 --- a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java @@ -46,6 +46,7 @@ public abstract class AbstractLocalProcessor implements Processor this.log = log; } + /** * Epoch returned by processor in the Result is _not_ guaranteed to be visible by the Follower by * the time when this method returns. @@ -59,7 +60,7 @@ public abstract class AbstractLocalProcessor implements Processor while (!retryPolicy.hasExpired()) { ClusterMetadata previous = log.waitForHighestConsecutive(); - if (!previous.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort())) + if (!acceptCommit(previous)) { String msg = String.format("Node %s is not a CMS member in epoch %s; members=%s", FBUtilities.getBroadcastAddressAndPort(), @@ -228,4 +229,5 @@ public abstract class AbstractLocalProcessor implements Processor public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy); protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch); + protected abstract boolean acceptCommit(ClusterMetadata metadata); } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java index bbbe4fa806..fd0c92c181 100644 --- a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java @@ -59,6 +59,18 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor this.epochHolder = new AtomicLong(epoch.getEpoch()); } + @Override + public boolean acceptCommit(ClusterMetadata metadata) + { + // AtomicLongBackedProcessor is only for use in tests and offline tools and it should be safe to assume that it + // is always allowed to process commit requests. In a non-test setup, when the CMS is initialized the initiating + // node is also registered. This is not the case in tests (see initCMS/recreateCMS in ServerTestUtils for more + // detail), as this simplifies setting up/tearing down topologies in test cases. The processor intended for use + // in real clusters (PaxosBackedProcessor) performs some actual validation here to ensure that the node is + // really a CMS member. + return true; + } + @Override protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch) { diff --git a/src/java/org/apache/cassandra/tcm/CMSMembership.java b/src/java/org/apache/cassandra/tcm/CMSMembership.java new file mode 100644 index 0000000000..67c30dc860 --- /dev/null +++ b/src/java/org/apache/cassandra/tcm/CMSMembership.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.MetaStrategy; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.tcm.membership.Directory; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.ownership.DataPlacement; +import org.apache.cassandra.tcm.ownership.VersionedEndpoints; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.utils.btree.BTreeSet; + +public class CMSMembership implements MetadataValue +{ + public static final Serializer serializer = new Serializer(); + public static final CMSMembership EMPTY = new CMSMembership(); + + private final Epoch lastModified; + private final BTreeSet fullMembers; + private final BTreeSet joiningMembers; + + /** + * Used to derive a CMSMembership when deserializing a ClusterMetadata instance written with a metadata version + * prior to V9. At that time, CMS membership was always inferred from the data placements of the distributed + * cluster metadata keyspace. Read replicas are full members of the CMS and write-only replicas are in the process + * of joining. Note: every read replica must also be a write replica, leaving the CMS is atomic in respect of the + * placements. + * @param placement + * @param directory + * @return + */ + public static CMSMembership reconstruct(DataPlacement placement, Directory directory) + { + BTreeSet.Builder fullMembersBuilder = BTreeSet.builder(NodeId::compareTo); + BTreeSet.Builder joiningMembersBuilder = BTreeSet.builder(NodeId::compareTo); + Epoch lm = Epoch.EMPTY; + for (VersionedEndpoints.ForRange endpoints : placement.reads.endpoints) + { + lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm; + endpoints.get().endpoints().forEach(e -> fullMembersBuilder.add(directory.peerId(e))); + } + BTreeSet full = fullMembersBuilder.build(); + + for (VersionedEndpoints.ForRange endpoints : placement.writes.endpoints) + { + lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm; + endpoints.get().endpoints().forEach(e -> { + NodeId id = directory.peerId(e); + if (!full.contains(id)) + joiningMembersBuilder.add(id); + }); + } + BTreeSet joining = joiningMembersBuilder.build(); + + return new CMSMembership(lm, full, joining); + } + + public DataPlacement toPlacement(Directory directory) + { + DataPlacement.Builder builder = DataPlacement.builder(); + for (NodeId id : fullMembers) + { + Replica replica = MetaStrategy.replica(directory.endpoint(id)); + builder.withReadReplica(lastModified, replica); + builder.withWriteReplica(lastModified, replica); + } + for(NodeId id : joiningMembers) + { + builder.withWriteReplica(lastModified, MetaStrategy.replica(directory.endpoint(id))); + } + return builder.build(); + } + + private CMSMembership() + { + this(Epoch.EMPTY, + BTreeSet.empty(NodeId::compareTo), + BTreeSet.empty(NodeId::compareTo)); + } + + private CMSMembership(Epoch lastModified, BTreeSet fullMembers, BTreeSet joiningMembers) + { + this.lastModified = lastModified; + this.fullMembers = fullMembers; + this.joiningMembers = joiningMembers; + } + + + @Override + public CMSMembership withLastModified(Epoch epoch) + { + return lastModified.is(epoch) ? this : new CMSMembership(epoch, fullMembers, joiningMembers); + } + + @Override + public Epoch lastModified() + { + return lastModified; + } + + public Set joiningMembers() + { + return joiningMembers; + } + + public Set fullMembers() + { + return fullMembers; + } + + public CMSMembership startJoining(NodeId id) + { + if (joiningMembers.contains(id)) + throw new IllegalStateException(id + " is already joining the CMS"); + if (fullMembers.contains(id)) + throw new IllegalStateException(id + " has already fully joined the CMS"); + + return new CMSMembership(lastModified, fullMembers, joiningMembers.with(id)); + } + + public CMSMembership cancelJoining(NodeId id) + { + if (!joiningMembers.contains(id)) + throw new IllegalStateException(id + " is not currently joining the CMS"); + if (fullMembers.contains(id)) + throw new IllegalStateException(id + " has already fully joined the CMS"); + + return new CMSMembership(lastModified, fullMembers, joiningMembers.without(id)); + } + + public CMSMembership finishJoining(NodeId id) + { + if (!joiningMembers.contains(id)) + throw new IllegalStateException(id + " is not currently joining the CMS"); + if (fullMembers.contains(id)) + throw new IllegalStateException(id + " has already fully joined the CMS"); + + return new CMSMembership(lastModified, fullMembers.with(id), joiningMembers.without(id)); + } + + public CMSMembership leave(NodeId id) + { + if (joiningMembers.contains(id)) + throw new IllegalStateException(id + " is currently joining the CMS, "); + if (!fullMembers.contains(id)) + throw new IllegalStateException(id + " is not a CMS member"); + + return new CMSMembership(lastModified, fullMembers.without(id), joiningMembers); + } + + @Override + public String toString() + { + return "CMSMembership{" + + "lastModified=" + lastModified + + ", fullMembers=" + fullMembers + + ", joiningMembers=" + joiningMembers + + '}'; + } + + @Override + public final boolean equals(Object o) + { + if (!(o instanceof CMSMembership)) return false; + + CMSMembership that = (CMSMembership) o; + return Objects.equals(lastModified, that.lastModified) && + Objects.equals(fullMembers, that.fullMembers) && + Objects.equals(joiningMembers, that.joiningMembers); + } + + @Override + public int hashCode() + { + int result = Objects.hashCode(lastModified); + result = 31 * result + Objects.hashCode(fullMembers); + result = 31 * result + Objects.hashCode(joiningMembers); + return result; + } + + public static class Serializer implements MetadataSerializer + { + @Override + public void serialize(CMSMembership t, DataOutputPlus out, Version version) throws IOException + { + Epoch.serializer.serialize(t.lastModified, out); + + out.writeUnsignedVInt32(t.fullMembers.size()); + for (NodeId id : t.fullMembers) + NodeId.serializer.serialize(id, out, version); + + out.writeUnsignedVInt32(t.joiningMembers.size()); + for (NodeId id : t.joiningMembers) + NodeId.serializer.serialize(id, out, version); + } + + @Override + public CMSMembership deserialize(DataInputPlus in, Version version) throws IOException + { + Epoch lastModified = Epoch.serializer.deserialize(in, version); + + int fullMemberCount = in.readUnsignedVInt32(); + BTreeSet.Builder fullMembers = BTreeSet.builder(NodeId::compareTo); + for (int i = 0; i < fullMemberCount; i++) + fullMembers.add(NodeId.serializer.deserialize(in, version)); + + int joiningMemberCount = in.readUnsignedVInt32(); + BTreeSet.Builder joiningMembers = BTreeSet.builder(NodeId::compareTo); + for (int i = 0; i < joiningMemberCount; i++) + joiningMembers.add(NodeId.serializer.deserialize(in, version)); + + return new CMSMembership(lastModified, fullMembers.build(), joiningMembers.build()) ; + } + + @Override + public long serializedSize(CMSMembership t, Version version) + { + long size = Epoch.serializer.serializedSize(t.lastModified); + + size += TypeSizes.sizeofUnsignedVInt(t.fullMembers.size()); + for (NodeId id : t.fullMembers) + size += NodeId.serializer.serializedSize(id, version); + + size += TypeSizes.sizeofUnsignedVInt(t.joiningMembers.size()); + for (NodeId id : t.joiningMembers) + size += NodeId.serializer.serializedSize(id, version); + + return size; + } + } + +} diff --git a/src/java/org/apache/cassandra/tcm/CMSOperations.java b/src/java/org/apache/cassandra/tcm/CMSOperations.java index 460513fbf5..60f4e67d1a 100644 --- a/src/java/org/apache/cassandra/tcm/CMSOperations.java +++ b/src/java/org/apache/cassandra/tcm/CMSOperations.java @@ -48,7 +48,6 @@ import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.tcm.transformations.Unregister; import org.apache.cassandra.tcm.transformations.cms.AdvanceCMSReconfiguration; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MBeanWrapper; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -217,7 +216,7 @@ public class CMSOperations implements CMSOperationsMBean String members = metadata.fullCMSMembers().stream().sorted().map(Object::toString).collect(Collectors.joining(",")); info.put(MEMBERS, members); info.put(NEEDS_RECONFIGURATION, Boolean.toString(metadata.epoch.isBefore(Epoch.FIRST) || needsReconfiguration(metadata))); - info.put(IS_MEMBER, Boolean.toString(cms.isCurrentMember(FBUtilities.getBroadcastAddressAndPort()))); + info.put(IS_MEMBER, Boolean.toString(metadata.isCMSMember())); info.put(SERVICE_STATE, ClusterMetadataService.state(metadata).toString()); info.put(IS_MIGRATING, Boolean.toString(cms.isMigrating())); info.put(EPOCH, Long.toString(metadata.epoch.getEpoch())); diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index be6d0bd212..0224ca5bfb 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -28,7 +28,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -77,6 +79,7 @@ import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.membership.NodeVersion; +import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator; import org.apache.cassandra.tcm.ownership.ReplicaGroups; @@ -89,7 +92,6 @@ import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -import static com.google.common.collect.ImmutableSet.toImmutableSet; import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR; import static org.apache.cassandra.db.TypeSizes.sizeof; import static org.apache.cassandra.tcm.serialization.Version.MIN_ACCORD_VERSION; @@ -107,13 +109,14 @@ public class ClusterMetadata public final DistributedSchema schema; public final Directory directory; public final TokenMap tokenMap; - public final DataPlacements placements; + private final DataPlacements placements; public final AccordFastPath accordFastPath; public final LockedRanges lockedRanges; public final InProgressSequences inProgressSequences; public final ConsensusMigrationState consensusMigrationState; public final ImmutableMap, ExtensionValue> extensions; public final AccordStaleReplicas accordStaleReplicas; + public final CMSMembership cmsMembership; // This isn't serialized as part of ClusterMetadata it's really just a view over the Directory. public final Locator locator; @@ -121,9 +124,10 @@ public class ClusterMetadata // These fields are lazy but only for the test purposes, since their computation requires initialization of the log ks private EndpointsForRange fullCMSReplicas; private Set fullCMSEndpoints; - private Set fullCMSIds; private volatile Map localRangesAllSettled = null; private static final RangesAtEndpoint EMPTY_LOCAL_RANGES = RangesAtEndpoint.empty(FBUtilities.getBroadcastAddressAndPort()); + private DataPlacement cmsDataPlacement; + private final NodeId localNodeId; public ClusterMetadata(IPartitioner partitioner) { @@ -151,7 +155,8 @@ public class ClusterMetadata InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public ClusterMetadata(Epoch epoch, @@ -165,7 +170,8 @@ public class ClusterMetadata InProgressSequences inProgressSequences, ConsensusMigrationState consensusMigrationState, Map, ExtensionValue> extensions, - AccordStaleReplicas accordStaleReplicas) + AccordStaleReplicas accordStaleReplicas, + CMSMembership cmsMembership) { this(EMPTY_METADATA_IDENTIFIER, epoch, @@ -179,22 +185,25 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas); + accordStaleReplicas, + cmsMembership); } + private ClusterMetadata(int metadataIdentifier, - Epoch epoch, - IPartitioner partitioner, - DistributedSchema schema, - Directory directory, - TokenMap tokenMap, - DataPlacements placements, - AccordFastPath accordFastPath, - LockedRanges lockedRanges, - InProgressSequences inProgressSequences, - ConsensusMigrationState consensusMigrationState, - Map, ExtensionValue> extensions, - AccordStaleReplicas accordStaleReplicas) + Epoch epoch, + IPartitioner partitioner, + DistributedSchema schema, + Directory directory, + TokenMap tokenMap, + DataPlacements placements, + AccordFastPath accordFastPath, + LockedRanges lockedRanges, + InProgressSequences inProgressSequences, + ConsensusMigrationState consensusMigrationState, + Map, ExtensionValue> extensions, + AccordStaleReplicas accordStaleReplicas, + CMSMembership cmsMembership) { // TODO: token map is a feature of the specific placement strategy, and so may not be a relevant component of // ClusterMetadata in the long term. We need to consider how the actual components of metadata can be evolved @@ -206,14 +215,42 @@ public class ClusterMetadata this.schema = schema; this.directory = directory; this.tokenMap = tokenMap; - this.placements = placements; this.accordFastPath = accordFastPath; + this.placements = placements; this.lockedRanges = lockedRanges; this.inProgressSequences = inProgressSequences; this.consensusMigrationState = consensusMigrationState; this.extensions = ImmutableMap.copyOf(extensions); this.locator = Locator.usingDirectory(directory); this.accordStaleReplicas = accordStaleReplicas; + this.cmsMembership = cmsMembership; + this.cmsDataPlacement = calculateCMSPlacement(placements, cmsMembership); + + InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); + this.localNodeId = directory.allAddresses().contains(broadcastAddress) + ? directory.peerId(broadcastAddress) + : NodeId.UNREGISTERED; + } + + public Set fullCMSMemberIds() + { + return cmsMembership.fullMembers(); + } + + public boolean isCMSMember() + { + if (epoch.isEqualOrBefore(Epoch.FIRST)) + return isCMSMember(FBUtilities.getBroadcastAddressAndPort()); + + return fullCMSMemberIds().contains(localNodeId); + } + + public boolean isCMSMember(InetAddressAndPort endpoint) + { + if (epoch.isEqualOrBefore(Epoch.FIRST)) + return cmsDataPlacement.reads.byEndpoint().keySet().contains(endpoint); + + return fullCMSMembers().contains(endpoint); } public Set fullCMSMembers() @@ -223,27 +260,14 @@ public class ClusterMetadata if (fullCMSEndpoints == null) { - if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) - return Collections.emptySet(); - this.fullCMSEndpoints = ImmutableSet.copyOf(placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet()); + fullCMSEndpoints = ImmutableSet.copyOf(cmsMembership.fullMembers() + .stream() + .map(directory::endpoint) + .collect(Collectors.toSet())); } return fullCMSEndpoints; } - public Set fullCMSMemberIds() - { - if (epoch.isBefore(Epoch.FIRST)) - return Collections.emptySet(); - - if (fullCMSIds == null) - { - if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) - return Collections.emptySet(); - this.fullCMSIds = placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet().stream().map(directory::peerId).collect(toImmutableSet()); - } - return fullCMSIds; - } - public EndpointsForRange fullCMSMembersAsReplicas() { if (epoch.isBefore(Epoch.FIRST)) @@ -251,16 +275,70 @@ public class ClusterMetadata if (fullCMSReplicas == null) { - if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) - return EndpointsForRange.empty(MetaStrategy.entireRange); - fullCMSReplicas = placements.get(ReplicationParams.meta(this)).reads.forRange(MetaStrategy.entireRange).get(); + EndpointsForRange.Builder builder = EndpointsForRange.builder(MetaStrategy.entireRange); + for (NodeId nodeId : fullCMSMemberIds()) + builder.add(MetaStrategy.replica(directory.endpoint(nodeId))); + fullCMSReplicas = builder.build(); } return fullCMSReplicas; } - public boolean isCMSMember(InetAddressAndPort endpoint) + private DataPlacement calculateCMSPlacement(DataPlacements placements, CMSMembership cms) { - return fullCMSMembers().contains(endpoint); + if (epoch.isBefore(Epoch.FIRST) || schema.getKeyspaces().get(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty()) + return DataPlacement.empty(); + + if (directory.isEmpty()) + { + if (epoch.is(Epoch.FIRST)) + { + // PRE_INITIALIZE_CMS: placements need to be hardcoded to the local address so that the subsequent + // INITIALIZE_CMS can be committed + Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort()); + return DataPlacement.builder() + .withReadReplica(Epoch.FIRST, localReplica) + .withWriteReplica(Epoch.FIRST, localReplica) + .build(); + } + else + { + // This cluster did not previously upgrade from a gossip based version (i.e. pre-6.0) but did at some point + // run a version prior to MetadataVersion.V9 where we started to encode CMS membership directly. This + // condition implies that we are reconstructing a serialized cluster metadata during replay or else the + // directory should not be empty after Epoch.FIRST as the base state in INITIALIZE_CMS now includes the + // first CMS node. Similarly, if the cluster had previously been running a gossip-based version, the + // directory would contain entries for each of the live nodes at the time of upgrade. + // Given this state, the very next transformation that is/was applied will be to register the node that + // committed the PRE_INITIALIZE_CMS and INTIALIZE_CMS transformations. So we just extract the placements + // associated with the MetaStrategy params as they will already contain that node as an endpoint. + for (ReplicationParams params : placements.keys()) + if (params.isMeta()) + return placements.get(params); + + // This point can only be reached in tests. + // TODO enforce that invariant somehow + Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort()); + return DataPlacement.builder() + .withReadReplica(epoch, localReplica) + .withWriteReplica(epoch, localReplica) + .build(); + } + } + else + { + // Build a placement based on the CMS membership + return cms.toPlacement(directory); + } + } + + public DataPlacement placement(ReplicationParams params) + { + return params.isMeta() ? cmsDataPlacement : placements.get(params); + } + + public DataPlacements placements() + { + return placements; } public Transformer transformer() @@ -291,10 +369,42 @@ public class ClusterMetadata capLastModified(inProgressSequences, epoch), capLastModified(consensusMigrationState, epoch), capLastModified(extensions, epoch), - capLastModified(accordStaleReplicas, epoch)); + capLastModified(accordStaleReplicas, epoch), + capLastModified(cmsMembership, epoch)); } - public ClusterMetadata initializeClusterIdentifier(int clusterIdentifier) + /** + * To be used only during the execute of PRE_INITIALIZE_CMS, this sets the DataPlacement for the metadata keyspace + * so that the global log can be initialized and the subsequent entry containing the INITIALIZE_CMS transformation + * can be committed. + * @param initialCMSPlacement Expected to be a singleton placement identifying the local node as the sole replica + * for the metadata keyspace + * @return ClusterMetadata instance in the correct state to constitute the result of the PRE_INITIALIZE_CMS + * transformation + */ + public ClusterMetadata forcePreInitializedState(DataPlacement initialCMSPlacement) + { + assert epoch.isEqualOrBefore(Epoch.FIRST); + // double check that all metadata elements have had the last modified epoch set correctly + ClusterMetadata initial = forceEpoch(Epoch.FIRST); + initial.cmsDataPlacement = initialCMSPlacement; + return initial; + } + + /** + * Produce a ClusterMetadata suitable for use as the base state in the INITIALIZE_CMS transformation. This should + * only be used on the first CMS node when bootstrapping the CMS after upgrade or in a brand new cluster. + * @param clusterIdentifier Unique identifier for split brain detection & protection + * @param addresses The NodeAddresses of the first CMS node. + * @param version Version info for the first CMS node. + * @param location The rack & DC of the first CMS node. + * @return ClusterMetadata instance in the correct state to constitute the base state of the INITIALIZE_CMS + * transformation + */ + public ClusterMetadata forceInitializedState(int clusterIdentifier, + NodeAddresses addresses, + NodeVersion version, + Location location) { if (this.metadataIdentifier != EMPTY_METADATA_IDENTIFIER) throw new IllegalStateException(String.format("Can only initialize cluster identifier once, but it was already set to %d", this.metadataIdentifier)); @@ -302,11 +412,21 @@ public class ClusterMetadata if (clusterIdentifier == EMPTY_METADATA_IDENTIFIER) throw new IllegalArgumentException("Can not initialize cluster with empty cluster identifier"); + if (this.epoch.isAfter(Epoch.FIRST)) + throw new IllegalStateException(String.format("Can only initialize cluster identifier during epoch %d, but current epoch is %d", Epoch.FIRST.getEpoch(), epoch.getEpoch())); + + // Maybe register the first CMS node. If upgrading from gossip, this should be a no-op + Directory withRegistered = directory.with(addresses, location, version); + NodeId firstNode = withRegistered.peerId(addresses.broadcastAddress); + if (firstNode == null) + throw new IllegalStateException("Failed to find first CMS node in directory"); + + CMSMembership initialCMS = cmsMembership.startJoining(firstNode).finishJoining(firstNode); return new ClusterMetadata(clusterIdentifier, epoch, partitioner, schema, - directory, + withRegistered, tokenMap, placements, accordFastPath, @@ -314,7 +434,8 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas); + accordStaleReplicas, + initialCMS); } private static Map, ExtensionValue> capLastModified(Map, ExtensionValue> original, Epoch maxEpoch) @@ -413,8 +534,8 @@ public class ClusterMetadata // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges public boolean hasPendingRangesFor(KeyspaceMetadata ksm, Token token) { - ReplicaGroups writes = placements.get(ksm.params.replication).writes; - ReplicaGroups reads = placements.get(ksm.params.replication).reads; + ReplicaGroups writes = placement(ksm.params.replication).writes; + ReplicaGroups reads = placement(ksm.params.replication).reads; if (ksm.params.replication.isMeta()) return !reads.equals(writes); return !reads.forToken(token).equals(writes.forToken(token)); @@ -423,8 +544,8 @@ public class ClusterMetadata // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges public boolean hasPendingRangesFor(KeyspaceMetadata ksm, InetAddressAndPort endpoint) { - ReplicaGroups writes = placements.get(ksm.params.replication).writes; - ReplicaGroups reads = placements.get(ksm.params.replication).reads; + ReplicaGroups writes = placement(ksm.params.replication).writes; + ReplicaGroups reads = placement(ksm.params.replication).reads; return !writes.byEndpoint().get(endpoint).equals(reads.byEndpoint().get(endpoint)); } @@ -435,22 +556,22 @@ public class ClusterMetadata public RangesAtEndpoint writeRanges(KeyspaceMetadata metadata, InetAddressAndPort peer) { - return placements.get(metadata.params.replication).writes.byEndpoint().get(peer); + return placement(metadata.params.replication).writes.byEndpoint().get(peer); } // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges public Map, VersionedEndpoints.ForRange> pendingRanges(KeyspaceMetadata metadata) { Map, VersionedEndpoints.ForRange> map = new HashMap<>(); - ReplicaGroups writes = placements.get(metadata.params.replication).writes; - ReplicaGroups reads = placements.get(metadata.params.replication).reads; + ReplicaGroups writes = placement(metadata.params.replication).writes; + ReplicaGroups reads = placement(metadata.params.replication).reads; // first, pending ranges as the result of range splitting or merging // i.e. new ranges being created through join/leave List> pending = new ArrayList<>(writes.ranges()); pending.removeAll(reads.ranges()); for (Range p : pending) - map.put(p, placements.get(metadata.params.replication).writes.forRange(p)); + map.put(p, placement(metadata.params.replication).writes.forRange(p)); // next, ranges where the ranges themselves are not changing, but the replicas are // i.e. replacement or RF increase @@ -467,8 +588,8 @@ public class ClusterMetadata // TODO Remove this as it isn't really an equivalent to the previous concept of pending endpoints public VersionedEndpoints.ForToken pendingEndpointsFor(KeyspaceMetadata metadata, Token t) { - VersionedEndpoints.ForToken writeEndpoints = placements.get(metadata.params.replication).writes.forToken(t); - VersionedEndpoints.ForToken readEndpoints = placements.get(metadata.params.replication).reads.forToken(t); + VersionedEndpoints.ForToken writeEndpoints = placement(metadata.params.replication).writes.forToken(t); + VersionedEndpoints.ForToken readEndpoints = placement(metadata.params.replication).reads.forToken(t); EndpointsForToken.Builder endpointsForToken = writeEndpoints.get().newBuilder(writeEndpoints.size() - readEndpoints.size()); for (Replica writeReplica : writeEndpoints.get()) @@ -495,6 +616,7 @@ public class ClusterMetadata private final Map, ExtensionValue> extensions; private final Set modifiedKeys; private AccordStaleReplicas accordStaleReplicas; + private CMSMembership cmsMembership; private Transformer(ClusterMetadata metadata, Epoch epoch) { @@ -512,6 +634,7 @@ public class ClusterMetadata extensions = new HashMap<>(metadata.extensions); modifiedKeys = new HashSet<>(); accordStaleReplicas = metadata.accordStaleReplicas; + cmsMembership = metadata.cmsMembership; } public Epoch epoch() @@ -637,6 +760,30 @@ public class ClusterMetadata return this; } + public Transformer startJoiningCMS(NodeId id) + { + cmsMembership = cmsMembership.startJoining(id); + return this; + } + + public Transformer finishJoiningCMS(NodeId id) + { + cmsMembership = cmsMembership.finishJoining(id); + return this; + } + + public Transformer cancelJoiningCMS(NodeId id) + { + cmsMembership = cmsMembership.cancelJoining(id); + return this; + } + + public Transformer leaveCMS(NodeId id) + { + cmsMembership = cmsMembership.leave(id); + return this; + } + public Transformer with(DataPlacements placements) { this.placements = placements; @@ -822,6 +969,12 @@ public class ClusterMetadata consensusMigrationState.validateAgainstSchema(schema); } + if (cmsMembership != base.cmsMembership) + { + modifiedKeys.add(MetadataKeys.CMS_MEMBERSHIP); + cmsMembership = cmsMembership.withLastModified(epoch); + } + return new Transformed(new ClusterMetadata(base.metadataIdentifier, epoch, partitioner, @@ -834,7 +987,8 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas), + accordStaleReplicas, + cmsMembership), ImmutableSet.copyOf(modifiedKeys)); } @@ -852,7 +1006,8 @@ public class ClusterMetadata inProgressSequences, consensusMigrationState, extensions, - accordStaleReplicas); + accordStaleReplicas, + cmsMembership); } @Override @@ -871,6 +1026,7 @@ public class ClusterMetadata ", inProgressSequences=" + inProgressSequences + ", consensusMigrationState=" + consensusMigrationState + ", extensions=" + extensions + + ", cmsMembership=" + cmsMembership + ", modifiedKeys=" + modifiedKeys + '}'; } @@ -966,6 +1122,9 @@ public class ClusterMetadata ", placements=" + placements + ", lockedRanges=" + lockedRanges + ", consensusMigrationState=" + lockedRanges + + ", inProgressSequences=" + inProgressSequences + + ", extensions=" + extensions + + ", cmsMembership=" + cmsMembership + '}'; } @@ -994,7 +1153,8 @@ public class ClusterMetadata inProgressSequences.equals(that.inProgressSequences) && consensusMigrationState.equals(that.consensusMigrationState) && accordStaleReplicas.equals(that.accordStaleReplicas) && - extensions.equals(that.extensions); + extensions.equals(that.extensions) && + cmsMembership.equals(that.cmsMembership); } private static final Logger logger = LoggerFactory.getLogger(ClusterMetadata.class); @@ -1037,12 +1197,16 @@ public class ClusterMetadata { logger.warn("Extensions differ: {} != {}", extensions, other.extensions); } + if (!cmsMembership.equals(other.cmsMembership)) + { + logger.warn("CMS Membership differ: {} != {}", cmsMembership, other.cmsMembership); + } } @Override public int hashCode() { - return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions); + return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions, cmsMembership); } public static ClusterMetadata current() @@ -1083,13 +1247,13 @@ public class ClusterMetadata public NodeId myNodeId() { - return directory.peerId(FBUtilities.getBroadcastAddressAndPort()); + return localNodeId; } public NodeState myNodeState() { NodeId nodeId = myNodeId(); - if (myNodeId() != null) + if (nodeId != NodeId.UNREGISTERED) return directory.peerState(nodeId); return null; } @@ -1113,7 +1277,10 @@ public class ClusterMetadata DistributedSchema.serializer.serialize(metadata.schema, out, version); Directory.serializer.serialize(metadata.directory, out, version); TokenMap.serializer.serialize(metadata.tokenMap, out, version); - DataPlacements.serializer.serialize(metadata.placements, out, version); + // Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements + // so when targetting such a version, emulate that. + DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements; + DataPlacements.serializer.serialize(placements, out, version); if (version.isAtLeast(MIN_ACCORD_VERSION)) { AccordFastPath.serializer.serialize(metadata.accordFastPath, out, version); @@ -1132,6 +1299,9 @@ public class ClusterMetadata assert key.valueType.isInstance(value); value.serialize(out, version); } + // From V9 CMS membership is directly encoded in ClusterMetadata + if (version.isAtLeast(Version.V9)) + CMSMembership.serializer.serialize(metadata.cmsMembership, out, version); } @Override @@ -1188,6 +1358,42 @@ public class ClusterMetadata value.deserialize(in, version); extensions.put(key, value); } + + CMSMembership cmsMembership = CMSMembership.EMPTY; + if (version.isAtLeast(Version.V9)) + cmsMembership = CMSMembership.serializer.deserialize(in, version); + else + { + Optional metadataKs = schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); + if (metadataKs.isPresent()) + { + // Pre-V9 the membership of the CMS was always inferred from the placement of the distributed + // metadata keyspace. + // If the directory is not empty the endpoints in the placement must belong to registered nodes, + // so we can derive the CMSMembership using the data placement and directory. + + // If the directory is empty, then the cluster metadata must be the payload of an INITIALIZE_CMS + // transformation of a cluster that began on a post-6.0, pre-MetadataVersion.V9 version. + // In this case, we can and must assume that the initial CMS membership will consist of a single + // node, with the node_id 1. + // Note: the only route to arrive at this scenario is if a cluster is initialized on a post-6.0, + // pre-V9 version and then upgraded to a post-V9 version without any metadata snapshots being taken. + // If there is a snapshot available locally, when the upgraded node starts up it will replay its + // local log from that point. The INITIALIZE_CMS transform will not be replayed. + if (!dir.isEmpty()) + { + DataPlacement placement = placements.get(metadataKs.get().params.replication); + cmsMembership = CMSMembership.reconstruct(placement, dir); + } + else + { + NodeId id = new NodeId(1); + cmsMembership = CMSMembership.EMPTY.startJoining(id).finishJoining(id); + } + placements = placements.unbuild().without(metadataKs.get().params.replication).build(); + } + } + return new ClusterMetadata(clusterIdentifier, epoch, partitioner, @@ -1200,7 +1406,8 @@ public class ClusterMetadata ips, consensusMigrationState, extensions, - staleReplicas); + staleReplicas, + cmsMembership); } private DistributedSchema deduplicateReplicationParams(DistributedSchema schema, DataPlacements placements) @@ -1235,8 +1442,7 @@ public class ClusterMetadata sizeof(metadata.partitioner.getClass().getCanonicalName()) + DistributedSchema.serializer.serializedSize(metadata.schema, version) + Directory.serializer.serializedSize(metadata.directory, version) + - TokenMap.serializer.serializedSize(metadata.tokenMap, version) + - DataPlacements.serializer.serializedSize(metadata.placements, version); + TokenMap.serializer.serializedSize(metadata.tokenMap, version); if (version.isAtLeast(MIN_ACCORD_VERSION)) { @@ -1248,9 +1454,27 @@ public class ClusterMetadata size += LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) + InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version); + // Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements + // so when targetting such a version, emulate that. + DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements; + size += DataPlacements.serializer.serializedSize(placements, version); + // From V9 CMS membership is directly encoded in ClusterMetadata + if (version.isAtLeast(Version.V9)) + size += CMSMembership.serializer.serializedSize(metadata.cmsMembership, version); + return size; } + private DataPlacements preV9Placements(ClusterMetadata metadata) + { + if (metadata.cmsDataPlacement.isEmpty()) + return metadata.placements; + + return metadata.placements.unbuild() + .with(ReplicationParams.meta(metadata), metadata.cmsDataPlacement) + .build(); + } + public static IPartitioner getPartitioner(DataInputPlus in, Version version) throws IOException { if (version.isAtLeast(Version.V1)) diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java index 6d1b054bec..08fadb2aa8 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java @@ -117,7 +117,7 @@ public class ClusterMetadataService trace); instance = newInstance; RegistrationStatus.instance.onInitialized(); - if (newInstance.metadata().myNodeId() != null) + if (newInstance.metadata().myNodeId() != NodeId.UNREGISTERED) RegistrationStatus.instance.onRegistration(); trace = new RuntimeException("Previously initialized trace"); DatabaseDescriptor.applyLocator(); @@ -176,8 +176,9 @@ public class ClusterMetadataService // The node is a full member of the CMS if it has started participating in reads for distributed metadata table (which // implies it is a write replica as well). In other words, it's a fully joined member of the replica set responsible for // the distributed metadata table. - if (ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())) + if (metadata.epoch.isEqualOrBefore(Epoch.FIRST) || metadata.isCMSMember()) return LOCAL; + return REMOTE; } @@ -327,14 +328,15 @@ public class ClusterMetadataService DatabaseDescriptor.getPartitioner(), new DistributedSchema(keyspaces), Directory.EMPTY, - new TokenMap(DatabaseDescriptor.getPartitioner()), - DataPlacements.empty(), - AccordFastPath.EMPTY, - LockedRanges.EMPTY, - InProgressSequences.EMPTY, - ConsensusMigrationState.EMPTY, - Collections.emptyMap(), - AccordStaleReplicas.EMPTY); + new TokenMap(DatabaseDescriptor.getPartitioner()), + DataPlacements.empty(), + AccordFastPath.EMPTY, + LockedRanges.EMPTY, + InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, + Collections.emptyMap(), + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); LocalLog.LogSpec logSpec = LocalLog.logSpec() @@ -360,6 +362,24 @@ public class ClusterMetadataService ClusterMetadataService.setInstance(cms); } + @SuppressWarnings("resource") + public static void initializeForClients() + { + if (instance != null) + return; + + ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools()); + } + + public static void initializeForClients(DistributedSchema initialSchema) + { + if (instance != null) + return; + + + ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema)); + } + /* * Hook to be executed when the LocalLog is bootstrapped with the PreInitialize transformation. This is done on * the first CMS member to set up the initial replication and data placements for the metadata keyspace. @@ -391,25 +411,8 @@ public class ClusterMetadataService }; } // otherwise, this is a noop. - return preInit -> {}; - } - - @SuppressWarnings("resource") - public static void initializeForClients() - { - if (instance != null) - return; - - ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools()); - } - - public static void initializeForClients(DistributedSchema initialSchema) - { - if (instance != null) - return; - - - ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema)); + return preInit -> { + }; } public boolean isCurrentMember(InetAddressAndPort peer) @@ -1100,6 +1103,6 @@ public class ClusterMetadataService public enum State { - LOCAL, REMOTE, GOSSIP, RESET + LOCAL, REMOTE, GOSSIP, RESET, OFFLINE_TOOL } } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/MetadataKeys.java b/src/java/org/apache/cassandra/tcm/MetadataKeys.java index 0be621b201..f788f1a449 100644 --- a/src/java/org/apache/cassandra/tcm/MetadataKeys.java +++ b/src/java/org/apache/cassandra/tcm/MetadataKeys.java @@ -45,18 +45,20 @@ public class MetadataKeys public static final MetadataKey LOCKED_RANGES = make(CORE_NS, "sequences", "locked_ranges"); public static final MetadataKey IN_PROGRESS_SEQUENCES = make(CORE_NS, "sequences", "in_progress"); public static final MetadataKey CONSENSUS_MIGRATION_STATE = make(CORE_NS, "consensus", "migration_state"); + public static final MetadataKey CMS_MEMBERSHIP = make(CORE_NS, "cms_membership", "cms_membership"); public static final ImmutableMap>> CORE_METADATA = ImmutableMap.>>builder() .put(SCHEMA, cm -> cm.schema) .put(NODE_DIRECTORY, cm -> cm.directory) .put(TOKEN_MAP, cm -> cm.tokenMap) - .put(DATA_PLACEMENTS, cm -> cm.placements) + .put(DATA_PLACEMENTS, ClusterMetadata::placements) .put(LOCKED_RANGES, cm -> cm.lockedRanges) .put(IN_PROGRESS_SEQUENCES, cm -> cm.inProgressSequences) .put(ACCORD_FAST_PATH, cm -> cm.accordFastPath) .put(ACCORD_STALE_REPLICAS, cm -> cm.accordStaleReplicas) .put(CONSENSUS_MIGRATION_STATE, cm -> cm.consensusMigrationState) + .put(CMS_MEMBERSHIP, cm -> cm.cmsMembership) .build(); public static MetadataKey make(String...parts) diff --git a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java index 4e18964e4d..4e36cf1066 100644 --- a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java @@ -63,6 +63,12 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor super(log); } + @Override + protected boolean acceptCommit(ClusterMetadata metadata) + { + return metadata.isCMSMember(); + } + @Override protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch) { diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index 57169d9876..f9a4313afa 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -63,8 +63,11 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; +import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.migration.CMSInitializationException; import org.apache.cassandra.tcm.migration.CMSInitializationRequest; import org.apache.cassandra.tcm.migration.Election; @@ -148,19 +151,24 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; */ public static void initializeAsFirstCMSNode() { - InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort(); - String datacenter = DatabaseDescriptor.getLocator().local().datacenter; + NodeAddresses addresses = NodeAddresses.current(); + Location location = DatabaseDescriptor.getLocator().local(); ClusterMetadataService cms = ClusterMetadataService.instance(); - cms.log().bootstrap(addr, datacenter, cms.logBootstrapCallback()); - ClusterMetadata metadata = ClusterMetadata.current(); - assert ClusterMetadataService.state() == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata); - Initialize initialize = new Initialize(metadata.initializeClusterIdentifier(addr.hashCode())); - ClusterMetadataService.instance().commit(initialize, - m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;}, - (code, message) -> { - logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message); - throw new CMSInitializationException(); - }); + cms.log().bootstrap(addresses.broadcastAddress, location.datacenter, cms.logBootstrapCallback()); + ClusterMetadata metadata = ClusterMetadata.current().forceInitializedState(addresses.broadcastAddress.hashCode(), + addresses, + NodeVersion.CURRENT, + location); + assert ClusterMetadataService.state(metadata) == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata); + Initialize initialize = new Initialize(metadata); + ClusterMetadata initialized = ClusterMetadataService.instance().commit(initialize, + m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;}, + (code, message) -> { + logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message); + throw new CMSInitializationException(); + }); + NodeId id = initialized.myNodeId(); + SystemKeyspace.setLocalHostId(id.toUUID()); } public static void initializeAsNonCmsNode(Function wrapProcessor) throws StartupException @@ -178,7 +186,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; NodeId nodeId = ClusterMetadata.current().myNodeId(); UUID currentHostId = SystemKeyspace.getLocalHostId(); - if (nodeId != null && !Objects.equals(nodeId.toUUID(), currentHostId)) + if (nodeId != NodeId.UNREGISTERED && !Objects.equals(nodeId.toUUID(), currentHostId)) { if (currentHostId == null) { @@ -394,7 +402,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; DatabaseDescriptor.getPartitioner().getClass().getCanonicalName(), metadata.partitioner.getClass().getCanonicalName())); - if (!metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort())) + if (!metadata.isCMSMember()) throw new IllegalStateException("When reinitializing with cluster metadata, we must be in the CMS"); metadata = metadata.forceEpoch(metadata.epoch.nextEpoch()); diff --git a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java index a7956cd21a..6e4a201613 100644 --- a/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java @@ -184,7 +184,8 @@ public class StubClusterMetadataService extends ClusterMetadataService InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } return new StubClusterMetadataService(new UniformRangePlacement(), snapshots != null ? snapshots : MetadataSnapshots.NO_OP, diff --git a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java index 965751c554..c603737620 100644 --- a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java +++ b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java @@ -58,6 +58,7 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; @@ -306,7 +307,8 @@ public class GossipHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, Collections.emptyMap(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public static ClusterMetadata fromEndpointStates(DistributedSchema schema, Map epStates) @@ -396,7 +398,8 @@ public class GossipHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, extensions, - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); DataPlacements placements = new UniformRangePlacement().calculatePlacements(Epoch.UPGRADE_GOSSIP, forPlacementCalculation, schema.getKeyspaces()); @@ -411,7 +414,8 @@ public class GossipHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, extensions, - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public static boolean isValidForClusterMetadata(Map epstates) diff --git a/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java b/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java index d8b9dd746e..db1c250d01 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java @@ -82,9 +82,9 @@ public class LegacyStateListener implements ChangeListener } } - // next.myNodeId() can be null during replay (before we have registered) but if it is present and + // next.myNodeId() can be UNREGISTERED during replay (before we have registered) but if not and // there is a relevant change to the state of the local node, process that synchronously. - if (next.myNodeId() != null && changed.contains(next.myNodeId())) + if (next.myNodeId() != NodeId.UNREGISTERED && changed.contains(next.myNodeId())) { // Default is to process updates for the local node synchronously, overridable via config/hotprop if (DatabaseDescriptor.getLegacyStateListenerSyncLocalUpdates()) diff --git a/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java b/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java index 605b526378..8fd4bf87c8 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java @@ -18,23 +18,41 @@ package org.apache.cassandra.tcm.listeners; +import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; public class PlacementsChangeListener implements ChangeListener { + private final Runnable onChange; + + @VisibleForTesting + public PlacementsChangeListener(Runnable onChange) + { + this.onChange = onChange; + } + + public PlacementsChangeListener() + { + this(StorageService.instance::invalidateLocalRanges); + } + @Override public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot) { if (shouldInvalidate(prev, next)) - StorageService.instance.invalidateLocalRanges(); + onChange.run(); } private boolean shouldInvalidate(ClusterMetadata prev, ClusterMetadata next) { - if (!prev.placements.lastModified().equals(next.placements.lastModified()) && - !prev.placements.equivalentTo(next.placements)) // <- todo should we update lastModified if the result is the same? + if (!prev.placements().lastModified().equals(next.placements().lastModified()) && + !prev.placements().equivalentTo(next.placements())) // <- todo should we update lastModified if the result is the same? + return true; + + if (!prev.cmsMembership.equals(next.cmsMembership)) return true; if (prev.schema.getKeyspaces().size() != next.schema.getKeyspaces().size()) diff --git a/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java b/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java index feb191d32a..0be6243e53 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java @@ -47,17 +47,23 @@ public class UpgradeMigrationListener implements ChangeListener logger.info("Detected upgrade from gossip mode"); return; } - else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty()) // directory is non-empty after initialization during gossip upgrade + else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty()) { NodeId localId = next.myNodeId(); - if (localId != null) + if (localId != NodeId.UNREGISTERED) { - logger.info("Initialized CMS, updating local host id to {}", next.myNodeId()); - SystemKeyspace.setLocalHostId(next.myNodeId().toUUID()); - Gossiper.instance.mergeNodeToGossip(next.myNodeId(), next); + // assigning the local node id is done in Epoch.FIRST in one of two scenarios: + // * during an upgrade from gossip, as all pre-existing nodes will have an id assigned + // * during CMS initialization for a brand new cluster iff the local node is the first CMS member + logger.info("Initialized CMS, updating local host id to {}", localId); + SystemKeyspace.setLocalHostId(localId.toUUID()); + if (Gossiper.instance.isEnabled()) + { + Gossiper.instance.mergeNodeToGossip(localId, next); + if (Gossiper.instance.getQuarantineDisabled()) + Gossiper.instance.clearQuarantinedEndpoints(); + } } - if (Gossiper.instance.getQuarantineDisabled()) - Gossiper.instance.clearQuarantinedEndpoints(); } CassandraVersion prevMinVersion = prev.directory.clusterMinVersion.cassandraVersion; diff --git a/src/java/org/apache/cassandra/tcm/log/LogState.java b/src/java/org/apache/cassandra/tcm/log/LogState.java index 62e20ceddf..e18854f161 100644 --- a/src/java/org/apache/cassandra/tcm/log/LogState.java +++ b/src/java/org/apache/cassandra/tcm/log/LogState.java @@ -306,7 +306,7 @@ public class LogState if (metadata != null) { NodeId mynodeId = metadata.myNodeId(); - if (mynodeId != null) + if (mynodeId != NodeId.UNREGISTERED) SystemKeyspace.setLocalHostId(mynodeId.toUUID()); } } diff --git a/src/java/org/apache/cassandra/tcm/membership/Directory.java b/src/java/org/apache/cassandra/tcm/membership/Directory.java index f86481f141..8d444c6443 100644 --- a/src/java/org/apache/cassandra/tcm/membership/Directory.java +++ b/src/java/org/apache/cassandra/tcm/membership/Directory.java @@ -467,7 +467,7 @@ public class Directory implements MetadataValue public NodeState peerState(NodeId peer) { - return states.get(peer); + return peer == NodeId.UNREGISTERED ? null : states.get(peer); } public NodeVersion version(NodeId peer) diff --git a/src/java/org/apache/cassandra/tcm/membership/NodeId.java b/src/java/org/apache/cassandra/tcm/membership/NodeId.java index b73c4f00c7..1d5d3c9cd4 100644 --- a/src/java/org/apache/cassandra/tcm/membership/NodeId.java +++ b/src/java/org/apache/cassandra/tcm/membership/NodeId.java @@ -32,6 +32,7 @@ import org.apache.cassandra.tcm.serialization.Version; public class NodeId implements Comparable, MultiStepOperation.SequenceKey { + public static final NodeId UNREGISTERED = new NodeId(-1); private final static long NODE_ID_UUID_MAGIC = 7861390860069061072L; public static final Serializer serializer = new Serializer(); diff --git a/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java b/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java index de1bc71250..f3b78a1ee9 100644 --- a/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java +++ b/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java @@ -36,7 +36,7 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public class NodeVersion implements Comparable { public static final Serializer serializer = new Serializer(); - public static final Version CURRENT_METADATA_VERSION = Version.V8; + public static final Version CURRENT_METADATA_VERSION = Version.V9; public static final NodeVersion CURRENT = new NodeVersion(new CassandraVersion(FBUtilities.getReleaseVersionString()), CURRENT_METADATA_VERSION); private static final CassandraVersion SINCE_VERSION = CassandraVersion.CASSANDRA_5_1; diff --git a/src/java/org/apache/cassandra/tcm/migration/Election.java b/src/java/org/apache/cassandra/tcm/migration/Election.java index 906b647488..4e1cb5d4df 100644 --- a/src/java/org/apache/cassandra/tcm/migration/Election.java +++ b/src/java/org/apache/cassandra/tcm/migration/Election.java @@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; -import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; @@ -54,7 +53,6 @@ import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.ownership.TokenMap; -import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -157,9 +155,6 @@ public class Election initiator.compareAndSet(currentInitiator, MIGRATING)) { Startup.initializeAsFirstCMSNode(); - Register.maybeRegister(); - SystemKeyspace.setLocalHostId(ClusterMetadata.current().myNodeId().toUUID()); - updateInitiator(MIGRATING, MIGRATED); MessageDelivery.fanoutAndWait(messaging, sendTo, Verb.TCM_NOTIFY_REQ, DistributedMetadataLogKeyspace.getLogState(Epoch.EMPTY, false)); } diff --git a/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java b/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java index fd1a2618db..6c0ff2012c 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java +++ b/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java @@ -106,6 +106,11 @@ public class DataPlacement return EMPTY; } + public boolean isEmpty() + { + return reads.isEmpty() && writes.isEmpty(); + } + public static Builder builder() { return new Builder(ReplicaGroups.builder(), diff --git a/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java b/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java index d813e8c287..6bc836c770 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java +++ b/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java @@ -46,6 +46,7 @@ import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.db.TypeSizes.sizeof; +import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public class DataPlacements extends ReplicationMap implements MetadataValue { @@ -62,21 +63,6 @@ public class DataPlacements extends ReplicationMap implements Met this.lastModified = lastModified; } - public DataPlacements replaceParams(Epoch lastModified, ReplicationParams oldParams, ReplicationParams newParams) - { - Map newMap = Maps.newHashMapWithExpectedSize(map.size()); - assert map.containsKey(oldParams) : String.format("Can't replace key %s, since map doesn't contain it: %s", oldParams, map); - for (Map.Entry e : map.entrySet()) - { - if (e.getKey().equals(oldParams)) - newMap.put(newParams, e.getValue()); - else - newMap.put(e.getKey(), e.getValue()); - } - - return new DataPlacements(lastModified, newMap); - } - protected DataPlacement defaultValue() { return DataPlacement.empty(); @@ -256,7 +242,11 @@ public class DataPlacements extends ReplicationMap implements Met public void serialize(DataPlacements t, DataOutputPlus out, Version version) throws IOException { Map map = t.asMap(); - out.writeInt(map.size()); + if (version.isBefore(Version.V9)) + out.writeInt(map.size()); + else + out.writeUnsignedVInt32(map.size()); + for (Map.Entry entry : map.entrySet()) { ReplicationParams.serializer.serialize(entry.getKey(), out, version); @@ -267,7 +257,7 @@ public class DataPlacements extends ReplicationMap implements Met public DataPlacements deserialize(DataInputPlus in, Version version) throws IOException { - int size = in.readInt(); + int size = version.isBefore(Version.V9) ? in.readInt() : in.readUnsignedVInt32(); Map map = Maps.newHashMapWithExpectedSize(size); for (int i = 0; i < size; i++) { @@ -280,8 +270,9 @@ public class DataPlacements extends ReplicationMap implements Met public long serializedSize(DataPlacements t, Version version) { - long size = sizeof(t.size()); - for (Map.Entry entry : t.asMap().entrySet()) + Map map = t.asMap(); + long size = version.isBefore(Version.V9) ? sizeof(map.size()) : sizeofUnsignedVInt(map.size()); + for (Map.Entry entry : map.entrySet()) { size += ReplicationParams.serializer.serializedSize(entry.getKey(), version); size += DataPlacement.serializerFor(entry.getKey()).serializedSize(entry.getValue(), version); diff --git a/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java b/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java index 3d3b4da374..6fa832f8d4 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java +++ b/src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java @@ -329,9 +329,13 @@ public class UniformRangePlacement implements PlacementProvider logger.trace("Calculating data placements for {}", ksMetadata.name); AbstractReplicationStrategy replication = ksMetadata.replicationStrategy; ReplicationParams params = ksMetadata.params.replication; - if (params.isMeta() || params.isLocal()) + if (params.isMeta()) { - placements.put(params, metadata.placements.get(params)); + // don't calculate meta strategy placements, these are derived from ClusterMetadata.cmsMembership + } + else if (params.isLocal()) + { + placements.put(params, metadata.placement(params)); } else { diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java index 43891d5abc..07147fac17 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java @@ -326,7 +326,7 @@ public class BootstrapAndJoin extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { // need to undo MID_JOIN and START_JOIN, then merge the ranges split by PrepareJoin @@ -357,7 +357,7 @@ public class BootstrapAndJoin extends MultiStepOperation @VisibleForTesting public Pair getMovementMaps(ClusterMetadata metadata) { - MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements, startJoin.delta()); + MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements(), startJoin.delta()); MovementMap strictMovementMap = toStrict(movementMap, finishJoin.delta()); return Pair.create(movementMap, strictMovementMap); } diff --git a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java index 552818a774..bd6d3672f9 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java +++ b/src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java @@ -318,7 +318,7 @@ public class BootstrapAndReplace extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { // need to undo MID_REPLACE and START_REPLACE, but PREPARE_REPLACE doesn't affect placements @@ -355,7 +355,7 @@ public class BootstrapAndReplace extends MultiStepOperation private static MovementMap movementMap(InetAddressAndPort beingReplaced, PlacementDeltas startDelta) { MovementMap.Builder movementMapBuilder = MovementMap.builder(); - DataPlacements placements = ClusterMetadata.current().placements; + DataPlacements placements = ClusterMetadata.current().placements(); startDelta.forEach((params, delta) -> { EndpointsByReplica.Builder movements = new EndpointsByReplica.Builder(); DataPlacement originalPlacements = placements.get(params); @@ -451,6 +451,8 @@ public class BootstrapAndReplace extends MultiStepOperation public static void gossipStateToHibernate(ClusterMetadata metadata, NodeId nodeId) { + if (nodeId == NodeId.UNREGISTERED) + return; // order is important here, the gossiper can fire in between adding these two states. It's ok to send TOKENS without STATUS, but *not* vice versa. List> states = new ArrayList<>(); VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory; @@ -462,6 +464,8 @@ public class BootstrapAndReplace extends MultiStepOperation public static void gossipStateToNormal(ClusterMetadata metadata, NodeId nodeId) { + if (nodeId == NodeId.UNREGISTERED) + return; List> states = new ArrayList<>(); VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory; Collection tokens = metadata.tokenMap.tokens(nodeId); diff --git a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java index 4139110236..3dbd9311bd 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java @@ -25,25 +25,20 @@ import java.util.Map; import org.apache.cassandra.exceptions.ExceptionCode; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.accord.topology.FastPathStrategy; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.Directory; -import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; - public class CancelCMSReconfiguration implements Transformation { public static final Serializer serializer = new Serializer(); @@ -66,53 +61,46 @@ public class CancelCMSReconfiguration implements Transformation if (reconfigureCMS == null) return new Rejected(ExceptionCode.INVALID, "Can not cancel reconfiguration since there does not seem to be any in-flight"); - ReplicationParams metaParams = ReplicationParams.meta(prev); ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement placement = prev.placements.get(metaParams); // Reset any partially completed transition by removing the pending replica from the write group if (reconfigureCMS.next.activeTransition != null) { - InetAddressAndPort pendingEndpoint = prev.directory.endpoint(reconfigureCMS.next.activeTransition.nodeId); - Replica pendingReplica = new Replica(pendingEndpoint, entireRange, true); - placement = placement.unbuild() - .withoutWriteReplica(prev.nextEpoch(), pendingReplica) - .build(); + // see what the placements for the meta keyspace will be after cancelling the active join + CMSMembership cms = prev.cmsMembership.cancelJoining(reconfigureCMS.next.activeTransition.nodeId); + if (!cms.joiningMembers().isEmpty()) + return new Rejected(ExceptionCode.INVALID, + String.format("Placements will be inconsistent if this transformation is applied:" + + "\nFull members %s\nJoining members: %s", + cms.fullMembers(), + cms.joiningMembers())); + + // if all is good, actually cancel the joining member + transformer = transformer.cancelJoiningCMS(reconfigureCMS.next.activeTransition.nodeId); + // Recalculate the replication params for the meta keyspace based on the actual placement as it will be + ReplicationParams recalculated = getAccurateReplication(prev.directory, cms); + + // If they no longer match the replication params in schema, i.e. the transitions completed so far did not + // bring the membership/placements into line with configuration, update schema to match what we actually have + if (!recalculated.equals(ReplicationParams.meta(prev))) + { + KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); + KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, + recalculated, + FastPathStrategy.simple())); + transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))); + } } - if (!placement.reads.equivalentTo(placement.writes)) - return new Rejected(ExceptionCode.INVALID, String.format("Placements will be inconsistent if this transformation is applied:\nReads %s\nWrites: %s", - placement.reads, - placement.writes)); - - // Reset the replication params for the meta keyspace based on the actual placement in case they no longer match - ReplicationParams fromPlacement = getAccurateReplication(prev.directory, placement); - - // If they no longer match, i.e. the transitions completed so far did not bring the placements into line with - // the configuration, remove the entry keyed by the existing configured params. - DataPlacements.Builder builder = prev.placements.unbuild(); - if (!metaParams.equals(fromPlacement)) - { - builder = builder.without(metaParams); - - // Also update schema with the corrected params - KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); - KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple())); - transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))); - } - - // finally, add the possibly corrected placement keyed by the possibly corrected params - builder = builder.with(fromPlacement, placement); - transformer = transformer.with(builder.build()); return Transformation.success(transformer.with(prev.inProgressSequences.without(ReconfigureCMS.SequenceKey.instance)) .with(prev.lockedRanges.unlock(reconfigureCMS.next.lockKey)), MetaStrategy.affectedRanges(prev)); } - private ReplicationParams getAccurateReplication(Directory directory, DataPlacement placement) + private ReplicationParams getAccurateReplication(Directory directory, CMSMembership membership) { Map replicasPerDc = new HashMap<>(); - placement.writes.byEndpoint().keySet().forEach(i -> { - String dc = directory.location(directory.peerId(i)).datacenter; + membership.fullMembers().forEach(id -> { + String dc = directory.location(id).datacenter; int count = replicasPerDc.getOrDefault(dc, 0); replicasPerDc.put(dc, ++count); }); diff --git a/src/java/org/apache/cassandra/tcm/sequences/Move.java b/src/java/org/apache/cassandra/tcm/sequences/Move.java index cf81e5a3ce..a9f68812f2 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/Move.java +++ b/src/java/org/apache/cassandra/tcm/sequences/Move.java @@ -321,7 +321,7 @@ public class Move extends MultiStepOperation StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION); Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces(); Map movementMap = movementMap(FailureDetector.instance, - metadata.placements, + metadata.placements(), toSplitRanges, startMove.delta(), midMove.delta(), @@ -430,7 +430,7 @@ public class Move extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { diff --git a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java index db728ef67e..09950edfe8 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java @@ -58,6 +58,7 @@ import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Retry; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.concurrent.AsyncPromise; @@ -170,8 +171,9 @@ public class ProgressBarrier Set> ranges = e.getValue(); for (Range range : ranges) { - EndpointsForRange writes = metadata.placements.get(params).writes.matchRange(range).get().filter(r -> filter.test(r.endpoint())); - EndpointsForRange reads = metadata.placements.get(params).reads.matchRange(range).get().filter(r -> filter.test(r.endpoint())); + DataPlacement placement = metadata.placement(params); + EndpointsForRange writes = placement.writes.matchRange(range).get().filter(r -> filter.test(r.endpoint())); + EndpointsForRange reads = placement.reads.matchRange(range).get().filter(r -> filter.test(r.endpoint())); // Affected ranges can contain ranges which are the results of merging or splitting and may not exist // as keys in the existing ReplicaGroups. As such, no replicas will be found for these ranges and so no // WaitFor is necessary. diff --git a/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java b/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java index d6e8ca3127..bd09f8aaca 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java +++ b/src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java @@ -121,7 +121,7 @@ public class RemoveNodeStreams implements LeaveStreams RangesByEndpoint startWriteAdditions = startDelta.get(params).writes.additions; RangesByEndpoint startWriteRemovals = startDelta.get(params).writes.removals; // find current placements from the metadata, we need to stream from replicas that are not changed and are therefore not in the deltas - ReplicaGroups currentPlacements = metadata.placements.get(params).reads; + ReplicaGroups currentPlacements = metadata.placement(params).reads; startWriteAdditions.flattenValues() .forEach(newReplica -> { EndpointsForRange.Builder candidateBuilder = new EndpointsForRange.Builder(newReplica.range()); diff --git a/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java b/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java index 4580d716a1..ab819c5f96 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java @@ -44,7 +44,7 @@ public class ReplaceSameAddress { MovementMap.Builder builder = MovementMap.builder(); InetAddressAndPort addr = metadata.directory.endpoint(nodeId); - metadata.placements.forEach((params, placement) -> { + metadata.placements().forEach((params, placement) -> { EndpointsByReplica.Builder sources = new EndpointsByReplica.Builder(); placement.reads.byEndpoint().get(addr).forEach(destination -> { placement.reads.forRange(destination.range()).forEach(potentialSource -> { diff --git a/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java b/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java index 876caa3d96..011d35f866 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java +++ b/src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java @@ -253,7 +253,7 @@ public class UnbootstrapAndLeave extends MultiStepOperation @Override public ClusterMetadata.Transformer cancel(ClusterMetadata metadata) { - DataPlacements placements = metadata.placements; + DataPlacements placements = metadata.placements(); switch (next) { // need to undo MID_LEAVE and START_LEAVE, but PrepareLeave doesn't affect placement diff --git a/src/java/org/apache/cassandra/tcm/serialization/Version.java b/src/java/org/apache/cassandra/tcm/serialization/Version.java index 9cbcb8587a..fb6e59cfd6 100644 --- a/src/java/org/apache/cassandra/tcm/serialization/Version.java +++ b/src/java/org/apache/cassandra/tcm/serialization/Version.java @@ -38,13 +38,13 @@ public enum Version */ V0(0), /** - * - Moved Partitioner in ClusterMetadata serializer to be the first field - * - Added a counter to Directory serializer to keep track of NodeIds + * - Moved Partitioner in ClusterMetadata serializer to be the first field + * - Added a counter to Directory serializer to keep track of NodeIds */ V1(1), /** - * - Added version to PlacementForRange serializer - * - Serialize MemtableParams when serializing TableParams + * - Added version to PlacementForRange serializer + * - Serialize MemtableParams when serializing TableParams */ V2(2), /** @@ -78,6 +78,11 @@ public enum Version * - Comments and security labels for schema elements (keyspaces, tables, columns, UDTs, and UDT fields) */ V8(8), + /** + * - DataPlacements don't include MetaStrategy, replaced by ClusterMetadata.CMSMembership + * - Size of DataPlacements is encoded as vint + */ + V9(9), UNKNOWN(Integer.MAX_VALUE); @@ -95,6 +100,7 @@ public enum Version } private final int version; + Version(int version) { this.version = version; diff --git a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java index 1e3aef5817..491a3a2118 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java @@ -78,7 +78,7 @@ public class AccordMarkStale implements Transformation for (KeyspaceMetadata keyspace : prev.schema.getKeyspaces().without(SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES)) { - List shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements, prev.directory); + List shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements(), prev.directory); for (AccordTopology.KeyspaceShard shard : shards) { diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java index 087ebfcc61..807fed2b17 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java @@ -244,7 +244,7 @@ public class AlterSchema implements Transformation DataPlacements.Builder newPlacementsBuilder = DataPlacements.builder(calculatedPlacements.size()); calculatedPlacements.forEach((params, newPlacement) -> { - DataPlacement previousPlacement = prev.placements.get(params); + DataPlacement previousPlacement = prev.placement(params); // Preserve placement versioning that has resulted from natural application where possible if (previousPlacement.equivalentTo(newPlacement)) newPlacementsBuilder.with(params, previousPlacement); diff --git a/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java b/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java index e247f66a88..4eb3d7b364 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java +++ b/src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java @@ -130,11 +130,11 @@ public class AlterTopology implements Transformation for (Map.Entry update : updates.entrySet()) updated = updated.withUpdatedRackAndDc(update.getKey(), update.getValue()); ClusterMetadata proposed = prev.transformer().with(updated).build().metadata; - DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements.lastModified(), + DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements().lastModified(), proposed.tokenMap.toRanges(), proposed, proposed.schema.getKeyspaces()); - if (!proposedPlacements.equivalentTo(prev.placements)) + if (!proposedPlacements.equivalentTo(prev.placements())) { logger.info("Rejecting topology modifications which would materially change data placements: {}", updates); return new Rejected(INVALID, "Proposed updates modify data placements, violating consistency guarantees"); diff --git a/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java b/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java index 10f48609d4..384e04dc83 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java +++ b/src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java @@ -82,7 +82,7 @@ public abstract class ApplyPlacementDeltas implements Transformation ClusterMetadata.Transformer next = prev.transformer(); if (!delta.isEmpty()) - next = next.with(delta.apply(prev.nextEpoch(), prev.placements)); + next = next.with(delta.apply(prev.nextEpoch(), prev.placements())); next = transform(prev, next); diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java index d8986e6829..08f9f10233 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java @@ -89,7 +89,7 @@ import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; */ public class PrepareJoin implements Transformation { - public static final Serializer serializer = new Serializer() + public static final Serializer serializer = new Serializer<>() { public PrepareJoin construct(NodeId nodeId, Set tokens, PlacementProvider placementProvider, boolean joinTokenRing, boolean streamData) { @@ -168,10 +168,10 @@ public class PrepareJoin implements Transformation startJoin, midJoin, finishJoin, joinTokenRing, streamData); if (!prev.tokenMap.isEmpty()) - assertPreExistingWriteReplica(prev.placements, transitionPlan); + assertPreExistingWriteReplica(prev.placements(), transitionPlan); LockedRanges newLockedRanges = prev.lockedRanges.lock(lockKey, rangesToLock); - DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements); + DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements()); ClusterMetadata.Transformer proposed = prev.transformer() .with(newLockedRanges) .with(startingPlacements) diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java index 0ad37f8bcd..74d2c2c810 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java @@ -115,7 +115,7 @@ public class PrepareLeave implements Transformation PlacementDeltas startDelta = transitionPlan.addToWrites(); PlacementDeltas midDelta = transitionPlan.moveReads(); PlacementDeltas finishDelta = transitionPlan.removeFromWrites(); - transitionPlan.assertPreExistingWriteReplica(prev.placements); + transitionPlan.assertPreExistingWriteReplica(prev.placements()); LockedRanges.Key unlockKey = LockedRanges.keyFor(proposed.epoch); diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java index e7e278d0d2..16dcdfca08 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java @@ -109,7 +109,7 @@ public class PrepareMove implements Transformation StartMove startMove = new StartMove(nodeId, transitionPlan.addToWrites(), lockKey); MidMove midMove = new MidMove(nodeId, transitionPlan.moveReads(), lockKey); FinishMove finishMove = new FinishMove(nodeId, tokens, transitionPlan.removeFromWrites(), lockKey); - transitionPlan.assertPreExistingWriteReplica(prev.placements); + transitionPlan.assertPreExistingWriteReplica(prev.placements()); Move sequence = Move.newSequence(prev.nextEpoch(), lockKey, @@ -123,7 +123,7 @@ public class PrepareMove implements Transformation return Transformation.success(prev.transformer() .withNodeState(nodeId, NodeState.MOVING) .with(prev.lockedRanges.lock(lockKey, rangesToLock)) - .with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements)) + .with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements())) .with(prev.inProgressSequences.with(nodeId, sequence)), rangesToLock); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java b/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java index 8b1577822b..94bf3508c0 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java +++ b/src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java @@ -109,7 +109,7 @@ public class PrepareReplace implements Transformation StartReplace start = new StartReplace(replaced, replacement, transitionPlan.addToWrites(), unlockKey); MidReplace mid = new MidReplace(replaced, replacement, transitionPlan.moveReads(), unlockKey); FinishReplace finish = new FinishReplace(replaced, replacement, transitionPlan.removeFromWrites(), unlockKey); - transitionPlan.assertPreExistingWriteReplica(prev.placements); + transitionPlan.assertPreExistingWriteReplica(prev.placements()); Set tokens = new HashSet<>(prev.tokenMap.tokens(replaced)); BootstrapAndReplace plan = BootstrapAndReplace.newSequence(prev.nextEpoch(), diff --git a/src/java/org/apache/cassandra/tcm/transformations/Register.java b/src/java/org/apache/cassandra/tcm/transformations/Register.java index a7551c7069..b2fce47011 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/Register.java +++ b/src/java/org/apache/cassandra/tcm/transformations/Register.java @@ -109,7 +109,7 @@ public class Register implements Transformation if (isReplacingSameAddress()) { NodeId self = ClusterMetadata.current().myNodeId(); - if (self == null) + if (self == NodeId.UNREGISTERED) throw new IllegalStateException("Tried to replace same address, but node does not seem to be registered"); return self; diff --git a/src/java/org/apache/cassandra/tcm/transformations/Startup.java b/src/java/org/apache/cassandra/tcm/transformations/Startup.java index 7b8c4cff7e..00b4023525 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/Startup.java +++ b/src/java/org/apache/cassandra/tcm/transformations/Startup.java @@ -24,10 +24,6 @@ import java.util.Objects; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.Keyspaces; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Transformation; @@ -35,14 +31,12 @@ import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; public class Startup implements Transformation { @@ -96,32 +90,13 @@ public class Startup implements Transformation if (!nodeId.equals(existingNodeId) && addresses.conflictsWith(existingAddresses)) return new Rejected(INVALID, String.format("New addresses %s conflicts with existing node %s with addresses %s", addresses, entry.getKey(), existingAddresses)); } - next = next.withNewAddresses(nodeId, addresses); - Keyspaces allKeyspaces = prev.schema.getKeyspaces().withAddedOrReplaced(prev.schema.getKeyspaces()); - DataPlacements newPlacement = ClusterMetadataService.instance() .placementProvider() .calculatePlacements(prev.nextEpoch(), prev.tokenMap.toRanges(), next.build().metadata, - allKeyspaces); - - if (prev.isCMSMember(prev.directory.endpoint(nodeId))) - { - ReplicationParams metaParams = ReplicationParams.meta(prev); - InetAddressAndPort endpoint = prev.directory.endpoint(nodeId); - Replica leavingReplica = new Replica(endpoint, entireRange, true); - Replica joiningReplica = new Replica(addresses.broadcastAddress, entireRange, true); - - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild(); - builder.reads.withoutReplica(prev.nextEpoch(), leavingReplica); - builder.writes.withoutReplica(prev.nextEpoch(), leavingReplica); - builder.reads.withReplica(prev.nextEpoch(), joiningReplica); - builder.writes.withReplica(prev.nextEpoch(), joiningReplica); - newPlacement = newPlacement.unbuild().with(metaParams, builder.build()).build(); - } - + prev.schema.getKeyspaces()); next = next.with(newPlacement); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java index 614dd8a4ed..dd4b49127b 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java @@ -29,15 +29,12 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.RangesByEndpoint; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; @@ -45,7 +42,6 @@ import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; import static org.apache.cassandra.tcm.MultiStepOperation.Kind.RECONFIGURE_CMS; /** @@ -154,38 +150,26 @@ public class AdvanceCMSReconfiguration implements Transformation { // Pop the next node to be added from the list diff.additions NodeId addition = diff.additions.get(0); - InetAddressAndPort endpoint = prev.directory.endpoint(addition); - Replica replica = new Replica(endpoint, entireRange, true); - List newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size())); // Check that the candidate is not already a CMS member - ReplicationParams metaParams = ReplicationParams.meta(prev); - RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint(); - RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint(); - if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint)) - return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS"); + CMSMembership cms = prev.cmsMembership; + if (cms.joiningMembers().contains(addition) || cms.fullMembers().contains(addition)) + return new Transformation.Rejected(INVALID, "Endpoint is already a full or joining member of the CMS: " + prev.directory.endpoint(addition)); - - ClusterMetadata.Transformer transformer = prev.transformer(); - // Add the candidate as a write replica - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild() - .withWriteReplica(prev.nextEpoch(), replica); - transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()); + // Add the candidate as a joining member + ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(addition); // Construct a set of sources for the new member to stream log tables from (essentially this is the existing members) - Set streamCandidates = new HashSet<>(); - for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues()) - { - if (!replica.equals(r)) - streamCandidates.add(r.endpoint()); - } + Set streamCandidates = prev.fullCMSMembers(); // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence, // including the remaining add/remove operations and the streaming that needs to be done by the joining node + List newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size())); AdvanceCMSReconfiguration next = next(prev.nextEpoch(), newAdditions, diff.removals, new ReconfigureCMS.ActiveTransition(addition, streamCandidates)); + // Create a new sequence instance with the next step to reflect that the state has progressed. ReconfigureCMS advanced = sequence.advance(next); // Finally, replace the existing reconfiguration sequence with this updated one. @@ -206,21 +190,21 @@ public class AdvanceCMSReconfiguration implements Transformation */ private Transformation.Result finishAdd(ClusterMetadata prev, ReconfigureCMS sequence, NodeId addition) { + // Check that the candidate is already a joining CMS member + CMSMembership cms = prev.cmsMembership; + if (!cms.joiningMembers().contains(addition)) + return new Transformation.Rejected(INVALID, "Endpoint is not a in the process of joining the CMS: " + prev.directory.endpoint(addition)); + // Add the new member as a full read replica, able to participate in quorums for log updates - ReplicationParams metaParams = ReplicationParams.meta(prev); - InetAddressAndPort endpoint = prev.directory.endpoint(addition); - Replica replica = new Replica(endpoint, entireRange, true); - ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement.Builder builder = prev.placements.get(metaParams) - .unbuild() - .withReadReplica(prev.nextEpoch(), replica); - transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()); + ClusterMetadata.Transformer transformer = prev.transformer().finishJoiningCMS(addition); // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence, // which includes the remaining add/remove operations AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, diff.removals, null); + // Create a new sequence instance with the next step to reflect that the state has progressed. ReconfigureCMS advanced = sequence.advance(next); + // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); @@ -238,29 +222,25 @@ public class AdvanceCMSReconfiguration implements Transformation List newRemovals = new ArrayList<>(diff.removals.subList(1, diff.removals.size())); // Check that the candidate is actually a CMS member - ClusterMetadata.Transformer transformer = prev.transformer(); + Set cms = prev.fullCMSMemberIds(); InetAddressAndPort endpoint = prev.directory.endpoint(removal); - Replica replica = new Replica(endpoint, entireRange, true); - ReplicationParams metaParams = ReplicationParams.meta(prev); - if (!prev.fullCMSMembers().contains(endpoint)) + if (!cms.contains(removal)) return new Transformation.Rejected(INVALID, String.format("%s is not currently a CMS member, cannot remove it", endpoint)); // Check that the candidate is not the only CMS member - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild(); - builder.reads.withoutReplica(prev.nextEpoch(), replica); - builder.writes.withoutReplica(prev.nextEpoch(), replica); - DataPlacement proposed = builder.build(); - if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty()) + if (cms.size() == 1) return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint)); - // Actually remove the candidate - transformer = transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()); + // Remove the CMS member + ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(removal); // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence, // which includes the remaining add/remove operations AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, newRemovals, null); + // Create a new sequence instance with the next step to reflect that the state has progressed. ReconfigureCMS advanced = sequence.advance(next); + // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java index a1bed8ed2a..9cef32aef5 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java @@ -26,7 +26,6 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.AddToCMS; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; @@ -84,12 +83,9 @@ public class FinishAddToCMS extends BaseMembershipTransformation InetAddressAndPort endpoint = prev.directory.endpoint(targetNode); Replica replica = new Replica(endpoint, entireRange, true); - ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement.Builder builder = prev.placements.get(metaParams) - .unbuild() - .withReadReplica(prev.nextEpoch(), replica); - transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()) - .with(prev.inProgressSequences.without(targetNode)); + ClusterMetadata.Transformer transformer = prev.transformer() + .finishJoiningCMS(targetNode) + .with(prev.inProgressSequences.without(targetNode)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java index 7286743f91..2725fffebf 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java @@ -20,6 +20,8 @@ package org.apache.cassandra.tcm.transformations.cms; import java.io.IOException; +import com.google.common.collect.ImmutableSet; + import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; @@ -29,12 +31,10 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.Keyspaces; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -72,7 +72,6 @@ public class PreInitialize implements Transformation assert metadata.epoch.isBefore(Epoch.FIRST); ClusterMetadata.Transformer transformer = metadata.transformer(); - // This null check is a leftover from previous implementations. In earlier versions the address and datacenter // were not be included in the serialized form of this transform and so were not written to the local or // distributed logs nor included in the log entries sent over the wire between peers. @@ -84,30 +83,33 @@ public class PreInitialize implements Transformation // PRE_INITIALIZE_CMS becomes irrelevant. if (addr != null) { - DataPlacement.Builder dataPlacementBuilder = DataPlacement.builder(); - Replica replica = new Replica(addr, - MetaStrategy.partitioner.getMinimumToken(), - MetaStrategy.partitioner.getMinimumToken(), - true); - dataPlacementBuilder.reads.withReplica(Epoch.FIRST, replica); - dataPlacementBuilder.writes.withReplica(Epoch.FIRST, replica); - DataPlacements initialPlacement = metadata.placements.unbuild() - .with(ReplicationParams.simpleMeta(1, datacenter), - dataPlacementBuilder.build()).build(); - - transformer.with(initialPlacement); // create the distributed metadata keyspace in schema with replication settings based on the DC of the // initial CMS node Keyspaces updated = metadata.schema.getKeyspaces() .withAddedOrReplaced(DistributedMetadataLogKeyspace.initialMetadata(datacenter)); transformer.with(new DistributedSchema(updated, Epoch.FIRST)); + ClusterMetadata.Transformer.Transformed transformed = transformer.build(); + + // This is required to bootstrap the initialization process. Because committing to the metadata log uses the + // previous ClusterMetadata to identify CMS members who form the consensus group, the first CMS member must + // be routable at that point. After the INITIALIZE_CMS is committed, and on instances other than the first + // CMS member, this placement is derived from the CMS membership list of node ids. + DataPlacement.Builder initialPlacement = DataPlacement.builder(); + Replica replica = new Replica(addr, + MetaStrategy.partitioner.getMinimumToken(), + MetaStrategy.partitioner.getMinimumToken(), + true); + initialPlacement.reads.withReplica(Epoch.FIRST, replica); + initialPlacement.writes.withReplica(Epoch.FIRST, replica); + metadata = transformed.metadata.forcePreInitializedState(initialPlacement.build()); + } + else + { + metadata = metadata.forceEpoch(Epoch.FIRST); } - ClusterMetadata.Transformer.Transformed transformed = transformer.build(); - metadata = transformed.metadata.forceEpoch(Epoch.FIRST); assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch; - - return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, transformed.modifiedKeys); + return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, ImmutableSet.of()); } @Override diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java index cfb7db66dd..5eec85e833 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +82,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation logger.info("Proposed CMS reconfiguration resulted in {}", diff); LockedRanges.Key lockKey = LockedRanges.keyFor(prev.nextEpoch()); - Set cms = prev.fullCMSMembers().stream().map(prev.directory::peerId).collect(Collectors.toSet()); + Set cms = prev.fullCMSMemberIds(); Set tmp = new HashSet<>(cms); tmp.addAll(diff.additions); tmp.removeAll(diff.removals); @@ -267,9 +266,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple())); - return executeInternal(prev, - transformer -> transformer.with(prev.placements.replaceParams(prev.nextEpoch(), ReplicationParams.meta(prev), replicationParams)) - .with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)))); + return executeInternal(prev, transformer -> transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)))); } public String toString() @@ -312,7 +309,6 @@ public abstract class PrepareCMSReconfiguration implements Transformation public static Diff diff(Set currentCms, Set newCms) { - assert !currentCms.contains(null) : "Current CMS contains a null value " + currentCms; assert !newCms.contains(null) : "New CMS contains a null value " + newCms; List additions = new ArrayList<>(); @@ -355,10 +351,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation public static boolean needsReconfiguration(ClusterMetadata metadata) { Map dcRf = extractRf(ReplicationParams.meta(metadata)); - Set currentCms = metadata.fullCMSMembers() - .stream() - .map(metadata.directory::peerId) - .collect(Collectors.toSet()); + Set currentCms = metadata.fullCMSMemberIds(); int expectedSize = dcRf.values().stream().mapToInt(Integer::intValue).sum(); if (currentCms.size() != expectedSize) return true; diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java index 8e37875e94..d958309ed8 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java @@ -29,20 +29,16 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, StartAddToCMS & FinishAddToCMS, contain a high degree of duplication with their intended @@ -87,11 +83,7 @@ public class RemoveFromCMS extends BaseMembershipTransformation if (sequence != null) return new Transformation.Rejected(INVALID, String.format("Can't remove %s from CMS as there are ongoing range movements on it", endpoint)); - ReplicationParams metaParams = ReplicationParams.meta(prev); - DataPlacement placements = prev.placements.get(metaParams); - - int minProposedSize = (int) Math.min(placements.reads.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count(), - placements.writes.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count()); + int minProposedSize = prev.fullCMSMemberIds().size() - 1; if (minProposedSize < MIN_SAFE_CMS_SIZE) { logger.warn("Removing {} from CMS members would reduce the service size to {} which is below the " + @@ -109,19 +101,8 @@ public class RemoveFromCMS extends BaseMembershipTransformation if (minProposedSize == 0) return new Transformation.Rejected(INVALID, String.format("Removing %s from the CMS would leave no members in CMS.", endpoint)); - ClusterMetadata.Transformer transformer = prev.transformer(); - Replica replica = new Replica(endpoint, entireRange, true); - - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild(); - builder.reads.withoutReplica(prev.nextEpoch(), replica); - builder.writes.withoutReplica(prev.nextEpoch(), replica); - DataPlacement proposed = builder.build(); - - if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty()) - return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint)); - - return Transformation.success(transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()), - MetaStrategy.affectedRanges(prev)); + ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(nodeId); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } @Override diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java index 90f3056f47..efa92c042a 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java @@ -18,25 +18,20 @@ package org.apache.cassandra.tcm.transformations.cms; -import java.util.HashSet; import java.util.Set; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.RangesByEndpoint; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.sequences.AddToCMS; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, FinishAddToCMS & RemoveFromCMS, contain a high degree of duplication with their intended @@ -76,27 +71,13 @@ public class StartAddToCMS extends BaseMembershipTransformation if (prev.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance) != null) return new Rejected(INVALID, String.format("Cannot add node to CMS as a CMS reconfiguration is currently active")); - Replica replica = new Replica(endpoint, entireRange, true); - ReplicationParams metaParams = ReplicationParams.meta(prev); - RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint(); - RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint(); - - if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint)) + CMSMembership cms = prev.cmsMembership; + if (cms.joiningMembers().contains(nodeId) || cms.fullMembers().contains(nodeId)) return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS"); - ClusterMetadata.Transformer transformer = prev.transformer(); - DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild() - .withWriteReplica(prev.nextEpoch(), replica); - - transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()); - - Set streamCandidates = new HashSet<>(); - for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues()) - { - if (!replica.equals(r)) - streamCandidates.add(r.endpoint()); - } + ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(nodeId); + Set streamCandidates = prev.fullCMSMembers(); AddToCMS joinSequence = new AddToCMS(prev.nextEpoch(), nodeId, streamCandidates, new FinishAddToCMS(endpoint)); transformer = transformer.with(prev.inProgressSequences.with(nodeId, joinSequence)); return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); diff --git a/src/java/org/apache/cassandra/tools/CMSOfflineTool.java b/src/java/org/apache/cassandra/tools/CMSOfflineTool.java index 90d44e3fc0..d42c91955b 100644 --- a/src/java/org/apache/cassandra/tools/CMSOfflineTool.java +++ b/src/java/org/apache/cassandra/tools/CMSOfflineTool.java @@ -37,10 +37,9 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.MultiStepOperation; @@ -441,24 +440,15 @@ public class CMSOfflineTool implements Runnable nodeState + " state. Only a JOINED node can be set as CMS member."); } InetAddressAndPort endpoint = metadata.directory.getNodeAddresses(nodeId).broadcastAddress; - ReplicationParams metaParams = ReplicationParams.meta(metadata); - Iterable currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues(); - DataPlacement.Builder placementBuilder = metadata.placements.get(metaParams).unbuild(); - for (Replica replica : currentReplicas) - { - placementBuilder.withoutReadReplica(metadata.epoch, replica) - .withoutWriteReplica(metadata.epoch, replica); - } + CMSMembership cms = metadata.cmsMembership; + ClusterMetadata.Transformer transformer = metadata.transformer(); + for (NodeId id : cms.fullMembers()) + transformer = transformer.leaveCMS(id); + for (NodeId id : cms.joiningMembers()) + transformer = transformer.cancelJoiningCMS(id); - Replica newCMS = MetaStrategy.replica(endpoint); - placementBuilder.withReadReplica(metadata.epoch, newCMS) - .withWriteReplica(metadata.epoch, newCMS); - - return metadata.transformer() - .with(metadata.placements.unbuild() - .with(metaParams, placementBuilder.build()) - .build()) - .build().metadata; + transformer = transformer.startJoiningCMS(nodeId).finishJoiningCMS(nodeId); + return transformer.build().metadata; } } @@ -619,7 +609,7 @@ public class CMSOfflineTool implements Runnable output.out.printf("Cluster Metadata Service:%n"); output.out.printf("Members: %s%n", members); output.out.printf("Needs reconfiguration: %s%n", needsReconfiguration(metadata)); - output.out.printf("Service State: %s%n", ClusterMetadataService.state(metadata)); + output.out.printf("Service State: %s%n", ClusterMetadataService.State.OFFLINE_TOOL); output.out.printf("Epoch: %s%n", metadata.epoch.getEpoch()); output.out.printf("Replication factor: %s%n", ReplicationParams.meta(metadata).toString()); } @@ -782,7 +772,7 @@ public class CMSOfflineTool implements Runnable throw new IllegalArgumentException("Keyspace " + keyspace + " not found in cluster metadata."); } - DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication); + DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication); List rows = new ArrayList<>(); rows.addAll(replicaGroupsToRows(placement.reads, "read")); rows.addAll(replicaGroupsToRows(placement.writes, "write")); diff --git a/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java b/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java index 59102ff27d..3c9d5a1ef9 100644 --- a/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java +++ b/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java @@ -27,13 +27,11 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -60,9 +58,9 @@ public class TransformClusterMetadataHelper DatabaseDescriptor.setPartitionerUnsafe(partitioner); ClusterMetadataService.initializeForTools(false); ClusterMetadata metadata = ClusterMetadataService.deserializeClusterMetadata(sourceFile); - System.out.println("Old CMS: " + metadata.placements.get(ReplicationParams.meta(metadata))); + System.out.println("Old CMS: " + metadata.placement(ReplicationParams.meta(metadata))); metadata = makeCMS(metadata, InetAddressAndPort.getByNameUnchecked(args[1])); - System.out.println("New CMS: " + metadata.placements.get(ReplicationParams.meta(metadata))); + System.out.println("New CMS: " + metadata.placement(ReplicationParams.meta(metadata))); Path p = Files.createTempFile("clustermetadata", "dump"); try (FileOutputStreamPlus out = new FileOutputStreamPlus(p)) { @@ -73,20 +71,9 @@ public class TransformClusterMetadataHelper public static ClusterMetadata makeCMS(ClusterMetadata metadata, InetAddressAndPort endpoint) { - ReplicationParams metaParams = ReplicationParams.meta(metadata); - Iterable currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues(); - DataPlacement.Builder builder = metadata.placements.get(metaParams).unbuild(); - for (Replica replica : currentReplicas) - { - builder.withoutReadReplica(metadata.epoch, replica) - .withoutWriteReplica(metadata.epoch, replica); - } - Replica newCMS = MetaStrategy.replica(endpoint); - builder.withReadReplica(metadata.epoch, newCMS) - .withWriteReplica(metadata.epoch, newCMS); - return metadata.transformer().with(metadata.placements.unbuild().with(metaParams, - builder.build()) - .build()) - .build().metadata; + NodeId id = metadata.directory.peerId(endpoint); + if (id == null) + throw new IllegalStateException("No node id found for endpoint: " + endpoint); + return metadata.transformer().startJoiningCMS(id).finishJoiningCMS(id).build().metadata; } } diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index ba47bda366..c3b0e3ebd4 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -839,7 +839,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { CommitLog.instance.recoverSegmentsOnDisk(); NodeId self = ClusterMetadata.current().myNodeId(); - if (self != null) + if (self != NodeId.UNREGISTERED) AccordService.localStartup(self); } catch (IOException e) diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index 19986671ec..006379ae3b 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -444,8 +444,8 @@ public class ClusterUtils for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces()) { List[] placements = new List[2]; - placements[0] = metadata.placements.get(keyspace.params.replication).reads.toReplicaStringList(); - placements[1] = metadata.placements.get(keyspace.params.replication).writes.toReplicaStringList(); + placements[0] = metadata.placement(keyspace.params.replication).reads.toReplicaStringList(); + placements[1] = metadata.placement(keyspace.params.replication).writes.toReplicaStringList(); byKeyspace.put(keyspace.name, placements); } return byKeyspace; @@ -465,10 +465,10 @@ public class ClusterUtils StringBuilder builder = new StringBuilder(); builder.append("'keyspace' { 'name':").append(keyspace.name).append("', "); builder.append("'reads':['"); - ReplicaGroups placement = metadata.placements.get(keyspace.params.replication).reads; + ReplicaGroups placement = metadata.placement(keyspace.params.replication).reads; builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString()); builder.append("'], 'writes':['"); - placement = metadata.placements.get(keyspace.params.replication).writes; + placement = metadata.placement(keyspace.params.replication).writes; builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString()); builder.append("']}"); keyspaces.add(builder.toString()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java index e84b51ff80..f39c16ecd7 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java @@ -27,7 +27,6 @@ import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; @@ -46,8 +45,8 @@ public class AssassinateCMSNodeTest extends TestBaseImpl InetSocketAddress toAssassinate = cluster.get(2).broadcastAddress(); cluster.get(2).shutdown().get(); cluster.get(1).nodetoolResult("assassinate", toAssassinate.getHostString()).asserts().success(); - cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()))); - cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()))); + cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember())); + cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember())); cluster.get(1).nodetoolResult("cms").asserts().success(); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java index bcc02b89a6..a76d0d8d42 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java @@ -32,14 +32,11 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.MetaStrategy; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.CMSOperations; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; -import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import static org.apache.cassandra.distributed.shared.ClusterUtils.start; @@ -108,15 +105,10 @@ public class BootWithMetadataTest extends TestBaseImpl try { ClusterMetadata metadata = ClusterMetadata.current(); - Replica oldCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.1")); - Replica newCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.2")); + NodeId oldCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.1")); + NodeId newCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.2")); ClusterMetadata.Transformer transformer = metadata.transformer(); - DataPlacement.Builder builder = metadata.placements.get(ReplicationParams.meta(metadata)).unbuild() - .withoutReadReplica(metadata.nextEpoch(), oldCMS) - .withoutWriteReplica(metadata.nextEpoch(), oldCMS) - .withWriteReplica(metadata.nextEpoch(), newCMS) - .withReadReplica(metadata.nextEpoch(), newCMS); - transformer = transformer.with(metadata.placements.unbuild().with(ReplicationParams.meta(metadata), builder.build()).build()); + transformer.leaveCMS(oldCMS).startJoiningCMS(newCMS).finishJoiningCMS(newCMS); ClusterMetadata toDump = transformer.build().metadata.forceEpoch(Epoch.create(1000)); Path p = Files.createTempFile("clustermetadata", "dump"); try (FileOutputStreamPlus out = new FileOutputStreamPlus(p)) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java index 7731bde594..c0a4466e78 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java @@ -66,7 +66,8 @@ public class ClusterMetadataDumpTest extends TestBaseImpl epochsSeen++; } assertEquals(3, unsafeJoinSeen); - assertEquals(3, registerSeen); + // Only 2 REGISTER transforms are expected as the first CMS node is registered implicitly by INITIALIZE_CMS + assertEquals(2, registerSeen); assertTrue(epochsSeen > 15); res = cluster.get(1).nodetoolResult("cms", "dumplog", "--start", "10", "--end", "15"); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java index f1c669efa7..d0ea424f24 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java @@ -64,6 +64,7 @@ import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; import org.apache.cassandra.tcm.AtomicLongBackedProcessor; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Commit; @@ -159,7 +160,8 @@ public class ClusterMetadataTestHelper public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner) { - return new ClusterMetadata(epoch, Murmur3Partitioner.instance, + return new ClusterMetadata(epoch, + Murmur3Partitioner.instance, DistributedSchema.empty(), Directory.EMPTY, new TokenMap(partitioner), @@ -169,7 +171,8 @@ public class ClusterMetadataTestHelper InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } public static ClusterMetadata minimalForTesting(IPartitioner partitioner) @@ -177,6 +180,11 @@ public class ClusterMetadataTestHelper return minimalForTesting(Epoch.EMPTY, partitioner); } + public static ClusterMetadata minimalForTesting(Epoch e, Keyspaces keyspaces, CMSMembership cms) + { + return minimalForTesting(e, Murmur3Partitioner.instance, new DistributedSchema(keyspaces), cms); + } + public static ClusterMetadata minimalForTesting(Keyspaces keyspaces) { return minimalForTesting(Murmur3Partitioner.instance).transformer() @@ -185,6 +193,28 @@ public class ClusterMetadataTestHelper .metadata.forceEpoch(Epoch.EMPTY); } + public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema) + { + return minimalForTesting(epoch, partitioner, schema, CMSMembership.EMPTY); + } + + public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema, CMSMembership cms) + { + return new ClusterMetadata(epoch, + Murmur3Partitioner.instance, + schema, + Directory.EMPTY, + new TokenMap(partitioner), + DataPlacements.empty(), + AccordFastPath.EMPTY, + LockedRanges.EMPTY, + InProgressSequences.EMPTY, + ConsensusMigrationState.EMPTY, + ImmutableMap.of(), + AccordStaleReplicas.EMPTY, + cms); + } + public static ClusterMetadataService syncInstanceForTest() { LocalLog log = LocalLog.logSpec() @@ -1057,7 +1087,7 @@ public class ClusterMetadataTestHelper public static VersionedEndpoints.ForToken getNaturalReplicasForToken(ClusterMetadata metadata, String keyspace, Token searchPosition) { KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace); - return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(searchPosition); + return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(searchPosition); } public static BootstrapAndJoin getBootstrapPlan(int idx) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java index b13cab2660..67331f0cb3 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java @@ -508,7 +508,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase Set bouncing = new HashSet<>(); Set replicasFromBouncedReplicaSets = new HashSet<>(); outer: - for (VersionedEndpoints.ForRange placements : sut.service.metadata().placements.get(rf.asKeyspaceParams().replication).writes.endpoints) + for (VersionedEndpoints.ForRange placements : sut.service.metadata().placement(rf.asKeyspaceParams().replication).writes.endpoints) { List replicas = new ArrayList<>(metadata.directory.toNodeIds(placements.get().endpoints())); List bounceCandidates = new ArrayList<>(); @@ -545,8 +545,8 @@ public class MetadataChangeSimulationTest extends CMSTestBase ClusterMetadata actualMetadata = sut.service.metadata(); ReplicationParams replication = actualMetadata.schema.getKeyspaces().get("test").get().params.replication; Assert.assertEquals(replication, sut.rf.asKeyspaceParams().replication); - match(actualMetadata.placements.get(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); - match(actualMetadata.placements.get(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); + match(actualMetadata.placement(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); + match(actualMetadata.placement(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap()); } public static void validatePlacements(CMSTestBase.CMSSut sut, ModelState modelState) throws Throwable @@ -559,7 +559,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase Assert.assertEquals(modelState.simulatedPlacements.nodes.stream().map(Node::token).collect(Collectors.toSet()), actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).getLongValue()).collect(Collectors.toSet())); - for (Map.Entry e : actualMetadata.placements.asMap().entrySet()) + for (Map.Entry e : actualMetadata.placements().asMap().entrySet()) { if (!e.getKey().equals(replication)) continue; @@ -569,7 +569,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase match(placement.reads, modelState.simulatedPlacements.readPlacements); } - validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements); + validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements()); } public static ModelChecker.Pair registerNewNode(ModelState state, CMSSut sut, int dcIdx, int rackIdx) @@ -953,7 +953,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase validatePlacements(sut, state); } // Finally verify that the predicted placements match the actual ones - Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placements.get(ksm.params.replication))); + Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placement(ksm.params.replication))); } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java index 2f3e9f81ef..af4354a133 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java @@ -116,8 +116,7 @@ public class OperationalEquivalenceTest extends CMSTestBase withMove = ClusterMetadata.current(); } - assertPlacements(simulateAndCompare(rf, equivalentNodes).placements, - withMove.placements); + assertPlacements(simulateAndCompare(rf, equivalentNodes).placements(), withMove.placements()); } private static ClusterMetadata simulateAndCompare(ReplicationFactor rf, List nodes) throws Exception diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java index ff1b7d50ac..3b99de22b3 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java @@ -41,7 +41,6 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.ReplicationParams; -import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.PaxosRepairHistory; import org.apache.cassandra.tcm.ClusterMetadata; @@ -57,6 +56,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart; import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack; import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology; +import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.psjava.util.AssertStatus.assertTrue; @@ -85,10 +85,10 @@ public class ReconfigureCMSTest extends FuzzTestBase ClusterMetadata metadata = ClusterMetadata.current(); assertEquals(5, metadata.fullCMSMembers().size()); assertEquals(ReplicationParams.simpleMeta(5, metadata.directory.knownDatacenters()), - metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get()); + metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication); }); cluster.stream().forEach(i -> { - Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0); + Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0); }); cluster.get(nodeSelector.get()).nodetoolResult("cms", "reconfigure", "1").asserts().success(); @@ -96,7 +96,7 @@ public class ReconfigureCMSTest extends FuzzTestBase ClusterMetadata metadata = ClusterMetadata.current(); assertEquals(1, metadata.fullCMSMembers().size()); assertEquals(ReplicationParams.simpleMeta(1, metadata.directory.knownDatacenters()), - metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get()); + metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication); }); } } @@ -136,7 +136,7 @@ public class ReconfigureCMSTest extends FuzzTestBase Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance)); assertEquals(2, metadata.fullCMSMembers().size()); ReplicationParams params = ReplicationParams.meta(metadata); - DataPlacement placements = metadata.placements.get(params); + DataPlacement placements = metadata.placements().get(params); assertTrue(placements.reads.equivalentTo(placements.writes)); assertEquals(metadata.fullCMSMembers().size(), Integer.parseInt(params.asMap().get("dc0"))); }); @@ -161,7 +161,7 @@ public class ReconfigureCMSTest extends FuzzTestBase Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance)); Assert.assertTrue(metadata.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort())); assertEquals(3, metadata.fullCMSMembers().size()); - DataPlacement placements = metadata.placements.get(ReplicationParams.meta(metadata)); + DataPlacement placements = metadata.placements().get(ReplicationParams.meta(metadata)); Assert.assertTrue(placements.reads.equivalentTo(placements.writes)); }); } @@ -216,7 +216,7 @@ public class ReconfigureCMSTest extends FuzzTestBase awaitRingJoin(replacingNode, cluster.get(1)); replacingNode.runOnInstance(() -> { ClusterMetadata metadata = ClusterMetadata.current(); - assertTrue(metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort())); + assertTrue(metadata.isCMSMember()); assertEquals(3, metadata.fullCMSMembers().size()); }); } @@ -326,11 +326,11 @@ public class ReconfigureCMSTest extends FuzzTestBase Object[][] rows = instance.executeInternal("select points from system.paxos_repair_history " + "where keyspace_name = ? " + "and table_name = ?", - SchemaConstants.METADATA_KEYSPACE_NAME, + METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME); if (rows.length == 0) - return PaxosRepairHistory.empty(SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME); + return PaxosRepairHistory.empty(METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME); assertEquals(1, rows.length); //noinspection unchecked List points = (List)rows[0][0]; diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java index 81006bbfc0..d12dddeef5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java @@ -19,12 +19,13 @@ package org.apache.cassandra.distributed.test.log; import java.io.IOException; -import java.net.UnknownHostException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.EnumSet; import org.junit.Test; +import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.IInstanceConfig; @@ -33,9 +34,11 @@ import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.io.util.DataInputBuffer; -import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.Location; @@ -45,15 +48,21 @@ import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.PlacementProvider; import org.apache.cassandra.tcm.sequences.LeaveStreams; +import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave; +import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; +import org.apache.cassandra.tcm.transformations.CustomTransformation; import org.apache.cassandra.tcm.transformations.PrepareLeave; import org.apache.cassandra.tcm.transformations.Register; import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.tcm.transformations.Unregister; -import org.apache.cassandra.utils.CassandraVersion; +import org.apache.cassandra.utils.vint.VIntCoding; +import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.addr; +import static org.apache.cassandra.tcm.membership.NodeVersion.CURRENT_METADATA_VERSION; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; public class RegisterTest extends TestBaseImpl { @@ -106,57 +115,42 @@ public class RegisterTest extends TestBaseImpl { cluster.get(1).startup(); cluster.get(1).runOnInstance(() -> { + + // Run a custom transformation to inject a fake node into the directory with a known id and an + // artificially lowered max supported serialization version + CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer); + CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME, + new RegisterNodeWithOldVersion()); + ClusterMetadataService.instance().commit(injectOldNode); + + // Doesn't matter which specific Transformation we use here, we're testing that the serializer uses + // the correct lower bound + Transformation t = new Register(NodeAddresses.current(), TEST_LOCATION, NodeVersion.CURRENT); try { - // Unregister to make directory empty - ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(), - EnumSet.allOf(NodeState.class), - ClusterMetadataService.instance().placementProvider())); - - // Register a ghost node with V0 (bypasses version check because directory is now empty). - // In a real world cluster we will always be upgrading from a smaller version. - ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")), - TEST_LOCATION, - new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0))); - NodeId oldNode = ClusterMetadata.current().directory.peerId(InetAddressAndPort.getByName("127.0.0.100")); - - // Register a node with upgraded version - CassandraVersion currentVersion = NodeVersion.CURRENT.cassandraVersion; - NodeVersion upgraded = new NodeVersion(new CassandraVersion(String.format("%d.%d.%d", currentVersion.major + 1, 0, 0)), - NodeVersion.CURRENT_METADATA_VERSION); - ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.200")), TEST_LOCATION, upgraded)); - - // Doesn't matter which specific Transformation we use here, we're testing that the serializer uses - // the correct lower bound - Transformation t = new Register(NodeAddresses.current(), new Location("DC", "RACK"), NodeVersion.CURRENT); - try + assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, RegisterNodeWithOldVersion.METADATA_VERSION); + ByteBuffer bytes = t.kind().toVersionedBytes(t); + try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) { - assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion, - Version.V0.asInt()); - ByteBuffer bytes = t.kind().toVersionedBytes(t); - try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) - { - // Because ClusterMetadata.current().directory still contains oldNode we must serialize at - // the version it supports - assertEquals(Version.V0, Version.fromInt(buf.readUnsignedVInt32())); - } - - // If we unregister oldNode, then the ceiling for serialization version will rise - ClusterMetadataService.instance().commit(new Unregister(oldNode, EnumSet.allOf(NodeState.class), ClusterMetadataService.instance().placementProvider())); - assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion, - NodeVersion.CURRENT_METADATA_VERSION.asInt()); - bytes = t.kind().toVersionedBytes(t); - try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) - { - assertEquals(NodeVersion.CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32())); - } + // Because ClusterMetadata.current().directory still contains the fake old node we must + // serialize at the version _it_ supports + assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32())); } - catch (IOException e) + + // If we unregister the fake node, then the ceiling for serialization version will rise + Unregister unregisterOldNode = new Unregister(RegisterNodeWithOldVersion.NODE_ID, + EnumSet.allOf(NodeState.class), + ClusterMetadataService.instance().placementProvider()); + ClusterMetadataService.instance().commit(unregisterOldNode); + + assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, CURRENT_METADATA_VERSION); + bytes = t.kind().toVersionedBytes(t); + try (DataInputBuffer buf = new DataInputBuffer(bytes, true)) { - throw new RuntimeException(e); + assertEquals(CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32())); } } - catch (UnknownHostException e) + catch (IOException e) { throw new RuntimeException(e); } @@ -172,28 +166,70 @@ public class RegisterTest extends TestBaseImpl { cluster.get(1).startup(); cluster.get(1).runOnInstance(() -> { + // Run a custom transformation to inject a fake node into the directory with a known id and an + // artificially lowered max supported serialization version + CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer); + CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME, + new RegisterNodeWithOldVersion()); + ClusterMetadataService.instance().commit(injectOldNode); + // Now trigger a snapshot which must be written to the snapshot using the old serialization version + Epoch epoch = ClusterMetadataService.instance().commit(TriggerSnapshot.instance).epoch; + // fetch the raw bytes of the snapshot we just serialized + ByteBuffer bytes = SystemKeyspace.getSnapshot(epoch); + assertNotNull(bytes); + // assert the prepended version matches + Version writtenVersion = null; try { - // Unregister to make directory empty - ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(), - EnumSet.allOf(NodeState.class), - ClusterMetadataService.instance().placementProvider())); - - // Register a ghost node with V0 (bypasses version check because directory is now empty). - // In a real world cluster we will always be upgrading from a smaller version. - ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")), - TEST_LOCATION, - new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0))); + writtenVersion = Version.fromInt(VIntCoding.readUnsignedVInt32(new DataInputBuffer(bytes, false))); } - catch (UnknownHostException e) + catch (IOException e) { throw new RuntimeException(e); } - ClusterMetadataService.instance().commit(TriggerSnapshot.instance); - + assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, writtenVersion); + // load the snapshot using the standard mechanism and assert it matches current cluster metadata ClusterMetadata cm = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots().getSnapshot(ClusterMetadata.current().epoch); cm.equals(ClusterMetadata.current()); }); } } + + // Custom transforms to lock/unlock an arbitrary set of ranges to + // avoid having to actually initiate some range movement + public static class RegisterNodeWithOldVersion implements Transformation, Serializable + { + public static final AsymmetricMetadataSerializer serializer = new AsymmetricMetadataSerializer() + { + @Override + public void serialize(Transformation t, DataOutputPlus out, Version version){} + @Override + public RegisterNodeWithOldVersion deserialize(DataInputPlus in, Version version) {return new RegisterNodeWithOldVersion();} + @Override + public long serializedSize(Transformation t, Version version) {return 0;} + }; + + public static final String NAME = "TestRegisterNodeWithOldVersion"; + public static final NodeId NODE_ID = new NodeId(99); + public static final Version METADATA_VERSION = Version.V0; + + @Override + public Kind kind() + { + return Kind.CUSTOM; + } + + @Override + public Result execute(ClusterMetadata metadata) + { + ClusterMetadata.Transformer transformer = metadata.transformer() + .unsafeRegisterForTesting(NODE_ID, + new NodeAddresses(addr(99)), + TEST_LOCATION, + new NodeVersion(NodeVersion.CURRENT.cassandraVersion, + METADATA_VERSION)); + return Transformation.success(transformer, LockedRanges.AffectedRanges.EMPTY); + } + } + } diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java index 07ec054d54..51ccb4d7ea 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java @@ -146,12 +146,12 @@ public class ResumableStartupTest extends FuzzTestBase KeyspaceMetadata ksm = metadata.schema.getKeyspaceMetadata(keyspace); boolean isWriteReplica = false; boolean isReadReplica = false; - for (InetAddressAndPort readReplica : metadata.placements.get(ksm.params.replication).reads.byEndpoint().keySet()) + for (InetAddressAndPort readReplica : metadata.placement(ksm.params.replication).reads.byEndpoint().keySet()) { if (readReplica.getHostAddressAndPort().equals(newAddress)) isReadReplica = true; } - for (InetAddressAndPort writeReplica : metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet()) + for (InetAddressAndPort writeReplica : metadata.placement(ksm.params.replication).writes.byEndpoint().keySet()) { if (writeReplica.getHostAddressAndPort().equals(newAddress)) isWriteReplica = true; diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java b/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java index 5275471476..d4131f060d 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java @@ -177,8 +177,8 @@ public abstract class SimulatedOperation sutActions.next(); ClusterMetadata m2 = ClusterMetadata.current(); - Map, VersionedEndpoints.ForRange> after = m2.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.asMap(); - m1.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> { + Map, VersionedEndpoints.ForRange> after = m2.placement(simulatedState.rf.asKeyspaceParams().replication).reads.asMap(); + m1.placement(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> { if (after.containsKey(k)) { VersionedEndpoints.ForRange afterPlacements = after.get(k); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java index e355fdccc8..62416825bb 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java @@ -69,7 +69,7 @@ public class SnapshotTest extends TestBaseImpl ClusterMetadata before = ClusterMetadata.current(); ClusterMetadata after = ClusterMetadataService.instance().triggerSnapshot(); ClusterMetadata serialized = ClusterMetadataService.instance().snapshotManager().getSnapshot(after.epoch); - assertEquals(before.placements, serialized.placements); + assertEquals(before.placements(), serialized.placements()); assertEquals(before.tokenMap, serialized.tokenMap); assertEquals(before.directory, serialized.directory); assertEquals(before.schema, serialized.schema); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java b/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java index 35c81a53bb..1b7eda5ace 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java @@ -54,7 +54,7 @@ public class RangeVersioningTest extends FuzzTestBase for (int i = 1; i <= 4; i++) { Epoch smallestSeen = null; - for (VersionedEndpoints.ForRange fr : metadata.placements.get(ReplicationParams.simple(i)).writes.endpoints) + for (VersionedEndpoints.ForRange fr : metadata.placement(ReplicationParams.simple(i)).writes.endpoints) { if (smallestSeen == null || fr.lastModified().isBefore(smallestSeen)) smallestSeen = fr.lastModified(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java b/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java index 4a78f8805a..7722969a6a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java @@ -34,7 +34,6 @@ import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.distributed.shared.ClusterUtils.addInstance; import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin; @@ -80,7 +79,7 @@ public class CMSPlacementAfterReplacementTest extends TestBaseImpl IInvokableInstance nodeToRemove = cluster.get(2); cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success(); cluster.get(2).runOnInstance(() -> { - assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())); + assertTrue(ClusterMetadata.current().isCMSMember()); }); nodeToRemove.shutdown().get(); IInvokableInstance replacingNode = addInstance(cluster, nodeToRemove.config(), diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java index c020fdcc66..d5875e70fe 100644 --- a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java @@ -60,7 +60,7 @@ public class ClusterMetadataUpgradeAssassinateTest extends UpgradeTestBase ((IInvokableInstance) i).runOnInstance(() -> { ClusterMetadata metadata = ClusterMetadata.current(); InetAddressAndPort ep = InetAddressAndPort.getByNameUnchecked(host); - metadata.placements.asMap().forEach((key, value) -> { + metadata.placements().forEach((key, value) -> { if (key.isMeta()) return; boolean existsInPlacements = Streams.concat(value.reads.endpoints.stream(), diff --git a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java index e1fffc74b7..b406ae914c 100644 --- a/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java +++ b/test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java @@ -181,7 +181,7 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase try { new ByteBuddy().rebase(ClusterMetadata.class) - .method(named("initializeClusterIdentifier")) + .method(named("forceInitializedState")) .intercept(MethodDelegation.to(ClusterMetadataUpgradeDelayedInitializeTest.BBInterceptor.class)) .make() .load(classLoader, ClassLoadingStrategy.Default.INJECTION); @@ -201,13 +201,13 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase public static class BBInterceptor { @SuppressWarnings("unused") - public static ClusterMetadata initializeClusterIdentifier(@SuperCall Callable zuper) + public static ClusterMetadata forceInitializedState(@SuperCall Callable zuper) { try { - logger.info("initializeClusterIdentifier waiting..."); + logger.info("forceInitializedState waiting..."); BBState.latch.await(60, TimeUnit.SECONDS); - logger.info("initializeClusterIdentifier continuing..."); + logger.info("forceInitializedState continuing..."); return zuper.call(); } catch (Throwable e) diff --git a/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java b/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java index 481392149d..8cccb595e1 100644 --- a/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java +++ b/test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java @@ -945,7 +945,7 @@ public abstract class TopologyMixupTestBase { ClusterMetadata current = ClusterMetadata.current(); - Set members = current.placements.get(ReplicationParams.meta(current)).writes.byEndpoint().keySet(); + Set members = current.placement(ReplicationParams.meta(current)).writes.byEndpoint().keySet(); // Why not just use 'current.fullCMSMembers()'? That uses the "read" replicas, so "could" have less endpoints // It would be more consistent to use fullCMSMembers but thought process is knowing the full set is better // than the coordination set. diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java index 140da9f18e..9718f86ab8 100644 --- a/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java +++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java @@ -78,7 +78,7 @@ class OnClusterReplace extends OnClusterChangeTopology List> repairRanges = actions.cluster.get(leaving).unsafeApplyOnThisThread( (String keyspaceName) -> { ClusterMetadata metadata = ClusterMetadata.current(); - return metadata.placements.get(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication) + return metadata.placement(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication) .writes.ranges() .stream() .map(OnClusterReplace::toStringEntry) @@ -93,7 +93,7 @@ class OnClusterReplace extends OnClusterChangeTopology (String keyspaceName, String tk) -> { ClusterMetadata metadata = ClusterMetadata.current(); KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspaceName); - return metadata.placements.get(keyspaceMetadata.params.replication).reads + return metadata.placement(keyspaceMetadata.params.replication).reads .forToken(Utils.parseToken(tk)) .get() .stream().map(Replica::endpoint) diff --git a/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java b/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java index f0060dae93..58c3038229 100644 --- a/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java +++ b/test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java @@ -18,6 +18,7 @@ package org.apache.cassandra.auth; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -34,6 +35,8 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.SystemKeyspace; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; @@ -59,6 +62,11 @@ public class GrantAndRevokeTest extends CQLTester ServerTestUtils.daemonInitialization(); DatabaseDescriptor.setPermissionsValidity(0); DatabaseDescriptor.setRolesValidity(0); + // for the tables in the distributed metadata keyspace to be queryable, there needs to be a valid placement + // which is derived from the CMS membership. Most unit tests don't actually need to query those dist tables + // which is why this isn't done as a matter of routine + ClusterMetadataTestHelper.reconfigureCms(ReplicationParams.ntsMeta(Collections.singletonMap(DatabaseDescriptor.getLocalDataCenter(), 1))); + ServerTestUtils.markCMS(); requireAuthentication(); requireNetwork(); CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces(); diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index 32e040f965..b1d4827253 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -204,7 +204,7 @@ public class BootStrapperTest ClusterMetadata metadata = ClusterMetadata.current(); Pair movements = Pair.create(MovementMap.empty(), MovementMap.empty()); - if (metadata.myNodeId() == null) + if (metadata.myNodeId() == NodeId.UNREGISTERED) { Token myToken = metadata.partitioner.getRandomToken(); InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1"); @@ -265,5 +265,4 @@ public class BootStrapperTest ClusterMetadataTestHelper.addEndpoint(addr, tokens); } } - } diff --git a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java index 5e3cdd4c42..ba30b10971 100644 --- a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java @@ -36,6 +36,7 @@ import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.Directory; @@ -96,7 +97,8 @@ public class MetaStrategyTest InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } @Test diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java index d2a59bdc71..25854eca3e 100644 --- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java @@ -372,6 +372,6 @@ public class SimpleStrategyTest extends CassandraTestBase ReplicationParams replicationParams, Token token) { - return metadata.placements.get(replicationParams).writes.forToken(token).get(); + return metadata.placement(replicationParams).writes.forToken(token).get(); } } diff --git a/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java b/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java index 8b6b9ddb3c..22a62ac84c 100644 --- a/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java +++ b/test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java @@ -105,7 +105,7 @@ public class AccordTopologyUtils { ReplicationParams replication = keyspace.params.replication; AbstractReplicationStrategy strategy = AbstractReplicationStrategy.createReplicationStrategy(keyspace.name, replication); - DataPlacements.Builder placements = metadata.placements.unbuild(); + DataPlacements.Builder placements = metadata.placements().unbuild(); DataPlacement placement = strategy.calculateDataPlacement(Epoch.EMPTY, metadata.tokenMap.toRanges(), metadata); placements.with(replication, placement); metadata = transformer.with(placements.build()).build().metadata; diff --git a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java index 5ca0272c6e..0c43c74ea8 100644 --- a/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java +++ b/test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java @@ -344,7 +344,7 @@ public class EpochSyncTest private static boolean joined(ClusterMetadata metadata, Node.Id id) { NodeAddresses address = metadata.directory.getNodeAddresses(new NodeId(id.id)); - return metadata.placements.get(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress); + return metadata.placement(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress); } public enum EpochTracker { topologyManager, accordSyncPropagator } @@ -615,7 +615,7 @@ public class EpochSyncTest { Topology t = AccordTopology.createAccordTopology(current); Ranges ranges = t.ranges().mergeTouching(); - if (!current.placements.get(replication_params).reads.isEmpty()) + if (!current.placement(replication_params).reads.isEmpty()) Assertions.assertThat(ranges).hasSize(1); cms.setMetadata(current); for (Node.Id id : status(s -> s != Status.Removed)) @@ -721,7 +721,7 @@ public class EpochSyncTest case Registered: Invariants.require(!t.nodes().contains(id), "Node was in Init state but present in the Topology!"); Invariants.require(current.directory.peerId(address(id)) != null, "Node exists but not in TCM"); - if (current.placements.get(replication_params).writes.byEndpoint().keySet().contains(address(id))) + if (current.placement(replication_params).writes.byEndpoint().keySet().contains(address(id))) status = Status.Joining; break; case Joining: diff --git a/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java b/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java index 6b0d119448..fe9c6f0264 100644 --- a/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java +++ b/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java @@ -41,6 +41,7 @@ import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.MembershipUtils; +import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.DataPlacements; @@ -110,7 +111,10 @@ public class BootWithMetadataTest Directory directory = first.directory; int nodeCount = 10; int tokensPerNode = 5; - for (int i = 0; i < nodeCount; i++) + // Ensure that the "local" node is registered as it being member of the CMS is a precondition of booting from + // a ClusterMetadata. + directory = directory.with(NodeAddresses.current(), new Location("DC1", "RACK1")); + for (int i = 0; i < nodeCount-1; i++) directory = directory.with(nodeAddresses(random), new Location("DC1", "RACK1")); t = t.with(directory); @@ -156,7 +160,7 @@ public class BootWithMetadataTest assertEquals(toWrite.schema, fromRead.schema); assertEquals(toWrite.directory, fromRead.directory); assertEquals(toWrite.tokenMap, fromRead.tokenMap); - assertEquals(toWrite.placements, fromRead.placements); + assertEquals(toWrite.placements(), fromRead.placements()); assertEquals(toWrite.lockedRanges, fromRead.lockedRanges); assertEquals(toWrite.inProgressSequences, fromRead.inProgressSequences); assertEquals(toWrite.extensions, fromRead.extensions); diff --git a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java index 528dbae477..235a9735e8 100644 --- a/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java +++ b/test/unit/org/apache/cassandra/tcm/ClusterMetadataTransformationTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.tcm; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Random; import java.util.concurrent.ThreadLocalRandom; @@ -33,6 +34,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata.Transformer.Transformed; @@ -55,6 +57,7 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import static org.apache.cassandra.tcm.MetadataKeys.ACCORD_FAST_PATH; import static org.apache.cassandra.tcm.MetadataKeys.ACCORD_STALE_REPLICAS; +import static org.apache.cassandra.tcm.MetadataKeys.CMS_MEMBERSHIP; import static org.apache.cassandra.tcm.MetadataKeys.CONSENSUS_MIGRATION_STATE; import static org.apache.cassandra.tcm.MetadataKeys.DATA_PLACEMENTS; import static org.apache.cassandra.tcm.MetadataKeys.IN_PROGRESS_SEQUENCES; @@ -137,6 +140,42 @@ public class ClusterMetadataTransformationTest assertModifications(transformed, NODE_DIRECTORY, TOKEN_MAP); } + @Test + public void testModifyCMSMembership() + { + ClusterMetadata metadata = new ClusterMetadata(Murmur3Partitioner.instance, Directory.EMPTY, DistributedSchema.empty()); + Transformed transformed = metadata.transformer().build(); + assertTrue(transformed.modifiedKeys.isEmpty()); + + List endpoints = MembershipUtils.uniqueEndpoints(random, 2); + NodeAddresses a1 = new NodeAddresses(endpoints.get(0)); + NodeAddresses a2 = new NodeAddresses(endpoints.get(1)); + + transformed = metadata.transformer() + .register(a1, new Location("dc1", "rack1"), NodeVersion.CURRENT) + .register(a2, new Location("dc1", "rack1"), NodeVersion.CURRENT) + .build(); + assertModifications(transformed, NODE_DIRECTORY); + NodeId n1 = transformed.metadata.directory.peerId(a1.broadcastAddress); + NodeId n2 = transformed.metadata.directory.peerId(a2.broadcastAddress); + + transformed = metadata.transformer().startJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + transformed = transformed.metadata.transformer().cancelJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + + transformed = transformed.metadata.transformer().startJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + transformed = transformed.metadata.transformer().finishJoiningCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + + transformed = transformed.metadata.transformer().startJoiningCMS(n2).finishJoiningCMS(n2).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + + transformed = transformed.metadata.transformer().leaveCMS(n1).build(); + assertModifications(transformed, CMS_MEMBERSHIP); + } + @Test public void testModifySchema() { @@ -309,7 +348,7 @@ public class ClusterMetadataTransformationTest else if (key == TOKEN_MAP) return metadata.tokenMap; else if (key == DATA_PLACEMENTS) - return metadata.placements; + return metadata.placements(); else if (key == LOCKED_RANGES) return metadata.lockedRanges; else if (key == IN_PROGRESS_SEQUENCES) @@ -320,6 +359,8 @@ public class ClusterMetadataTransformationTest return metadata.consensusMigrationState; else if (key == ACCORD_STALE_REPLICAS) return metadata.accordStaleReplicas; + else if (key == CMS_MEMBERSHIP) + return metadata.cmsMembership; throw new IllegalArgumentException("Unknown metadata key " + key); } diff --git a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java index 167aa3992b..93ae4f1249 100644 --- a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java @@ -41,9 +41,12 @@ import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; +import org.apache.cassandra.tcm.membership.Location; +import org.apache.cassandra.tcm.membership.NodeAddresses; +import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; import org.apache.cassandra.tcm.transformations.CustomTransformation; -import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.tcm.transformations.cms.Initialize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -79,7 +82,17 @@ public class GetLogStateTest ClusterMetadataService.unsetInstance(); ClusterMetadataService.setInstance(cms); log.readyUnchecked(); - log.unsafeBootstrapForTesting(FBUtilities.getBroadcastAddressAndPort()); + + NodeAddresses addresses = NodeAddresses.current(); + Location location = DatabaseDescriptor.getInitialLocationProvider().initialLocation(); + log.unsafeBootstrapForTesting(addresses.broadcastAddress); + // register first node & make it CMS as this affects how the meta strategy placements are calculated + ClusterMetadata metadata = ClusterMetadata.current().forceInitializedState(addresses.broadcastAddress.hashCode(), + addresses, + NodeVersion.CURRENT, + location); + Initialize initialize = new Initialize(metadata); + cms.commit(initialize); } @Test diff --git a/test/unit/org/apache/cassandra/tcm/UnregisterTest.java b/test/unit/org/apache/cassandra/tcm/UnregisterTest.java index 62a6b160a2..3861eefd6b 100644 --- a/test/unit/org/apache/cassandra/tcm/UnregisterTest.java +++ b/test/unit/org/apache/cassandra/tcm/UnregisterTest.java @@ -110,7 +110,7 @@ public class UnregisterTest assertFalse(metadata.directory.allJoinedEndpoints().contains(ep)); assertFalse(metadata.directory.allDatacenterRacks().containsKey("dc2")); assertFalse(metadata.directory.knownDatacenters().contains("dc2")); - metadata.placements.asMap().forEach((params, placement) -> { + metadata.placements().forEach((params, placement) -> { assertFalse(Streams.concat(placement.writes.endpoints.stream(), placement.reads.endpoints.stream()).anyMatch((fr) -> fr.endpoints().contains(ep))); }); } diff --git a/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java b/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java index cac53d553f..17d515e1ff 100644 --- a/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java +++ b/test/unit/org/apache/cassandra/tcm/compatibility/GossipHelperTest.java @@ -104,7 +104,7 @@ public class GossipHelperTest assertEquals(internal, metadata.directory.addresses.get(nodeId).localAddress); assertEquals(nativeAddress, metadata.directory.addresses.get(nodeId).nativeAddress); - DataPlacements dp = metadata.placements; + DataPlacements dp = metadata.placements(); assertEquals(1, dp.get(KSM.params.replication).reads.forToken(token).get().size()); assertTrue(dp.get(KSM.params.replication).reads.forToken(token).get().contains(endpoint)); assertEquals(1, dp.get(KSM.params.replication).writes.forToken(token).get().size()); @@ -196,8 +196,8 @@ public class GossipHelperTest assertEquals(entry.getValue(), metadata.tokenMap.tokens(nodeId).iterator().next()); } - ReplicaGroups reads = metadata.placements.get(KSM_NTS.params.replication).reads; - ReplicaGroups writes = metadata.placements.get(KSM_NTS.params.replication).writes; + ReplicaGroups reads = metadata.placement(KSM_NTS.params.replication).reads; + ReplicaGroups writes = metadata.placement(KSM_NTS.params.replication).writes; assertEquals(reads, writes); // tokens are // dc1: 1: 1000, 3: 3000, 5: 5000, 6: 7000, 7: 9000 diff --git a/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java b/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java index 46adc7c67a..05bc00cadb 100644 --- a/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java +++ b/test/unit/org/apache/cassandra/tcm/listeners/MetadataSnapshotListenerTest.java @@ -109,7 +109,7 @@ public class MetadataSnapshotListenerTest listener.notify(entry, result); ClusterMetadata snapshot = snapshots.getSnapshot(nextEpoch); assertEquals(nextEpoch, snapshot.epoch); - assertEquals(toSnapshot.placements, snapshot.placements); + assertEquals(toSnapshot.placements(), snapshot.placements()); } private MetadataSnapshots init() diff --git a/test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java b/test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java new file mode 100644 index 0000000000..70705ff4f9 --- /dev/null +++ b/test/unit/org/apache/cassandra/tcm/listeners/PlacementsChangeListenerTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tcm.listeners; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.CassandraRelevantProperties; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.DistributedSchema; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Keyspaces; +import org.apache.cassandra.tcm.CMSMembership; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.tcm.membership.MembershipUtils; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.tcm.ownership.DataPlacement; +import org.apache.cassandra.tcm.ownership.DataPlacements; +import org.apache.cassandra.tcm.ownership.OwnershipUtils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.psjava.util.AssertStatus.assertTrue; + +public class PlacementsChangeListenerTest +{ + private static final Logger logger = LoggerFactory.getLogger(PlacementsChangeListenerTest.class); + static Random random; + static Epoch e; + static NodeId node1; + static KeyspaceMetadata ks1; + + @BeforeClass + public static void setupClass() + { + long seed = System.nanoTime(); + logger.info("Seed: {}", seed); + random = new Random(seed); + e = Epoch.create(10); + node1 = new NodeId(1); + ks1 = KeyspaceMetadata.create("ks1", KeyspaceParams.simple(1)); + CassandraRelevantProperties.TCM_SORT_REPLICA_GROUPS.setBoolean(false); + } + + @Test + public void testPlacementChange() + { + DataPlacements before = OwnershipUtils.randomPlacements(random).withLastModified(e); + DataPlacements.Builder builder = before.unbuild(); + before.forEach((params, placement) -> { + Replica remove = placement.writes.byEndpoint().flattenValues().iterator().next(); + Replica add = Replica.fullReplica(MembershipUtils.endpoint(99), remove.range()); + DataPlacement newPlacement = placement.unbuild() + .withoutWriteReplica(e.nextEpoch(), remove) + .withWriteReplica(e.nextEpoch(), add).build(); + builder.with(params, newPlacement); + }); + DataPlacements after = builder.build().withLastModified(e.nextEpoch()); + + // only placements are different + ClusterMetadata prev = metadata(e, before, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e.nextEpoch(), after, Keyspaces.of(ks1), node1); + assertNotEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertFalse(prev.placements().equivalentTo(next.placements())); + assertEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + @Test + public void testKeyspaceCountChange() + { + DataPlacements placements = OwnershipUtils.randomPlacements(random).withLastModified(e); + KeyspaceMetadata ks2 = KeyspaceMetadata.create("ks2", KeyspaceParams.simple(1)); + + // only keyspace counts are different + ClusterMetadata prev = metadata(e, placements, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e, placements, Keyspaces.of(ks1, ks2), node1); + assertEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertTrue(prev.placements().equivalentTo(next.placements())); + assertNotEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + @Test + public void testKeyspaceParamsChange() + { + DataPlacements placements = OwnershipUtils.randomPlacements(random).withLastModified(e); + KeyspaceMetadata ks1a = KeyspaceMetadata.create("ks1", KeyspaceParams.simple(2)); + + // only keyspace params are different + ClusterMetadata prev = metadata(e, placements, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e, placements, Keyspaces.of(ks1a), node1); + assertEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertTrue(prev.placements().equivalentTo(next.placements())); + assertEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertNotEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + @Test + public void testCMSMembershipChange() + { + DataPlacements placements = OwnershipUtils.randomPlacements(random).withLastModified(e); + NodeId node2 = new NodeId(2); + + // only cms memberships are different + ClusterMetadata prev = metadata(e, placements, Keyspaces.of(ks1), node1); + ClusterMetadata next = metadata(e, placements, Keyspaces.of(ks1), node1, node2); + assertEquals(prev.placements().lastModified(), next.placements().lastModified()); + assertTrue(prev.placements().equivalentTo(next.placements())); + assertEquals(prev.schema.getKeyspaces().size(), next.schema.getKeyspaces().size()); + assertEquals(prev.schema.getKeyspaceMetadata("ks1").params, next.schema.getKeyspaceMetadata("ks1").params); + assertNotEquals(prev.cmsMembership, next.cmsMembership); + + assertOnChangeEvent(prev, next); + } + + private static ClusterMetadata metadata(Epoch epoch, + DataPlacements placements, + Keyspaces keyspaces, + NodeId...cmsNode) + { + CMSMembership cms = CMSMembership.EMPTY; + for (NodeId n : cmsNode) + cms = cms.startJoining(n).finishJoining(n); + + ClusterMetadata.Transformer t = ClusterMetadataTestHelper.minimalForTesting(epoch, + Murmur3Partitioner.instance, + new DistributedSchema(keyspaces, epoch), + cms) + .forceEpoch(epoch) + .transformer() + .with(placements); + + return t.build().metadata; + } + + private static void assertOnChangeEvent(ClusterMetadata prev, ClusterMetadata next) + { + AtomicInteger cnt = new AtomicInteger(0); + PlacementsChangeListener listener = new PlacementsChangeListener(cnt::incrementAndGet); + listener.notifyPostCommit(prev, next, false); + assertEquals(1, cnt.get()); + } +} diff --git a/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java b/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java index a4c884406c..fc8c0559e1 100644 --- a/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java @@ -62,8 +62,9 @@ public class DistributedLogStateTest extends LogStateTestBase { return new LogStateSUT() { - - // start test entries at FIRST + 1 as the pre-init transform is automatically inserted with Epoch.FIRST + // we start test entries at FIRST, but in a real log the PRE_INITIALIZE_CMS transform is automatically + // inserted with Epoch.FIRST, followed by INITIALIZE_CMS so the next entry to be committed would be at + // epoch 3 Epoch currentEpoch = Epoch.FIRST; Epoch nextEpoch; boolean applied; diff --git a/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java b/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java index 91e1f1e1c3..3820c8f8d4 100644 --- a/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java +++ b/test/unit/org/apache/cassandra/tcm/membership/MembershipUtils.java @@ -19,8 +19,8 @@ package org.apache.cassandra.tcm.membership; import java.net.UnknownHostException; +import java.util.List; import java.util.Random; -import java.util.Set; import java.util.stream.Collectors; import org.apache.cassandra.locator.InetAddressAndPort; @@ -38,13 +38,13 @@ public class MembershipUtils return endpoint(random.nextInt(254) + 1); } - public static Set uniqueEndpoints(Random random, int count) + public static List uniqueEndpoints(Random random, int count) { return random.ints(1, 255) .distinct() .limit(count) .mapToObj(MembershipUtils::endpoint) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); } public static InetAddressAndPort endpoint(int i) diff --git a/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java b/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java index f9e8104ca9..07a3dc4b03 100644 --- a/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java +++ b/test/unit/org/apache/cassandra/tcm/ownership/LocalRangesAllSettledTest.java @@ -97,7 +97,7 @@ public class LocalRangesAllSettledTest AllLocalRanges proposed = snapshotAllLocalRanges(LocalRangeStatus.SETTLED, INITIAL_NODES); assertEquals(initial, proposed); // Check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, initial, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), initial, INITIAL_NODES); // Initiate an operation which affects ownership. This will add the MultiStepOperation which encodes any // necessary range movements so subsequent calls to ClusterMetadata::localRangesAllSettled @@ -123,7 +123,7 @@ public class LocalRangesAllSettledTest assertEquals(proposed, finalized); // Finally, check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, finalized, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), finalized, INITIAL_NODES); } @Test @@ -134,7 +134,7 @@ public class LocalRangesAllSettledTest AllLocalRanges proposed = snapshotAllLocalRanges(LocalRangeStatus.SETTLED, INITIAL_NODES); assertEquals(initial, proposed); // Check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, initial, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), initial, INITIAL_NODES); // Initiate an operation which affects ownership. This will add the MultiStepOperation which encodes any // necessary range movements so subsequent calls to ClusterMetadata::localRangesAllSettled @@ -161,7 +161,7 @@ public class LocalRangesAllSettledTest assertEquals(proposed, finalized); // Finally, check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, finalized, expandedNodes); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), finalized, expandedNodes); } @Test @@ -172,7 +172,7 @@ public class LocalRangesAllSettledTest AllLocalRanges proposed = snapshotAllLocalRanges(LocalRangeStatus.SETTLED, INITIAL_NODES); assertEquals(initial, proposed); // Check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, initial, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), initial, INITIAL_NODES); // Initiate an operation which affects ownership. This will add the MultiStepOperation which encodes any // necessary range movements so subsequent calls to ClusterMetadata::localRangesAllSettled @@ -201,7 +201,7 @@ public class LocalRangesAllSettledTest assertEquals(proposed, finalized); // Finally, check against the actual write placements - assertLocalRangesMatchPlacements(ClusterMetadata.current().placements, finalized, INITIAL_NODES); + assertLocalRangesMatchPlacements(ClusterMetadata.current().placements(), finalized, INITIAL_NODES); } private void assertLocalRangesMatchPlacements(DataPlacements placements, diff --git a/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java b/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java index 897dea5e5e..4b8b4e4164 100644 --- a/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java +++ b/test/unit/org/apache/cassandra/tcm/ownership/OwnershipUtils.java @@ -254,6 +254,6 @@ public class OwnershipUtils assert result.isSuccess(); workingMetadata = result.success().metadata; } - return workingMetadata.placements; + return workingMetadata.placements(); } } diff --git a/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java b/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java index d4e1b4df29..50946d7ac4 100644 --- a/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java +++ b/test/unit/org/apache/cassandra/tcm/sequences/InProgressSequenceCancellationTest.java @@ -105,11 +105,13 @@ public class InProgressSequenceCancellationTest LockedRanges locked = lockedRanges(placements, random); // state of metadata before starting the sequence + // note: don't allow epoch to be Epoch.FIRST as this is a + // special case for calculating meta strategy placements. ClusterMetadata before = metadata(directory).transformer() .with(placements) .withNodeState(nodeId, NodeState.REGISTERED) .with(locked) - .build().metadata; + .build().metadata.forceEpoch(epoch(random)); // Placements after PREPARE_JOIN DataPlacements afterPrepare = placements(ranges(random), replication, random); @@ -179,11 +181,13 @@ public class InProgressSequenceCancellationTest // Ranges locked by other operations LockedRanges locked = lockedRanges(placements, random); // state of metadata before starting the sequence + // note: don't allow epoch to be Epoch.FIRST as this is a + // special case for calculating meta strategy placements. ClusterMetadata before = metadata(directory).transformer() .with(placements) .withNodeState(nodeId, NodeState.JOINED) .with(locked) - .build().metadata; + .build().metadata.forceEpoch(epoch(random)); // PREPARE_LEAVE does not modify placements, so first transformation is START_LEAVE @@ -301,7 +305,7 @@ public class InProgressSequenceCancellationTest private void assertRelevantMetadata(ClusterMetadata first, ClusterMetadata second) { - assertTrue(first.placements.equivalentTo(second.placements)); + assertTrue(first.placements().equivalentTo(second.placements())); assertTrue(first.directory.equivalentTo(second.directory)); assertTrue(first.tokenMap.equivalentTo(second.tokenMap)); assertEquals(first.lockedRanges.locked.keySet(), second.lockedRanges.locked.keySet()); diff --git a/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java b/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java index cb4f15e5ef..2f3a3a0fd0 100644 --- a/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java +++ b/test/unit/org/apache/cassandra/tcm/sequences/ProgressBarrierTest.java @@ -113,7 +113,7 @@ public class ProgressBarrierTest extends CMSTestBase // Internally affectedRanges::toPeers uses the same logic as // the progress barrier does to identify the consensus group Set consensusGroup = leave.barrier().affectedRanges.toPeers(ReplicationParams.simple(1), - sut.service.metadata().placements, + sut.service.metadata().placements(), sut.service.metadata().directory); assertEquals(Set.of(node2.nodeId(), node3.nodeId()), consensusGroup); } @@ -217,7 +217,7 @@ public class ProgressBarrierTest extends CMSTestBase case ALL: { Set replicas = metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) .collect(Collectors.toSet()); @@ -235,7 +235,7 @@ public class ProgressBarrierTest extends CMSTestBase case QUORUM: { Set replicas = metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) .collect(Collectors.toSet()); @@ -253,7 +253,7 @@ public class ProgressBarrierTest extends CMSTestBase case LOCAL_QUORUM: { List replicas = new ArrayList<>(metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .filter((n) -> metadata.directory.location(n).datacenter.equals(dc)) .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) @@ -277,7 +277,7 @@ public class ProgressBarrierTest extends CMSTestBase { Map byDc = new HashMap<>(); metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .forEach(n -> byDc.compute(metadata.directory.location(n).datacenter, (k, v) -> v == null ? 1 : v + 1)); @@ -306,7 +306,7 @@ public class ProgressBarrierTest extends CMSTestBase } case ONE: Set replicas = metadata.lockedRanges.locked.get(LockedRanges.keyFor(metadata.epoch)) - .toPeers(rf.asKeyspaceParams().replication, metadata.placements, metadata.directory) + .toPeers(rf.asKeyspaceParams().replication, metadata.placements(), metadata.directory) .stream() .map(n -> metadata.directory.getNodeAddresses(n).broadcastAddress) .collect(Collectors.toSet()); diff --git a/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java b/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java index 11fd03c696..0f9115805f 100644 --- a/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java +++ b/test/unit/org/apache/cassandra/tcm/transformations/EventsMetadataTest.java @@ -95,8 +95,8 @@ public class EventsMetadataTest // should not be in tokenMap (no tokens yet) assertTrue(metadata.tokenMap.tokens(nodeId).isEmpty()); - assertTrue(metadata.placements.get(KSM.params.replication).writes.byEndpoint().isEmpty()); - assertTrue(metadata.placements.get(KSM.params.replication).reads.byEndpoint().isEmpty()); + assertTrue(metadata.placement(KSM.params.replication).writes.byEndpoint().isEmpty()); + assertTrue(metadata.placement(KSM.params.replication).reads.byEndpoint().isEmpty()); assertTrue(metadata.lockedRanges.locked.isEmpty()); } @@ -130,9 +130,9 @@ public class EventsMetadataTest assertTrue(ClusterMetadata.current().tokenMap.tokens(nodeId).isEmpty()); assertEquals(NodeState.BOOTSTRAPPING, ClusterMetadata.current().directory.peerState(nodeId)); - assertTrue(ClusterMetadata.current().placements.get(KSM.params.replication).writes.byEndpoint().containsKey(node1)); + assertTrue(ClusterMetadata.current().placement(KSM.params.replication).writes.byEndpoint().containsKey(node1)); // the first joined node gets added to the read endpoints immediately - assertTrue(ClusterMetadata.current().placements.get(KSM.params.replication).reads.byEndpoint().containsKey(node1)); + assertTrue(ClusterMetadata.current().placement(KSM.params.replication).reads.byEndpoint().containsKey(node1)); ClusterMetadataService.instance().commit(plan.midJoin); ClusterMetadataService.instance().commit(plan.finishJoin); @@ -152,8 +152,8 @@ public class EventsMetadataTest assertTrue(ClusterMetadata.current().tokenMap.tokens(nodeId).isEmpty()); assertEquals(NodeState.BOOTSTRAPPING, ClusterMetadata.current().directory.peerState(nodeId)); - assertTrue(ClusterMetadata.current().placements.get(KSM.params.replication).writes.byEndpoint().containsKey(node2)); - assertFalse(ClusterMetadata.current().placements.get(KSM.params.replication).reads.byEndpoint().containsKey(node2)); + assertTrue(ClusterMetadata.current().placement(KSM.params.replication).writes.byEndpoint().containsKey(node2)); + assertFalse(ClusterMetadata.current().placement(KSM.params.replication).reads.byEndpoint().containsKey(node2)); } @Test @@ -178,7 +178,7 @@ public class EventsMetadataTest // no change in metadata after prepareLeave; assertEquals(before.directory, after.directory); assertEquals(before.tokenMap, after.tokenMap); - assertEquals(before.placements, after.placements); + assertEquals(before.placements(), after.placements()); assertEquals(before.schema, after.schema); ClusterMetadataService.instance().commit(leave.startLeave); diff --git a/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java b/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java index f5044a00fd..bbf58eda54 100644 --- a/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java +++ b/test/unit/org/apache/cassandra/tcm/transformations/PrepareLeaveTest.java @@ -84,10 +84,11 @@ public class PrepareLeaveTest public void testCheckRF_Simple() throws Throwable { Keyspaces kss = Keyspaces.of(DistributedMetadataLogKeyspace.initialMetadata(Sets.newHashSet(hostDc.values())), KSM); + // should be accepted (2 nodes in dc1 where we remove the host): ClusterMetadata metadata = prepMetadata(kss, 2, 2); assertTrue(executeLeave(metadata)); - // should be rejected: - metadata = prepMetadata(kss, 1, 2); + // should be rejected because only 1 node in dc2: + metadata = prepMetadata(kss, 2, 1); assertFalse(executeLeave(metadata)); } @@ -97,17 +98,17 @@ public class PrepareLeaveTest Keyspaces kss = Keyspaces.of(DistributedMetadataLogKeyspace.initialMetadata(Sets.newHashSet(hostDc.values())), KSM_NTS); ClusterMetadata metadata = prepMetadata(kss, 4, 4); assertTrue(executeLeave(metadata)); - // should be accepted (4 nodes in dc1 where we remove the host): - metadata = prepMetadata(kss, 4, 2); + // should be accepted (4 nodes in dc2 where we remove the host): + metadata = prepMetadata(kss, 2, 4); assertTrue(executeLeave(metadata)); - // should be rejected - metadata = prepMetadata(kss, 3, 4); + // should be rejected because there are already only 3 replicas in dc2 + metadata = prepMetadata(kss, 4, 3); assertFalse(executeLeave(metadata)); } private boolean executeLeave(ClusterMetadata metadata) throws Throwable { - PrepareLeave prepareLeave = new PrepareLeave(metadata.directory.peerId(InetAddressAndPort.getByName("127.0.0.1")), + PrepareLeave prepareLeave = new PrepareLeave(metadata.directory.peerId(InetAddressAndPort.getByName("127.0.0.11")), false, dummyPlacementProvider, LeaveStreams.Kind.UNBOOTSTRAP); diff --git a/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java b/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java index 57d87d22e7..04209514bb 100644 --- a/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java +++ b/test/unit/org/apache/cassandra/tools/CMSOfflineToolTest.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -55,6 +54,7 @@ import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.accord.topology.AccordFastPath; import org.apache.cassandra.service.accord.topology.AccordStaleReplicas; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; @@ -110,7 +110,8 @@ public class CMSOfflineToolTest extends OfflineToolUtils InProgressSequences.EMPTY, ConsensusMigrationState.EMPTY, ImmutableMap.of(), - AccordStaleReplicas.EMPTY); + AccordStaleReplicas.EMPTY, + CMSMembership.EMPTY); } @Before @@ -669,7 +670,7 @@ public class CMSOfflineToolTest extends OfflineToolUtils "Cluster Metadata Service:\n" + "Members: /127.0.0.1:" + storagePort + ",/127.0.0.2:" + storagePort + ",/127.0.0.3:" + storagePort + '\n' + "Needs reconfiguration: false\n" + - "Service State: LOCAL\n" + + "Service State: " + ClusterMetadataService.State.OFFLINE_TOOL + '\n' + "Epoch: 2\n" + "Replication factor: ReplicationParams{class=org.apache.cassandra.locator.MetaStrategy, datacenter1=3}\n"; assertThat(result.getStdout()).isEqualTo(expectedOutput); @@ -1668,10 +1669,7 @@ public class CMSOfflineToolTest extends OfflineToolUtils KeyspaceMetadata normalKeyspace = KeyspaceMetadata.create("ks", KeyspaceParams.simple(3)); Keyspaces keyspaces = Keyspaces.none().with(metaKeyspace).with(normalKeyspace); - ClusterMetadata clusterMetadata = getClusterMetadata(keyspaces, partitioner, directory); - - - ClusterMetadata metadata = clusterMetadata + ClusterMetadata metadata = getClusterMetadata(keyspaces, partitioner, directory) .transformer() .with(directory) .join(nodeId1) @@ -1680,12 +1678,12 @@ public class CMSOfflineToolTest extends OfflineToolUtils .proposeToken(nodeId2, getRandomTokens(partitioner, tokenSize)) .join(nodeId3) .proposeToken(nodeId3, getRandomTokens(partitioner, tokenSize)) + .startJoiningCMS(nodeId1).finishJoiningCMS(nodeId1) + .startJoiningCMS(nodeId2).finishJoiningCMS(nodeId2) + .startJoiningCMS(nodeId3).finishJoiningCMS(nodeId3) .build().metadata; - // Create replicas for the metadata keyspace on all three nodes - ReplicationParams metaParams = ReplicationParams.ntsMeta(Collections.singletonMap(DC, 3)); DataPlacements placements = DataPlacements.empty().unbuild() - .with(metaParams, getCMSMemberPlacement(metadata, List.of(addr1, addr2, addr3))) .with(ReplicationParams.simple(3), getKeyspacePlacement(metadata, normalKeyspace)) .build(); @@ -1799,6 +1797,11 @@ public class CMSOfflineToolTest extends OfflineToolUtils ClusterMetadata startReplacing(NodeId oldNodeId, NodeId newNodeId, ClusterMetadata clusterMetadata) { + // In a real cluster, a node being replaced is removed from the CMS prior to the replacement starting + // i.e. before the PrepareReplace is committed + if (clusterMetadata.fullCMSMemberIds().contains(oldNodeId)) + clusterMetadata = clusterMetadata.transformer().leaveCMS(oldNodeId).build().metadata; + Register register = new Register(getNodeAddresses(newNodeId.id()), new Location(DC, "rack" + newNodeId.id()), NodeVersion.CURRENT); diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index 4c0a6758ec..1438fc8b62 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -140,6 +140,7 @@ import org.apache.cassandra.service.accord.topology.SimpleFastPathStrategy; import org.apache.cassandra.service.accord.topology.UpFastPathStrategy; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState; +import org.apache.cassandra.tcm.CMSMembership; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.extensions.ExtensionKey; @@ -1955,7 +1956,8 @@ public final class CassandraGenerators ConsensusMigrationState consensusMigrationState = ConsensusMigrationState.EMPTY; Map, ExtensionValue> extensions = ImmutableMap.of(); AccordStaleReplicas accordStaleReplicas = accordStaleReplicasGen.generate(rnd); - return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions, accordStaleReplicas); + CMSMembership cms = CMSMembership.EMPTY; + return new ClusterMetadata(epoch, partitioner, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, extensions, accordStaleReplicas, cms); }; } }