Add CMS membership directly to ClusterMetadata

Patch by Sam Tunnicliffe; reviewed by Marcus Eriksson for
CASSANDRA-20736

CHANGES.txt

@@ -1,4 +1,5 @@
 6.0-alpha2
+ * Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736)
  * Fix maven remote publishing of Accord artifacts (CASSANDRA-21261)
  * Move long running TCM operations to a longer timout (CASSANDRA-21453)
  * Offline nodetool commands should not print network options in help (CASSANDRA-20876)

src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java

@@ -96,7 +96,7 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement
         // as we have as keys in metadata.placements to have a fast map lookup
         // ReplicationParams are immutable, so it is a safe optimization
         KeyspaceParams keyspaceParams = attrs.asNewKeyspaceParams();
-        ReplicationParams replicationParams = metadata.placements.deduplicateReplicationParams(keyspaceParams.replication);
+        ReplicationParams replicationParams = metadata.placements().deduplicateReplicationParams(keyspaceParams.replication);
         keyspaceParams = keyspaceParams.withSwapped(replicationParams);
         KeyspaceMetadata keyspaceMetadata = KeyspaceMetadata.create(keyspaceName, keyspaceParams);
 

src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java

@@ -195,6 +195,6 @@ public abstract class AbstractMutationVerbHandler<T extends IMutation> implement
 
     private static VersionedEndpoints.ForToken writePlacements(ClusterMetadata metadata, String keyspace, DecoratedKey key)
     {
-        return metadata.placements.get(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken());
+        return metadata.placement(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken());
     }
 }

src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java

@@ -240,8 +240,8 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
 
     private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace)
     {
-        return metadata.placements
+        return metadata
-               .get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication)
+               .placement(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication)
                .reads
                .forToken(token)
                .get()

src/java/org/apache/cassandra/db/compaction/CompactionManager.java

@@ -780,7 +780,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
         // we only consider write placements during cleanup as range movements always ensure
         // overlap between new replicas accepting reads and old replicas accepting writes
         ClusterMetadata cm = ClusterMetadata.current();
-        DataPlacement placement = cm.placements.get(keyspace.getMetadata().params.replication);
+        DataPlacement placement = cm.placement(keyspace.getMetadata().params.replication);
         InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
         RangesAtEndpoint localWrites = placement.writes.byEndpoint().get(local);
         // TODO review: Hack to get local partitioner not to fail out because it's handled very poorly with data placements

src/java/org/apache/cassandra/db/view/ViewUtils.java

@@ -30,6 +30,7 @@ import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.membership.Location;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
 
 public final class ViewUtils
 {
@@ -64,8 +65,9 @@ public final class ViewUtils
         Location local = metadata.locator.local();
         KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
 
-        EndpointsForToken naturalBaseReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(baseToken).get();
+        DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication);
-        EndpointsForToken naturalViewReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(viewToken).get();
+        EndpointsForToken naturalBaseReplicas = placement.reads.forToken(baseToken).get();
+        EndpointsForToken naturalViewReplicas = placement.reads.forToken(viewToken).get();
 
         Optional<Replica> localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isSelf).toJavaUtil();
         if (localReplica.isPresent())

src/java/org/apache/cassandra/gms/Gossiper.java

@@ -1764,7 +1764,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
         register(DatabaseDescriptor.getLocalAddressReconnectionHelper());
 
         ClusterMetadata metadata = ClusterMetadata.current();
-        if (mergeLocalStates && metadata.myNodeId() != null)
+        if (mergeLocalStates && metadata.myNodeId() != NodeId.UNREGISTERED)
             mergeNodeToGossip(metadata.myNodeId(), metadata);
 
         shutdownAnnounced.set(false);
@@ -2234,7 +2234,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
     {
         checkProperThreadForStateMutation();
         assert !endpoint.equals(getBroadcastAddressAndPort()) || epstate.getHeartBeatState().getGeneration() > 0 :
-        "We should not update epstates with generation = 0 for the local host";
+        String.format("We should not update epstates with generation = 0 for the local host " +
+                      "(endpoint: %s, broadcast: %s, generation: %s)",
+                      endpoint, getBroadcastAddressAndPort(), epstate.getHeartBeatState().getGeneration());
         EndpointState old = endpointStateMap.get(endpoint);
         if (old == null)
             endpointStateMap.put(endpoint, epstate);
@@ -2278,6 +2280,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
         taskLock.lock();
         try
         {
+            if (nodeId == NodeId.UNREGISTERED)
+                return;
+
             boolean isLocal = nodeId.equals(metadata.myNodeId());
             IPartitioner partitioner = metadata.tokenMap.partitioner();
             NodeAddresses addresses = metadata.directory.getNodeAddresses(nodeId);

src/java/org/apache/cassandra/hints/HintsDispatcher.java

@@ -276,7 +276,7 @@ final class HintsDispatcher implements AutoCloseable
             // Also may need to apply locally because it's possible this is from the batchlog
             // and we never applied it locally
             // TODO (review): Additional error handling necessary? Hints are lossy
-            DataPlacement dataPlacement = cm.placements.get(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication);
+            DataPlacement dataPlacement = cm.placement(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication);
             VersionedEndpoints.ForToken forToken = dataPlacement.writes.forToken(mutation.key().getToken());
             Replica self = forToken.get().selfIfPresent();
             if (self != null)

src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java

@@ -90,7 +90,8 @@ public class CMSPlacementStrategy
 
         // Although MetaStrategy has its own entireRange, it uses a custom partitioner which isn't compatible with
         // regular, non-CMS placements. For that reason, we select replicas here using tokens provided by the
-        // globally configured partitioner.
+        // globally configured partitioner. This also has the benefit of making concurrent operations, such as
+        // bounces/upgrades/etc, safe for the CMS if they are replica aware.
         Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken();
         EndpointsForRange endpoints = NetworkTopologyStrategy.calculateNaturalReplicas(minToken,
                                                                                        new Range<>(minToken, minToken),

src/java/org/apache/cassandra/locator/EndpointsForToken.java

@@ -164,7 +164,7 @@ public class EndpointsForToken extends Endpoints<EndpointsForToken>
 
     public static VersionedEndpoints.ForToken natural(Keyspace keyspace, Token token)
     {
-        return ClusterMetadata.current().placements.get(keyspace.getMetadata().params.replication).reads.forToken(token);
+        return ClusterMetadata.current().placement(keyspace.getMetadata().params.replication).reads.forToken(token);
     }
 
 }

src/java/org/apache/cassandra/locator/MetaStrategy.java

@@ -82,13 +82,13 @@ public class MetaStrategy extends SystemStrategy
     @Override
     public EndpointsForRange calculateNaturalReplicas(Token token, ClusterMetadata metadata)
     {
-        return metadata.placements.get(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get();
+        return metadata.placement(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get();
     }
 
     @Override
     public DataPlacement calculateDataPlacement(Epoch epoch, List<Range<Token>> ranges, ClusterMetadata metadata)
     {
-        return metadata.placements.get(ReplicationParams.meta(metadata));
+        return metadata.placement(ReplicationParams.meta(metadata));
     }
 
     @Override

src/java/org/apache/cassandra/locator/ReplicaLayout.java

@@ -239,7 +239,7 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
         {
             // todo deduplicate so that "pending" contains "read - write",
             // which is a hack until we revisit how consistency level handles pending
-            DataPlacement dataPlacement = metadata.placements.get(ks.params.replication);
+            DataPlacement dataPlacement = metadata.placement(ks.params.replication);
             natural = forNonLocalStrategyTokenRead(dataPlacement, token);
             // perf optimization to avoid double endpoints search and filtering for a typical case
             // DataPlacement constructor does a deduplication of reads/writes, so we can use cheap == comparision here
@@ -394,12 +394,12 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
 
     static EndpointsForRange forNonLocalStategyRangeRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, AbstractBounds<PartitionPosition> range)
     {
-        return metadata.placements.get(keyspace.params.replication).reads.forRange(range.right.getToken()).get();
+        return metadata.placement(keyspace.params.replication).reads.forRange(range.right.getToken()).get();
     }
 
     public static EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token)
     {
-        return forNonLocalStrategyTokenRead(metadata.placements.get(keyspace.params.replication), token);
+        return forNonLocalStrategyTokenRead(metadata.placement(keyspace.params.replication), token);
     }
 
     public static EndpointsForToken forNonLocalStrategyTokenRead(DataPlacement dataPlacement, Token token)
@@ -407,17 +407,11 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
         return dataPlacement.reads.forToken(token).get();
     }
 
-    static EndpointsForToken forNonLocalStrategyTokenWrite(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token)
+    private static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token)
-    {
-        return forNonLocalStrategyTokenWrite(metadata.placements.get(keyspace.params.replication), token);
-    }
-
-    static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token)
     {
         return dataPlacement.writes.forToken(token).get();
     }
 
-
     static EndpointsForRange forLocalStrategyRange(ClusterMetadata metadata, AbstractReplicationStrategy replicationStrategy, AbstractBounds<PartitionPosition> range)
     {
         return replicationStrategy.calculateNaturalReplicas(range.right.getToken(), metadata);

src/java/org/apache/cassandra/locator/ReplicaPlans.java

@@ -234,7 +234,7 @@ public class ReplicaPlans
         NodeProximity proximity = DatabaseDescriptor.getNodeProximity();
         AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
 
-        EndpointsForToken replicas = metadata.placements.get(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get();
+        EndpointsForToken replicas = metadata.placement(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get();
 
         // CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping
         replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint()));

src/java/org/apache/cassandra/metrics/TCMMetrics.java

@@ -29,7 +29,6 @@ import org.apache.cassandra.gms.FailureDetector;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.EpochAwareDebounce;
-import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
 import static org.apache.cassandra.tcm.transformations.cms.PrepareCMSReconfiguration.needsReconfiguration;
@@ -94,7 +93,7 @@ public class TCMMetrics
 
         isCMSMember = Metrics.register(factory.createMetricName("IsCMSMember"), () -> {
             ClusterMetadata metadata =  ClusterMetadata.currentNullable();
-            return metadata != null && metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) ? 1 : 0;
+            return metadata != null && metadata.isCMSMember() ? 1 : 0;
         });
 
         needsCMSReconfiguration = Metrics.register(factory.createMetricName("NeedsCMSReconfiguration"), () -> {

src/java/org/apache/cassandra/net/ResponseVerbHandler.java

@@ -30,7 +30,6 @@ import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tracing.Tracing;
-import org.apache.cassandra.utils.FBUtilities;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND;
@@ -98,7 +97,7 @@ class ResponseVerbHandler implements IVerbHandler
         if (SKIP_CATCHUP_FOR.contains(message.verb()))
             return;
 
-        if (metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) && CMS_SKIP_CATCHUP_FOR.contains(message.verb()))
+        if (metadata.isCMSMember() && CMS_SKIP_CATCHUP_FOR.contains(message.verb()))
             return;
 
         // Gossip stage is single-threaded, so we may end up in a deadlock with after-commit hook

src/java/org/apache/cassandra/repair/RepairCoordinator.java

@@ -81,7 +81,6 @@ import org.apache.cassandra.tracing.TraceState;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.transport.Dispatcher;
 import org.apache.cassandra.transport.messages.ResultMessage;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 import org.apache.cassandra.utils.Throwables;
 import org.apache.cassandra.utils.TimeUUID;
@@ -402,7 +401,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai
         //calculation multiple times
         Iterable<Range<Token>> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges();
         boolean isMeta = Keyspace.open(state.keyspace).getMetadata().params.replication.isMeta();
-        boolean isCMS = ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort());
+        boolean isCMS = ClusterMetadata.current().isCMSMember();
         for (Range<Token> range : state.options.getRanges())
         {
             EndpointsForRange allForRange = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range);

src/java/org/apache/cassandra/service/ActiveRepairService.java

@@ -1213,7 +1213,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                 // are based on the system partitioner
                 EndpointsForRange endpoints = replication.isMeta()
                                               ? ClusterMetadata.current().fullCMSMembersAsReplicas()
-                                              : ClusterMetadata.current().placements.get(replication).reads.forRange(range).get();
+                                              : ClusterMetadata.current().placement(replication).reads.forRange(range).get();
 
                 Set<InetAddressAndPort> liveEndpoints = endpoints.filter(FailureDetector.isReplicaAlive).endpoints();
                 if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, liveEndpoints))

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -360,7 +360,7 @@ public class CassandraDaemon
         {
             CommitLog.instance.recoverSegmentsOnDisk();
             NodeId self = ClusterMetadata.current().myNodeId();
-            if (self != null)
+            if (self != NodeId.UNREGISTERED)
                 AccordService.localStartup(self);
         }
         catch (IOException e)

src/java/org/apache/cassandra/service/Rebuild.java

@@ -52,12 +52,12 @@ import org.apache.cassandra.streaming.StreamOperation;
 import org.apache.cassandra.streaming.StreamResultFuture;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ownership.DataPlacement;
-import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.MovementMap;
 import org.apache.cassandra.utils.concurrent.Future;
 import org.apache.cassandra.utils.concurrent.FutureCombiner;
 import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
 
+import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME;
 import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
 
 public class Rebuild
@@ -232,21 +232,23 @@ public class Rebuild
     private static MovementMap movementMap(ClusterMetadata metadata, String keyspace, String tokens)
     {
         MovementMap.Builder movementMapBuilder = MovementMap.builder();
-        DataPlacements placements = metadata.placements;
         if (keyspace == null)
         {
-            placements.forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null)));
+            metadata.placements().forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null)));
+            // Special case to include the system metadata keyspace
+            ReplicationParams metaParams = Keyspace.open(METADATA_KEYSPACE_NAME).getMetadata().params.replication;
+            movementMapBuilder.put(metaParams, addMovementsForParams(metadata.placement(metaParams), null));
         }
         else if (tokens == null)
         {
             ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication;
-            movementMapBuilder.put(params, addMovementsForParams(placements.get(params), null));
+            movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), null));
         }
         else
         {
             ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication;
             RangesAtEndpoint ranges = rangesForRebuildWithTokens(tokens, keyspace);
-            movementMapBuilder.put(params, addMovementsForParams(placements.get(params), ranges));
+            movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), ranges));
         }
         return movementMapBuilder.build();
     }

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -175,6 +175,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator;
 import org.apache.cassandra.service.reads.range.RangeCommands;
 import org.apache.cassandra.service.reads.repair.ReadRepair;
 import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
 import org.apache.cassandra.tracing.Tracing;
@@ -2197,7 +2198,7 @@ public class StorageProxy implements StorageProxyMBean
         if (metadata == null)
             return false;
 
-        if (metadata.myNodeId() == null)
+        if (metadata.myNodeId() == NodeId.UNREGISTERED)
             return false;
 
         return metadata.myNodeState() == NodeState.JOINED;

src/java/org/apache/cassandra/service/StorageService.java

@@ -2057,9 +2057,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         // creation and initialization of cluster metadata service. Metadata collector does accept
         // null localhost ID values, it's just that TokenMetadata was created earlier.
         ClusterMetadata metadata = ClusterMetadata.currentNullable();
-        if (metadata == null || metadata.directory.peerId(getBroadcastAddressAndPort()) == null)
+        if (metadata == null || metadata.myNodeId() == NodeId.UNREGISTERED)
+        {
+            // this condition is to prevent accessing the tables when the node is not started yet, and in particular,
+            // when it is not going to be started at all (e.g. when running some unit tests or client tools).
+            if ((DatabaseDescriptor.isDaemonInitialized() || DatabaseDescriptor.isToolInitialized()) && CommitLog.instance.isStarted())
+                return SystemKeyspace.getLocalHostId();
+            else
                 return null;
-        return metadata.directory.peerId(getBroadcastAddressAndPort()).toUUID();
+        }
+        return metadata.myNodeId().toUUID();
     }
 
     public Map<String, String> getHostIdMap()
@@ -2067,7 +2074,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         return getEndpointToHostId();
     }
 
-
     public Map<String, String> getEndpointToHostId()
     {
         return getEndpointToHostId(false);
@@ -2119,7 +2125,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         {
             if (keyspaceMetadata.params.replication.isMeta())
             {
-                DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication);
+                DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication);
                 // May be empty if mid-upgrade and CMS is not yet initialized
                 if (!placement.reads.isEmpty())
                     rangeToEndpointMap.put(MetaStrategy.entireRange, placement.reads.forRange(MetaStrategy.entireRange).get());
@@ -2129,8 +2135,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
                 TokenMap tokenMap = metadata.tokenMap;
                 for (Range<Token> range : ranges)
                 {
-                    Token token = tokenMap.nextToken(tokenMap.tokens(), range.right.getToken());
+                    Token token = TokenMap.nextToken(tokenMap.tokens(), range.right.getToken());
-                    rangeToEndpointMap.put(range, metadata.placements.get(keyspaceMetadata.params.replication)
+                    rangeToEndpointMap.put(range, metadata.placement(keyspaceMetadata.params.replication)
                                                   .reads.forRange(token).get());
                 }
             }
@@ -3447,14 +3453,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
             token = MetaStrategy.partitioner.getToken(key);
         else
             token = metadata.partitioner.getToken(key);
-        return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(token).get();
+        return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(token).get();
     }
 
     public boolean isEndpointValidForWrite(String keyspace, Token token)
     {
         ClusterMetadata metadata = ClusterMetadata.current();
         KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
-        return keyspaceMetadata != null && metadata.placements.get(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf();
+        return keyspaceMetadata != null && metadata.placement(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf();
     }
 
     public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception
@@ -4262,7 +4268,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
         if (replicationParams.isMeta())
         {
             LinkedHashMap<InetAddressAndPort, Float> ownership = Maps.newLinkedHashMap();
-            metadata.placements.get(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> {
+            metadata.placement(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> {
                 ownership.put(r.endpoint(), 1.0f);
             });
             return ownership;

src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java

@@ -312,7 +312,7 @@ public class AccordTopology
 
     public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup)
     {
-        return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas);
+        return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements(), metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas);
     }
 
     public static Topology createAccordTopology(ClusterMetadata metadata, Topology current)

src/java/org/apache/cassandra/service/paxos/Paxos.java

@@ -277,7 +277,7 @@ public class Paxos
             final Token token = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : key.getToken();
             ClusterMetadata metadata = ClusterMetadata.current();
             Keyspace keyspace = Keyspace.open(table.keyspace);
-            DataPlacement placement = metadata.placements.get(keyspace.getMetadata().params.replication);
+            DataPlacement placement = metadata.placement(keyspace.getMetadata().params.replication);
             Epoch epoch = placement.writes.forToken(token).lastModified();
             ForTokenWrite electorate = forTokenWriteLiveAndDown(metadata, keyspace, token);
             if (consistency == LOCAL_SERIAL)

src/java/org/apache/cassandra/service/paxos/PaxosRepair.java

@@ -586,7 +586,7 @@ public class PaxosRepair extends AbstractPaxosRepair
         ClusterMetadata metadata = ClusterMetadata.current();
         Collection<InetAddressAndPort> allEndpoints = replication.isMeta()
                                                       ? metadata.fullCMSMembers()
-                                                      : metadata.placements.get(replication).reads.forRange(range).endpoints();
+                                                      : metadata.placement(replication).reads.forRange(range).endpoints();
         return hasSufficientLiveNodesForTopologyChange(allEndpoints,
                                                        liveEndpoints,
                                                        ep -> metadata.locator.location(ep).datacenter,

src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java

@@ -50,7 +50,7 @@ public class DataMovementVerbHandler implements IVerbHandler<DataMovement>
         StreamPlan streamPlan = new StreamPlan(StreamOperation.fromString(message.payload.streamOperation));
         ClusterMetadata metadata = ClusterMetadata.current();
         Schema.instance.getNonLocalStrategyKeyspaces().stream().forEach((ksm) -> {
-            if (metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1)
+            if (metadata.placement(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1)
                 return;
 
             message.payload.movements.get(ksm.params.replication).asMap().forEach((local, endpoints) -> {

src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java

@@ -46,6 +46,7 @@ public abstract class AbstractLocalProcessor implements Processor
         this.log = log;
     }
 
+
     /**
      * Epoch returned by processor in the Result is _not_ guaranteed to be visible by the Follower by
      * the time when this method returns.
@@ -59,7 +60,7 @@ public abstract class AbstractLocalProcessor implements Processor
         while (!retryPolicy.hasExpired())
         {
             ClusterMetadata previous = log.waitForHighestConsecutive();
-            if (!previous.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort()))
+            if (!acceptCommit(previous))
             {
                 String msg = String.format("Node %s is not a CMS member in epoch %s; members=%s",
                                            FBUtilities.getBroadcastAddressAndPort(),
@@ -228,4 +229,5 @@ public abstract class AbstractLocalProcessor implements Processor
 
     public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy);
     protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch);
+    protected abstract boolean acceptCommit(ClusterMetadata metadata);
 }

src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java

@@ -59,6 +59,18 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor
         this.epochHolder = new AtomicLong(epoch.getEpoch());
     }
 
+    @Override
+    public boolean acceptCommit(ClusterMetadata metadata)
+    {
+        // AtomicLongBackedProcessor is only for use in tests and offline tools and it should be safe to assume that it
+        // is always allowed to process commit requests. In a non-test setup, when the CMS is initialized the initiating
+        // node is also registered. This is not the case in tests (see initCMS/recreateCMS in ServerTestUtils for more
+        // detail), as this simplifies setting up/tearing down topologies in test cases. The processor intended for use
+        // in real clusters (PaxosBackedProcessor) performs some actual validation here to ensure that the node is
+        // really a CMS member.
+        return true;
+    }
+
     @Override
     protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch)
     {

src/java/org/apache/cassandra/tcm/CMSMembership.java

@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tcm;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.cassandra.db.TypeSizes;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.locator.MetaStrategy;
+import org.apache.cassandra.locator.Replica;
+import org.apache.cassandra.tcm.membership.Directory;
+import org.apache.cassandra.tcm.membership.NodeId;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
+import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
+import org.apache.cassandra.tcm.serialization.MetadataSerializer;
+import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.utils.btree.BTreeSet;
+
+public class CMSMembership implements MetadataValue<CMSMembership>
+{
+    public static final Serializer serializer = new Serializer();
+    public static final CMSMembership EMPTY = new CMSMembership();
+
+    private final Epoch lastModified;
+    private final BTreeSet<NodeId> fullMembers;
+    private final BTreeSet<NodeId> joiningMembers;
+
+    /**
+     * Used to derive a CMSMembership when deserializing a ClusterMetadata instance written with a metadata version
+     * prior to V9. At that time, CMS membership was always inferred from the data placements of the distributed
+     * cluster metadata keyspace. Read replicas are full members of the CMS and write-only replicas are in the process
+     * of joining. Note: every read replica must also be a write replica, leaving the CMS is atomic in respect of the
+     * placements.
+     * @param placement
+     * @param directory
+     * @return
+     */
+    public static CMSMembership reconstruct(DataPlacement placement, Directory directory)
+    {
+        BTreeSet.Builder<NodeId> fullMembersBuilder = BTreeSet.builder(NodeId::compareTo);
+        BTreeSet.Builder<NodeId> joiningMembersBuilder = BTreeSet.builder(NodeId::compareTo);
+        Epoch lm = Epoch.EMPTY;
+        for (VersionedEndpoints.ForRange endpoints : placement.reads.endpoints)
+        {
+            lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm;
+            endpoints.get().endpoints().forEach(e -> fullMembersBuilder.add(directory.peerId(e)));
+        }
+        BTreeSet<NodeId> full = fullMembersBuilder.build();
+
+        for (VersionedEndpoints.ForRange endpoints : placement.writes.endpoints)
+        {
+            lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm;
+            endpoints.get().endpoints().forEach(e -> {
+                NodeId id = directory.peerId(e);
+                if (!full.contains(id))
+                    joiningMembersBuilder.add(id);
+            });
+        }
+        BTreeSet<NodeId> joining = joiningMembersBuilder.build();
+
+        return new CMSMembership(lm, full, joining);
+    }
+
+    public DataPlacement toPlacement(Directory directory)
+    {
+        DataPlacement.Builder builder = DataPlacement.builder();
+        for (NodeId id : fullMembers)
+        {
+            Replica replica = MetaStrategy.replica(directory.endpoint(id));
+            builder.withReadReplica(lastModified, replica);
+            builder.withWriteReplica(lastModified, replica);
+        }
+        for(NodeId id : joiningMembers)
+        {
+            builder.withWriteReplica(lastModified, MetaStrategy.replica(directory.endpoint(id)));
+        }
+        return builder.build();
+    }
+
+    private CMSMembership()
+    {
+        this(Epoch.EMPTY,
+             BTreeSet.empty(NodeId::compareTo),
+             BTreeSet.empty(NodeId::compareTo));
+    }
+
+    private CMSMembership(Epoch lastModified, BTreeSet<NodeId> fullMembers, BTreeSet<NodeId> joiningMembers)
+    {
+        this.lastModified = lastModified;
+        this.fullMembers = fullMembers;
+        this.joiningMembers = joiningMembers;
+    }
+
+
+    @Override
+    public CMSMembership withLastModified(Epoch epoch)
+    {
+        return lastModified.is(epoch) ? this : new CMSMembership(epoch, fullMembers, joiningMembers);
+    }
+
+    @Override
+    public Epoch lastModified()
+    {
+        return lastModified;
+    }
+
+    public Set<NodeId> joiningMembers()
+    {
+        return joiningMembers;
+    }
+
+    public Set<NodeId> fullMembers()
+    {
+        return fullMembers;
+    }
+
+    public CMSMembership startJoining(NodeId id)
+    {
+        if (joiningMembers.contains(id))
+            throw new IllegalStateException(id + " is already joining the CMS");
+        if (fullMembers.contains(id))
+            throw new IllegalStateException(id + " has already fully joined the CMS");
+
+        return new CMSMembership(lastModified, fullMembers, joiningMembers.with(id));
+    }
+
+    public CMSMembership cancelJoining(NodeId id)
+    {
+        if (!joiningMembers.contains(id))
+            throw new IllegalStateException(id + " is not currently joining the CMS");
+        if (fullMembers.contains(id))
+            throw new IllegalStateException(id + " has already fully joined the CMS");
+
+        return new CMSMembership(lastModified, fullMembers, joiningMembers.without(id));
+    }
+
+    public CMSMembership finishJoining(NodeId id)
+    {
+        if (!joiningMembers.contains(id))
+            throw new IllegalStateException(id + " is not currently joining the CMS");
+        if (fullMembers.contains(id))
+            throw new IllegalStateException(id + " has already fully joined the CMS");
+
+        return new CMSMembership(lastModified, fullMembers.with(id), joiningMembers.without(id));
+    }
+
+    public CMSMembership leave(NodeId id)
+    {
+        if (joiningMembers.contains(id))
+            throw new IllegalStateException(id + " is currently joining the CMS, ");
+        if (!fullMembers.contains(id))
+            throw new IllegalStateException(id + " is not a CMS member");
+
+        return new CMSMembership(lastModified, fullMembers.without(id), joiningMembers);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "CMSMembership{" +
+               "lastModified=" + lastModified +
+               ", fullMembers=" + fullMembers +
+               ", joiningMembers=" + joiningMembers +
+               '}';
+    }
+
+    @Override
+    public final boolean equals(Object o)
+    {
+        if (!(o instanceof CMSMembership)) return false;
+
+        CMSMembership that = (CMSMembership) o;
+        return Objects.equals(lastModified, that.lastModified) &&
+               Objects.equals(fullMembers, that.fullMembers) &&
+               Objects.equals(joiningMembers, that.joiningMembers);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int result = Objects.hashCode(lastModified);
+        result = 31 * result + Objects.hashCode(fullMembers);
+        result = 31 * result + Objects.hashCode(joiningMembers);
+        return result;
+    }
+
+    public static class Serializer implements MetadataSerializer<CMSMembership>
+    {
+        @Override
+        public void serialize(CMSMembership t, DataOutputPlus out, Version version) throws IOException
+        {
+            Epoch.serializer.serialize(t.lastModified, out);
+
+            out.writeUnsignedVInt32(t.fullMembers.size());
+            for (NodeId id : t.fullMembers)
+                NodeId.serializer.serialize(id, out, version);
+
+            out.writeUnsignedVInt32(t.joiningMembers.size());
+            for (NodeId id : t.joiningMembers)
+                NodeId.serializer.serialize(id, out, version);
+        }
+
+        @Override
+        public CMSMembership deserialize(DataInputPlus in, Version version) throws IOException
+        {
+            Epoch lastModified = Epoch.serializer.deserialize(in, version);
+
+            int fullMemberCount = in.readUnsignedVInt32();
+            BTreeSet.Builder<NodeId> fullMembers = BTreeSet.builder(NodeId::compareTo);
+            for (int i = 0; i < fullMemberCount; i++)
+                fullMembers.add(NodeId.serializer.deserialize(in, version));
+
+            int joiningMemberCount = in.readUnsignedVInt32();
+            BTreeSet.Builder<NodeId> joiningMembers = BTreeSet.builder(NodeId::compareTo);
+            for (int i = 0; i < joiningMemberCount; i++)
+                joiningMembers.add(NodeId.serializer.deserialize(in, version));
+
+            return new CMSMembership(lastModified, fullMembers.build(), joiningMembers.build()) ;
+        }
+
+        @Override
+        public long serializedSize(CMSMembership t, Version version)
+        {
+            long size = Epoch.serializer.serializedSize(t.lastModified);
+
+            size += TypeSizes.sizeofUnsignedVInt(t.fullMembers.size());
+            for (NodeId id : t.fullMembers)
+                size += NodeId.serializer.serializedSize(id, version);
+
+            size += TypeSizes.sizeofUnsignedVInt(t.joiningMembers.size());
+            for (NodeId id : t.joiningMembers)
+                size += NodeId.serializer.serializedSize(id, version);
+
+            return size;
+        }
+    }
+
+}

src/java/org/apache/cassandra/tcm/CMSOperations.java

@@ -48,7 +48,6 @@ import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
 import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.tcm.transformations.Unregister;
 import org.apache.cassandra.tcm.transformations.cms.AdvanceCMSReconfiguration;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.MBeanWrapper;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -217,7 +216,7 @@ public class CMSOperations implements CMSOperationsMBean
         String members = metadata.fullCMSMembers().stream().sorted().map(Object::toString).collect(Collectors.joining(","));
         info.put(MEMBERS, members);
         info.put(NEEDS_RECONFIGURATION, Boolean.toString(metadata.epoch.isBefore(Epoch.FIRST) || needsReconfiguration(metadata)));
-        info.put(IS_MEMBER, Boolean.toString(cms.isCurrentMember(FBUtilities.getBroadcastAddressAndPort())));
+        info.put(IS_MEMBER, Boolean.toString(metadata.isCMSMember()));
         info.put(SERVICE_STATE, ClusterMetadataService.state(metadata).toString());
         info.put(IS_MIGRATING, Boolean.toString(cms.isMigrating()));
         info.put(EPOCH, Long.toString(metadata.epoch.getEpoch()));

src/java/org/apache/cassandra/tcm/ClusterMetadata.java

@@ -28,7 +28,9 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
@@ -77,6 +79,7 @@ import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.tcm.membership.NodeVersion;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator;
 import org.apache.cassandra.tcm.ownership.ReplicaGroups;
@@ -89,7 +92,6 @@ import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
 import static org.apache.cassandra.db.TypeSizes.sizeof;
 import static org.apache.cassandra.tcm.serialization.Version.MIN_ACCORD_VERSION;
@@ -107,13 +109,14 @@ public class ClusterMetadata
     public final DistributedSchema schema;
     public final Directory directory;
     public final TokenMap tokenMap;
-    public final DataPlacements placements;
+    private final DataPlacements placements;
     public final AccordFastPath accordFastPath;
     public final LockedRanges lockedRanges;
     public final InProgressSequences inProgressSequences;
     public final ConsensusMigrationState consensusMigrationState;
     public final ImmutableMap<ExtensionKey<?,?>, ExtensionValue<?>> extensions;
     public final AccordStaleReplicas accordStaleReplicas;
+    public final CMSMembership cmsMembership;
 
     // This isn't serialized as part of ClusterMetadata it's really just a view over the Directory.
     public final Locator locator;
@@ -121,9 +124,10 @@ public class ClusterMetadata
     // These fields are lazy but only for the test purposes, since their computation requires initialization of the log ks
     private EndpointsForRange fullCMSReplicas;
     private Set<InetAddressAndPort> fullCMSEndpoints;
-    private Set<NodeId> fullCMSIds;
     private volatile Map<ReplicationParams, RangesAtEndpoint> localRangesAllSettled = null;
     private static final RangesAtEndpoint EMPTY_LOCAL_RANGES = RangesAtEndpoint.empty(FBUtilities.getBroadcastAddressAndPort());
+    private DataPlacement cmsDataPlacement;
+    private final NodeId localNodeId;
 
     public ClusterMetadata(IPartitioner partitioner)
     {
@@ -151,7 +155,8 @@ public class ClusterMetadata
              InProgressSequences.EMPTY,
              ConsensusMigrationState.EMPTY,
              ImmutableMap.of(),
-             AccordStaleReplicas.EMPTY);
+             AccordStaleReplicas.EMPTY,
+             CMSMembership.EMPTY);
     }
 
     public ClusterMetadata(Epoch epoch,
@@ -165,7 +170,8 @@ public class ClusterMetadata
                            InProgressSequences inProgressSequences,
                            ConsensusMigrationState consensusMigrationState,
                            Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions,
-                           AccordStaleReplicas accordStaleReplicas)
+                           AccordStaleReplicas accordStaleReplicas,
+                           CMSMembership cmsMembership)
     {
         this(EMPTY_METADATA_IDENTIFIER,
              epoch,
@@ -179,9 +185,11 @@ public class ClusterMetadata
              inProgressSequences,
              consensusMigrationState,
              extensions,
-             accordStaleReplicas);
+             accordStaleReplicas,
+             cmsMembership);
     }
 
+
     private ClusterMetadata(int metadataIdentifier,
                             Epoch epoch,
                             IPartitioner partitioner,
@@ -194,7 +202,8 @@ public class ClusterMetadata
                             InProgressSequences inProgressSequences,
                             ConsensusMigrationState consensusMigrationState,
                             Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions,
-                           AccordStaleReplicas accordStaleReplicas)
+                            AccordStaleReplicas accordStaleReplicas,
+                            CMSMembership cmsMembership)
     {
         // TODO: token map is a feature of the specific placement strategy, and so may not be a relevant component of
         //  ClusterMetadata in the long term. We need to consider how the actual components of metadata can be evolved
@@ -206,14 +215,42 @@ public class ClusterMetadata
         this.schema = schema;
         this.directory = directory;
         this.tokenMap = tokenMap;
-        this.placements = placements;
         this.accordFastPath = accordFastPath;
+        this.placements = placements;
         this.lockedRanges = lockedRanges;
         this.inProgressSequences = inProgressSequences;
         this.consensusMigrationState = consensusMigrationState;
         this.extensions = ImmutableMap.copyOf(extensions);
         this.locator = Locator.usingDirectory(directory);
         this.accordStaleReplicas = accordStaleReplicas;
+        this.cmsMembership = cmsMembership;
+        this.cmsDataPlacement = calculateCMSPlacement(placements, cmsMembership);
+
+        InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
+        this.localNodeId = directory.allAddresses().contains(broadcastAddress)
+                           ? directory.peerId(broadcastAddress)
+                           : NodeId.UNREGISTERED;
+    }
+
+    public Set<NodeId> fullCMSMemberIds()
+    {
+        return cmsMembership.fullMembers();
+    }
+
+    public boolean isCMSMember()
+    {
+        if (epoch.isEqualOrBefore(Epoch.FIRST))
+            return isCMSMember(FBUtilities.getBroadcastAddressAndPort());
+
+        return fullCMSMemberIds().contains(localNodeId);
+    }
+
+    public boolean isCMSMember(InetAddressAndPort endpoint)
+    {
+        if (epoch.isEqualOrBefore(Epoch.FIRST))
+            return cmsDataPlacement.reads.byEndpoint().keySet().contains(endpoint);
+
+        return fullCMSMembers().contains(endpoint);
     }
 
     public Set<InetAddressAndPort> fullCMSMembers()
@@ -223,27 +260,14 @@ public class ClusterMetadata
 
         if (fullCMSEndpoints == null)
         {
-            if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
+            fullCMSEndpoints = ImmutableSet.copyOf(cmsMembership.fullMembers()
-                return Collections.emptySet();
+                                                                .stream()
-            this.fullCMSEndpoints = ImmutableSet.copyOf(placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet());
+                                                                .map(directory::endpoint)
+                                                                .collect(Collectors.toSet()));
         }
         return fullCMSEndpoints;
     }
 
-    public Set<NodeId> fullCMSMemberIds()
-    {
-        if (epoch.isBefore(Epoch.FIRST))
-            return Collections.emptySet();
-
-        if (fullCMSIds == null)
-        {
-            if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
-                return Collections.emptySet();
-            this.fullCMSIds = placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet().stream().map(directory::peerId).collect(toImmutableSet());
-        }
-        return fullCMSIds;
-    }
-
     public EndpointsForRange fullCMSMembersAsReplicas()
     {
         if (epoch.isBefore(Epoch.FIRST))
@@ -251,16 +275,70 @@ public class ClusterMetadata
 
         if (fullCMSReplicas == null)
         {
-            if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
+            EndpointsForRange.Builder builder = EndpointsForRange.builder(MetaStrategy.entireRange);
-                return EndpointsForRange.empty(MetaStrategy.entireRange);
+            for (NodeId nodeId : fullCMSMemberIds())
-            fullCMSReplicas = placements.get(ReplicationParams.meta(this)).reads.forRange(MetaStrategy.entireRange).get();
+                builder.add(MetaStrategy.replica(directory.endpoint(nodeId)));
+            fullCMSReplicas = builder.build();
         }
         return fullCMSReplicas;
     }
 
-    public boolean isCMSMember(InetAddressAndPort endpoint)
+    private DataPlacement calculateCMSPlacement(DataPlacements placements, CMSMembership cms)
     {
-        return fullCMSMembers().contains(endpoint);
+        if (epoch.isBefore(Epoch.FIRST) || schema.getKeyspaces().get(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
+            return DataPlacement.empty();
+
+        if (directory.isEmpty())
+        {
+            if (epoch.is(Epoch.FIRST))
+            {
+                // PRE_INITIALIZE_CMS: placements need to be hardcoded to the local address so that the subsequent
+                // INITIALIZE_CMS can be committed
+                Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort());
+                return DataPlacement.builder()
+                                    .withReadReplica(Epoch.FIRST, localReplica)
+                                    .withWriteReplica(Epoch.FIRST, localReplica)
+                                    .build();
+            }
+            else
+            {
+                // This cluster did not previously upgrade from a gossip based version (i.e. pre-6.0) but did at some point
+                // run a version prior to MetadataVersion.V9 where we started to encode CMS membership directly. This
+                // condition implies that we are reconstructing a serialized cluster metadata during replay or else the
+                // directory should not be empty after Epoch.FIRST as the base state in INITIALIZE_CMS now includes the
+                // first CMS node. Similarly, if the cluster had previously been running a gossip-based version, the
+                // directory would contain entries for each of the live nodes at the time of upgrade.
+                // Given this state, the very next transformation that is/was applied will be to register the node that
+                // committed the PRE_INITIALIZE_CMS and INTIALIZE_CMS transformations. So we just extract the placements
+                // associated with the MetaStrategy params as they will already contain that node as an endpoint.
+                for (ReplicationParams params : placements.keys())
+                    if (params.isMeta())
+                        return placements.get(params);
+
+                // This point can only be reached in tests.
+                // TODO enforce that invariant somehow
+                Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort());
+                return DataPlacement.builder()
+                                    .withReadReplica(epoch, localReplica)
+                                    .withWriteReplica(epoch, localReplica)
+                                    .build();
+            }
+        }
+        else
+        {
+            // Build a placement based on the CMS membership
+            return cms.toPlacement(directory);
+        }
+    }
+
+    public DataPlacement placement(ReplicationParams params)
+    {
+        return params.isMeta() ? cmsDataPlacement : placements.get(params);
+    }
+
+    public DataPlacements placements()
+    {
+        return placements;
     }
 
     public Transformer transformer()
@@ -291,10 +369,42 @@ public class ClusterMetadata
                                    capLastModified(inProgressSequences, epoch),
                                    capLastModified(consensusMigrationState, epoch),
                                    capLastModified(extensions, epoch),
-                                   capLastModified(accordStaleReplicas, epoch));
+                                   capLastModified(accordStaleReplicas, epoch),
+                                   capLastModified(cmsMembership, epoch));
     }
 
-    public ClusterMetadata initializeClusterIdentifier(int clusterIdentifier)
+    /**
+     * To be used only during the execute of PRE_INITIALIZE_CMS, this sets the DataPlacement for the metadata keyspace
+     * so that the global log can be initialized and the subsequent entry containing the INITIALIZE_CMS transformation
+     * can be committed.
+     * @param initialCMSPlacement Expected to be a singleton placement identifying the local node as the sole replica
+     *                            for the metadata keyspace
+     * @return ClusterMetadata instance in the correct state to constitute the result of the PRE_INITIALIZE_CMS
+     *         transformation
+     */
+    public ClusterMetadata forcePreInitializedState(DataPlacement initialCMSPlacement)
+    {
+        assert epoch.isEqualOrBefore(Epoch.FIRST);
+        // double check that all metadata elements have had the last modified epoch set correctly
+        ClusterMetadata initial = forceEpoch(Epoch.FIRST);
+        initial.cmsDataPlacement = initialCMSPlacement;
+        return initial;
+    }
+
+    /**
+     * Produce a ClusterMetadata suitable for use as the base state in the INITIALIZE_CMS transformation. This should
+     * only be used on the first CMS node when bootstrapping the CMS after upgrade or in a brand new cluster.
+     * @param clusterIdentifier Unique identifier for split brain detection & protection
+     * @param addresses The NodeAddresses of the first CMS node.
+     * @param version Version info for the first CMS node.
+     * @param location The rack & DC of the first CMS node.
+     * @return ClusterMetadata instance in the correct state to constitute the base state of the INITIALIZE_CMS
+     *         transformation
+     */
+    public ClusterMetadata forceInitializedState(int clusterIdentifier,
+                                                 NodeAddresses addresses,
+                                                 NodeVersion version,
+                                                 Location location)
     {
         if (this.metadataIdentifier != EMPTY_METADATA_IDENTIFIER)
             throw new IllegalStateException(String.format("Can only initialize cluster identifier once, but it was already set to %d", this.metadataIdentifier));
@@ -302,11 +412,21 @@ public class ClusterMetadata
         if (clusterIdentifier == EMPTY_METADATA_IDENTIFIER)
             throw new IllegalArgumentException("Can not initialize cluster with empty cluster identifier");
 
+        if (this.epoch.isAfter(Epoch.FIRST))
+            throw new IllegalStateException(String.format("Can only initialize cluster identifier during epoch %d, but current epoch is %d", Epoch.FIRST.getEpoch(), epoch.getEpoch()));
+
+        // Maybe register the first CMS node. If upgrading from gossip, this should be a no-op
+        Directory withRegistered = directory.with(addresses, location, version);
+        NodeId firstNode = withRegistered.peerId(addresses.broadcastAddress);
+        if (firstNode == null)
+            throw new IllegalStateException("Failed to find first CMS node in directory");
+
+        CMSMembership initialCMS = cmsMembership.startJoining(firstNode).finishJoining(firstNode);
         return new ClusterMetadata(clusterIdentifier,
                                    epoch,
                                    partitioner,
                                    schema,
-                                   directory,
+                                   withRegistered,
                                    tokenMap,
                                    placements,
                                    accordFastPath,
@@ -314,7 +434,8 @@ public class ClusterMetadata
                                    inProgressSequences,
                                    consensusMigrationState,
                                    extensions,
-                                   accordStaleReplicas);
+                                   accordStaleReplicas,
+                                   initialCMS);
     }
 
     private static Map<ExtensionKey<?,?>, ExtensionValue<?>> capLastModified(Map<ExtensionKey<?,?>, ExtensionValue<?>> original, Epoch maxEpoch)
@@ -413,8 +534,8 @@ public class ClusterMetadata
     // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges
     public boolean hasPendingRangesFor(KeyspaceMetadata ksm, Token token)
     {
-        ReplicaGroups writes = placements.get(ksm.params.replication).writes;
+        ReplicaGroups writes = placement(ksm.params.replication).writes;
-        ReplicaGroups reads = placements.get(ksm.params.replication).reads;
+        ReplicaGroups reads = placement(ksm.params.replication).reads;
         if (ksm.params.replication.isMeta())
             return !reads.equals(writes);
         return !reads.forToken(token).equals(writes.forToken(token));
@@ -423,8 +544,8 @@ public class ClusterMetadata
     // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges
     public boolean hasPendingRangesFor(KeyspaceMetadata ksm, InetAddressAndPort endpoint)
     {
-        ReplicaGroups writes = placements.get(ksm.params.replication).writes;
+        ReplicaGroups writes = placement(ksm.params.replication).writes;
-        ReplicaGroups reads = placements.get(ksm.params.replication).reads;
+        ReplicaGroups reads = placement(ksm.params.replication).reads;
         return !writes.byEndpoint().get(endpoint).equals(reads.byEndpoint().get(endpoint));
     }
 
@@ -435,22 +556,22 @@ public class ClusterMetadata
 
     public RangesAtEndpoint writeRanges(KeyspaceMetadata metadata, InetAddressAndPort peer)
     {
-        return placements.get(metadata.params.replication).writes.byEndpoint().get(peer);
+        return placement(metadata.params.replication).writes.byEndpoint().get(peer);
     }
 
     // TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges
     public Map<Range<Token>, VersionedEndpoints.ForRange> pendingRanges(KeyspaceMetadata metadata)
     {
         Map<Range<Token>, VersionedEndpoints.ForRange> map = new HashMap<>();
-        ReplicaGroups writes = placements.get(metadata.params.replication).writes;
+        ReplicaGroups writes = placement(metadata.params.replication).writes;
-        ReplicaGroups reads = placements.get(metadata.params.replication).reads;
+        ReplicaGroups reads = placement(metadata.params.replication).reads;
 
         // first, pending ranges as the result of range splitting or merging
         // i.e. new ranges being created through join/leave
         List<Range<Token>> pending = new ArrayList<>(writes.ranges());
         pending.removeAll(reads.ranges());
         for (Range<Token> p : pending)
-            map.put(p, placements.get(metadata.params.replication).writes.forRange(p));
+            map.put(p, placement(metadata.params.replication).writes.forRange(p));
 
         // next, ranges where the ranges themselves are not changing, but the replicas are
         // i.e. replacement or RF increase
@@ -467,8 +588,8 @@ public class ClusterMetadata
     // TODO Remove this as it isn't really an equivalent to the previous concept of pending endpoints
     public VersionedEndpoints.ForToken pendingEndpointsFor(KeyspaceMetadata metadata, Token t)
     {
-        VersionedEndpoints.ForToken writeEndpoints = placements.get(metadata.params.replication).writes.forToken(t);
+        VersionedEndpoints.ForToken writeEndpoints = placement(metadata.params.replication).writes.forToken(t);
-        VersionedEndpoints.ForToken readEndpoints = placements.get(metadata.params.replication).reads.forToken(t);
+        VersionedEndpoints.ForToken readEndpoints = placement(metadata.params.replication).reads.forToken(t);
         EndpointsForToken.Builder endpointsForToken = writeEndpoints.get().newBuilder(writeEndpoints.size() - readEndpoints.size());
 
         for (Replica writeReplica : writeEndpoints.get())
@@ -495,6 +616,7 @@ public class ClusterMetadata
         private final Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions;
         private final Set<MetadataKey> modifiedKeys;
         private AccordStaleReplicas accordStaleReplicas;
+        private CMSMembership cmsMembership;
 
         private Transformer(ClusterMetadata metadata, Epoch epoch)
         {
@@ -512,6 +634,7 @@ public class ClusterMetadata
             extensions = new HashMap<>(metadata.extensions);
             modifiedKeys = new HashSet<>();
             accordStaleReplicas = metadata.accordStaleReplicas;
+            cmsMembership = metadata.cmsMembership;
         }
 
         public Epoch epoch()
@@ -637,6 +760,30 @@ public class ClusterMetadata
             return this;
         }
 
+        public Transformer startJoiningCMS(NodeId id)
+        {
+            cmsMembership = cmsMembership.startJoining(id);
+            return this;
+        }
+
+        public Transformer finishJoiningCMS(NodeId id)
+        {
+            cmsMembership = cmsMembership.finishJoining(id);
+            return this;
+        }
+
+        public Transformer cancelJoiningCMS(NodeId id)
+        {
+            cmsMembership = cmsMembership.cancelJoining(id);
+            return this;
+        }
+
+        public Transformer leaveCMS(NodeId id)
+        {
+            cmsMembership = cmsMembership.leave(id);
+            return this;
+        }
+
         public Transformer with(DataPlacements placements)
         {
             this.placements = placements;
@@ -822,6 +969,12 @@ public class ClusterMetadata
                 consensusMigrationState.validateAgainstSchema(schema);
             }
 
+            if (cmsMembership != base.cmsMembership)
+            {
+                modifiedKeys.add(MetadataKeys.CMS_MEMBERSHIP);
+                cmsMembership = cmsMembership.withLastModified(epoch);
+            }
+
             return new Transformed(new ClusterMetadata(base.metadataIdentifier,
                                                        epoch,
                                                        partitioner,
@@ -834,7 +987,8 @@ public class ClusterMetadata
                                                        inProgressSequences,
                                                        consensusMigrationState,
                                                        extensions,
-                                                       accordStaleReplicas),
+                                                       accordStaleReplicas,
+                                                       cmsMembership),
                                    ImmutableSet.copyOf(modifiedKeys));
         }
 
@@ -852,7 +1006,8 @@ public class ClusterMetadata
                                        inProgressSequences,
                                        consensusMigrationState,
                                        extensions,
-                    accordStaleReplicas);
+                                       accordStaleReplicas,
+                                       cmsMembership);
         }
 
         @Override
@@ -871,6 +1026,7 @@ public class ClusterMetadata
                    ", inProgressSequences=" + inProgressSequences +
                    ", consensusMigrationState=" + consensusMigrationState +
                    ", extensions=" + extensions +
+                   ", cmsMembership=" + cmsMembership +
                    ", modifiedKeys=" + modifiedKeys +
                    '}';
         }
@@ -966,6 +1122,9 @@ public class ClusterMetadata
                ", placements=" + placements +
                ", lockedRanges=" + lockedRanges +
                ", consensusMigrationState=" + lockedRanges +
+               ", inProgressSequences=" + inProgressSequences +
+               ", extensions=" + extensions +
+               ", cmsMembership=" + cmsMembership +
                '}';
     }
 
@@ -994,7 +1153,8 @@ public class ClusterMetadata
                inProgressSequences.equals(that.inProgressSequences) &&
                consensusMigrationState.equals(that.consensusMigrationState) &&
                accordStaleReplicas.equals(that.accordStaleReplicas) &&
-               extensions.equals(that.extensions);
+               extensions.equals(that.extensions) &&
+               cmsMembership.equals(that.cmsMembership);
     }
 
     private static final Logger logger = LoggerFactory.getLogger(ClusterMetadata.class);
@@ -1037,12 +1197,16 @@ public class ClusterMetadata
         {
             logger.warn("Extensions differ: {} != {}", extensions, other.extensions);
         }
+        if (!cmsMembership.equals(other.cmsMembership))
+        {
+            logger.warn("CMS Membership differ: {} != {}", cmsMembership, other.cmsMembership);
+        }
     }
 
     @Override
     public int hashCode()
     {
-        return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions);
+        return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions, cmsMembership);
     }
 
     public static ClusterMetadata current()
@@ -1083,13 +1247,13 @@ public class ClusterMetadata
 
     public NodeId myNodeId()
     {
-        return directory.peerId(FBUtilities.getBroadcastAddressAndPort());
+        return localNodeId;
     }
 
     public NodeState myNodeState()
     {
         NodeId nodeId = myNodeId();
-        if (myNodeId() != null)
+        if (nodeId != NodeId.UNREGISTERED)
             return directory.peerState(nodeId);
         return null;
     }
@@ -1113,7 +1277,10 @@ public class ClusterMetadata
             DistributedSchema.serializer.serialize(metadata.schema, out, version);
             Directory.serializer.serialize(metadata.directory, out, version);
             TokenMap.serializer.serialize(metadata.tokenMap, out, version);
-            DataPlacements.serializer.serialize(metadata.placements, out, version);
+            // Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements
+            // so when targetting such a version, emulate that.
+            DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements;
+            DataPlacements.serializer.serialize(placements, out, version);
             if (version.isAtLeast(MIN_ACCORD_VERSION))
             {
                 AccordFastPath.serializer.serialize(metadata.accordFastPath, out, version);
@@ -1132,6 +1299,9 @@ public class ClusterMetadata
                 assert key.valueType.isInstance(value);
                 value.serialize(out, version);
             }
+            // From V9 CMS membership is directly encoded in ClusterMetadata
+            if (version.isAtLeast(Version.V9))
+                CMSMembership.serializer.serialize(metadata.cmsMembership, out, version);
         }
 
         @Override
@@ -1188,6 +1358,42 @@ public class ClusterMetadata
                 value.deserialize(in, version);
                 extensions.put(key, value);
             }
+
+            CMSMembership cmsMembership = CMSMembership.EMPTY;
+            if (version.isAtLeast(Version.V9))
+                cmsMembership = CMSMembership.serializer.deserialize(in, version);
+            else
+            {
+                Optional<KeyspaceMetadata> metadataKs = schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
+                if (metadataKs.isPresent())
+                {
+                    // Pre-V9 the membership of the CMS was always inferred from the placement of the distributed
+                    // metadata keyspace.
+                    // If the directory is not empty the endpoints in the placement must belong to registered nodes,
+                    // so we can derive the CMSMembership using the data placement and directory.
+
+                    // If the directory is empty, then the cluster metadata must be the payload of an INITIALIZE_CMS
+                    // transformation of a cluster that began on a post-6.0, pre-MetadataVersion.V9 version.
+                    // In this case, we can and must assume that the initial CMS membership will consist of a single
+                    // node, with the node_id 1.
+                    // Note: the only route to arrive at this scenario is if a cluster is initialized on a post-6.0,
+                    // pre-V9 version and then upgraded to a post-V9 version without any metadata snapshots being taken.
+                    // If there is a snapshot available locally, when the upgraded node starts up it will replay its
+                    // local log from that point. The INITIALIZE_CMS transform will not be replayed.
+                    if (!dir.isEmpty())
+                    {
+                        DataPlacement placement = placements.get(metadataKs.get().params.replication);
+                        cmsMembership = CMSMembership.reconstruct(placement, dir);
+                    }
+                    else
+                    {
+                        NodeId id = new NodeId(1);
+                        cmsMembership = CMSMembership.EMPTY.startJoining(id).finishJoining(id);
+                    }
+                    placements = placements.unbuild().without(metadataKs.get().params.replication).build();
+                }
+            }
+
             return new ClusterMetadata(clusterIdentifier,
                                        epoch,
                                        partitioner,
@@ -1200,7 +1406,8 @@ public class ClusterMetadata
                                        ips,
                                        consensusMigrationState,
                                        extensions,
-                                       staleReplicas);
+                                       staleReplicas,
+                                       cmsMembership);
         }
 
         private DistributedSchema deduplicateReplicationParams(DistributedSchema schema, DataPlacements placements)
@@ -1235,8 +1442,7 @@ public class ClusterMetadata
                     sizeof(metadata.partitioner.getClass().getCanonicalName()) +
                     DistributedSchema.serializer.serializedSize(metadata.schema, version) +
                     Directory.serializer.serializedSize(metadata.directory, version) +
-                    TokenMap.serializer.serializedSize(metadata.tokenMap, version) +
+                    TokenMap.serializer.serializedSize(metadata.tokenMap, version);
-                    DataPlacements.serializer.serializedSize(metadata.placements, version);
 
             if (version.isAtLeast(MIN_ACCORD_VERSION))
             {
@@ -1248,9 +1454,27 @@ public class ClusterMetadata
             size += LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) +
                     InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version);
 
+            // Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements
+            // so when targetting such a version, emulate that.
+            DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements;
+            size += DataPlacements.serializer.serializedSize(placements, version);
+            // From V9 CMS membership is directly encoded in ClusterMetadata
+            if (version.isAtLeast(Version.V9))
+                size += CMSMembership.serializer.serializedSize(metadata.cmsMembership, version);
+
             return size;
         }
 
+        private DataPlacements preV9Placements(ClusterMetadata metadata)
+        {
+            if (metadata.cmsDataPlacement.isEmpty())
+                return metadata.placements;
+
+            return metadata.placements.unbuild()
+                                      .with(ReplicationParams.meta(metadata), metadata.cmsDataPlacement)
+                                      .build();
+        }
+
         public static IPartitioner getPartitioner(DataInputPlus in, Version version) throws IOException
         {
             if (version.isAtLeast(Version.V1))

src/java/org/apache/cassandra/tcm/ClusterMetadataService.java

@@ -117,7 +117,7 @@ public class ClusterMetadataService
                                             trace);
         instance = newInstance;
         RegistrationStatus.instance.onInitialized();
-        if (newInstance.metadata().myNodeId() != null)
+        if (newInstance.metadata().myNodeId() != NodeId.UNREGISTERED)
             RegistrationStatus.instance.onRegistration();
         trace = new RuntimeException("Previously initialized trace");
         DatabaseDescriptor.applyLocator();
@@ -176,8 +176,9 @@ public class ClusterMetadataService
         // The node is a full member of the CMS if it has started participating in reads for distributed metadata table (which
         // implies it is a write replica as well). In other words, it's a fully joined member of the replica set responsible for
         // the distributed metadata table.
-        if (ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()))
+        if (metadata.epoch.isEqualOrBefore(Epoch.FIRST) || metadata.isCMSMember())
             return LOCAL;
+
         return REMOTE;
     }
 
@@ -334,7 +335,8 @@ public class ClusterMetadataService
                                                     InProgressSequences.EMPTY,
                                                     ConsensusMigrationState.EMPTY,
                                                     Collections.emptyMap(),
-                                   AccordStaleReplicas.EMPTY);
+                                                    AccordStaleReplicas.EMPTY,
+                                                    CMSMembership.EMPTY);
 
 
         LocalLog.LogSpec logSpec = LocalLog.logSpec()
@@ -360,6 +362,24 @@ public class ClusterMetadataService
         ClusterMetadataService.setInstance(cms);
     }
 
+    @SuppressWarnings("resource")
+    public static void initializeForClients()
+    {
+        if (instance != null)
+            return;
+
+        ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools());
+    }
+
+    public static void initializeForClients(DistributedSchema initialSchema)
+    {
+        if (instance != null)
+            return;
+
+
+        ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema));
+    }
+
     /*
      * Hook to be executed when the LocalLog is bootstrapped with the PreInitialize transformation. This is done on
      * the first CMS member to set up the initial replication and data placements for the metadata keyspace.
@@ -391,25 +411,8 @@ public class ClusterMetadataService
             };
         }
         // otherwise, this is a noop.
-        return preInit -> {};
+        return preInit -> {
-    }
+        };
-
-    @SuppressWarnings("resource")
-    public static void initializeForClients()
-    {
-        if (instance != null)
-            return;
-
-        ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools());
-    }
-
-    public static void initializeForClients(DistributedSchema initialSchema)
-    {
-        if (instance != null)
-            return;
-
-
-        ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema));
     }
 
     public boolean isCurrentMember(InetAddressAndPort peer)
@@ -1100,6 +1103,6 @@ public class ClusterMetadataService
 
     public enum State
     {
-        LOCAL, REMOTE, GOSSIP, RESET
+        LOCAL, REMOTE, GOSSIP, RESET, OFFLINE_TOOL
     }
 }

src/java/org/apache/cassandra/tcm/MetadataKeys.java

@@ -45,18 +45,20 @@ public class MetadataKeys
     public static final MetadataKey LOCKED_RANGES           = make(CORE_NS, "sequences", "locked_ranges");
     public static final MetadataKey IN_PROGRESS_SEQUENCES   = make(CORE_NS, "sequences", "in_progress");
     public static final MetadataKey CONSENSUS_MIGRATION_STATE = make(CORE_NS, "consensus", "migration_state");
+    public static final MetadataKey CMS_MEMBERSHIP          = make(CORE_NS, "cms_membership", "cms_membership");
 
     public static final ImmutableMap<MetadataKey, Function<ClusterMetadata, MetadataValue<?>>> CORE_METADATA
     = ImmutableMap.<MetadataKey, Function<ClusterMetadata, MetadataValue<?>>>builder()
                   .put(SCHEMA, cm -> cm.schema)
                   .put(NODE_DIRECTORY, cm -> cm.directory)
                   .put(TOKEN_MAP, cm -> cm.tokenMap)
-                  .put(DATA_PLACEMENTS, cm -> cm.placements)
+                  .put(DATA_PLACEMENTS, ClusterMetadata::placements)
                   .put(LOCKED_RANGES, cm -> cm.lockedRanges)
                   .put(IN_PROGRESS_SEQUENCES, cm -> cm.inProgressSequences)
                   .put(ACCORD_FAST_PATH, cm -> cm.accordFastPath)
                   .put(ACCORD_STALE_REPLICAS, cm -> cm.accordStaleReplicas)
                   .put(CONSENSUS_MIGRATION_STATE, cm -> cm.consensusMigrationState)
+                  .put(CMS_MEMBERSHIP, cm -> cm.cmsMembership)
                   .build();
 
     public static MetadataKey make(String...parts)

src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java

@@ -63,6 +63,12 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
         super(log);
     }
 
+    @Override
+    protected boolean acceptCommit(ClusterMetadata metadata)
+    {
+        return metadata.isCMSMember();
+    }
+
     @Override
     protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch)
     {

src/java/org/apache/cassandra/tcm/Startup.java

@@ -63,8 +63,11 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tcm.log.LocalLog;
 import org.apache.cassandra.tcm.log.LogStorage;
 import org.apache.cassandra.tcm.log.SystemKeyspaceStorage;
+import org.apache.cassandra.tcm.membership.Location;
+import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeState;
+import org.apache.cassandra.tcm.membership.NodeVersion;
 import org.apache.cassandra.tcm.migration.CMSInitializationException;
 import org.apache.cassandra.tcm.migration.CMSInitializationRequest;
 import org.apache.cassandra.tcm.migration.Election;
@@ -148,19 +151,24 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
      */
     public static void initializeAsFirstCMSNode()
     {
-        InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
+        NodeAddresses addresses = NodeAddresses.current();
-        String datacenter = DatabaseDescriptor.getLocator().local().datacenter;
+        Location location = DatabaseDescriptor.getLocator().local();
         ClusterMetadataService cms = ClusterMetadataService.instance();
-        cms.log().bootstrap(addr, datacenter, cms.logBootstrapCallback());
+        cms.log().bootstrap(addresses.broadcastAddress, location.datacenter, cms.logBootstrapCallback());
-        ClusterMetadata metadata =  ClusterMetadata.current();
+        ClusterMetadata metadata =  ClusterMetadata.current().forceInitializedState(addresses.broadcastAddress.hashCode(),
-        assert ClusterMetadataService.state() == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(),  metadata);
+                                                                                    addresses,
-        Initialize initialize = new Initialize(metadata.initializeClusterIdentifier(addr.hashCode()));
+                                                                                    NodeVersion.CURRENT,
-        ClusterMetadataService.instance().commit(initialize,
+                                                                                    location);
+        assert ClusterMetadataService.state(metadata) == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(),  metadata);
+        Initialize initialize = new Initialize(metadata);
+        ClusterMetadata initialized = ClusterMetadataService.instance().commit(initialize,
                                                                                m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;},
                                                                                (code, message) -> {
                                                                                    logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message);
                                                                                    throw new CMSInitializationException();
                                                                                });
+        NodeId id = initialized.myNodeId();
+        SystemKeyspace.setLocalHostId(id.toUUID());
     }
 
     public static void initializeAsNonCmsNode(Function<Processor, Processor> wrapProcessor) throws StartupException
@@ -178,7 +186,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
 
         NodeId nodeId = ClusterMetadata.current().myNodeId();
         UUID currentHostId = SystemKeyspace.getLocalHostId();
-        if (nodeId != null && !Objects.equals(nodeId.toUUID(), currentHostId))
+        if (nodeId != NodeId.UNREGISTERED && !Objects.equals(nodeId.toUUID(), currentHostId))
         {
             if (currentHostId == null)
             {
@@ -394,7 +402,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
                                                           DatabaseDescriptor.getPartitioner().getClass().getCanonicalName(),
                                                           metadata.partitioner.getClass().getCanonicalName()));
 
-        if (!metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()))
+        if (!metadata.isCMSMember())
             throw new IllegalStateException("When reinitializing with cluster metadata, we must be in the CMS");
 
         metadata = metadata.forceEpoch(metadata.epoch.nextEpoch());

src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java

@@ -184,7 +184,8 @@ public class StubClusterMetadataService extends ClusterMetadataService
                                               InProgressSequences.EMPTY,
                                               ConsensusMigrationState.EMPTY,
                                               ImmutableMap.of(),
-                                              AccordStaleReplicas.EMPTY);
+                                              AccordStaleReplicas.EMPTY,
+                                              CMSMembership.EMPTY);
             }
             return new StubClusterMetadataService(new UniformRangePlacement(),
                                                   snapshots != null ? snapshots : MetadataSnapshots.NO_OP,

src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java

@@ -58,6 +58,7 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.service.accord.topology.AccordFastPath;
 import org.apache.cassandra.service.accord.topology.AccordStaleReplicas;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
+import org.apache.cassandra.tcm.CMSMembership;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.MultiStepOperation;
@@ -306,7 +307,8 @@ public class GossipHelper
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,
                                    Collections.emptyMap(),
-                                   AccordStaleReplicas.EMPTY);
+                                   AccordStaleReplicas.EMPTY,
+                                   CMSMembership.EMPTY);
     }
 
     public static ClusterMetadata fromEndpointStates(DistributedSchema schema, Map<InetAddressAndPort, EndpointState> epStates)
@@ -396,7 +398,8 @@ public class GossipHelper
                                                                       InProgressSequences.EMPTY,
                                                                       ConsensusMigrationState.EMPTY,
                                                                       extensions,
-                                                                      AccordStaleReplicas.EMPTY);
+                                                                      AccordStaleReplicas.EMPTY,
+                                                                      CMSMembership.EMPTY);
         DataPlacements placements = new UniformRangePlacement().calculatePlacements(Epoch.UPGRADE_GOSSIP,
                                                                                     forPlacementCalculation,
                                                                                     schema.getKeyspaces());
@@ -411,7 +414,8 @@ public class GossipHelper
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,
                                    extensions,
-                                   AccordStaleReplicas.EMPTY);
+                                   AccordStaleReplicas.EMPTY,
+                                   CMSMembership.EMPTY);
     }
 
     public static boolean isValidForClusterMetadata(Map<InetAddressAndPort, EndpointState> epstates)

src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java

@@ -82,9 +82,9 @@ public class LegacyStateListener implements ChangeListener
             }
         }
 
-        // next.myNodeId() can be null during replay (before we have registered) but if it is present and
+        // next.myNodeId() can be UNREGISTERED during replay (before we have registered) but if not and
         // there is a relevant change to the state of the local node, process that synchronously.
-        if (next.myNodeId() != null && changed.contains(next.myNodeId()))
+        if (next.myNodeId() != NodeId.UNREGISTERED && changed.contains(next.myNodeId()))
         {
             // Default is to process updates for the local node synchronously, overridable via config/hotprop
             if (DatabaseDescriptor.getLegacyStateListenerSyncLocalUpdates())

src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java

@@ -18,23 +18,41 @@
 
 package org.apache.cassandra.tcm.listeners;
 
+import com.google.common.annotations.VisibleForTesting;
+
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.tcm.ClusterMetadata;
 
 public class PlacementsChangeListener implements ChangeListener
 {
+    private final Runnable onChange;
+
+    @VisibleForTesting
+    public PlacementsChangeListener(Runnable onChange)
+    {
+        this.onChange = onChange;
+    }
+
+    public PlacementsChangeListener()
+    {
+        this(StorageService.instance::invalidateLocalRanges);
+    }
+
     @Override
     public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
     {
         if (shouldInvalidate(prev, next))
-            StorageService.instance.invalidateLocalRanges();
+            onChange.run();
     }
 
     private boolean shouldInvalidate(ClusterMetadata prev, ClusterMetadata next)
     {
-        if (!prev.placements.lastModified().equals(next.placements.lastModified()) &&
+        if (!prev.placements().lastModified().equals(next.placements().lastModified()) &&
-            !prev.placements.equivalentTo(next.placements)) // <- todo should we update lastModified if the result is the same?
+            !prev.placements().equivalentTo(next.placements())) // <- todo should we update lastModified if the result is the same?
+            return true;
+
+        if (!prev.cmsMembership.equals(next.cmsMembership))
             return true;
 
         if (prev.schema.getKeyspaces().size() != next.schema.getKeyspaces().size())

src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java

@@ -47,18 +47,24 @@ public class UpgradeMigrationListener implements ChangeListener
             logger.info("Detected upgrade from gossip mode");
             return;
         }
-        else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty()) // directory is non-empty after initialization during gossip upgrade
+        else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty())
         {
             NodeId localId = next.myNodeId();
-            if (localId != null)
+            if (localId != NodeId.UNREGISTERED)
             {
-                logger.info("Initialized CMS, updating local host id to {}", next.myNodeId());
+                // assigning the local node id is done in Epoch.FIRST in one of two scenarios:
-                SystemKeyspace.setLocalHostId(next.myNodeId().toUUID());
+                // * during an upgrade from gossip, as all pre-existing nodes will have an id assigned
-                Gossiper.instance.mergeNodeToGossip(next.myNodeId(), next);
+                // * during CMS initialization for a brand new cluster iff the local node is the first CMS member
-            }
+                logger.info("Initialized CMS, updating local host id to {}", localId);
+                SystemKeyspace.setLocalHostId(localId.toUUID());
+                if (Gossiper.instance.isEnabled())
+                {
+                    Gossiper.instance.mergeNodeToGossip(localId, next);
                     if (Gossiper.instance.getQuarantineDisabled())
                         Gossiper.instance.clearQuarantinedEndpoints();
                 }
+            }
+        }
 
         CassandraVersion prevMinVersion = prev.directory.clusterMinVersion.cassandraVersion;
         CassandraVersion minVersion = next.directory.clusterMinVersion.cassandraVersion;

src/java/org/apache/cassandra/tcm/log/LogState.java

@@ -306,7 +306,7 @@ public class LogState
                 if (metadata != null)
                 {
                     NodeId mynodeId = metadata.myNodeId();
-                    if (mynodeId != null)
+                    if (mynodeId != NodeId.UNREGISTERED)
                         SystemKeyspace.setLocalHostId(mynodeId.toUUID());
                 }
             }

src/java/org/apache/cassandra/tcm/membership/Directory.java

@@ -467,7 +467,7 @@ public class Directory implements MetadataValue<Directory>
 
     public NodeState peerState(NodeId peer)
     {
-        return states.get(peer);
+        return peer == NodeId.UNREGISTERED ? null : states.get(peer);
     }
 
     public NodeVersion version(NodeId peer)

src/java/org/apache/cassandra/tcm/membership/NodeId.java

@@ -32,6 +32,7 @@ import org.apache.cassandra.tcm.serialization.Version;
 
 public class NodeId implements Comparable<NodeId>, MultiStepOperation.SequenceKey
 {
+    public static final NodeId UNREGISTERED = new NodeId(-1);
     private final static long NODE_ID_UUID_MAGIC = 7861390860069061072L;
     public static final Serializer serializer = new Serializer();
 

src/java/org/apache/cassandra/tcm/membership/NodeVersion.java

@@ -36,7 +36,7 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
 public class NodeVersion implements Comparable<NodeVersion>
 {
     public static final Serializer serializer = new Serializer();
-    public static final Version CURRENT_METADATA_VERSION = Version.V8;
+    public static final Version CURRENT_METADATA_VERSION = Version.V9;
     public static final NodeVersion CURRENT = new NodeVersion(new CassandraVersion(FBUtilities.getReleaseVersionString()), CURRENT_METADATA_VERSION);
     private static final CassandraVersion SINCE_VERSION = CassandraVersion.CASSANDRA_5_1;
 

src/java/org/apache/cassandra/tcm/migration/Election.java

@@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.Mutation;
-import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.net.IVerbHandler;
 import org.apache.cassandra.net.Message;
@@ -54,7 +53,6 @@ import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.tcm.ownership.TokenMap;
-import org.apache.cassandra.tcm.transformations.Register;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.Pair;
 
@@ -157,9 +155,6 @@ public class Election
             initiator.compareAndSet(currentInitiator, MIGRATING))
         {
             Startup.initializeAsFirstCMSNode();
-            Register.maybeRegister();
-            SystemKeyspace.setLocalHostId(ClusterMetadata.current().myNodeId().toUUID());
-
             updateInitiator(MIGRATING, MIGRATED);
             MessageDelivery.fanoutAndWait(messaging, sendTo, Verb.TCM_NOTIFY_REQ, DistributedMetadataLogKeyspace.getLogState(Epoch.EMPTY, false));
         }

src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java

@@ -106,6 +106,11 @@ public class DataPlacement
         return EMPTY;
     }
 
+    public boolean isEmpty()
+    {
+        return reads.isEmpty() && writes.isEmpty();
+    }
+
     public static Builder builder()
     {
         return new Builder(ReplicaGroups.builder(),

src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java

@@ -46,6 +46,7 @@ import org.apache.cassandra.tcm.serialization.Version;
 import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.db.TypeSizes.sizeof;
+import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
 
 public class DataPlacements extends ReplicationMap<DataPlacement> implements MetadataValue<DataPlacements>
 {
@@ -62,21 +63,6 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
         this.lastModified = lastModified;
     }
 
-    public DataPlacements replaceParams(Epoch lastModified, ReplicationParams oldParams, ReplicationParams newParams)
-    {
-        Map<ReplicationParams, DataPlacement> newMap = Maps.newHashMapWithExpectedSize(map.size());
-        assert map.containsKey(oldParams) : String.format("Can't replace key %s, since map doesn't contain it: %s", oldParams, map);
-        for (Map.Entry<ReplicationParams, DataPlacement> e : map.entrySet())
-        {
-            if (e.getKey().equals(oldParams))
-                newMap.put(newParams, e.getValue());
-            else
-                newMap.put(e.getKey(), e.getValue());
-        }
-
-        return new DataPlacements(lastModified, newMap);
-    }
-
     protected DataPlacement defaultValue()
     {
         return DataPlacement.empty();
@@ -256,7 +242,11 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
         public void serialize(DataPlacements t, DataOutputPlus out, Version version) throws IOException
         {
             Map<ReplicationParams, DataPlacement> map = t.asMap();
+            if (version.isBefore(Version.V9))
                 out.writeInt(map.size());
+            else
+                out.writeUnsignedVInt32(map.size());
+
             for (Map.Entry<ReplicationParams, DataPlacement> entry : map.entrySet())
             {
                 ReplicationParams.serializer.serialize(entry.getKey(), out, version);
@@ -267,7 +257,7 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
 
         public DataPlacements deserialize(DataInputPlus in, Version version) throws IOException
         {
-            int size = in.readInt();
+            int size = version.isBefore(Version.V9) ? in.readInt() : in.readUnsignedVInt32();
             Map<ReplicationParams, DataPlacement> map = Maps.newHashMapWithExpectedSize(size);
             for (int i = 0; i < size; i++)
             {
@@ -280,8 +270,9 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
 
         public long serializedSize(DataPlacements t, Version version)
         {
-            long size = sizeof(t.size());
+            Map<ReplicationParams, DataPlacement> map = t.asMap();
-            for (Map.Entry<ReplicationParams, DataPlacement> entry : t.asMap().entrySet())
+            long size = version.isBefore(Version.V9) ? sizeof(map.size()) :  sizeofUnsignedVInt(map.size());
+            for (Map.Entry<ReplicationParams, DataPlacement> entry : map.entrySet())
             {
                 size += ReplicationParams.serializer.serializedSize(entry.getKey(), version);
                 size += DataPlacement.serializerFor(entry.getKey()).serializedSize(entry.getValue(), version);

src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java

@@ -329,9 +329,13 @@ public class UniformRangePlacement implements PlacementProvider
             logger.trace("Calculating data placements for {}", ksMetadata.name);
             AbstractReplicationStrategy replication = ksMetadata.replicationStrategy;
             ReplicationParams params = ksMetadata.params.replication;
-            if (params.isMeta() || params.isLocal())
+            if (params.isMeta())
             {
-                placements.put(params, metadata.placements.get(params));
+                // don't calculate meta strategy placements, these are derived from ClusterMetadata.cmsMembership
+            }
+            else if (params.isLocal())
+            {
+                placements.put(params, metadata.placement(params));
             }
             else
             {

src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java

@@ -326,7 +326,7 @@ public class BootstrapAndJoin extends MultiStepOperation<Epoch>
     @Override
     public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
     {
-        DataPlacements placements = metadata.placements;
+        DataPlacements placements = metadata.placements();
         switch (next)
         {
             // need to undo MID_JOIN and START_JOIN, then merge the ranges split by PrepareJoin
@@ -357,7 +357,7 @@ public class BootstrapAndJoin extends MultiStepOperation<Epoch>
     @VisibleForTesting
     public Pair<MovementMap, MovementMap> getMovementMaps(ClusterMetadata metadata)
     {
-        MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements, startJoin.delta());
+        MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements(), startJoin.delta());
         MovementMap strictMovementMap = toStrict(movementMap, finishJoin.delta());
         return Pair.create(movementMap, strictMovementMap);
     }

src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java

@@ -318,7 +318,7 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
     @Override
     public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
     {
-        DataPlacements placements = metadata.placements;
+        DataPlacements placements = metadata.placements();
         switch (next)
         {
             // need to undo MID_REPLACE and START_REPLACE, but PREPARE_REPLACE doesn't affect placements
@@ -355,7 +355,7 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
     private static MovementMap movementMap(InetAddressAndPort beingReplaced, PlacementDeltas startDelta)
     {
         MovementMap.Builder movementMapBuilder = MovementMap.builder();
-        DataPlacements placements = ClusterMetadata.current().placements;
+        DataPlacements placements = ClusterMetadata.current().placements();
         startDelta.forEach((params, delta) -> {
             EndpointsByReplica.Builder movements = new EndpointsByReplica.Builder();
             DataPlacement originalPlacements = placements.get(params);
@@ -451,6 +451,8 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
 
     public static void gossipStateToHibernate(ClusterMetadata metadata, NodeId nodeId)
     {
+        if (nodeId == NodeId.UNREGISTERED)
+            return;
         // order is important here, the gossiper can fire in between adding these two states.  It's ok to send TOKENS without STATUS, but *not* vice versa.
         List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<>();
         VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory;
@@ -462,6 +464,8 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
 
     public static void gossipStateToNormal(ClusterMetadata metadata, NodeId nodeId)
     {
+        if (nodeId == NodeId.UNREGISTERED)
+            return;
         List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<>();
         VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory;
         Collection<Token> tokens = metadata.tokenMap.tokens(nodeId);

src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java

@@ -25,25 +25,20 @@ import java.util.Map;
 import org.apache.cassandra.exceptions.ExceptionCode;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.DistributedSchema;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.KeyspaceParams;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.accord.topology.FastPathStrategy;
+import org.apache.cassandra.tcm.CMSMembership;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.Directory;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
-import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
-import static org.apache.cassandra.locator.MetaStrategy.entireRange;
-
 public class CancelCMSReconfiguration implements Transformation
 {
     public static final Serializer serializer = new Serializer();
@@ -66,53 +61,46 @@ public class CancelCMSReconfiguration implements Transformation
         if (reconfigureCMS == null)
             return new Rejected(ExceptionCode.INVALID, "Can not cancel reconfiguration since there does not seem to be any in-flight");
 
-        ReplicationParams metaParams = ReplicationParams.meta(prev);
         ClusterMetadata.Transformer transformer = prev.transformer();
-        DataPlacement placement = prev.placements.get(metaParams);
         // Reset any partially completed transition by removing the pending replica from the write group
         if (reconfigureCMS.next.activeTransition != null)
         {
-            InetAddressAndPort pendingEndpoint = prev.directory.endpoint(reconfigureCMS.next.activeTransition.nodeId);
+            // see what the placements for the meta keyspace will be after cancelling the active join
-            Replica pendingReplica = new Replica(pendingEndpoint, entireRange, true);
+            CMSMembership cms = prev.cmsMembership.cancelJoining(reconfigureCMS.next.activeTransition.nodeId);
-            placement = placement.unbuild()
+            if (!cms.joiningMembers().isEmpty())
-                                 .withoutWriteReplica(prev.nextEpoch(), pendingReplica)
+                return new Rejected(ExceptionCode.INVALID,
-                                 .build();
+                                    String.format("Placements will be inconsistent if this transformation is applied:" +
-        }
+                                                  "\nFull members %s\nJoining members: %s",
-        if (!placement.reads.equivalentTo(placement.writes))
+                                                  cms.fullMembers(),
-            return new Rejected(ExceptionCode.INVALID, String.format("Placements will be inconsistent if this transformation is applied:\nReads %s\nWrites: %s",
+                                                  cms.joiningMembers()));
-                                                                     placement.reads,
-                                                                     placement.writes));
 
-        // Reset the replication params for the meta keyspace based on the actual placement in case they no longer match
+            // if all is good, actually cancel the joining member
-        ReplicationParams fromPlacement = getAccurateReplication(prev.directory, placement);
+            transformer = transformer.cancelJoiningCMS(reconfigureCMS.next.activeTransition.nodeId);
+            // Recalculate the replication params for the meta keyspace based on the actual placement as it will be
+            ReplicationParams recalculated = getAccurateReplication(prev.directory, cms);
 
-        // If they no longer match, i.e. the transitions completed so far did not bring the placements into line with
+            // If they no longer match the replication params in schema, i.e. the transitions completed so far did not
-        // the configuration, remove the entry keyed by the existing configured params.
+            // bring the membership/placements into line with configuration, update schema to match what we actually have
-        DataPlacements.Builder builder = prev.placements.unbuild();
+            if (!recalculated.equals(ReplicationParams.meta(prev)))
-        if (!metaParams.equals(fromPlacement))
             {
-            builder = builder.without(metaParams);
-
-            // Also update schema with the corrected params
                 KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
-            KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple()));
+                KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites,
+                                                                                       recalculated,
+                                                                                       FastPathStrategy.simple()));
                 transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)));
             }
-
+        }
-        // finally, add the possibly corrected placement keyed by the possibly corrected params
-        builder = builder.with(fromPlacement, placement);
-        transformer = transformer.with(builder.build());
 
         return Transformation.success(transformer.with(prev.inProgressSequences.without(ReconfigureCMS.SequenceKey.instance))
                                                  .with(prev.lockedRanges.unlock(reconfigureCMS.next.lockKey)),
                                       MetaStrategy.affectedRanges(prev));
     }
 
-    private ReplicationParams getAccurateReplication(Directory directory, DataPlacement placement)
+    private ReplicationParams getAccurateReplication(Directory directory, CMSMembership membership)
     {
         Map<String, Integer> replicasPerDc = new HashMap<>();
-        placement.writes.byEndpoint().keySet().forEach(i -> {
+        membership.fullMembers().forEach(id -> {
-            String dc = directory.location(directory.peerId(i)).datacenter;
+            String dc = directory.location(id).datacenter;
             int count = replicasPerDc.getOrDefault(dc, 0);
             replicasPerDc.put(dc, ++count);
         });

src/java/org/apache/cassandra/tcm/sequences/Move.java

@@ -321,7 +321,7 @@ public class Move extends MultiStepOperation<Epoch>
                     StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION);
                     Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
                     Map<ReplicationParams, EndpointsByReplica> movementMap = movementMap(FailureDetector.instance,
-                                                                                         metadata.placements,
+                                                                                         metadata.placements(),
                                                                                          toSplitRanges,
                                                                                          startMove.delta(),
                                                                                          midMove.delta(),
@@ -430,7 +430,7 @@ public class Move extends MultiStepOperation<Epoch>
     @Override
     public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
     {
-        DataPlacements placements = metadata.placements;
+        DataPlacements placements = metadata.placements();
 
         switch (next)
         {

src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java

@@ -58,6 +58,7 @@ import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.Retry;
 import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.Location;
+import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.utils.Clock;
 import org.apache.cassandra.utils.concurrent.AsyncPromise;
 
@@ -170,8 +171,9 @@ public class ProgressBarrier
             Set<Range<Token>> ranges = e.getValue();
             for (Range<Token> range : ranges)
             {
-                EndpointsForRange writes = metadata.placements.get(params).writes.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
+                DataPlacement placement = metadata.placement(params);
-                EndpointsForRange reads = metadata.placements.get(params).reads.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
+                EndpointsForRange writes = placement.writes.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
+                EndpointsForRange reads = placement.reads.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
                 // Affected ranges can contain ranges which are the results of merging or splitting and may not exist
                 // as keys in the existing ReplicaGroups. As such, no replicas will be found for these ranges and so no
                 // WaitFor is necessary.

src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java

@@ -121,7 +121,7 @@ public class RemoveNodeStreams implements LeaveStreams
             RangesByEndpoint startWriteAdditions = startDelta.get(params).writes.additions;
             RangesByEndpoint startWriteRemovals = startDelta.get(params).writes.removals;
             // find current placements from the metadata, we need to stream from replicas that are not changed and are therefore not in the deltas
-            ReplicaGroups currentPlacements = metadata.placements.get(params).reads;
+            ReplicaGroups currentPlacements = metadata.placement(params).reads;
             startWriteAdditions.flattenValues()
                                .forEach(newReplica -> {
                                    EndpointsForRange.Builder candidateBuilder = new EndpointsForRange.Builder(newReplica.range());

src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java

@@ -44,7 +44,7 @@ public class ReplaceSameAddress
     {
         MovementMap.Builder builder = MovementMap.builder();
         InetAddressAndPort addr = metadata.directory.endpoint(nodeId);
-        metadata.placements.forEach((params, placement) -> {
+        metadata.placements().forEach((params, placement) -> {
             EndpointsByReplica.Builder sources = new EndpointsByReplica.Builder();
             placement.reads.byEndpoint().get(addr).forEach(destination -> {
                 placement.reads.forRange(destination.range()).forEach(potentialSource -> {

src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java

@@ -253,7 +253,7 @@ public class UnbootstrapAndLeave extends MultiStepOperation<Epoch>
     @Override
     public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
     {
-        DataPlacements placements = metadata.placements;
+        DataPlacements placements = metadata.placements();
         switch (next)
         {
             // need to undo MID_LEAVE and START_LEAVE, but PrepareLeave doesn't affect placement

src/java/org/apache/cassandra/tcm/serialization/Version.java

@@ -78,6 +78,11 @@ public enum Version
      *  - Comments and security labels for schema elements (keyspaces, tables, columns, UDTs, and UDT fields)
      */
     V8(8),
+    /**
+     * - DataPlacements don't include MetaStrategy, replaced by ClusterMetadata.CMSMembership
+     * - Size of DataPlacements is encoded as vint
+     */
+    V9(9),
 
     UNKNOWN(Integer.MAX_VALUE);
 
@@ -95,6 +100,7 @@ public enum Version
     }
 
     private final int version;
+
     Version(int version)
     {
         this.version = version;

src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java

@@ -78,7 +78,7 @@ public class AccordMarkStale implements Transformation
 
         for (KeyspaceMetadata keyspace : prev.schema.getKeyspaces().without(SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES))
         {
-            List<AccordTopology.KeyspaceShard> shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements, prev.directory);
+            List<AccordTopology.KeyspaceShard> shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements(), prev.directory);
             
             for (AccordTopology.KeyspaceShard shard : shards)
             {

src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java

@@ -244,7 +244,7 @@ public class AlterSchema implements Transformation
 
             DataPlacements.Builder newPlacementsBuilder = DataPlacements.builder(calculatedPlacements.size());
             calculatedPlacements.forEach((params, newPlacement) -> {
-                DataPlacement previousPlacement = prev.placements.get(params);
+                DataPlacement previousPlacement = prev.placement(params);
                 // Preserve placement versioning that has resulted from natural application where possible
                 if (previousPlacement.equivalentTo(newPlacement))
                     newPlacementsBuilder.with(params, previousPlacement);

src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java

@@ -130,11 +130,11 @@ public class AlterTopology implements Transformation
         for (Map.Entry<NodeId, Location> update : updates.entrySet())
             updated = updated.withUpdatedRackAndDc(update.getKey(), update.getValue());
         ClusterMetadata proposed = prev.transformer().with(updated).build().metadata;
-        DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements.lastModified(),
+        DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements().lastModified(),
                                                                                   proposed.tokenMap.toRanges(),
                                                                                   proposed,
                                                                                   proposed.schema.getKeyspaces());
-        if (!proposedPlacements.equivalentTo(prev.placements))
+        if (!proposedPlacements.equivalentTo(prev.placements()))
         {
             logger.info("Rejecting topology modifications which would materially change data placements: {}", updates);
             return new Rejected(INVALID, "Proposed updates modify data placements, violating consistency guarantees");

src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java

@@ -82,7 +82,7 @@ public abstract class ApplyPlacementDeltas implements Transformation
         ClusterMetadata.Transformer next = prev.transformer();
 
         if (!delta.isEmpty())
-            next = next.with(delta.apply(prev.nextEpoch(), prev.placements));
+            next = next.with(delta.apply(prev.nextEpoch(), prev.placements()));
 
         next = transform(prev, next);
 

src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java

@@ -89,7 +89,7 @@ import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
  */
 public class PrepareJoin implements Transformation
 {
-    public static final Serializer<PrepareJoin> serializer = new Serializer<PrepareJoin>()
+    public static final Serializer<PrepareJoin> serializer = new Serializer<>()
     {
         public PrepareJoin construct(NodeId nodeId, Set<Token> tokens, PlacementProvider placementProvider, boolean joinTokenRing, boolean streamData)
         {
@@ -168,10 +168,10 @@ public class PrepareJoin implements Transformation
                                                              startJoin, midJoin, finishJoin,
                                                              joinTokenRing, streamData);
         if (!prev.tokenMap.isEmpty())
-            assertPreExistingWriteReplica(prev.placements, transitionPlan);
+            assertPreExistingWriteReplica(prev.placements(), transitionPlan);
 
         LockedRanges newLockedRanges = prev.lockedRanges.lock(lockKey, rangesToLock);
-        DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements);
+        DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements());
         ClusterMetadata.Transformer proposed = prev.transformer()
                                                    .with(newLockedRanges)
                                                    .with(startingPlacements)

src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java

@@ -115,7 +115,7 @@ public class PrepareLeave implements Transformation
         PlacementDeltas startDelta = transitionPlan.addToWrites();
         PlacementDeltas midDelta = transitionPlan.moveReads();
         PlacementDeltas finishDelta = transitionPlan.removeFromWrites();
-        transitionPlan.assertPreExistingWriteReplica(prev.placements);
+        transitionPlan.assertPreExistingWriteReplica(prev.placements());
 
         LockedRanges.Key unlockKey = LockedRanges.keyFor(proposed.epoch);
 

src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java

@@ -109,7 +109,7 @@ public class PrepareMove implements Transformation
         StartMove startMove = new StartMove(nodeId, transitionPlan.addToWrites(), lockKey);
         MidMove midMove = new MidMove(nodeId, transitionPlan.moveReads(), lockKey);
         FinishMove finishMove = new FinishMove(nodeId, tokens, transitionPlan.removeFromWrites(), lockKey);
-        transitionPlan.assertPreExistingWriteReplica(prev.placements);
+        transitionPlan.assertPreExistingWriteReplica(prev.placements());
 
         Move sequence = Move.newSequence(prev.nextEpoch(),
                                          lockKey,
@@ -123,7 +123,7 @@ public class PrepareMove implements Transformation
         return Transformation.success(prev.transformer()
                                           .withNodeState(nodeId, NodeState.MOVING)
                                           .with(prev.lockedRanges.lock(lockKey, rangesToLock))
-                                          .with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements))
+                                          .with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements()))
                                           .with(prev.inProgressSequences.with(nodeId, sequence)),
                                       rangesToLock);
     }

src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java

@@ -109,7 +109,7 @@ public class PrepareReplace implements Transformation
         StartReplace start = new StartReplace(replaced, replacement, transitionPlan.addToWrites(), unlockKey);
         MidReplace mid = new MidReplace(replaced, replacement, transitionPlan.moveReads(), unlockKey);
         FinishReplace finish = new FinishReplace(replaced, replacement, transitionPlan.removeFromWrites(), unlockKey);
-        transitionPlan.assertPreExistingWriteReplica(prev.placements);
+        transitionPlan.assertPreExistingWriteReplica(prev.placements());
 
         Set<Token> tokens = new HashSet<>(prev.tokenMap.tokens(replaced));
         BootstrapAndReplace plan = BootstrapAndReplace.newSequence(prev.nextEpoch(),

src/java/org/apache/cassandra/tcm/transformations/Register.java

@@ -109,7 +109,7 @@ public class Register implements Transformation
         if (isReplacingSameAddress())
         {
             NodeId self = ClusterMetadata.current().myNodeId();
-            if (self == null)
+            if (self == NodeId.UNREGISTERED)
                 throw new IllegalStateException("Tried to replace same address, but node does not seem to be registered");
 
             return self;

src/java/org/apache/cassandra/tcm/transformations/Startup.java

@@ -24,10 +24,6 @@ import java.util.Objects;
 
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
-import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.Keyspaces;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Transformation;
@@ -35,14 +31,12 @@ import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeVersion;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.sequences.LockedRanges;
 import org.apache.cassandra.tcm.serialization.MetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
 import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
-import static org.apache.cassandra.locator.MetaStrategy.entireRange;
 
 public class Startup implements Transformation
 {
@@ -96,32 +90,13 @@ public class Startup implements Transformation
                 if (!nodeId.equals(existingNodeId) && addresses.conflictsWith(existingAddresses))
                     return new Rejected(INVALID, String.format("New addresses %s conflicts with existing node %s with addresses %s", addresses, entry.getKey(), existingAddresses));
             }
-
             next = next.withNewAddresses(nodeId, addresses);
-            Keyspaces allKeyspaces = prev.schema.getKeyspaces().withAddedOrReplaced(prev.schema.getKeyspaces());
-
             DataPlacements newPlacement = ClusterMetadataService.instance()
                                                                 .placementProvider()
                                                                 .calculatePlacements(prev.nextEpoch(),
                                                                                      prev.tokenMap.toRanges(),
                                                                                      next.build().metadata,
-                                                                                     allKeyspaces);
+                                                                                     prev.schema.getKeyspaces());
-
-            if (prev.isCMSMember(prev.directory.endpoint(nodeId)))
-            {
-                ReplicationParams metaParams = ReplicationParams.meta(prev);
-                InetAddressAndPort endpoint = prev.directory.endpoint(nodeId);
-                Replica leavingReplica = new Replica(endpoint, entireRange, true);
-                Replica joiningReplica = new Replica(addresses.broadcastAddress, entireRange, true);
-
-                DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild();
-                builder.reads.withoutReplica(prev.nextEpoch(), leavingReplica);
-                builder.writes.withoutReplica(prev.nextEpoch(), leavingReplica);
-                builder.reads.withReplica(prev.nextEpoch(), joiningReplica);
-                builder.writes.withReplica(prev.nextEpoch(), joiningReplica);
-                newPlacement = newPlacement.unbuild().with(metaParams, builder.build()).build();
-            }
-
             next = next.with(newPlacement);
         }
 

src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java

@@ -29,15 +29,12 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.RangesByEndpoint;
+import org.apache.cassandra.tcm.CMSMembership;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.MultiStepOperation;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.sequences.InProgressSequences;
 import org.apache.cassandra.tcm.sequences.LockedRanges;
 import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
@@ -45,7 +42,6 @@ import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
 import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
-import static org.apache.cassandra.locator.MetaStrategy.entireRange;
 import static org.apache.cassandra.tcm.MultiStepOperation.Kind.RECONFIGURE_CMS;
 
 /**
@@ -154,38 +150,26 @@ public class AdvanceCMSReconfiguration implements Transformation
     {
         // Pop the next node to be added from the list diff.additions
         NodeId addition = diff.additions.get(0);
-        InetAddressAndPort endpoint = prev.directory.endpoint(addition);
-        Replica replica = new Replica(endpoint, entireRange, true);
-        List<NodeId> newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size()));
 
         // Check that the candidate is not already a CMS member
-        ReplicationParams metaParams = ReplicationParams.meta(prev);
+        CMSMembership cms = prev.cmsMembership;
-        RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint();
+        if (cms.joiningMembers().contains(addition) || cms.fullMembers().contains(addition))
-        RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint();
+            return new Transformation.Rejected(INVALID, "Endpoint is already a full or joining member of the CMS: " + prev.directory.endpoint(addition));
-        if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint))
-            return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS");
 
-
+        // Add the candidate as a joining member
-        ClusterMetadata.Transformer transformer = prev.transformer();
+        ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(addition);
-        // Add the candidate as a write replica
-        DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild()
-                                                       .withWriteReplica(prev.nextEpoch(), replica);
-        transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build());
 
         // Construct a set of sources for the new member to stream log tables from (essentially this is the existing members)
-        Set<InetAddressAndPort> streamCandidates = new HashSet<>();
+        Set<InetAddressAndPort> streamCandidates = prev.fullCMSMembers();
-        for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues())
-        {
-            if (!replica.equals(r))
-                streamCandidates.add(r.endpoint());
-        }
 
         // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence,
         // including the remaining add/remove operations and the streaming that needs to be done by the joining node
+        List<NodeId> newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size()));
         AdvanceCMSReconfiguration next = next(prev.nextEpoch(),
                                               newAdditions,
                                               diff.removals,
                                               new ReconfigureCMS.ActiveTransition(addition, streamCandidates));
+
         // Create a new sequence instance with the next step to reflect that the state has progressed.
         ReconfigureCMS advanced = sequence.advance(next);
         // Finally, replace the existing reconfiguration sequence with this updated one.
@@ -206,21 +190,21 @@ public class AdvanceCMSReconfiguration implements Transformation
      */
     private Transformation.Result finishAdd(ClusterMetadata prev, ReconfigureCMS sequence, NodeId addition)
     {
+        // Check that the candidate is already a joining CMS member
+        CMSMembership cms = prev.cmsMembership;
+        if (!cms.joiningMembers().contains(addition))
+            return new Transformation.Rejected(INVALID, "Endpoint is not a in the process of joining the CMS: " + prev.directory.endpoint(addition));
+
         // Add the new member as a full read replica, able to participate in quorums for log updates
-        ReplicationParams metaParams = ReplicationParams.meta(prev);
+        ClusterMetadata.Transformer transformer = prev.transformer().finishJoiningCMS(addition);
-        InetAddressAndPort endpoint = prev.directory.endpoint(addition);
-        Replica replica = new Replica(endpoint, entireRange, true);
-        ClusterMetadata.Transformer transformer = prev.transformer();
-        DataPlacement.Builder builder = prev.placements.get(metaParams)
-                                                       .unbuild()
-                                                       .withReadReplica(prev.nextEpoch(), replica);
-        transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build());
 
         // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence,
         // which includes the remaining add/remove operations
         AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, diff.removals, null);
+
         // Create a new sequence instance with the next step to reflect that the state has progressed.
         ReconfigureCMS advanced = sequence.advance(next);
+
         // Finally, replace the existing reconfiguration sequence with this updated one.
         transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced));
         return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));
@@ -238,29 +222,25 @@ public class AdvanceCMSReconfiguration implements Transformation
         List<NodeId> newRemovals = new ArrayList<>(diff.removals.subList(1, diff.removals.size()));
 
         // Check that the candidate is actually a CMS member
-        ClusterMetadata.Transformer transformer = prev.transformer();
+        Set<NodeId> cms = prev.fullCMSMemberIds();
         InetAddressAndPort endpoint = prev.directory.endpoint(removal);
-        Replica replica = new Replica(endpoint, entireRange, true);
+        if (!cms.contains(removal))
-        ReplicationParams metaParams = ReplicationParams.meta(prev);
-        if (!prev.fullCMSMembers().contains(endpoint))
             return new Transformation.Rejected(INVALID, String.format("%s is not currently a CMS member, cannot remove it", endpoint));
 
         // Check that the candidate is not the only CMS member
-        DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild();
+        if (cms.size() == 1)
-        builder.reads.withoutReplica(prev.nextEpoch(), replica);
-        builder.writes.withoutReplica(prev.nextEpoch(), replica);
-        DataPlacement proposed = builder.build();
-        if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty())
             return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint));
 
-        // Actually remove the candidate
+        // Remove the CMS member
-        transformer = transformer.with(prev.placements.unbuild().with(metaParams, proposed).build());
+        ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(removal);
 
         // Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence,
         // which includes the remaining add/remove operations
         AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, newRemovals, null);
+
         // Create a new sequence instance with the next step to reflect that the state has progressed.
         ReconfigureCMS advanced = sequence.advance(next);
+
         // Finally, replace the existing reconfiguration sequence with this updated one.
         transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced));
         return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));

src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java

@@ -26,7 +26,6 @@ import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.MultiStepOperation;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.sequences.AddToCMS;
 import org.apache.cassandra.tcm.sequences.InProgressSequences;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
@@ -84,11 +83,8 @@ public class FinishAddToCMS extends BaseMembershipTransformation
         InetAddressAndPort endpoint = prev.directory.endpoint(targetNode);
         Replica replica = new Replica(endpoint, entireRange, true);
 
-        ClusterMetadata.Transformer transformer = prev.transformer();
+        ClusterMetadata.Transformer transformer = prev.transformer()
-        DataPlacement.Builder builder = prev.placements.get(metaParams)
+                                                      .finishJoiningCMS(targetNode)
-                                                       .unbuild()
-                                                       .withReadReplica(prev.nextEpoch(), replica);
-        transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build())
                                                       .with(prev.inProgressSequences.without(targetNode));
         return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));
     }

src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java

@@ -20,6 +20,8 @@ package org.apache.cassandra.tcm.transformations.cms;
 
 import java.io.IOException;
 
+import com.google.common.collect.ImmutableSet;
+
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
@@ -29,12 +31,10 @@ import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.DistributedMetadataLogKeyspace;
 import org.apache.cassandra.schema.DistributedSchema;
 import org.apache.cassandra.schema.Keyspaces;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.ownership.DataPlacement;
-import org.apache.cassandra.tcm.ownership.DataPlacements;
 import org.apache.cassandra.tcm.sequences.LockedRanges;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
@@ -72,7 +72,6 @@ public class PreInitialize implements Transformation
         assert metadata.epoch.isBefore(Epoch.FIRST);
 
         ClusterMetadata.Transformer transformer = metadata.transformer();
-
         // This null check is a leftover from previous implementations. In earlier versions the address and datacenter
         // were not be included in the serialized form of this transform and so were not written to the local or
         // distributed logs nor included in the log entries sent over the wire between peers.
@@ -84,30 +83,33 @@ public class PreInitialize implements Transformation
         // PRE_INITIALIZE_CMS becomes irrelevant.
         if (addr != null)
         {
-            DataPlacement.Builder dataPlacementBuilder = DataPlacement.builder();
-            Replica replica = new Replica(addr,
-                                          MetaStrategy.partitioner.getMinimumToken(),
-                                          MetaStrategy.partitioner.getMinimumToken(),
-                                          true);
-            dataPlacementBuilder.reads.withReplica(Epoch.FIRST, replica);
-            dataPlacementBuilder.writes.withReplica(Epoch.FIRST, replica);
-            DataPlacements initialPlacement = metadata.placements.unbuild()
-                                                                 .with(ReplicationParams.simpleMeta(1, datacenter),
-                                                                       dataPlacementBuilder.build()).build();
-
-            transformer.with(initialPlacement);
             // create the distributed metadata keyspace in schema with replication settings based on the DC of the
             // initial CMS node
             Keyspaces updated = metadata.schema.getKeyspaces()
                                                .withAddedOrReplaced(DistributedMetadataLogKeyspace.initialMetadata(datacenter));
             transformer.with(new DistributedSchema(updated, Epoch.FIRST));
+            ClusterMetadata.Transformer.Transformed transformed = transformer.build();
+
+            // This is required to bootstrap the initialization process. Because committing to the metadata log uses the
+            // previous ClusterMetadata to identify CMS members who form the consensus group, the first CMS member must
+            // be routable at that point. After the INITIALIZE_CMS is committed, and on instances other than the first
+            // CMS member, this placement is derived from the CMS membership list of node ids.
+            DataPlacement.Builder initialPlacement = DataPlacement.builder();
+            Replica replica = new Replica(addr,
+                                          MetaStrategy.partitioner.getMinimumToken(),
+                                          MetaStrategy.partitioner.getMinimumToken(),
+                                          true);
+            initialPlacement.reads.withReplica(Epoch.FIRST, replica);
+            initialPlacement.writes.withReplica(Epoch.FIRST, replica);
+            metadata = transformed.metadata.forcePreInitializedState(initialPlacement.build());
+        }
+        else
+        {
+            metadata = metadata.forceEpoch(Epoch.FIRST);
         }
 
-        ClusterMetadata.Transformer.Transformed transformed = transformer.build();
-        metadata = transformed.metadata.forceEpoch(Epoch.FIRST);
         assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch;
-
+        return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, ImmutableSet.of());
-        return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, transformed.modifiedKeys);
     }
 
     @Override

src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java

@@ -29,7 +29,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -83,7 +82,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
 
         logger.info("Proposed CMS reconfiguration resulted in {}", diff);
         LockedRanges.Key lockKey = LockedRanges.keyFor(prev.nextEpoch());
-        Set<NodeId> cms = prev.fullCMSMembers().stream().map(prev.directory::peerId).collect(Collectors.toSet());
+        Set<NodeId> cms = prev.fullCMSMemberIds();
         Set<NodeId> tmp = new HashSet<>(cms);
         tmp.addAll(diff.additions);
         tmp.removeAll(diff.removals);
@@ -267,9 +266,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
             KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
             KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple()));
 
-            return executeInternal(prev,
+            return executeInternal(prev, transformer -> transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))));
-                                   transformer -> transformer.with(prev.placements.replaceParams(prev.nextEpoch(), ReplicationParams.meta(prev), replicationParams))
-                                                             .with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))));
         }
 
         public String toString()
@@ -312,7 +309,6 @@ public abstract class PrepareCMSReconfiguration implements Transformation
 
     public static Diff diff(Set<NodeId> currentCms, Set<NodeId> newCms)
     {
-        assert !currentCms.contains(null) : "Current CMS contains a null value " + currentCms;
         assert !newCms.contains(null) : "New CMS contains a null value " + newCms;
 
         List<NodeId> additions = new ArrayList<>();
@@ -355,10 +351,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
     public static boolean needsReconfiguration(ClusterMetadata metadata)
     {
         Map<String, Integer> dcRf = extractRf(ReplicationParams.meta(metadata));
-        Set<NodeId> currentCms = metadata.fullCMSMembers()
+        Set<NodeId> currentCms = metadata.fullCMSMemberIds();
-                                         .stream()
-                                         .map(metadata.directory::peerId)
-                                         .collect(Collectors.toSet());
         int expectedSize = dcRf.values().stream().mapToInt(Integer::intValue).sum();
         if (currentCms.size() != expectedSize)
             return true;

src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java

@@ -29,20 +29,16 @@ import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.MultiStepOperation;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.sequences.InProgressSequences;
 import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
 import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
-import static org.apache.cassandra.locator.MetaStrategy.entireRange;
 
 /**
  * This class along with AddToCMS, StartAddToCMS & FinishAddToCMS, contain a high degree of duplication with their intended
@@ -87,11 +83,7 @@ public class RemoveFromCMS extends BaseMembershipTransformation
         if (sequence != null)
             return new Transformation.Rejected(INVALID, String.format("Can't remove %s from CMS as there are ongoing range movements on it", endpoint));
 
-        ReplicationParams metaParams = ReplicationParams.meta(prev);
+        int minProposedSize = prev.fullCMSMemberIds().size() - 1;
-        DataPlacement placements = prev.placements.get(metaParams);
-
-        int minProposedSize = (int) Math.min(placements.reads.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count(),
-                                             placements.writes.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count());
         if (minProposedSize < MIN_SAFE_CMS_SIZE)
         {
             logger.warn("Removing {} from CMS members would reduce the service size to {} which is below the " +
@@ -109,19 +101,8 @@ public class RemoveFromCMS extends BaseMembershipTransformation
         if (minProposedSize == 0)
             return new Transformation.Rejected(INVALID, String.format("Removing %s from the CMS would leave no members in CMS.", endpoint));
 
-        ClusterMetadata.Transformer transformer = prev.transformer();
+        ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(nodeId);
-        Replica replica = new Replica(endpoint, entireRange, true);
+        return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));
-
-        DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild();
-        builder.reads.withoutReplica(prev.nextEpoch(), replica);
-        builder.writes.withoutReplica(prev.nextEpoch(), replica);
-        DataPlacement proposed = builder.build();
-
-        if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty())
-            return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint));
-
-        return Transformation.success(transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()),
-                                      MetaStrategy.affectedRanges(prev));
     }
 
     @Override

src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java

@@ -18,25 +18,20 @@
 
 package org.apache.cassandra.tcm.transformations.cms;
 
-import java.util.HashSet;
 import java.util.Set;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.RangesByEndpoint;
+import org.apache.cassandra.tcm.CMSMembership;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.MultiStepOperation;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.sequences.AddToCMS;
 import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
 import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 
 import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
-import static org.apache.cassandra.locator.MetaStrategy.entireRange;
 
 /**
  * This class along with AddToCMS, FinishAddToCMS & RemoveFromCMS, contain a high degree of duplication with their intended
@@ -76,27 +71,13 @@ public class StartAddToCMS extends BaseMembershipTransformation
         if (prev.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance) != null)
             return new Rejected(INVALID, String.format("Cannot add node to CMS as a CMS reconfiguration is currently active"));
 
-        Replica replica = new Replica(endpoint, entireRange, true);
+        CMSMembership cms = prev.cmsMembership;
-        ReplicationParams metaParams = ReplicationParams.meta(prev);
+        if (cms.joiningMembers().contains(nodeId) || cms.fullMembers().contains(nodeId))
-        RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint();
-        RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint();
-
-        if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint))
             return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS");
 
-        ClusterMetadata.Transformer transformer = prev.transformer();
+        ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(nodeId);
-        DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild()
-                                                       .withWriteReplica(prev.nextEpoch(), replica);
-
-        transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build());
-
-        Set<InetAddressAndPort> streamCandidates = new HashSet<>();
-        for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues())
-        {
-            if (!replica.equals(r))
-                streamCandidates.add(r.endpoint());
-        }
 
+        Set<InetAddressAndPort> streamCandidates = prev.fullCMSMembers();
         AddToCMS joinSequence = new AddToCMS(prev.nextEpoch(), nodeId, streamCandidates, new FinishAddToCMS(endpoint));
         transformer = transformer.with(prev.inProgressSequences.with(nodeId, joinSequence));
         return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));

src/java/org/apache/cassandra/tools/CMSOfflineTool.java

@@ -37,10 +37,9 @@ import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.KeyspaceMetadata;
 import org.apache.cassandra.schema.ReplicationParams;
+import org.apache.cassandra.tcm.CMSMembership;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.MultiStepOperation;
@@ -441,24 +440,15 @@ public class CMSOfflineTool implements Runnable
                                                    nodeState + " state. Only a JOINED node can be set as CMS member.");
             }
             InetAddressAndPort endpoint = metadata.directory.getNodeAddresses(nodeId).broadcastAddress;
-            ReplicationParams metaParams = ReplicationParams.meta(metadata);
+            CMSMembership cms = metadata.cmsMembership;
-            Iterable<Replica> currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues();
+            ClusterMetadata.Transformer transformer = metadata.transformer();
-            DataPlacement.Builder placementBuilder = metadata.placements.get(metaParams).unbuild();
+            for (NodeId id : cms.fullMembers())
-            for (Replica replica : currentReplicas)
+                transformer = transformer.leaveCMS(id);
-            {
+            for (NodeId id : cms.joiningMembers())
-                placementBuilder.withoutReadReplica(metadata.epoch, replica)
+                transformer = transformer.cancelJoiningCMS(id);
-                                .withoutWriteReplica(metadata.epoch, replica);
-            }
 
-            Replica newCMS = MetaStrategy.replica(endpoint);
+            transformer = transformer.startJoiningCMS(nodeId).finishJoiningCMS(nodeId);
-            placementBuilder.withReadReplica(metadata.epoch, newCMS)
+            return transformer.build().metadata;
-                            .withWriteReplica(metadata.epoch, newCMS);
-
-            return metadata.transformer()
-                           .with(metadata.placements.unbuild()
-                                                    .with(metaParams, placementBuilder.build())
-                                                    .build())
-                           .build().metadata;
         }
     }
 
@@ -619,7 +609,7 @@ public class CMSOfflineTool implements Runnable
             output.out.printf("Cluster Metadata Service:%n");
             output.out.printf("Members: %s%n", members);
             output.out.printf("Needs reconfiguration: %s%n", needsReconfiguration(metadata));
-            output.out.printf("Service State: %s%n", ClusterMetadataService.state(metadata));
+            output.out.printf("Service State: %s%n", ClusterMetadataService.State.OFFLINE_TOOL);
             output.out.printf("Epoch: %s%n", metadata.epoch.getEpoch());
             output.out.printf("Replication factor: %s%n", ReplicationParams.meta(metadata).toString());
         }
@@ -782,7 +772,7 @@ public class CMSOfflineTool implements Runnable
                 throw new IllegalArgumentException("Keyspace " + keyspace + " not found in cluster metadata.");
             }
 
-            DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication);
+            DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication);
             List<Object[]> rows = new ArrayList<>();
             rows.addAll(replicaGroupsToRows(placement.reads, "read"));
             rows.addAll(replicaGroupsToRows(placement.writes, "write"));

src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java

@@ -27,13 +27,11 @@ import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.io.util.FileInputStreamPlus;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.Replica;
 import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeVersion;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
 
@@ -60,9 +58,9 @@ public class TransformClusterMetadataHelper
         DatabaseDescriptor.setPartitionerUnsafe(partitioner);
         ClusterMetadataService.initializeForTools(false);
         ClusterMetadata metadata = ClusterMetadataService.deserializeClusterMetadata(sourceFile);
-        System.out.println("Old CMS: " + metadata.placements.get(ReplicationParams.meta(metadata)));
+        System.out.println("Old CMS: " + metadata.placement(ReplicationParams.meta(metadata)));
         metadata = makeCMS(metadata, InetAddressAndPort.getByNameUnchecked(args[1]));
-        System.out.println("New CMS: " + metadata.placements.get(ReplicationParams.meta(metadata)));
+        System.out.println("New CMS: " + metadata.placement(ReplicationParams.meta(metadata)));
         Path p = Files.createTempFile("clustermetadata", "dump");
         try (FileOutputStreamPlus out = new FileOutputStreamPlus(p))
         {
@@ -73,20 +71,9 @@ public class TransformClusterMetadataHelper
 
     public static ClusterMetadata makeCMS(ClusterMetadata metadata, InetAddressAndPort endpoint)
     {
-        ReplicationParams metaParams = ReplicationParams.meta(metadata);
+        NodeId id = metadata.directory.peerId(endpoint);
-        Iterable<Replica> currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues();
+        if (id == null)
-        DataPlacement.Builder builder = metadata.placements.get(metaParams).unbuild();
+            throw new IllegalStateException("No node id found for endpoint: " + endpoint);
-        for (Replica replica : currentReplicas)
+        return metadata.transformer().startJoiningCMS(id).finishJoiningCMS(id).build().metadata;
-        {
-            builder.withoutReadReplica(metadata.epoch, replica)
-                   .withoutWriteReplica(metadata.epoch, replica);
-        }
-        Replica newCMS = MetaStrategy.replica(endpoint);
-        builder.withReadReplica(metadata.epoch, newCMS)
-               .withWriteReplica(metadata.epoch, newCMS);
-        return metadata.transformer().with(metadata.placements.unbuild().with(metaParams,
-                                                                              builder.build())
-                                                              .build())
-                       .build().metadata;
     }
 }

test/distributed/org/apache/cassandra/distributed/impl/Instance.java

@@ -839,7 +839,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
         {
             CommitLog.instance.recoverSegmentsOnDisk();
             NodeId self = ClusterMetadata.current().myNodeId();
-            if (self != null)
+            if (self != NodeId.UNREGISTERED)
                 AccordService.localStartup(self);
         }
         catch (IOException e)

test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java

@@ -444,8 +444,8 @@ public class ClusterUtils
         for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces())
         {
             List[] placements = new List[2];
-            placements[0] = metadata.placements.get(keyspace.params.replication).reads.toReplicaStringList();
+            placements[0] = metadata.placement(keyspace.params.replication).reads.toReplicaStringList();
-            placements[1] = metadata.placements.get(keyspace.params.replication).writes.toReplicaStringList();
+            placements[1] = metadata.placement(keyspace.params.replication).writes.toReplicaStringList();
             byKeyspace.put(keyspace.name, placements);
         }
         return byKeyspace;
@@ -465,10 +465,10 @@ public class ClusterUtils
             StringBuilder builder = new StringBuilder();
             builder.append("'keyspace' { 'name':").append(keyspace.name).append("', ");
             builder.append("'reads':['");
-            ReplicaGroups placement = metadata.placements.get(keyspace.params.replication).reads;
+            ReplicaGroups placement = metadata.placement(keyspace.params.replication).reads;
             builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString());
             builder.append("'], 'writes':['");
-            placement = metadata.placements.get(keyspace.params.replication).writes;
+            placement = metadata.placement(keyspace.params.replication).writes;
             builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString());
             builder.append("']}");
             keyspaces.add(builder.toString());

test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java

@@ -27,7 +27,6 @@ import org.junit.Test;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.tcm.ClusterMetadata;
-import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
 import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@@ -46,8 +45,8 @@ public class AssassinateCMSNodeTest extends TestBaseImpl
             InetSocketAddress toAssassinate = cluster.get(2).broadcastAddress();
             cluster.get(2).shutdown().get();
             cluster.get(1).nodetoolResult("assassinate", toAssassinate.getHostString()).asserts().success();
-            cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())));
+            cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember()));
-            cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())));
+            cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember()));
             cluster.get(1).nodetoolResult("cms").asserts().success();
         }
     }

test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java

@@ -32,14 +32,11 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.locator.MetaStrategy;
-import org.apache.cassandra.locator.Replica;
-import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.tcm.CMSOperations;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
+import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeVersion;
-import org.apache.cassandra.tcm.ownership.DataPlacement;
 import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
 
 import static org.apache.cassandra.distributed.shared.ClusterUtils.start;
@@ -108,15 +105,10 @@ public class BootWithMetadataTest extends TestBaseImpl
                 try
                 {
                     ClusterMetadata metadata = ClusterMetadata.current();
-                    Replica oldCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.1"));
+                    NodeId oldCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.1"));
-                    Replica newCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.2"));
+                    NodeId newCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.2"));
                     ClusterMetadata.Transformer transformer = metadata.transformer();
-                    DataPlacement.Builder builder = metadata.placements.get(ReplicationParams.meta(metadata)).unbuild()
+                    transformer.leaveCMS(oldCMS).startJoiningCMS(newCMS).finishJoiningCMS(newCMS);
-                                                                       .withoutReadReplica(metadata.nextEpoch(), oldCMS)
-                                                                       .withoutWriteReplica(metadata.nextEpoch(), oldCMS)
-                                                                       .withWriteReplica(metadata.nextEpoch(), newCMS)
-                                                                       .withReadReplica(metadata.nextEpoch(), newCMS);
-                    transformer = transformer.with(metadata.placements.unbuild().with(ReplicationParams.meta(metadata), builder.build()).build());
                     ClusterMetadata toDump = transformer.build().metadata.forceEpoch(Epoch.create(1000));
                     Path p = Files.createTempFile("clustermetadata", "dump");
                     try (FileOutputStreamPlus out = new FileOutputStreamPlus(p))

test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java

@@ -66,7 +66,8 @@ public class ClusterMetadataDumpTest extends TestBaseImpl
                     epochsSeen++;
             }
             assertEquals(3, unsafeJoinSeen);
-            assertEquals(3, registerSeen);
+            // Only 2 REGISTER transforms are expected as the first CMS node is registered implicitly by INITIALIZE_CMS
+            assertEquals(2, registerSeen);
             assertTrue(epochsSeen > 15);
 
             res = cluster.get(1).nodetoolResult("cms", "dumplog", "--start", "10", "--end", "15");

test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java

@@ -64,6 +64,7 @@ import org.apache.cassandra.service.accord.topology.AccordFastPath;
 import org.apache.cassandra.service.accord.topology.AccordStaleReplicas;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
 import org.apache.cassandra.tcm.AtomicLongBackedProcessor;
+import org.apache.cassandra.tcm.CMSMembership;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.Commit;
@@ -159,7 +160,8 @@ public class ClusterMetadataTestHelper
 
     public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner)
     {
-        return new ClusterMetadata(epoch, Murmur3Partitioner.instance,
+        return new ClusterMetadata(epoch,
+                                   Murmur3Partitioner.instance,
                                    DistributedSchema.empty(),
                                    Directory.EMPTY,
                                    new TokenMap(partitioner),
@@ -169,7 +171,8 @@ public class ClusterMetadataTestHelper
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,
                                    ImmutableMap.of(),
-                                   AccordStaleReplicas.EMPTY);
+                                   AccordStaleReplicas.EMPTY,
+                                   CMSMembership.EMPTY);
     }
 
     public static ClusterMetadata minimalForTesting(IPartitioner partitioner)
@@ -177,6 +180,11 @@ public class ClusterMetadataTestHelper
         return minimalForTesting(Epoch.EMPTY, partitioner);
     }
 
+    public static ClusterMetadata minimalForTesting(Epoch e, Keyspaces keyspaces, CMSMembership cms)
+    {
+        return minimalForTesting(e, Murmur3Partitioner.instance, new DistributedSchema(keyspaces), cms);
+    }
+
     public static ClusterMetadata minimalForTesting(Keyspaces keyspaces)
     {
         return minimalForTesting(Murmur3Partitioner.instance).transformer()
@@ -185,6 +193,28 @@ public class ClusterMetadataTestHelper
                .metadata.forceEpoch(Epoch.EMPTY);
     }
 
+    public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema)
+    {
+        return minimalForTesting(epoch, partitioner, schema, CMSMembership.EMPTY);
+    }
+
+    public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema, CMSMembership cms)
+    {
+        return new ClusterMetadata(epoch,
+                                   Murmur3Partitioner.instance,
+                                   schema,
+                                   Directory.EMPTY,
+                                   new TokenMap(partitioner),
+                                   DataPlacements.empty(),
+                                   AccordFastPath.EMPTY,
+                                   LockedRanges.EMPTY,
+                                   InProgressSequences.EMPTY,
+                                   ConsensusMigrationState.EMPTY,
+                                   ImmutableMap.of(),
+                                   AccordStaleReplicas.EMPTY,
+                                   cms);
+    }
+
     public static ClusterMetadataService syncInstanceForTest()
     {
         LocalLog log = LocalLog.logSpec()
@@ -1057,7 +1087,7 @@ public class ClusterMetadataTestHelper
     public static VersionedEndpoints.ForToken getNaturalReplicasForToken(ClusterMetadata metadata, String keyspace, Token searchPosition)
     {
         KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
-        return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(searchPosition);
+        return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(searchPosition);
     }
 
     public static BootstrapAndJoin getBootstrapPlan(int idx)

test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java

@@ -508,7 +508,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
                 Set<NodeId> bouncing = new HashSet<>();
                 Set<NodeId> replicasFromBouncedReplicaSets = new HashSet<>();
                 outer:
-                for (VersionedEndpoints.ForRange placements : sut.service.metadata().placements.get(rf.asKeyspaceParams().replication).writes.endpoints)
+                for (VersionedEndpoints.ForRange placements : sut.service.metadata().placement(rf.asKeyspaceParams().replication).writes.endpoints)
                 {
                     List<NodeId> replicas = new ArrayList<>(metadata.directory.toNodeIds(placements.get().endpoints()));
                     List<NodeId> bounceCandidates = new ArrayList<>();
@@ -545,8 +545,8 @@ public class MetadataChangeSimulationTest extends CMSTestBase
         ClusterMetadata actualMetadata = sut.service.metadata();
         ReplicationParams replication = actualMetadata.schema.getKeyspaces().get("test").get().params.replication;
         Assert.assertEquals(replication, sut.rf.asKeyspaceParams().replication);
-        match(actualMetadata.placements.get(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
+        match(actualMetadata.placement(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
-        match(actualMetadata.placements.get(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
+        match(actualMetadata.placement(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
     }
 
     public static void validatePlacements(CMSTestBase.CMSSut sut, ModelState modelState) throws Throwable
@@ -559,7 +559,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
         Assert.assertEquals(modelState.simulatedPlacements.nodes.stream().map(Node::token).collect(Collectors.toSet()),
                             actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).getLongValue()).collect(Collectors.toSet()));
 
-        for (Map.Entry<ReplicationParams, DataPlacement> e : actualMetadata.placements.asMap().entrySet())
+        for (Map.Entry<ReplicationParams, DataPlacement> e : actualMetadata.placements().asMap().entrySet())
         {
             if (!e.getKey().equals(replication))
                 continue;
@@ -569,7 +569,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
             match(placement.reads, modelState.simulatedPlacements.readPlacements);
         }
 
-        validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements);
+        validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements());
     }
 
     public static ModelChecker.Pair<ModelState, Node> registerNewNode(ModelState state, CMSSut sut, int dcIdx, int rackIdx)
@@ -953,7 +953,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
                 validatePlacements(sut, state);
             }
             // Finally verify that the predicted placements match the actual ones
-            Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placements.get(ksm.params.replication)));
+            Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placement(ksm.params.replication)));
         }
     }
 }

test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java

@@ -116,8 +116,7 @@ public class OperationalEquivalenceTest extends CMSTestBase
             withMove = ClusterMetadata.current();
         }
 
-        assertPlacements(simulateAndCompare(rf, equivalentNodes).placements,
+        assertPlacements(simulateAndCompare(rf, equivalentNodes).placements(), withMove.placements());
-                         withMove.placements);
     }
 
     private static ClusterMetadata simulateAndCompare(ReplicationFactor rf, List<Node> nodes) throws Exception

test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java

@@ -41,7 +41,6 @@ import org.apache.cassandra.distributed.shared.NetworkTopology;
 import org.apache.cassandra.locator.MetaStrategy;
 import org.apache.cassandra.schema.DistributedMetadataLogKeyspace;
 import org.apache.cassandra.schema.ReplicationParams;
-import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.service.paxos.Ballot;
 import org.apache.cassandra.service.paxos.PaxosRepairHistory;
 import org.apache.cassandra.tcm.ClusterMetadata;
@@ -57,6 +56,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin
 import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart;
 import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack;
 import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology;
+import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.psjava.util.AssertStatus.assertTrue;
@@ -85,10 +85,10 @@ public class ReconfigureCMSTest extends FuzzTestBase
                 ClusterMetadata metadata = ClusterMetadata.current();
                 assertEquals(5, metadata.fullCMSMembers().size());
                 assertEquals(ReplicationParams.simpleMeta(5, metadata.directory.knownDatacenters()),
-                                    metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get());
+                             metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication);
             });
             cluster.stream().forEach(i -> {
-                Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0);
+                Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0);
             });
 
             cluster.get(nodeSelector.get()).nodetoolResult("cms", "reconfigure", "1").asserts().success();
@@ -96,7 +96,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
                 ClusterMetadata metadata = ClusterMetadata.current();
                 assertEquals(1, metadata.fullCMSMembers().size());
                 assertEquals(ReplicationParams.simpleMeta(1, metadata.directory.knownDatacenters()),
-                                    metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get());
+                             metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication);
             });
         }
     }
@@ -136,7 +136,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
                 Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance));
                 assertEquals(2, metadata.fullCMSMembers().size());
                 ReplicationParams params = ReplicationParams.meta(metadata);
-                DataPlacement placements = metadata.placements.get(params);
+                DataPlacement placements = metadata.placements().get(params);
                 assertTrue(placements.reads.equivalentTo(placements.writes));
                 assertEquals(metadata.fullCMSMembers().size(), Integer.parseInt(params.asMap().get("dc0")));
             });
@@ -161,7 +161,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
                 Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance));
                 Assert.assertTrue(metadata.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort()));
                 assertEquals(3, metadata.fullCMSMembers().size());
-                DataPlacement placements = metadata.placements.get(ReplicationParams.meta(metadata));
+                DataPlacement placements = metadata.placements().get(ReplicationParams.meta(metadata));
                 Assert.assertTrue(placements.reads.equivalentTo(placements.writes));
             });
         }
@@ -216,7 +216,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
             awaitRingJoin(replacingNode, cluster.get(1));
             replacingNode.runOnInstance(() -> {
                 ClusterMetadata metadata = ClusterMetadata.current();
-                assertTrue(metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()));
+                assertTrue(metadata.isCMSMember());
                 assertEquals(3, metadata.fullCMSMembers().size());
             });
         }
@@ -326,11 +326,11 @@ public class ReconfigureCMSTest extends FuzzTestBase
         Object[][] rows = instance.executeInternal("select points from system.paxos_repair_history " +
                                                    "where keyspace_name = ? " +
                                                    "and table_name = ?",
-                                                   SchemaConstants.METADATA_KEYSPACE_NAME,
+                                                   METADATA_KEYSPACE_NAME,
                                                    DistributedMetadataLogKeyspace.TABLE_NAME);
 
         if (rows.length == 0)
-            return PaxosRepairHistory.empty(SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME);
+            return PaxosRepairHistory.empty(METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME);
         assertEquals(1, rows.length);
         //noinspection unchecked
         List<ByteBuffer> points = (List<ByteBuffer>)rows[0][0];

test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java

@@ -19,12 +19,13 @@
 package org.apache.cassandra.distributed.test.log;
 
 import java.io.IOException;
-import java.net.UnknownHostException;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.EnumSet;
 
 import org.junit.Test;
 
+import org.apache.cassandra.db.SystemKeyspace;
 import org.apache.cassandra.distributed.Cluster;
 import org.apache.cassandra.distributed.api.Feature;
 import org.apache.cassandra.distributed.api.IInstanceConfig;
@@ -33,9 +34,11 @@ import org.apache.cassandra.distributed.api.TokenSupplier;
 import org.apache.cassandra.distributed.shared.NetworkTopology;
 import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.io.util.DataInputBuffer;
-import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.MetadataSnapshots;
 import org.apache.cassandra.tcm.Transformation;
 import org.apache.cassandra.tcm.membership.Location;
@@ -45,15 +48,21 @@ import org.apache.cassandra.tcm.membership.NodeState;
 import org.apache.cassandra.tcm.membership.NodeVersion;
 import org.apache.cassandra.tcm.ownership.PlacementProvider;
 import org.apache.cassandra.tcm.sequences.LeaveStreams;
+import org.apache.cassandra.tcm.sequences.LockedRanges;
 import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave;
+import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
 import org.apache.cassandra.tcm.serialization.Version;
+import org.apache.cassandra.tcm.transformations.CustomTransformation;
 import org.apache.cassandra.tcm.transformations.PrepareLeave;
 import org.apache.cassandra.tcm.transformations.Register;
 import org.apache.cassandra.tcm.transformations.TriggerSnapshot;
 import org.apache.cassandra.tcm.transformations.Unregister;
-import org.apache.cassandra.utils.CassandraVersion;
+import org.apache.cassandra.utils.vint.VIntCoding;
 
+import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.addr;
+import static org.apache.cassandra.tcm.membership.NodeVersion.CURRENT_METADATA_VERSION;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 
 public class RegisterTest extends TestBaseImpl
 {
@@ -106,60 +115,45 @@ public class RegisterTest extends TestBaseImpl
         {
             cluster.get(1).startup();
             cluster.get(1).runOnInstance(() -> {
-                try
-                {
-                    // Unregister to make directory empty
-                    ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(),
-                                                                            EnumSet.allOf(NodeState.class),
-                                                                            ClusterMetadataService.instance().placementProvider()));
 
-                    // Register a ghost node with V0 (bypasses version check because directory is now empty).
+                // Run a custom transformation to inject a fake node into the directory with a known id and an
-                    // In a real world cluster we will always be upgrading from a smaller version.
+                // artificially lowered max supported serialization version
-                    ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")),
+                CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer);
-                                                                          TEST_LOCATION,
+                CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME,
-                                                                          new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0)));
+                                                                              new RegisterNodeWithOldVersion());
-                    NodeId oldNode = ClusterMetadata.current().directory.peerId(InetAddressAndPort.getByName("127.0.0.100"));
+                ClusterMetadataService.instance().commit(injectOldNode);
-
-                    // Register a node with upgraded version
-                    CassandraVersion currentVersion = NodeVersion.CURRENT.cassandraVersion;
-                    NodeVersion upgraded = new NodeVersion(new CassandraVersion(String.format("%d.%d.%d", currentVersion.major + 1, 0, 0)),
-                                                            NodeVersion.CURRENT_METADATA_VERSION);
-                    ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.200")), TEST_LOCATION, upgraded));
 
                 // Doesn't matter which specific Transformation we use here, we're testing that the serializer uses
                 // the correct lower bound
-                    Transformation t = new Register(NodeAddresses.current(), new Location("DC", "RACK"), NodeVersion.CURRENT);
+                Transformation t = new Register(NodeAddresses.current(), TEST_LOCATION, NodeVersion.CURRENT);
                 try
                 {
-                        assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion,
+                    assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, RegisterNodeWithOldVersion.METADATA_VERSION);
-                                     Version.V0.asInt());
                     ByteBuffer bytes = t.kind().toVersionedBytes(t);
                     try (DataInputBuffer buf = new DataInputBuffer(bytes, true))
                     {
-                            // Because ClusterMetadata.current().directory still contains oldNode we must serialize at
+                        // Because ClusterMetadata.current().directory still contains the fake old node we must
-                            // the version it supports
+                        // serialize at the version _it_ supports
-                            assertEquals(Version.V0, Version.fromInt(buf.readUnsignedVInt32()));
+                        assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32()));
                     }
 
-                        // If we unregister oldNode, then the ceiling for serialization version will rise
+                    // If we unregister the fake node, then the ceiling for serialization version will rise
-                        ClusterMetadataService.instance().commit(new Unregister(oldNode, EnumSet.allOf(NodeState.class), ClusterMetadataService.instance().placementProvider()));
+                    Unregister unregisterOldNode = new Unregister(RegisterNodeWithOldVersion.NODE_ID,
-                        assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion,
+                                                                  EnumSet.allOf(NodeState.class),
-                                     NodeVersion.CURRENT_METADATA_VERSION.asInt());
+                                                                  ClusterMetadataService.instance().placementProvider());
+                    ClusterMetadataService.instance().commit(unregisterOldNode);
+
+                    assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, CURRENT_METADATA_VERSION);
                     bytes = t.kind().toVersionedBytes(t);
                     try (DataInputBuffer buf = new DataInputBuffer(bytes, true))
                     {
-                            assertEquals(NodeVersion.CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32()));
+                        assertEquals(CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32()));
                     }
                 }
                 catch (IOException e)
                 {
                     throw new RuntimeException(e);
                 }
-                }
-                catch (UnknownHostException e)
-                {
-                    throw new RuntimeException(e);
-                }
             });
         }
     }
@@ -172,28 +166,70 @@ public class RegisterTest extends TestBaseImpl
         {
             cluster.get(1).startup();
             cluster.get(1).runOnInstance(() -> {
+                // Run a custom transformation to inject a fake node into the directory with a known id and an
+                // artificially lowered max supported serialization version
+                CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer);
+                CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME,
+                                                                              new RegisterNodeWithOldVersion());
+                ClusterMetadataService.instance().commit(injectOldNode);
+                // Now trigger a snapshot which must be written to the snapshot using the old serialization version
+                Epoch epoch = ClusterMetadataService.instance().commit(TriggerSnapshot.instance).epoch;
+                // fetch the raw bytes of the snapshot we just serialized
+                ByteBuffer bytes = SystemKeyspace.getSnapshot(epoch);
+                assertNotNull(bytes);
+                // assert the prepended version matches
+                Version writtenVersion = null;
                 try
                 {
-                    // Unregister to make directory empty
+                    writtenVersion = Version.fromInt(VIntCoding.readUnsignedVInt32(new DataInputBuffer(bytes, false)));
-                    ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(),
-                                                                            EnumSet.allOf(NodeState.class),
-                                                                            ClusterMetadataService.instance().placementProvider()));
-
-                    // Register a ghost node with V0 (bypasses version check because directory is now empty).
-                    // In a real world cluster we will always be upgrading from a smaller version.
-                    ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")),
-                                                                          TEST_LOCATION,
-                                                                          new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0)));
                 }
-                catch (UnknownHostException e)
+                catch (IOException e)
                 {
                     throw new RuntimeException(e);
                 }
-                ClusterMetadataService.instance().commit(TriggerSnapshot.instance);
+                assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, writtenVersion);
-
+                // load the snapshot using the standard mechanism and assert it matches current cluster metadata
                 ClusterMetadata cm = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots().getSnapshot(ClusterMetadata.current().epoch);
                 cm.equals(ClusterMetadata.current());
             });
         }
     }
+
+    // Custom transforms to lock/unlock an arbitrary set of ranges to
+    // avoid having to actually initiate some range movement
+    public static class RegisterNodeWithOldVersion implements Transformation, Serializable
+    {
+        public static final AsymmetricMetadataSerializer<Transformation, RegisterNodeWithOldVersion> serializer = new AsymmetricMetadataSerializer<Transformation, RegisterNodeWithOldVersion>()
+        {
+            @Override
+            public void serialize(Transformation t, DataOutputPlus out, Version version){}
+            @Override
+            public RegisterNodeWithOldVersion deserialize(DataInputPlus in, Version version) {return new RegisterNodeWithOldVersion();}
+            @Override
+            public long serializedSize(Transformation t, Version version) {return 0;}
+        };
+
+        public static final String NAME = "TestRegisterNodeWithOldVersion";
+        public static final NodeId NODE_ID = new NodeId(99);
+        public static final Version METADATA_VERSION = Version.V0;
+
+        @Override
+        public Kind kind()
+        {
+            return Kind.CUSTOM;
+        }
+
+        @Override
+        public Result execute(ClusterMetadata metadata)
+        {
+            ClusterMetadata.Transformer transformer = metadata.transformer()
+                                                              .unsafeRegisterForTesting(NODE_ID,
+                                                                                        new NodeAddresses(addr(99)),
+                                                                                        TEST_LOCATION,
+                                                                                        new NodeVersion(NodeVersion.CURRENT.cassandraVersion,
+                                                                                                        METADATA_VERSION));
+            return Transformation.success(transformer, LockedRanges.AffectedRanges.EMPTY);
+        }
+    }
+
 }

test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java

@@ -146,12 +146,12 @@ public class ResumableStartupTest extends FuzzTestBase
                         KeyspaceMetadata ksm = metadata.schema.getKeyspaceMetadata(keyspace);
                         boolean isWriteReplica = false;
                         boolean isReadReplica = false;
-                        for (InetAddressAndPort readReplica : metadata.placements.get(ksm.params.replication).reads.byEndpoint().keySet())
+                        for (InetAddressAndPort readReplica : metadata.placement(ksm.params.replication).reads.byEndpoint().keySet())
                         {
                             if (readReplica.getHostAddressAndPort().equals(newAddress))
                                 isReadReplica = true;
                         }
-                        for (InetAddressAndPort writeReplica : metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet())
+                        for (InetAddressAndPort writeReplica : metadata.placement(ksm.params.replication).writes.byEndpoint().keySet())
                         {
                             if (writeReplica.getHostAddressAndPort().equals(newAddress))
                                 isWriteReplica = true;

test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java

@@ -177,8 +177,8 @@ public abstract class SimulatedOperation
         sutActions.next();
         ClusterMetadata m2 = ClusterMetadata.current();
 
-        Map<Range<Token>, VersionedEndpoints.ForRange> after = m2.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.asMap();
+        Map<Range<Token>, VersionedEndpoints.ForRange> after = m2.placement(simulatedState.rf.asKeyspaceParams().replication).reads.asMap();
-        m1.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> {
+        m1.placement(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> {
             if (after.containsKey(k))
             {
                 VersionedEndpoints.ForRange afterPlacements = after.get(k);

test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java

@@ -69,7 +69,7 @@ public class SnapshotTest extends TestBaseImpl
                 ClusterMetadata before = ClusterMetadata.current();
                 ClusterMetadata after = ClusterMetadataService.instance().triggerSnapshot();
                 ClusterMetadata serialized = ClusterMetadataService.instance().snapshotManager().getSnapshot(after.epoch);
-                assertEquals(before.placements, serialized.placements);
+                assertEquals(before.placements(), serialized.placements());
                 assertEquals(before.tokenMap, serialized.tokenMap);
                 assertEquals(before.directory, serialized.directory);
                 assertEquals(before.schema, serialized.schema);

test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java

@@ -54,7 +54,7 @@ public class RangeVersioningTest extends FuzzTestBase
                 for (int i = 1; i <= 4; i++)
                 {
                     Epoch smallestSeen = null;
-                    for (VersionedEndpoints.ForRange fr : metadata.placements.get(ReplicationParams.simple(i)).writes.endpoints)
+                    for (VersionedEndpoints.ForRange fr : metadata.placement(ReplicationParams.simple(i)).writes.endpoints)
                     {
                         if (smallestSeen == null || fr.lastModified().isBefore(smallestSeen))
                             smallestSeen = fr.lastModified();

test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java

@@ -34,7 +34,6 @@ import org.apache.cassandra.distributed.test.TestBaseImpl;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.membership.NodeId;
-import org.apache.cassandra.utils.FBUtilities;
 
 import static org.apache.cassandra.distributed.shared.ClusterUtils.addInstance;
 import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin;
@@ -80,7 +79,7 @@ public class CMSPlacementAfterReplacementTest extends TestBaseImpl
         IInvokableInstance nodeToRemove = cluster.get(2);
         cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success();
         cluster.get(2).runOnInstance(() -> {
-            assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()));
+            assertTrue(ClusterMetadata.current().isCMSMember());
         });
         nodeToRemove.shutdown().get();
         IInvokableInstance replacingNode = addInstance(cluster, nodeToRemove.config(),

test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java

@@ -60,7 +60,7 @@ public class ClusterMetadataUpgradeAssassinateTest extends UpgradeTestBase
         ((IInvokableInstance) i).runOnInstance(() -> {
             ClusterMetadata metadata = ClusterMetadata.current();
             InetAddressAndPort ep = InetAddressAndPort.getByNameUnchecked(host);
-            metadata.placements.asMap().forEach((key, value) -> {
+            metadata.placements().forEach((key, value) -> {
                 if (key.isMeta())
                     return;
                 boolean existsInPlacements = Streams.concat(value.reads.endpoints.stream(),

test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java

@@ -181,7 +181,7 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase
             try
             {
                 new ByteBuddy().rebase(ClusterMetadata.class)
-                               .method(named("initializeClusterIdentifier"))
+                               .method(named("forceInitializedState"))
                                .intercept(MethodDelegation.to(ClusterMetadataUpgradeDelayedInitializeTest.BBInterceptor.class))
                                .make()
                                .load(classLoader, ClassLoadingStrategy.Default.INJECTION);
@@ -201,13 +201,13 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase
     public static class BBInterceptor
     {
         @SuppressWarnings("unused")
-        public static ClusterMetadata initializeClusterIdentifier(@SuperCall Callable<ClusterMetadata> zuper)
+        public static ClusterMetadata forceInitializedState(@SuperCall Callable<ClusterMetadata> zuper)
         {
             try
             {
-                logger.info("initializeClusterIdentifier waiting...");
+                logger.info("forceInitializedState waiting...");
                 BBState.latch.await(60, TimeUnit.SECONDS);
-                logger.info("initializeClusterIdentifier continuing...");
+                logger.info("forceInitializedState continuing...");
                 return zuper.call();
             }
             catch (Throwable e)

test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java

@@ -945,7 +945,7 @@ public abstract class TopologyMixupTestBase<S extends TopologyMixupTestBase.Sche
         {
             return inst.callOnInstance(() -> {
                 ClusterMetadata current = ClusterMetadata.current();
-                Set<InetAddressAndPort> members = current.placements.get(ReplicationParams.meta(current)).writes.byEndpoint().keySet();
+                Set<InetAddressAndPort> members = current.placement(ReplicationParams.meta(current)).writes.byEndpoint().keySet();
                 // Why not just use 'current.fullCMSMembers()'?  That uses the "read" replicas, so "could" have less endpoints
                 // It would be more consistent to use fullCMSMembers but thought process is knowing the full set is better
                 // than the coordination set.

test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java

@@ -78,7 +78,7 @@ class OnClusterReplace extends OnClusterChangeTopology
             List<Map.Entry<String, String>> repairRanges = actions.cluster.get(leaving).unsafeApplyOnThisThread(
             (String keyspaceName) -> {
                 ClusterMetadata metadata = ClusterMetadata.current();
-                return metadata.placements.get(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication)
+                return metadata.placement(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication)
                        .writes.ranges()
                               .stream()
                               .map(OnClusterReplace::toStringEntry)
@@ -93,7 +93,7 @@ class OnClusterReplace extends OnClusterChangeTopology
                                       (String keyspaceName, String tk) -> {
                                           ClusterMetadata metadata = ClusterMetadata.current();
                                           KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspaceName);
-                                          return metadata.placements.get(keyspaceMetadata.params.replication).reads
+                                          return metadata.placement(keyspaceMetadata.params.replication).reads
                                                  .forToken(Utils.parseToken(tk))
                                                  .get()
                                                  .stream().map(Replica::endpoint)

test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java

@@ -18,6 +18,7 @@
 package org.apache.cassandra.auth;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -34,6 +35,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ConsistencyLevel;
 import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper;
+import org.apache.cassandra.schema.ReplicationParams;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.schema.TableMetadata;
@@ -59,6 +62,11 @@ public class GrantAndRevokeTest extends CQLTester
         ServerTestUtils.daemonInitialization();
         DatabaseDescriptor.setPermissionsValidity(0);
         DatabaseDescriptor.setRolesValidity(0);
+        // for the tables in the distributed metadata keyspace to be queryable, there needs to be a valid placement
+        // which is derived from the CMS membership. Most unit tests don't actually need to query those dist tables
+        // which is why this isn't done as a matter of routine
+        ClusterMetadataTestHelper.reconfigureCms(ReplicationParams.ntsMeta(Collections.singletonMap(DatabaseDescriptor.getLocalDataCenter(), 1)));
+        ServerTestUtils.markCMS();
         requireAuthentication();
         requireNetwork();
         CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces();

test/unit/org/apache/cassandra/dht/BootStrapperTest.java

@@ -204,7 +204,7 @@ public class BootStrapperTest
         ClusterMetadata metadata = ClusterMetadata.current();
         Pair<MovementMap, MovementMap> movements = Pair.create(MovementMap.empty(), MovementMap.empty());
 
-        if (metadata.myNodeId() == null)
+        if (metadata.myNodeId() == NodeId.UNREGISTERED)
         {
             Token myToken = metadata.partitioner.getRandomToken();
             InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1");
@@ -265,5 +265,4 @@ public class BootStrapperTest
             ClusterMetadataTestHelper.addEndpoint(addr, tokens);
         }
     }
-
 }

test/unit/org/apache/cassandra/locator/MetaStrategyTest.java

@@ -36,6 +36,7 @@ import org.apache.cassandra.schema.DistributedSchema;
 import org.apache.cassandra.service.accord.topology.AccordFastPath;
 import org.apache.cassandra.service.accord.topology.AccordStaleReplicas;
 import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
+import org.apache.cassandra.tcm.CMSMembership;
 import org.apache.cassandra.tcm.ClusterMetadata;
 import org.apache.cassandra.tcm.Epoch;
 import org.apache.cassandra.tcm.membership.Directory;
@@ -96,7 +97,8 @@ public class MetaStrategyTest
                                    InProgressSequences.EMPTY,
                                    ConsensusMigrationState.EMPTY,
                                    ImmutableMap.of(),
-                                   AccordStaleReplicas.EMPTY);
+                                   AccordStaleReplicas.EMPTY,
+                                   CMSMembership.EMPTY);
     }
 
     @Test

test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java

@@ -372,6 +372,6 @@ public class SimpleStrategyTest extends CassandraTestBase
                                                       ReplicationParams replicationParams,
                                                       Token token)
     {
-        return metadata.placements.get(replicationParams).writes.forToken(token).get();
+        return metadata.placement(replicationParams).writes.forToken(token).get();
     }
 }

test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java

@@ -105,7 +105,7 @@ public class AccordTopologyUtils
         {
             ReplicationParams replication = keyspace.params.replication;
             AbstractReplicationStrategy strategy = AbstractReplicationStrategy.createReplicationStrategy(keyspace.name, replication);
-            DataPlacements.Builder placements = metadata.placements.unbuild();
+            DataPlacements.Builder placements = metadata.placements().unbuild();
             DataPlacement placement = strategy.calculateDataPlacement(Epoch.EMPTY, metadata.tokenMap.toRanges(), metadata);
             placements.with(replication, placement);
             metadata = transformer.with(placements.build()).build().metadata;

test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java

@@ -344,7 +344,7 @@ public class EpochSyncTest
         private static boolean joined(ClusterMetadata metadata, Node.Id id)
         {
             NodeAddresses address = metadata.directory.getNodeAddresses(new NodeId(id.id));
-            return metadata.placements.get(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress);
+            return metadata.placement(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress);
         }
 
         public enum EpochTracker { topologyManager, accordSyncPropagator }
@@ -615,7 +615,7 @@ public class EpochSyncTest
         {
             Topology t = AccordTopology.createAccordTopology(current);
             Ranges ranges = t.ranges().mergeTouching();
-            if (!current.placements.get(replication_params).reads.isEmpty())
+            if (!current.placement(replication_params).reads.isEmpty())
                 Assertions.assertThat(ranges).hasSize(1);
             cms.setMetadata(current);
             for (Node.Id id : status(s -> s != Status.Removed))
@@ -721,7 +721,7 @@ public class EpochSyncTest
                     case Registered:
                         Invariants.require(!t.nodes().contains(id), "Node was in Init state but present in the Topology!");
                         Invariants.require(current.directory.peerId(address(id)) != null, "Node exists but not in TCM");
-                        if (current.placements.get(replication_params).writes.byEndpoint().keySet().contains(address(id)))
+                        if (current.placement(replication_params).writes.byEndpoint().keySet().contains(address(id)))
                             status = Status.Joining;
                         break;
                     case Joining:

test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java

@@ -41,6 +41,7 @@ import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.tcm.membership.Directory;
 import org.apache.cassandra.tcm.membership.Location;
 import org.apache.cassandra.tcm.membership.MembershipUtils;
+import org.apache.cassandra.tcm.membership.NodeAddresses;
 import org.apache.cassandra.tcm.membership.NodeId;
 import org.apache.cassandra.tcm.membership.NodeVersion;
 import org.apache.cassandra.tcm.ownership.DataPlacements;
@@ -110,7 +111,10 @@ public class BootWithMetadataTest
         Directory directory = first.directory;
         int nodeCount = 10;
         int tokensPerNode = 5;
-        for (int i = 0; i < nodeCount; i++)
+        // Ensure that the "local" node is registered as it being member of the CMS is a precondition of booting from
+        // a ClusterMetadata.
+        directory = directory.with(NodeAddresses.current(), new Location("DC1", "RACK1"));
+        for (int i = 0; i < nodeCount-1; i++)
             directory = directory.with(nodeAddresses(random), new Location("DC1", "RACK1"));
         t = t.with(directory);
 
@@ -156,7 +160,7 @@ public class BootWithMetadataTest
         assertEquals(toWrite.schema, fromRead.schema);
         assertEquals(toWrite.directory, fromRead.directory);
         assertEquals(toWrite.tokenMap, fromRead.tokenMap);
-        assertEquals(toWrite.placements, fromRead.placements);
+        assertEquals(toWrite.placements(), fromRead.placements());
         assertEquals(toWrite.lockedRanges, fromRead.lockedRanges);
         assertEquals(toWrite.inProgressSequences, fromRead.inProgressSequences);
         assertEquals(toWrite.extensions, fromRead.extensions);