natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add CMS membership directly to ClusterMetadata 

Patch by Sam Tunnicliffe; reviewed by Marcus Eriksson for
CASSANDRA-20736
This commit is contained in:
Sam Tunnicliffe 2026-07-06 11:52:18 +01:00
parent 15f139b91e
commit e1e56e5d5d
116 changed files with 1398 additions and 653 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
6.0-alpha2
* Add CMS membership as a field in ClusterMetadata (CASSANDRA-20736)
* Fix maven remote publishing of Accord artifacts (CASSANDRA-21261)
* Move long running TCM operations to a longer timout (CASSANDRA-21453)
* Offline nodetool commands should not print network options in help (CASSANDRA-20876)

2
src/java/org/apache/cassandra/cql3/statements/schema/CreateKeyspaceStatement.java
View File

@ -96,7 +96,7 @@ public final class CreateKeyspaceStatement extends AlterSchemaStatement
// as we have as keys in metadata.placements to have a fast map lookup
// ReplicationParams are immutable, so it is a safe optimization
KeyspaceParams keyspaceParams = attrs.asNewKeyspaceParams();
ReplicationParams replicationParams = metadata.placements.deduplicateReplicationParams(keyspaceParams.replication);
ReplicationParams replicationParams = metadata.placements().deduplicateReplicationParams(keyspaceParams.replication);
keyspaceParams = keyspaceParams.withSwapped(replicationParams);
KeyspaceMetadata keyspaceMetadata = KeyspaceMetadata.create(keyspaceName, keyspaceParams);

2
src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java
View File

@ -195,6 +195,6 @@ public abstract class AbstractMutationVerbHandler<T extends IMutation> implement
private static VersionedEndpoints.ForToken writePlacements(ClusterMetadata metadata, String keyspace, DecoratedKey key)
{
return metadata.placements.get(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken());
return metadata.placement(metadata.schema.getKeyspace(keyspace).getMetadata().params.replication).writes.forToken(key.getToken());
}
}

4
src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
View File

@ -240,8 +240,8 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace)
{
return metadata.placements
.get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication)
return metadata
.placement(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication)
.reads
.forToken(token)
.get()

2
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
View File

@ -780,7 +780,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
// we only consider write placements during cleanup as range movements always ensure
// overlap between new replicas accepting reads and old replicas accepting writes
ClusterMetadata cm = ClusterMetadata.current();
DataPlacement placement = cm.placements.get(keyspace.getMetadata().params.replication);
DataPlacement placement = cm.placement(keyspace.getMetadata().params.replication);
InetAddressAndPort local = FBUtilities.getBroadcastAddressAndPort();
RangesAtEndpoint localWrites = placement.writes.byEndpoint().get(local);
// TODO review: Hack to get local partitioner not to fail out because it's handled very poorly with data placements

6
src/java/org/apache/cassandra/db/view/ViewUtils.java
View File

@ -30,6 +30,7 @@ import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.ownership.DataPlacement;
public final class ViewUtils
{
@ -64,8 +65,9 @@ public final class ViewUtils
Location local = metadata.locator.local();
KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
EndpointsForToken naturalBaseReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(baseToken).get();
EndpointsForToken naturalViewReplicas = metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(viewToken).get();
DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication);
EndpointsForToken naturalBaseReplicas = placement.reads.forToken(baseToken).get();
EndpointsForToken naturalViewReplicas = placement.reads.forToken(viewToken).get();
Optional<Replica> localReplica = Iterables.tryFind(naturalViewReplicas, Replica::isSelf).toJavaUtil();
if (localReplica.isPresent())

9
src/java/org/apache/cassandra/gms/Gossiper.java
View File

@ -1764,7 +1764,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
register(DatabaseDescriptor.getLocalAddressReconnectionHelper());
ClusterMetadata metadata = ClusterMetadata.current();
if (mergeLocalStates && metadata.myNodeId() != null)
if (mergeLocalStates && metadata.myNodeId() != NodeId.UNREGISTERED)
mergeNodeToGossip(metadata.myNodeId(), metadata);
shutdownAnnounced.set(false);
@ -2234,7 +2234,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
{
checkProperThreadForStateMutation();
assert !endpoint.equals(getBroadcastAddressAndPort()) || epstate.getHeartBeatState().getGeneration() > 0 :
"We should not update epstates with generation = 0 for the local host";
String.format("We should not update epstates with generation = 0 for the local host " +
"(endpoint: %s, broadcast: %s, generation: %s)",
endpoint, getBroadcastAddressAndPort(), epstate.getHeartBeatState().getGeneration());
EndpointState old = endpointStateMap.get(endpoint);
if (old == null)
endpointStateMap.put(endpoint, epstate);
@ -2278,6 +2280,9 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean,
taskLock.lock();
try
{
if (nodeId == NodeId.UNREGISTERED)
return;
boolean isLocal = nodeId.equals(metadata.myNodeId());
IPartitioner partitioner = metadata.tokenMap.partitioner();
NodeAddresses addresses = metadata.directory.getNodeAddresses(nodeId);

2
src/java/org/apache/cassandra/hints/HintsDispatcher.java
View File

@ -276,7 +276,7 @@ final class HintsDispatcher implements AutoCloseable
// Also may need to apply locally because it's possible this is from the batchlog
// and we never applied it locally
// TODO (review): Additional error handling necessary? Hints are lossy
DataPlacement dataPlacement = cm.placements.get(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication);
DataPlacement dataPlacement = cm.placement(cm.schema.getKeyspace(mutation.getKeyspaceName()).getMetadata().params.replication);
VersionedEndpoints.ForToken forToken = dataPlacement.writes.forToken(mutation.key().getToken());
Replica self = forToken.get().selfIfPresent();
if (self != null)

3
src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java
View File

@ -90,7 +90,8 @@ public class CMSPlacementStrategy
// Although MetaStrategy has its own entireRange, it uses a custom partitioner which isn't compatible with
// regular, non-CMS placements. For that reason, we select replicas here using tokens provided by the
// globally configured partitioner.
// globally configured partitioner. This also has the benefit of making concurrent operations, such as
// bounces/upgrades/etc, safe for the CMS if they are replica aware.
Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken();
EndpointsForRange endpoints = NetworkTopologyStrategy.calculateNaturalReplicas(minToken,
new Range<>(minToken, minToken),

2
src/java/org/apache/cassandra/locator/EndpointsForToken.java
View File

@ -164,7 +164,7 @@ public class EndpointsForToken extends Endpoints<EndpointsForToken>
public static VersionedEndpoints.ForToken natural(Keyspace keyspace, Token token)
{
return ClusterMetadata.current().placements.get(keyspace.getMetadata().params.replication).reads.forToken(token);
return ClusterMetadata.current().placement(keyspace.getMetadata().params.replication).reads.forToken(token);
}
}

4
src/java/org/apache/cassandra/locator/MetaStrategy.java
View File

@ -82,13 +82,13 @@ public class MetaStrategy extends SystemStrategy
@Override
public EndpointsForRange calculateNaturalReplicas(Token token, ClusterMetadata metadata)
{
return metadata.placements.get(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get();
return metadata.placement(ReplicationParams.meta(metadata)).reads.forRange(entireRange).get();
}
@Override
public DataPlacement calculateDataPlacement(Epoch epoch, List<Range<Token>> ranges, ClusterMetadata metadata)
{
return metadata.placements.get(ReplicationParams.meta(metadata));
return metadata.placement(ReplicationParams.meta(metadata));
}
@Override

14
src/java/org/apache/cassandra/locator/ReplicaLayout.java
View File

@ -239,7 +239,7 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
{
// todo deduplicate so that "pending" contains "read - write",
// which is a hack until we revisit how consistency level handles pending
DataPlacement dataPlacement = metadata.placements.get(ks.params.replication);
DataPlacement dataPlacement = metadata.placement(ks.params.replication);
natural = forNonLocalStrategyTokenRead(dataPlacement, token);
// perf optimization to avoid double endpoints search and filtering for a typical case
// DataPlacement constructor does a deduplication of reads/writes, so we can use cheap == comparision here
@ -394,12 +394,12 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
static EndpointsForRange forNonLocalStategyRangeRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, AbstractBounds<PartitionPosition> range)
{
return metadata.placements.get(keyspace.params.replication).reads.forRange(range.right.getToken()).get();
return metadata.placement(keyspace.params.replication).reads.forRange(range.right.getToken()).get();
}
public static EndpointsForToken forNonLocalStrategyTokenRead(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token)
{
return forNonLocalStrategyTokenRead(metadata.placements.get(keyspace.params.replication), token);
return forNonLocalStrategyTokenRead(metadata.placement(keyspace.params.replication), token);
}
public static EndpointsForToken forNonLocalStrategyTokenRead(DataPlacement dataPlacement, Token token)
@ -407,17 +407,11 @@ public abstract class ReplicaLayout<E extends Endpoints<E>>
return dataPlacement.reads.forToken(token).get();
}
static EndpointsForToken forNonLocalStrategyTokenWrite(ClusterMetadata metadata, KeyspaceMetadata keyspace, Token token)
{
return forNonLocalStrategyTokenWrite(metadata.placements.get(keyspace.params.replication), token);
}
static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token)
private static EndpointsForToken forNonLocalStrategyTokenWrite(DataPlacement dataPlacement, Token token)
{
return dataPlacement.writes.forToken(token).get();
}
static EndpointsForRange forLocalStrategyRange(ClusterMetadata metadata, AbstractReplicationStrategy replicationStrategy, AbstractBounds<PartitionPosition> range)
{
return replicationStrategy.calculateNaturalReplicas(range.right.getToken(), metadata);

2
src/java/org/apache/cassandra/locator/ReplicaPlans.java
View File

@ -234,7 +234,7 @@ public class ReplicaPlans
NodeProximity proximity = DatabaseDescriptor.getNodeProximity();
AbstractReplicationStrategy replicationStrategy = keyspace.getReplicationStrategy();
EndpointsForToken replicas = metadata.placements.get(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get();
EndpointsForToken replicas = metadata.placement(keyspace.getMetadata().params.replication).reads.forToken(key.getToken()).get();
// CASSANDRA-13043: filter out those endpoints not accepting clients yet, maybe because still bootstrapping
replicas = replicas.filter(replica -> StorageService.instance.isRpcReady(replica.endpoint()));

3
src/java/org/apache/cassandra/metrics/TCMMetrics.java
View File

@ -29,7 +29,6 @@ import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.EpochAwareDebounce;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics;
import static org.apache.cassandra.tcm.transformations.cms.PrepareCMSReconfiguration.needsReconfiguration;
@ -94,7 +93,7 @@ public class TCMMetrics
isCMSMember = Metrics.register(factory.createMetricName("IsCMSMember"), () -> {
ClusterMetadata metadata = ClusterMetadata.currentNullable();
return metadata != null && metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) ? 1 : 0;
return metadata != null && metadata.isCMSMember() ? 1 : 0;
});
needsCMSReconfiguration = Metrics.register(factory.createMetricName("NeedsCMSReconfiguration"), () -> {

3
src/java/org/apache/cassandra/net/ResponseVerbHandler.java
View File

@ -30,7 +30,6 @@ import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.FBUtilities;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND;
@ -98,7 +97,7 @@ class ResponseVerbHandler implements IVerbHandler
if (SKIP_CATCHUP_FOR.contains(message.verb()))
return;
if (metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()) && CMS_SKIP_CATCHUP_FOR.contains(message.verb()))
if (metadata.isCMSMember() && CMS_SKIP_CATCHUP_FOR.contains(message.verb()))
return;
// Gossip stage is single-threaded, so we may end up in a deadlock with after-commit hook

3
src/java/org/apache/cassandra/repair/RepairCoordinator.java
View File

@ -81,7 +81,6 @@ import org.apache.cassandra.tracing.TraceState;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.transport.messages.ResultMessage;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
@ -402,7 +401,7 @@ public class RepairCoordinator implements Runnable, ProgressEventNotifier, Repai
//calculation multiple times
Iterable<Range<Token>> keyspaceLocalRanges = getLocalReplicas.apply(state.keyspace).ranges();
boolean isMeta = Keyspace.open(state.keyspace).getMetadata().params.replication.isMeta();
boolean isCMS = ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort());
boolean isCMS = ClusterMetadata.current().isCMSMember();
for (Range<Token> range : state.options.getRanges())
{
EndpointsForRange allForRange = ctx.repair().getNeighbors(state.keyspace, keyspaceLocalRanges, range);

2
src/java/org/apache/cassandra/service/ActiveRepairService.java
View File

@ -1213,7 +1213,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
// are based on the system partitioner
EndpointsForRange endpoints = replication.isMeta()
? ClusterMetadata.current().fullCMSMembersAsReplicas()
: ClusterMetadata.current().placements.get(replication).reads.forRange(range).get();
: ClusterMetadata.current().placement(replication).reads.forRange(range).get();
Set<InetAddressAndPort> liveEndpoints = endpoints.filter(FailureDetector.isReplicaAlive).endpoints();
if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, liveEndpoints))

2
src/java/org/apache/cassandra/service/CassandraDaemon.java
View File

@ -360,7 +360,7 @@ public class CassandraDaemon
{
CommitLog.instance.recoverSegmentsOnDisk();
NodeId self = ClusterMetadata.current().myNodeId();
if (self != null)
if (self != NodeId.UNREGISTERED)
AccordService.localStartup(self);
}
catch (IOException e)

12
src/java/org/apache/cassandra/service/Rebuild.java
View File

@ -52,12 +52,12 @@ import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.ownership.MovementMap;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException;
import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME;
import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
public class Rebuild
@ -232,21 +232,23 @@ public class Rebuild
private static MovementMap movementMap(ClusterMetadata metadata, String keyspace, String tokens)
{
MovementMap.Builder movementMapBuilder = MovementMap.builder();
DataPlacements placements = metadata.placements;
if (keyspace == null)
{
placements.forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null)));
metadata.placements().forEach((params, placement) -> movementMapBuilder.put(params, addMovementsForParams(placement, null)));
// Special case to include the system metadata keyspace
ReplicationParams metaParams = Keyspace.open(METADATA_KEYSPACE_NAME).getMetadata().params.replication;
movementMapBuilder.put(metaParams, addMovementsForParams(metadata.placement(metaParams), null));
}
else if (tokens == null)
{
ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication;
movementMapBuilder.put(params, addMovementsForParams(placements.get(params), null));
movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), null));
}
else
{
ReplicationParams params = Keyspace.open(keyspace).getMetadata().params.replication;
RangesAtEndpoint ranges = rangesForRebuildWithTokens(tokens, keyspace);
movementMapBuilder.put(params, addMovementsForParams(placements.get(params), ranges));
movementMapBuilder.put(params, addMovementsForParams(metadata.placement(params), ranges));
}
return movementMapBuilder.build();
}

3
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -175,6 +175,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator;
import org.apache.cassandra.service.reads.range.RangeCommands;
import org.apache.cassandra.service.reads.repair.ReadRepair;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeState;
import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
import org.apache.cassandra.tracing.Tracing;
@ -2197,7 +2198,7 @@ public class StorageProxy implements StorageProxyMBean
if (metadata == null)
return false;
if (metadata.myNodeId() == null)
if (metadata.myNodeId() == NodeId.UNREGISTERED)
return false;
return metadata.myNodeState() == NodeState.JOINED;

26
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -2057,9 +2057,16 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
// creation and initialization of cluster metadata service. Metadata collector does accept
// null localhost ID values, it's just that TokenMetadata was created earlier.
ClusterMetadata metadata = ClusterMetadata.currentNullable();
if (metadata == null || metadata.directory.peerId(getBroadcastAddressAndPort()) == null)
return null;
return metadata.directory.peerId(getBroadcastAddressAndPort()).toUUID();
if (metadata == null || metadata.myNodeId() == NodeId.UNREGISTERED)
{
// this condition is to prevent accessing the tables when the node is not started yet, and in particular,
// when it is not going to be started at all (e.g. when running some unit tests or client tools).
if ((DatabaseDescriptor.isDaemonInitialized() || DatabaseDescriptor.isToolInitialized()) && CommitLog.instance.isStarted())
return SystemKeyspace.getLocalHostId();
else
return null;
}
return metadata.myNodeId().toUUID();
}
public Map<String, String> getHostIdMap()
@ -2067,7 +2074,6 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return getEndpointToHostId();
}
public Map<String, String> getEndpointToHostId()
{
return getEndpointToHostId(false);
@ -2119,7 +2125,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
{
if (keyspaceMetadata.params.replication.isMeta())
{
DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication);
DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication);
// May be empty if mid-upgrade and CMS is not yet initialized
if (!placement.reads.isEmpty())
rangeToEndpointMap.put(MetaStrategy.entireRange, placement.reads.forRange(MetaStrategy.entireRange).get());
@ -2129,8 +2135,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
TokenMap tokenMap = metadata.tokenMap;
for (Range<Token> range : ranges)
{
Token token = tokenMap.nextToken(tokenMap.tokens(), range.right.getToken());
rangeToEndpointMap.put(range, metadata.placements.get(keyspaceMetadata.params.replication)
Token token = TokenMap.nextToken(tokenMap.tokens(), range.right.getToken());
rangeToEndpointMap.put(range, metadata.placement(keyspaceMetadata.params.replication)
.reads.forRange(token).get());
}
}
@ -3447,14 +3453,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
token = MetaStrategy.partitioner.getToken(key);
else
token = metadata.partitioner.getToken(key);
return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(token).get();
return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(token).get();
}
public boolean isEndpointValidForWrite(String keyspace, Token token)
{
ClusterMetadata metadata = ClusterMetadata.current();
KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
return keyspaceMetadata != null && metadata.placements.get(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf();
return keyspaceMetadata != null && metadata.placement(keyspaceMetadata.params.replication).writes.forToken(token).get().containsSelf();
}
public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception
@ -4262,7 +4268,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
if (replicationParams.isMeta())
{
LinkedHashMap<InetAddressAndPort, Float> ownership = Maps.newLinkedHashMap();
metadata.placements.get(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> {
metadata.placement(replicationParams).writes.byEndpoint().flattenValues().forEach((r) -> {
ownership.put(r.endpoint(), 1.0f);
});
return ownership;

2
src/java/org/apache/cassandra/service/accord/topology/AccordTopology.java
View File

@ -312,7 +312,7 @@ public class AccordTopology
public static Topology createAccordTopology(ClusterMetadata metadata, ShardLookup lookup)
{
return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements, metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas);
return createAccordTopology(metadata.epoch, metadata.schema, metadata.placements(), metadata.directory, metadata.accordFastPath, lookup, metadata.accordStaleReplicas);
}
public static Topology createAccordTopology(ClusterMetadata metadata, Topology current)

2
src/java/org/apache/cassandra/service/paxos/Paxos.java
View File

@ -277,7 +277,7 @@ public class Paxos
final Token token = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : key.getToken();
ClusterMetadata metadata = ClusterMetadata.current();
Keyspace keyspace = Keyspace.open(table.keyspace);
DataPlacement placement = metadata.placements.get(keyspace.getMetadata().params.replication);
DataPlacement placement = metadata.placement(keyspace.getMetadata().params.replication);
Epoch epoch = placement.writes.forToken(token).lastModified();
ForTokenWrite electorate = forTokenWriteLiveAndDown(metadata, keyspace, token);
if (consistency == LOCAL_SERIAL)

2
src/java/org/apache/cassandra/service/paxos/PaxosRepair.java
View File

@ -586,7 +586,7 @@ public class PaxosRepair extends AbstractPaxosRepair
ClusterMetadata metadata = ClusterMetadata.current();
Collection<InetAddressAndPort> allEndpoints = replication.isMeta()
? metadata.fullCMSMembers()
: metadata.placements.get(replication).reads.forRange(range).endpoints();
: metadata.placement(replication).reads.forRange(range).endpoints();
return hasSufficientLiveNodesForTopologyChange(allEndpoints,
liveEndpoints,
ep -> metadata.locator.location(ep).datacenter,

2
src/java/org/apache/cassandra/streaming/DataMovementVerbHandler.java
View File

@ -50,7 +50,7 @@ public class DataMovementVerbHandler implements IVerbHandler<DataMovement>
StreamPlan streamPlan = new StreamPlan(StreamOperation.fromString(message.payload.streamOperation));
ClusterMetadata metadata = ClusterMetadata.current();
Schema.instance.getNonLocalStrategyKeyspaces().stream().forEach((ksm) -> {
if (metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1)
if (metadata.placement(ksm.params.replication).writes.byEndpoint().keySet().size() <= 1)
return;
message.payload.movements.get(ksm.params.replication).asMap().forEach((local, endpoints) -> {

4
src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java
View File

@ -46,6 +46,7 @@ public abstract class AbstractLocalProcessor implements Processor
this.log = log;
}
/**
* Epoch returned by processor in the Result is _not_ guaranteed to be visible by the Follower by
* the time when this method returns.
@ -59,7 +60,7 @@ public abstract class AbstractLocalProcessor implements Processor
while (!retryPolicy.hasExpired())
{
ClusterMetadata previous = log.waitForHighestConsecutive();
if (!previous.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort()))
if (!acceptCommit(previous))
{
String msg = String.format("Node %s is not a CMS member in epoch %s; members=%s",
FBUtilities.getBroadcastAddressAndPort(),
@ -228,4 +229,5 @@ public abstract class AbstractLocalProcessor implements Processor
public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry retryPolicy);
protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch);
protected abstract boolean acceptCommit(ClusterMetadata metadata);
}

12
src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java
View File

@ -59,6 +59,18 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor
this.epochHolder = new AtomicLong(epoch.getEpoch());
}
@Override
public boolean acceptCommit(ClusterMetadata metadata)
{
// AtomicLongBackedProcessor is only for use in tests and offline tools and it should be safe to assume that it
// is always allowed to process commit requests. In a non-test setup, when the CMS is initialized the initiating
// node is also registered. This is not the case in tests (see initCMS/recreateCMS in ServerTestUtils for more
// detail), as this simplifies setting up/tearing down topologies in test cases. The processor intended for use
// in real clusters (PaxosBackedProcessor) performs some actual validation here to ensure that the node is
// really a CMS member.
return true;
}
@Override
protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch)
{

257
src/java/org/apache/cassandra/tcm/CMSMembership.java Normal file
View File

@ -0,0 +1,257 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tcm;
import java.io.IOException;
import java.util.Objects;
import java.util.Set;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.VersionedEndpoints;
import org.apache.cassandra.tcm.serialization.MetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.btree.BTreeSet;
public class CMSMembership implements MetadataValue<CMSMembership>
{
public static final Serializer serializer = new Serializer();
public static final CMSMembership EMPTY = new CMSMembership();
private final Epoch lastModified;
private final BTreeSet<NodeId> fullMembers;
private final BTreeSet<NodeId> joiningMembers;
/**
* Used to derive a CMSMembership when deserializing a ClusterMetadata instance written with a metadata version
* prior to V9. At that time, CMS membership was always inferred from the data placements of the distributed
* cluster metadata keyspace. Read replicas are full members of the CMS and write-only replicas are in the process
* of joining. Note: every read replica must also be a write replica, leaving the CMS is atomic in respect of the
* placements.
* @param placement
* @param directory
* @return
*/
public static CMSMembership reconstruct(DataPlacement placement, Directory directory)
{
BTreeSet.Builder<NodeId> fullMembersBuilder = BTreeSet.builder(NodeId::compareTo);
BTreeSet.Builder<NodeId> joiningMembersBuilder = BTreeSet.builder(NodeId::compareTo);
Epoch lm = Epoch.EMPTY;
for (VersionedEndpoints.ForRange endpoints : placement.reads.endpoints)
{
lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm;
endpoints.get().endpoints().forEach(e -> fullMembersBuilder.add(directory.peerId(e)));
}
BTreeSet<NodeId> full = fullMembersBuilder.build();
for (VersionedEndpoints.ForRange endpoints : placement.writes.endpoints)
{
lm = endpoints.lastModified().isAfter(lm) ? endpoints.lastModified() : lm;
endpoints.get().endpoints().forEach(e -> {
NodeId id = directory.peerId(e);
if (!full.contains(id))
joiningMembersBuilder.add(id);
});
}
BTreeSet<NodeId> joining = joiningMembersBuilder.build();
return new CMSMembership(lm, full, joining);
}
public DataPlacement toPlacement(Directory directory)
{
DataPlacement.Builder builder = DataPlacement.builder();
for (NodeId id : fullMembers)
{
Replica replica = MetaStrategy.replica(directory.endpoint(id));
builder.withReadReplica(lastModified, replica);
builder.withWriteReplica(lastModified, replica);
}
for(NodeId id : joiningMembers)
{
builder.withWriteReplica(lastModified, MetaStrategy.replica(directory.endpoint(id)));
}
return builder.build();
}
private CMSMembership()
{
this(Epoch.EMPTY,
BTreeSet.empty(NodeId::compareTo),
BTreeSet.empty(NodeId::compareTo));
}
private CMSMembership(Epoch lastModified, BTreeSet<NodeId> fullMembers, BTreeSet<NodeId> joiningMembers)
{
this.lastModified = lastModified;
this.fullMembers = fullMembers;
this.joiningMembers = joiningMembers;
}
@Override
public CMSMembership withLastModified(Epoch epoch)
{
return lastModified.is(epoch) ? this : new CMSMembership(epoch, fullMembers, joiningMembers);
}
@Override
public Epoch lastModified()
{
return lastModified;
}
public Set<NodeId> joiningMembers()
{
return joiningMembers;
}
public Set<NodeId> fullMembers()
{
return fullMembers;
}
public CMSMembership startJoining(NodeId id)
{
if (joiningMembers.contains(id))
throw new IllegalStateException(id + " is already joining the CMS");
if (fullMembers.contains(id))
throw new IllegalStateException(id + " has already fully joined the CMS");
return new CMSMembership(lastModified, fullMembers, joiningMembers.with(id));
}
public CMSMembership cancelJoining(NodeId id)
{
if (!joiningMembers.contains(id))
throw new IllegalStateException(id + " is not currently joining the CMS");
if (fullMembers.contains(id))
throw new IllegalStateException(id + " has already fully joined the CMS");
return new CMSMembership(lastModified, fullMembers, joiningMembers.without(id));
}
public CMSMembership finishJoining(NodeId id)
{
if (!joiningMembers.contains(id))
throw new IllegalStateException(id + " is not currently joining the CMS");
if (fullMembers.contains(id))
throw new IllegalStateException(id + " has already fully joined the CMS");
return new CMSMembership(lastModified, fullMembers.with(id), joiningMembers.without(id));
}
public CMSMembership leave(NodeId id)
{
if (joiningMembers.contains(id))
throw new IllegalStateException(id + " is currently joining the CMS, ");
if (!fullMembers.contains(id))
throw new IllegalStateException(id + " is not a CMS member");
return new CMSMembership(lastModified, fullMembers.without(id), joiningMembers);
}
@Override
public String toString()
{
return "CMSMembership{" +
"lastModified=" + lastModified +
", fullMembers=" + fullMembers +
", joiningMembers=" + joiningMembers +
'}';
}
@Override
public final boolean equals(Object o)
{
if (!(o instanceof CMSMembership)) return false;
CMSMembership that = (CMSMembership) o;
return Objects.equals(lastModified, that.lastModified) &&
Objects.equals(fullMembers, that.fullMembers) &&
Objects.equals(joiningMembers, that.joiningMembers);
}
@Override
public int hashCode()
{
int result = Objects.hashCode(lastModified);
result = 31 * result + Objects.hashCode(fullMembers);
result = 31 * result + Objects.hashCode(joiningMembers);
return result;
}
public static class Serializer implements MetadataSerializer<CMSMembership>
{
@Override
public void serialize(CMSMembership t, DataOutputPlus out, Version version) throws IOException
{
Epoch.serializer.serialize(t.lastModified, out);
out.writeUnsignedVInt32(t.fullMembers.size());
for (NodeId id : t.fullMembers)
NodeId.serializer.serialize(id, out, version);
out.writeUnsignedVInt32(t.joiningMembers.size());
for (NodeId id : t.joiningMembers)
NodeId.serializer.serialize(id, out, version);
}
@Override
public CMSMembership deserialize(DataInputPlus in, Version version) throws IOException
{
Epoch lastModified = Epoch.serializer.deserialize(in, version);
int fullMemberCount = in.readUnsignedVInt32();
BTreeSet.Builder<NodeId> fullMembers = BTreeSet.builder(NodeId::compareTo);
for (int i = 0; i < fullMemberCount; i++)
fullMembers.add(NodeId.serializer.deserialize(in, version));
int joiningMemberCount = in.readUnsignedVInt32();
BTreeSet.Builder<NodeId> joiningMembers = BTreeSet.builder(NodeId::compareTo);
for (int i = 0; i < joiningMemberCount; i++)
joiningMembers.add(NodeId.serializer.deserialize(in, version));
return new CMSMembership(lastModified, fullMembers.build(), joiningMembers.build()) ;
}
@Override
public long serializedSize(CMSMembership t, Version version)
{
long size = Epoch.serializer.serializedSize(t.lastModified);
size += TypeSizes.sizeofUnsignedVInt(t.fullMembers.size());
for (NodeId id : t.fullMembers)
size += NodeId.serializer.serializedSize(id, version);
size += TypeSizes.sizeofUnsignedVInt(t.joiningMembers.size());
for (NodeId id : t.joiningMembers)
size += NodeId.serializer.serializedSize(id, version);
return size;
}
}
}

3
src/java/org/apache/cassandra/tcm/CMSOperations.java
View File

@ -48,7 +48,6 @@ import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.Unregister;
import org.apache.cassandra.tcm.transformations.cms.AdvanceCMSReconfiguration;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.MBeanWrapper;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@ -217,7 +216,7 @@ public class CMSOperations implements CMSOperationsMBean
String members = metadata.fullCMSMembers().stream().sorted().map(Object::toString).collect(Collectors.joining(","));
info.put(MEMBERS, members);
info.put(NEEDS_RECONFIGURATION, Boolean.toString(metadata.epoch.isBefore(Epoch.FIRST) || needsReconfiguration(metadata)));
info.put(IS_MEMBER, Boolean.toString(cms.isCurrentMember(FBUtilities.getBroadcastAddressAndPort())));
info.put(IS_MEMBER, Boolean.toString(metadata.isCMSMember()));
info.put(SERVICE_STATE, ClusterMetadataService.state(metadata).toString());
info.put(IS_MIGRATING, Boolean.toString(cms.isMigrating()));
info.put(EPOCH, Long.toString(metadata.epoch.getEpoch()));

354
src/java/org/apache/cassandra/tcm/ClusterMetadata.java
View File

@ -28,7 +28,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
@ -77,6 +79,7 @@ import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeState;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.ownership.PrimaryRangeComparator;
import org.apache.cassandra.tcm.ownership.ReplicaGroups;
@ -89,7 +92,6 @@ import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
import static org.apache.cassandra.db.TypeSizes.sizeof;
import static org.apache.cassandra.tcm.serialization.Version.MIN_ACCORD_VERSION;
@ -107,13 +109,14 @@ public class ClusterMetadata
public final DistributedSchema schema;
public final Directory directory;
public final TokenMap tokenMap;
public final DataPlacements placements;
private final DataPlacements placements;
public final AccordFastPath accordFastPath;
public final LockedRanges lockedRanges;
public final InProgressSequences inProgressSequences;
public final ConsensusMigrationState consensusMigrationState;
public final ImmutableMap<ExtensionKey<?,?>, ExtensionValue<?>> extensions;
public final AccordStaleReplicas accordStaleReplicas;
public final CMSMembership cmsMembership;
// This isn't serialized as part of ClusterMetadata it's really just a view over the Directory.
public final Locator locator;
@ -121,9 +124,10 @@ public class ClusterMetadata
// These fields are lazy but only for the test purposes, since their computation requires initialization of the log ks
private EndpointsForRange fullCMSReplicas;
private Set<InetAddressAndPort> fullCMSEndpoints;
private Set<NodeId> fullCMSIds;
private volatile Map<ReplicationParams, RangesAtEndpoint> localRangesAllSettled = null;
private static final RangesAtEndpoint EMPTY_LOCAL_RANGES = RangesAtEndpoint.empty(FBUtilities.getBroadcastAddressAndPort());
private DataPlacement cmsDataPlacement;
private final NodeId localNodeId;
public ClusterMetadata(IPartitioner partitioner)
{
@ -151,7 +155,8 @@ public class ClusterMetadata
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
ImmutableMap.of(),
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
}
public ClusterMetadata(Epoch epoch,
@ -165,7 +170,8 @@ public class ClusterMetadata
InProgressSequences inProgressSequences,
ConsensusMigrationState consensusMigrationState,
Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions,
AccordStaleReplicas accordStaleReplicas)
AccordStaleReplicas accordStaleReplicas,
CMSMembership cmsMembership)
{
this(EMPTY_METADATA_IDENTIFIER,
epoch,
@ -179,22 +185,25 @@ public class ClusterMetadata
inProgressSequences,
consensusMigrationState,
extensions,
accordStaleReplicas);
accordStaleReplicas,
cmsMembership);
}
private ClusterMetadata(int metadataIdentifier,
Epoch epoch,
IPartitioner partitioner,
DistributedSchema schema,
Directory directory,
TokenMap tokenMap,
DataPlacements placements,
AccordFastPath accordFastPath,
LockedRanges lockedRanges,
InProgressSequences inProgressSequences,
ConsensusMigrationState consensusMigrationState,
Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions,
AccordStaleReplicas accordStaleReplicas)
Epoch epoch,
IPartitioner partitioner,
DistributedSchema schema,
Directory directory,
TokenMap tokenMap,
DataPlacements placements,
AccordFastPath accordFastPath,
LockedRanges lockedRanges,
InProgressSequences inProgressSequences,
ConsensusMigrationState consensusMigrationState,
Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions,
AccordStaleReplicas accordStaleReplicas,
CMSMembership cmsMembership)
{
// TODO: token map is a feature of the specific placement strategy, and so may not be a relevant component of
// ClusterMetadata in the long term. We need to consider how the actual components of metadata can be evolved
@ -206,14 +215,42 @@ public class ClusterMetadata
this.schema = schema;
this.directory = directory;
this.tokenMap = tokenMap;
this.placements = placements;
this.accordFastPath = accordFastPath;
this.placements = placements;
this.lockedRanges = lockedRanges;
this.inProgressSequences = inProgressSequences;
this.consensusMigrationState = consensusMigrationState;
this.extensions = ImmutableMap.copyOf(extensions);
this.locator = Locator.usingDirectory(directory);
this.accordStaleReplicas = accordStaleReplicas;
this.cmsMembership = cmsMembership;
this.cmsDataPlacement = calculateCMSPlacement(placements, cmsMembership);
InetAddressAndPort broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
this.localNodeId = directory.allAddresses().contains(broadcastAddress)
? directory.peerId(broadcastAddress)
: NodeId.UNREGISTERED;
}
public Set<NodeId> fullCMSMemberIds()
{
return cmsMembership.fullMembers();
}
public boolean isCMSMember()
{
if (epoch.isEqualOrBefore(Epoch.FIRST))
return isCMSMember(FBUtilities.getBroadcastAddressAndPort());
return fullCMSMemberIds().contains(localNodeId);
}
public boolean isCMSMember(InetAddressAndPort endpoint)
{
if (epoch.isEqualOrBefore(Epoch.FIRST))
return cmsDataPlacement.reads.byEndpoint().keySet().contains(endpoint);
return fullCMSMembers().contains(endpoint);
}
public Set<InetAddressAndPort> fullCMSMembers()
@ -223,27 +260,14 @@ public class ClusterMetadata
if (fullCMSEndpoints == null)
{
if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
return Collections.emptySet();
this.fullCMSEndpoints = ImmutableSet.copyOf(placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet());
fullCMSEndpoints = ImmutableSet.copyOf(cmsMembership.fullMembers()
.stream()
.map(directory::endpoint)
.collect(Collectors.toSet()));
}
return fullCMSEndpoints;
}
public Set<NodeId> fullCMSMemberIds()
{
if (epoch.isBefore(Epoch.FIRST))
return Collections.emptySet();
if (fullCMSIds == null)
{
if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
return Collections.emptySet();
this.fullCMSIds = placements.get(ReplicationParams.meta(this)).reads.byEndpoint().keySet().stream().map(directory::peerId).collect(toImmutableSet());
}
return fullCMSIds;
}
public EndpointsForRange fullCMSMembersAsReplicas()
{
if (epoch.isBefore(Epoch.FIRST))
@ -251,16 +275,70 @@ public class ClusterMetadata
if (fullCMSReplicas == null)
{
if (schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
return EndpointsForRange.empty(MetaStrategy.entireRange);
fullCMSReplicas = placements.get(ReplicationParams.meta(this)).reads.forRange(MetaStrategy.entireRange).get();
EndpointsForRange.Builder builder = EndpointsForRange.builder(MetaStrategy.entireRange);
for (NodeId nodeId : fullCMSMemberIds())
builder.add(MetaStrategy.replica(directory.endpoint(nodeId)));
fullCMSReplicas = builder.build();
}
return fullCMSReplicas;
}
public boolean isCMSMember(InetAddressAndPort endpoint)
private DataPlacement calculateCMSPlacement(DataPlacements placements, CMSMembership cms)
{
return fullCMSMembers().contains(endpoint);
if (epoch.isBefore(Epoch.FIRST) || schema.getKeyspaces().get(SchemaConstants.METADATA_KEYSPACE_NAME).isEmpty())
return DataPlacement.empty();
if (directory.isEmpty())
{
if (epoch.is(Epoch.FIRST))
{
// PRE_INITIALIZE_CMS: placements need to be hardcoded to the local address so that the subsequent
// INITIALIZE_CMS can be committed
Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort());
return DataPlacement.builder()
.withReadReplica(Epoch.FIRST, localReplica)
.withWriteReplica(Epoch.FIRST, localReplica)
.build();
}
else
{
// This cluster did not previously upgrade from a gossip based version (i.e. pre-6.0) but did at some point
// run a version prior to MetadataVersion.V9 where we started to encode CMS membership directly. This
// condition implies that we are reconstructing a serialized cluster metadata during replay or else the
// directory should not be empty after Epoch.FIRST as the base state in INITIALIZE_CMS now includes the
// first CMS node. Similarly, if the cluster had previously been running a gossip-based version, the
// directory would contain entries for each of the live nodes at the time of upgrade.
// Given this state, the very next transformation that is/was applied will be to register the node that
// committed the PRE_INITIALIZE_CMS and INTIALIZE_CMS transformations. So we just extract the placements
// associated with the MetaStrategy params as they will already contain that node as an endpoint.
for (ReplicationParams params : placements.keys())
if (params.isMeta())
return placements.get(params);
// This point can only be reached in tests.
// TODO enforce that invariant somehow
Replica localReplica = MetaStrategy.replica(FBUtilities.getBroadcastAddressAndPort());
return DataPlacement.builder()
.withReadReplica(epoch, localReplica)
.withWriteReplica(epoch, localReplica)
.build();
}
}
else
{
// Build a placement based on the CMS membership
return cms.toPlacement(directory);
}
}
public DataPlacement placement(ReplicationParams params)
{
return params.isMeta() ? cmsDataPlacement : placements.get(params);
}
public DataPlacements placements()
{
return placements;
}
public Transformer transformer()
@ -291,10 +369,42 @@ public class ClusterMetadata
capLastModified(inProgressSequences, epoch),
capLastModified(consensusMigrationState, epoch),
capLastModified(extensions, epoch),
capLastModified(accordStaleReplicas, epoch));
capLastModified(accordStaleReplicas, epoch),
capLastModified(cmsMembership, epoch));
}
public ClusterMetadata initializeClusterIdentifier(int clusterIdentifier)
/**
* To be used only during the execute of PRE_INITIALIZE_CMS, this sets the DataPlacement for the metadata keyspace
* so that the global log can be initialized and the subsequent entry containing the INITIALIZE_CMS transformation
* can be committed.
* @param initialCMSPlacement Expected to be a singleton placement identifying the local node as the sole replica
* for the metadata keyspace
* @return ClusterMetadata instance in the correct state to constitute the result of the PRE_INITIALIZE_CMS
* transformation
*/
public ClusterMetadata forcePreInitializedState(DataPlacement initialCMSPlacement)
{
assert epoch.isEqualOrBefore(Epoch.FIRST);
// double check that all metadata elements have had the last modified epoch set correctly
ClusterMetadata initial = forceEpoch(Epoch.FIRST);
initial.cmsDataPlacement = initialCMSPlacement;
return initial;
}
/**
* Produce a ClusterMetadata suitable for use as the base state in the INITIALIZE_CMS transformation. This should
* only be used on the first CMS node when bootstrapping the CMS after upgrade or in a brand new cluster.
* @param clusterIdentifier Unique identifier for split brain detection & protection
* @param addresses The NodeAddresses of the first CMS node.
* @param version Version info for the first CMS node.
* @param location The rack & DC of the first CMS node.
* @return ClusterMetadata instance in the correct state to constitute the base state of the INITIALIZE_CMS
* transformation
*/
public ClusterMetadata forceInitializedState(int clusterIdentifier,
NodeAddresses addresses,
NodeVersion version,
Location location)
{
if (this.metadataIdentifier != EMPTY_METADATA_IDENTIFIER)
throw new IllegalStateException(String.format("Can only initialize cluster identifier once, but it was already set to %d", this.metadataIdentifier));
@ -302,11 +412,21 @@ public class ClusterMetadata
if (clusterIdentifier == EMPTY_METADATA_IDENTIFIER)
throw new IllegalArgumentException("Can not initialize cluster with empty cluster identifier");
if (this.epoch.isAfter(Epoch.FIRST))
throw new IllegalStateException(String.format("Can only initialize cluster identifier during epoch %d, but current epoch is %d", Epoch.FIRST.getEpoch(), epoch.getEpoch()));
// Maybe register the first CMS node. If upgrading from gossip, this should be a no-op
Directory withRegistered = directory.with(addresses, location, version);
NodeId firstNode = withRegistered.peerId(addresses.broadcastAddress);
if (firstNode == null)
throw new IllegalStateException("Failed to find first CMS node in directory");
CMSMembership initialCMS = cmsMembership.startJoining(firstNode).finishJoining(firstNode);
return new ClusterMetadata(clusterIdentifier,
epoch,
partitioner,
schema,
directory,
withRegistered,
tokenMap,
placements,
accordFastPath,
@ -314,7 +434,8 @@ public class ClusterMetadata
inProgressSequences,
consensusMigrationState,
extensions,
accordStaleReplicas);
accordStaleReplicas,
initialCMS);
}
private static Map<ExtensionKey<?,?>, ExtensionValue<?>> capLastModified(Map<ExtensionKey<?,?>, ExtensionValue<?>> original, Epoch maxEpoch)
@ -413,8 +534,8 @@ public class ClusterMetadata
// TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges
public boolean hasPendingRangesFor(KeyspaceMetadata ksm, Token token)
{
ReplicaGroups writes = placements.get(ksm.params.replication).writes;
ReplicaGroups reads = placements.get(ksm.params.replication).reads;
ReplicaGroups writes = placement(ksm.params.replication).writes;
ReplicaGroups reads = placement(ksm.params.replication).reads;
if (ksm.params.replication.isMeta())
return !reads.equals(writes);
return !reads.forToken(token).equals(writes.forToken(token));
@ -423,8 +544,8 @@ public class ClusterMetadata
// TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges
public boolean hasPendingRangesFor(KeyspaceMetadata ksm, InetAddressAndPort endpoint)
{
ReplicaGroups writes = placements.get(ksm.params.replication).writes;
ReplicaGroups reads = placements.get(ksm.params.replication).reads;
ReplicaGroups writes = placement(ksm.params.replication).writes;
ReplicaGroups reads = placement(ksm.params.replication).reads;
return !writes.byEndpoint().get(endpoint).equals(reads.byEndpoint().get(endpoint));
}
@ -435,22 +556,22 @@ public class ClusterMetadata
public RangesAtEndpoint writeRanges(KeyspaceMetadata metadata, InetAddressAndPort peer)
{
return placements.get(metadata.params.replication).writes.byEndpoint().get(peer);
return placement(metadata.params.replication).writes.byEndpoint().get(peer);
}
// TODO Remove this as it isn't really an equivalent to the previous concept of pending ranges
public Map<Range<Token>, VersionedEndpoints.ForRange> pendingRanges(KeyspaceMetadata metadata)
{
Map<Range<Token>, VersionedEndpoints.ForRange> map = new HashMap<>();
ReplicaGroups writes = placements.get(metadata.params.replication).writes;
ReplicaGroups reads = placements.get(metadata.params.replication).reads;
ReplicaGroups writes = placement(metadata.params.replication).writes;
ReplicaGroups reads = placement(metadata.params.replication).reads;
// first, pending ranges as the result of range splitting or merging
// i.e. new ranges being created through join/leave
List<Range<Token>> pending = new ArrayList<>(writes.ranges());
pending.removeAll(reads.ranges());
for (Range<Token> p : pending)
map.put(p, placements.get(metadata.params.replication).writes.forRange(p));
map.put(p, placement(metadata.params.replication).writes.forRange(p));
// next, ranges where the ranges themselves are not changing, but the replicas are
// i.e. replacement or RF increase
@ -467,8 +588,8 @@ public class ClusterMetadata
// TODO Remove this as it isn't really an equivalent to the previous concept of pending endpoints
public VersionedEndpoints.ForToken pendingEndpointsFor(KeyspaceMetadata metadata, Token t)
{
VersionedEndpoints.ForToken writeEndpoints = placements.get(metadata.params.replication).writes.forToken(t);
VersionedEndpoints.ForToken readEndpoints = placements.get(metadata.params.replication).reads.forToken(t);
VersionedEndpoints.ForToken writeEndpoints = placement(metadata.params.replication).writes.forToken(t);
VersionedEndpoints.ForToken readEndpoints = placement(metadata.params.replication).reads.forToken(t);
EndpointsForToken.Builder endpointsForToken = writeEndpoints.get().newBuilder(writeEndpoints.size() - readEndpoints.size());
for (Replica writeReplica : writeEndpoints.get())
@ -495,6 +616,7 @@ public class ClusterMetadata
private final Map<ExtensionKey<?, ?>, ExtensionValue<?>> extensions;
private final Set<MetadataKey> modifiedKeys;
private AccordStaleReplicas accordStaleReplicas;
private CMSMembership cmsMembership;
private Transformer(ClusterMetadata metadata, Epoch epoch)
{
@ -512,6 +634,7 @@ public class ClusterMetadata
extensions = new HashMap<>(metadata.extensions);
modifiedKeys = new HashSet<>();
accordStaleReplicas = metadata.accordStaleReplicas;
cmsMembership = metadata.cmsMembership;
}
public Epoch epoch()
@ -637,6 +760,30 @@ public class ClusterMetadata
return this;
}
public Transformer startJoiningCMS(NodeId id)
{
cmsMembership = cmsMembership.startJoining(id);
return this;
}
public Transformer finishJoiningCMS(NodeId id)
{
cmsMembership = cmsMembership.finishJoining(id);
return this;
}
public Transformer cancelJoiningCMS(NodeId id)
{
cmsMembership = cmsMembership.cancelJoining(id);
return this;
}
public Transformer leaveCMS(NodeId id)
{
cmsMembership = cmsMembership.leave(id);
return this;
}
public Transformer with(DataPlacements placements)
{
this.placements = placements;
@ -822,6 +969,12 @@ public class ClusterMetadata
consensusMigrationState.validateAgainstSchema(schema);
}
if (cmsMembership != base.cmsMembership)
{
modifiedKeys.add(MetadataKeys.CMS_MEMBERSHIP);
cmsMembership = cmsMembership.withLastModified(epoch);
}
return new Transformed(new ClusterMetadata(base.metadataIdentifier,
epoch,
partitioner,
@ -834,7 +987,8 @@ public class ClusterMetadata
inProgressSequences,
consensusMigrationState,
extensions,
accordStaleReplicas),
accordStaleReplicas,
cmsMembership),
ImmutableSet.copyOf(modifiedKeys));
}
@ -852,7 +1006,8 @@ public class ClusterMetadata
inProgressSequences,
consensusMigrationState,
extensions,
accordStaleReplicas);
accordStaleReplicas,
cmsMembership);
}
@Override
@ -871,6 +1026,7 @@ public class ClusterMetadata
", inProgressSequences=" + inProgressSequences +
", consensusMigrationState=" + consensusMigrationState +
", extensions=" + extensions +
", cmsMembership=" + cmsMembership +
", modifiedKeys=" + modifiedKeys +
'}';
}
@ -966,6 +1122,9 @@ public class ClusterMetadata
", placements=" + placements +
", lockedRanges=" + lockedRanges +
", consensusMigrationState=" + lockedRanges +
", inProgressSequences=" + inProgressSequences +
", extensions=" + extensions +
", cmsMembership=" + cmsMembership +
'}';
}
@ -994,7 +1153,8 @@ public class ClusterMetadata
inProgressSequences.equals(that.inProgressSequences) &&
consensusMigrationState.equals(that.consensusMigrationState) &&
accordStaleReplicas.equals(that.accordStaleReplicas) &&
extensions.equals(that.extensions);
extensions.equals(that.extensions) &&
cmsMembership.equals(that.cmsMembership);
}
private static final Logger logger = LoggerFactory.getLogger(ClusterMetadata.class);
@ -1037,12 +1197,16 @@ public class ClusterMetadata
{
logger.warn("Extensions differ: {} != {}", extensions, other.extensions);
}
if (!cmsMembership.equals(other.cmsMembership))
{
logger.warn("CMS Membership differ: {} != {}", cmsMembership, other.cmsMembership);
}
}
@Override
public int hashCode()
{
return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions);
return Objects.hash(epoch, schema, directory, tokenMap, placements, accordFastPath, lockedRanges, inProgressSequences, consensusMigrationState, accordStaleReplicas, extensions, cmsMembership);
}
public static ClusterMetadata current()
@ -1083,13 +1247,13 @@ public class ClusterMetadata
public NodeId myNodeId()
{
return directory.peerId(FBUtilities.getBroadcastAddressAndPort());
return localNodeId;
}
public NodeState myNodeState()
{
NodeId nodeId = myNodeId();
if (myNodeId() != null)
if (nodeId != NodeId.UNREGISTERED)
return directory.peerState(nodeId);
return null;
}
@ -1113,7 +1277,10 @@ public class ClusterMetadata
DistributedSchema.serializer.serialize(metadata.schema, out, version);
Directory.serializer.serialize(metadata.directory, out, version);
TokenMap.serializer.serialize(metadata.tokenMap, out, version);
DataPlacements.serializer.serialize(metadata.placements, out, version);
// Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements
// so when targetting such a version, emulate that.
DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements;
DataPlacements.serializer.serialize(placements, out, version);
if (version.isAtLeast(MIN_ACCORD_VERSION))
{
AccordFastPath.serializer.serialize(metadata.accordFastPath, out, version);
@ -1132,6 +1299,9 @@ public class ClusterMetadata
assert key.valueType.isInstance(value);
value.serialize(out, version);
}
// From V9 CMS membership is directly encoded in ClusterMetadata
if (version.isAtLeast(Version.V9))
CMSMembership.serializer.serialize(metadata.cmsMembership, out, version);
}
@Override
@ -1188,6 +1358,42 @@ public class ClusterMetadata
value.deserialize(in, version);
extensions.put(key, value);
}
CMSMembership cmsMembership = CMSMembership.EMPTY;
if (version.isAtLeast(Version.V9))
cmsMembership = CMSMembership.serializer.deserialize(in, version);
else
{
Optional<KeyspaceMetadata> metadataKs = schema.maybeGetKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
if (metadataKs.isPresent())
{
// Pre-V9 the membership of the CMS was always inferred from the placement of the distributed
// metadata keyspace.
// If the directory is not empty the endpoints in the placement must belong to registered nodes,
// so we can derive the CMSMembership using the data placement and directory.
// If the directory is empty, then the cluster metadata must be the payload of an INITIALIZE_CMS
// transformation of a cluster that began on a post-6.0, pre-MetadataVersion.V9 version.
// In this case, we can and must assume that the initial CMS membership will consist of a single
// node, with the node_id 1.
// Note: the only route to arrive at this scenario is if a cluster is initialized on a post-6.0,
// pre-V9 version and then upgraded to a post-V9 version without any metadata snapshots being taken.
// If there is a snapshot available locally, when the upgraded node starts up it will replay its
// local log from that point. The INITIALIZE_CMS transform will not be replayed.
if (!dir.isEmpty())
{
DataPlacement placement = placements.get(metadataKs.get().params.replication);
cmsMembership = CMSMembership.reconstruct(placement, dir);
}
else
{
NodeId id = new NodeId(1);
cmsMembership = CMSMembership.EMPTY.startJoining(id).finishJoining(id);
}
placements = placements.unbuild().without(metadataKs.get().params.replication).build();
}
}
return new ClusterMetadata(clusterIdentifier,
epoch,
partitioner,
@ -1200,7 +1406,8 @@ public class ClusterMetadata
ips,
consensusMigrationState,
extensions,
staleReplicas);
staleReplicas,
cmsMembership);
}
private DistributedSchema deduplicateReplicationParams(DistributedSchema schema, DataPlacements placements)
@ -1235,8 +1442,7 @@ public class ClusterMetadata
sizeof(metadata.partitioner.getClass().getCanonicalName()) +
DistributedSchema.serializer.serializedSize(metadata.schema, version) +
Directory.serializer.serializedSize(metadata.directory, version) +
TokenMap.serializer.serializedSize(metadata.tokenMap, version) +
DataPlacements.serializer.serializedSize(metadata.placements, version);
TokenMap.serializer.serializedSize(metadata.tokenMap, version);
if (version.isAtLeast(MIN_ACCORD_VERSION))
{
@ -1248,9 +1454,27 @@ public class ClusterMetadata
size += LockedRanges.serializer.serializedSize(metadata.lockedRanges, version) +
InProgressSequences.serializer.serializedSize(metadata.inProgressSequences, version);
// Prior to V9, placements for the MetaStrategy keyspace were included in the main DataPlacements
// so when targetting such a version, emulate that.
DataPlacements placements = version.isBefore(Version.V9) ? preV9Placements(metadata) : metadata.placements;
size += DataPlacements.serializer.serializedSize(placements, version);
// From V9 CMS membership is directly encoded in ClusterMetadata
if (version.isAtLeast(Version.V9))
size += CMSMembership.serializer.serializedSize(metadata.cmsMembership, version);
return size;
}
private DataPlacements preV9Placements(ClusterMetadata metadata)
{
if (metadata.cmsDataPlacement.isEmpty())
return metadata.placements;
return metadata.placements.unbuild()
.with(ReplicationParams.meta(metadata), metadata.cmsDataPlacement)
.build();
}
public static IPartitioner getPartitioner(DataInputPlus in, Version version) throws IOException
{
if (version.isAtLeast(Version.V1))

63
src/java/org/apache/cassandra/tcm/ClusterMetadataService.java
View File

@ -117,7 +117,7 @@ public class ClusterMetadataService
trace);
instance = newInstance;
RegistrationStatus.instance.onInitialized();
if (newInstance.metadata().myNodeId() != null)
if (newInstance.metadata().myNodeId() != NodeId.UNREGISTERED)
RegistrationStatus.instance.onRegistration();
trace = new RuntimeException("Previously initialized trace");
DatabaseDescriptor.applyLocator();
@ -176,8 +176,9 @@ public class ClusterMetadataService
// The node is a full member of the CMS if it has started participating in reads for distributed metadata table (which
// implies it is a write replica as well). In other words, it's a fully joined member of the replica set responsible for
// the distributed metadata table.
if (ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()))
if (metadata.epoch.isEqualOrBefore(Epoch.FIRST) || metadata.isCMSMember())
return LOCAL;
return REMOTE;
}
@ -327,14 +328,15 @@ public class ClusterMetadataService
DatabaseDescriptor.getPartitioner(),
new DistributedSchema(keyspaces),
Directory.EMPTY,
new TokenMap(DatabaseDescriptor.getPartitioner()),
DataPlacements.empty(),
AccordFastPath.EMPTY,
LockedRanges.EMPTY,
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
Collections.emptyMap(),
AccordStaleReplicas.EMPTY);
new TokenMap(DatabaseDescriptor.getPartitioner()),
DataPlacements.empty(),
AccordFastPath.EMPTY,
LockedRanges.EMPTY,
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
Collections.emptyMap(),
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
LocalLog.LogSpec logSpec = LocalLog.logSpec()
@ -360,6 +362,24 @@ public class ClusterMetadataService
ClusterMetadataService.setInstance(cms);
}
@SuppressWarnings("resource")
public static void initializeForClients()
{
if (instance != null)
return;
ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools());
}
public static void initializeForClients(DistributedSchema initialSchema)
{
if (instance != null)
return;
ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema));
}
/*
* Hook to be executed when the LocalLog is bootstrapped with the PreInitialize transformation. This is done on
* the first CMS member to set up the initial replication and data placements for the metadata keyspace.
@ -391,25 +411,8 @@ public class ClusterMetadataService
};
}
// otherwise, this is a noop.
return preInit -> {};
}
@SuppressWarnings("resource")
public static void initializeForClients()
{
if (instance != null)
return;
ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools());
}
public static void initializeForClients(DistributedSchema initialSchema)
{
if (instance != null)
return;
ClusterMetadataService.setInstance(StubClusterMetadataService.forClientTools(initialSchema));
return preInit -> {
};
}
public boolean isCurrentMember(InetAddressAndPort peer)
@ -1100,6 +1103,6 @@ public class ClusterMetadataService
public enum State
{
LOCAL, REMOTE, GOSSIP, RESET
LOCAL, REMOTE, GOSSIP, RESET, OFFLINE_TOOL
}
}

4
src/java/org/apache/cassandra/tcm/MetadataKeys.java
View File

@ -45,18 +45,20 @@ public class MetadataKeys
public static final MetadataKey LOCKED_RANGES = make(CORE_NS, "sequences", "locked_ranges");
public static final MetadataKey IN_PROGRESS_SEQUENCES = make(CORE_NS, "sequences", "in_progress");
public static final MetadataKey CONSENSUS_MIGRATION_STATE = make(CORE_NS, "consensus", "migration_state");
public static final MetadataKey CMS_MEMBERSHIP = make(CORE_NS, "cms_membership", "cms_membership");
public static final ImmutableMap<MetadataKey, Function<ClusterMetadata, MetadataValue<?>>> CORE_METADATA
= ImmutableMap.<MetadataKey, Function<ClusterMetadata, MetadataValue<?>>>builder()
.put(SCHEMA, cm -> cm.schema)
.put(NODE_DIRECTORY, cm -> cm.directory)
.put(TOKEN_MAP, cm -> cm.tokenMap)
.put(DATA_PLACEMENTS, cm -> cm.placements)
.put(DATA_PLACEMENTS, ClusterMetadata::placements)
.put(LOCKED_RANGES, cm -> cm.lockedRanges)
.put(IN_PROGRESS_SEQUENCES, cm -> cm.inProgressSequences)
.put(ACCORD_FAST_PATH, cm -> cm.accordFastPath)
.put(ACCORD_STALE_REPLICAS, cm -> cm.accordStaleReplicas)
.put(CONSENSUS_MIGRATION_STATE, cm -> cm.consensusMigrationState)
.put(CMS_MEMBERSHIP, cm -> cm.cmsMembership)
.build();
public static MetadataKey make(String...parts)

6
src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java
View File

@ -63,6 +63,12 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor
super(log);
}
@Override
protected boolean acceptCommit(ClusterMetadata metadata)
{
return metadata.isCMSMember();
}
@Override
protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch)
{

36
src/java/org/apache/cassandra/tcm/Startup.java
View File

@ -63,8 +63,11 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.log.LocalLog;
import org.apache.cassandra.tcm.log.LogStorage;
import org.apache.cassandra.tcm.log.SystemKeyspaceStorage;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeState;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.migration.CMSInitializationException;
import org.apache.cassandra.tcm.migration.CMSInitializationRequest;
import org.apache.cassandra.tcm.migration.Election;
@ -148,19 +151,24 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
*/
public static void initializeAsFirstCMSNode()
{
InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
String datacenter = DatabaseDescriptor.getLocator().local().datacenter;
NodeAddresses addresses = NodeAddresses.current();
Location location = DatabaseDescriptor.getLocator().local();
ClusterMetadataService cms = ClusterMetadataService.instance();
cms.log().bootstrap(addr, datacenter, cms.logBootstrapCallback());
ClusterMetadata metadata = ClusterMetadata.current();
assert ClusterMetadataService.state() == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata);
Initialize initialize = new Initialize(metadata.initializeClusterIdentifier(addr.hashCode()));
ClusterMetadataService.instance().commit(initialize,
m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;},
(code, message) -> {
logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message);
throw new CMSInitializationException();
});
cms.log().bootstrap(addresses.broadcastAddress, location.datacenter, cms.logBootstrapCallback());
ClusterMetadata metadata = ClusterMetadata.current().forceInitializedState(addresses.broadcastAddress.hashCode(),
addresses,
NodeVersion.CURRENT,
location);
assert ClusterMetadataService.state(metadata) == LOCAL : String.format("Can't initialize as node hasn't transitioned to CMS state. State: %s.\n%s", ClusterMetadataService.state(), metadata);
Initialize initialize = new Initialize(metadata);
ClusterMetadata initialized = ClusterMetadataService.instance().commit(initialize,
m -> { logger.info("INITIALIZE_CMS committed successfully"); return m;},
(code, message) -> {
logger.info("INITIALIZE_CMS commit failure: ({}) {}", code, message);
throw new CMSInitializationException();
});
NodeId id = initialized.myNodeId();
SystemKeyspace.setLocalHostId(id.toUUID());
}
public static void initializeAsNonCmsNode(Function<Processor, Processor> wrapProcessor) throws StartupException
@ -178,7 +186,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
NodeId nodeId = ClusterMetadata.current().myNodeId();
UUID currentHostId = SystemKeyspace.getLocalHostId();
if (nodeId != null && !Objects.equals(nodeId.toUUID(), currentHostId))
if (nodeId != NodeId.UNREGISTERED && !Objects.equals(nodeId.toUUID(), currentHostId))
{
if (currentHostId == null)
{
@ -394,7 +402,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort;
DatabaseDescriptor.getPartitioner().getClass().getCanonicalName(),
metadata.partitioner.getClass().getCanonicalName()));
if (!metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()))
if (!metadata.isCMSMember())
throw new IllegalStateException("When reinitializing with cluster metadata, we must be in the CMS");
metadata = metadata.forceEpoch(metadata.epoch.nextEpoch());

3
src/java/org/apache/cassandra/tcm/StubClusterMetadataService.java
View File

@ -184,7 +184,8 @@ public class StubClusterMetadataService extends ClusterMetadataService
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
ImmutableMap.of(),
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
}
return new StubClusterMetadataService(new UniformRangePlacement(),
snapshots != null ? snapshots : MetadataSnapshots.NO_OP,

10
src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java
View File

@ -58,6 +58,7 @@ import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.service.accord.topology.AccordFastPath;
import org.apache.cassandra.service.accord.topology.AccordStaleReplicas;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.MultiStepOperation;
@ -306,7 +307,8 @@ public class GossipHelper
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
Collections.emptyMap(),
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
}
public static ClusterMetadata fromEndpointStates(DistributedSchema schema, Map<InetAddressAndPort, EndpointState> epStates)
@ -396,7 +398,8 @@ public class GossipHelper
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
extensions,
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
DataPlacements placements = new UniformRangePlacement().calculatePlacements(Epoch.UPGRADE_GOSSIP,
forPlacementCalculation,
schema.getKeyspaces());
@ -411,7 +414,8 @@ public class GossipHelper
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
extensions,
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
}
public static boolean isValidForClusterMetadata(Map<InetAddressAndPort, EndpointState> epstates)

4
src/java/org/apache/cassandra/tcm/listeners/LegacyStateListener.java
View File

@ -82,9 +82,9 @@ public class LegacyStateListener implements ChangeListener
}
}
// next.myNodeId() can be null during replay (before we have registered) but if it is present and
// next.myNodeId() can be UNREGISTERED during replay (before we have registered) but if not and
// there is a relevant change to the state of the local node, process that synchronously.
if (next.myNodeId() != null && changed.contains(next.myNodeId()))
if (next.myNodeId() != NodeId.UNREGISTERED && changed.contains(next.myNodeId()))
{
// Default is to process updates for the local node synchronously, overridable via config/hotprop
if (DatabaseDescriptor.getLegacyStateListenerSyncLocalUpdates())

24
src/java/org/apache/cassandra/tcm/listeners/PlacementsChangeListener.java
View File

@ -18,23 +18,41 @@
package org.apache.cassandra.tcm.listeners;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.ClusterMetadata;
public class PlacementsChangeListener implements ChangeListener
{
private final Runnable onChange;
@VisibleForTesting
public PlacementsChangeListener(Runnable onChange)
{
this.onChange = onChange;
}
public PlacementsChangeListener()
{
this(StorageService.instance::invalidateLocalRanges);
}
@Override
public void notifyPostCommit(ClusterMetadata prev, ClusterMetadata next, boolean fromSnapshot)
{
if (shouldInvalidate(prev, next))
StorageService.instance.invalidateLocalRanges();
onChange.run();
}
private boolean shouldInvalidate(ClusterMetadata prev, ClusterMetadata next)
{
if (!prev.placements.lastModified().equals(next.placements.lastModified()) &&
!prev.placements.equivalentTo(next.placements)) // <- todo should we update lastModified if the result is the same?
if (!prev.placements().lastModified().equals(next.placements().lastModified()) &&
!prev.placements().equivalentTo(next.placements())) // <- todo should we update lastModified if the result is the same?
return true;
if (!prev.cmsMembership.equals(next.cmsMembership))
return true;
if (prev.schema.getKeyspaces().size() != next.schema.getKeyspaces().size())

20
src/java/org/apache/cassandra/tcm/listeners/UpgradeMigrationListener.java
View File

@ -47,17 +47,23 @@ public class UpgradeMigrationListener implements ChangeListener
logger.info("Detected upgrade from gossip mode");
return;
}
else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty()) // directory is non-empty after initialization during gossip upgrade
else if (prev.epoch.equals(Epoch.FIRST) && !next.directory.isEmpty())
{
NodeId localId = next.myNodeId();
if (localId != null)
if (localId != NodeId.UNREGISTERED)
{
logger.info("Initialized CMS, updating local host id to {}", next.myNodeId());
SystemKeyspace.setLocalHostId(next.myNodeId().toUUID());
Gossiper.instance.mergeNodeToGossip(next.myNodeId(), next);
// assigning the local node id is done in Epoch.FIRST in one of two scenarios:
// * during an upgrade from gossip, as all pre-existing nodes will have an id assigned
// * during CMS initialization for a brand new cluster iff the local node is the first CMS member
logger.info("Initialized CMS, updating local host id to {}", localId);
SystemKeyspace.setLocalHostId(localId.toUUID());
if (Gossiper.instance.isEnabled())
{
Gossiper.instance.mergeNodeToGossip(localId, next);
if (Gossiper.instance.getQuarantineDisabled())
Gossiper.instance.clearQuarantinedEndpoints();
}
}
if (Gossiper.instance.getQuarantineDisabled())
Gossiper.instance.clearQuarantinedEndpoints();
}
CassandraVersion prevMinVersion = prev.directory.clusterMinVersion.cassandraVersion;

2
src/java/org/apache/cassandra/tcm/log/LogState.java
View File

@ -306,7 +306,7 @@ public class LogState
if (metadata != null)
{
NodeId mynodeId = metadata.myNodeId();
if (mynodeId != null)
if (mynodeId != NodeId.UNREGISTERED)
SystemKeyspace.setLocalHostId(mynodeId.toUUID());
}
}

2
src/java/org/apache/cassandra/tcm/membership/Directory.java
View File

@ -467,7 +467,7 @@ public class Directory implements MetadataValue<Directory>
public NodeState peerState(NodeId peer)
{
return states.get(peer);
return peer == NodeId.UNREGISTERED ? null : states.get(peer);
}
public NodeVersion version(NodeId peer)

1
src/java/org/apache/cassandra/tcm/membership/NodeId.java
View File

@ -32,6 +32,7 @@ import org.apache.cassandra.tcm.serialization.Version;
public class NodeId implements Comparable<NodeId>, MultiStepOperation.SequenceKey
{
public static final NodeId UNREGISTERED = new NodeId(-1);
private final static long NODE_ID_UUID_MAGIC = 7861390860069061072L;
public static final Serializer serializer = new Serializer();

2
src/java/org/apache/cassandra/tcm/membership/NodeVersion.java
View File

@ -36,7 +36,7 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
public class NodeVersion implements Comparable<NodeVersion>
{
public static final Serializer serializer = new Serializer();
public static final Version CURRENT_METADATA_VERSION = Version.V8;
public static final Version CURRENT_METADATA_VERSION = Version.V9;
public static final NodeVersion CURRENT = new NodeVersion(new CassandraVersion(FBUtilities.getReleaseVersionString()), CURRENT_METADATA_VERSION);
private static final CassandraVersion SINCE_VERSION = CassandraVersion.CASSANDRA_5_1;

5
src/java/org/apache/cassandra/tcm/migration/Election.java
View File

@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
@ -54,7 +53,6 @@ import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeState;
import org.apache.cassandra.tcm.ownership.TokenMap;
import org.apache.cassandra.tcm.transformations.Register;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
@ -157,9 +155,6 @@ public class Election
initiator.compareAndSet(currentInitiator, MIGRATING))
{
Startup.initializeAsFirstCMSNode();
Register.maybeRegister();
SystemKeyspace.setLocalHostId(ClusterMetadata.current().myNodeId().toUUID());
updateInitiator(MIGRATING, MIGRATED);
MessageDelivery.fanoutAndWait(messaging, sendTo, Verb.TCM_NOTIFY_REQ, DistributedMetadataLogKeyspace.getLogState(Epoch.EMPTY, false));
}

5
src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java
View File

@ -106,6 +106,11 @@ public class DataPlacement
return EMPTY;
}
public boolean isEmpty()
{
return reads.isEmpty() && writes.isEmpty();
}
public static Builder builder()
{
return new Builder(ReplicaGroups.builder(),

29
src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java
View File

@ -46,6 +46,7 @@ import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.db.TypeSizes.sizeof;
import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt;
public class DataPlacements extends ReplicationMap<DataPlacement> implements MetadataValue<DataPlacements>
{
@ -62,21 +63,6 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
this.lastModified = lastModified;
}
public DataPlacements replaceParams(Epoch lastModified, ReplicationParams oldParams, ReplicationParams newParams)
{
Map<ReplicationParams, DataPlacement> newMap = Maps.newHashMapWithExpectedSize(map.size());
assert map.containsKey(oldParams) : String.format("Can't replace key %s, since map doesn't contain it: %s", oldParams, map);
for (Map.Entry<ReplicationParams, DataPlacement> e : map.entrySet())
{
if (e.getKey().equals(oldParams))
newMap.put(newParams, e.getValue());
else
newMap.put(e.getKey(), e.getValue());
}
return new DataPlacements(lastModified, newMap);
}
protected DataPlacement defaultValue()
{
return DataPlacement.empty();
@ -256,7 +242,11 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
public void serialize(DataPlacements t, DataOutputPlus out, Version version) throws IOException
{
Map<ReplicationParams, DataPlacement> map = t.asMap();
out.writeInt(map.size());
if (version.isBefore(Version.V9))
out.writeInt(map.size());
else
out.writeUnsignedVInt32(map.size());
for (Map.Entry<ReplicationParams, DataPlacement> entry : map.entrySet())
{
ReplicationParams.serializer.serialize(entry.getKey(), out, version);
@ -267,7 +257,7 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
public DataPlacements deserialize(DataInputPlus in, Version version) throws IOException
{
int size = in.readInt();
int size = version.isBefore(Version.V9) ? in.readInt() : in.readUnsignedVInt32();
Map<ReplicationParams, DataPlacement> map = Maps.newHashMapWithExpectedSize(size);
for (int i = 0; i < size; i++)
{
@ -280,8 +270,9 @@ public class DataPlacements extends ReplicationMap<DataPlacement> implements Met
public long serializedSize(DataPlacements t, Version version)
{
long size = sizeof(t.size());
for (Map.Entry<ReplicationParams, DataPlacement> entry : t.asMap().entrySet())
Map<ReplicationParams, DataPlacement> map = t.asMap();
long size = version.isBefore(Version.V9) ? sizeof(map.size()) : sizeofUnsignedVInt(map.size());
for (Map.Entry<ReplicationParams, DataPlacement> entry : map.entrySet())
{
size += ReplicationParams.serializer.serializedSize(entry.getKey(), version);
size += DataPlacement.serializerFor(entry.getKey()).serializedSize(entry.getValue(), version);

8
src/java/org/apache/cassandra/tcm/ownership/UniformRangePlacement.java
View File

@ -329,9 +329,13 @@ public class UniformRangePlacement implements PlacementProvider
logger.trace("Calculating data placements for {}", ksMetadata.name);
AbstractReplicationStrategy replication = ksMetadata.replicationStrategy;
ReplicationParams params = ksMetadata.params.replication;
if (params.isMeta() || params.isLocal())
if (params.isMeta())
{
placements.put(params, metadata.placements.get(params));
// don't calculate meta strategy placements, these are derived from ClusterMetadata.cmsMembership
}
else if (params.isLocal())
{
placements.put(params, metadata.placement(params));
}
else
{

4
src/java/org/apache/cassandra/tcm/sequences/BootstrapAndJoin.java
View File

@ -326,7 +326,7 @@ public class BootstrapAndJoin extends MultiStepOperation<Epoch>
@Override
public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
{
DataPlacements placements = metadata.placements;
DataPlacements placements = metadata.placements();
switch (next)
{
// need to undo MID_JOIN and START_JOIN, then merge the ranges split by PrepareJoin
@ -357,7 +357,7 @@ public class BootstrapAndJoin extends MultiStepOperation<Epoch>
@VisibleForTesting
public Pair<MovementMap, MovementMap> getMovementMaps(ClusterMetadata metadata)
{
MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements, startJoin.delta());
MovementMap movementMap = movementMap(metadata.directory.endpoint(startJoin.nodeId()), metadata.placements(), startJoin.delta());
MovementMap strictMovementMap = toStrict(movementMap, finishJoin.delta());
return Pair.create(movementMap, strictMovementMap);
}

8
src/java/org/apache/cassandra/tcm/sequences/BootstrapAndReplace.java
View File

@ -318,7 +318,7 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
@Override
public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
{
DataPlacements placements = metadata.placements;
DataPlacements placements = metadata.placements();
switch (next)
{
// need to undo MID_REPLACE and START_REPLACE, but PREPARE_REPLACE doesn't affect placements
@ -355,7 +355,7 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
private static MovementMap movementMap(InetAddressAndPort beingReplaced, PlacementDeltas startDelta)
{
MovementMap.Builder movementMapBuilder = MovementMap.builder();
DataPlacements placements = ClusterMetadata.current().placements;
DataPlacements placements = ClusterMetadata.current().placements();
startDelta.forEach((params, delta) -> {
EndpointsByReplica.Builder movements = new EndpointsByReplica.Builder();
DataPlacement originalPlacements = placements.get(params);
@ -451,6 +451,8 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
public static void gossipStateToHibernate(ClusterMetadata metadata, NodeId nodeId)
{
if (nodeId == NodeId.UNREGISTERED)
return;
// order is important here, the gossiper can fire in between adding these two states. It's ok to send TOKENS without STATUS, but *not* vice versa.
List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<>();
VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory;
@ -462,6 +464,8 @@ public class BootstrapAndReplace extends MultiStepOperation<Epoch>
public static void gossipStateToNormal(ClusterMetadata metadata, NodeId nodeId)
{
if (nodeId == NodeId.UNREGISTERED)
return;
List<Pair<ApplicationState, VersionedValue>> states = new ArrayList<>();
VersionedValue.VersionedValueFactory valueFactory = StorageService.instance.valueFactory;
Collection<Token> tokens = metadata.tokenMap.tokens(nodeId);

68
src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java
View File

@ -25,25 +25,20 @@ import java.util.Map;
import org.apache.cassandra.exceptions.ExceptionCode;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.DistributedSchema;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.accord.topology.FastPathStrategy;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import static org.apache.cassandra.locator.MetaStrategy.entireRange;
public class CancelCMSReconfiguration implements Transformation
{
public static final Serializer serializer = new Serializer();
@ -66,53 +61,46 @@ public class CancelCMSReconfiguration implements Transformation
if (reconfigureCMS == null)
return new Rejected(ExceptionCode.INVALID, "Can not cancel reconfiguration since there does not seem to be any in-flight");
ReplicationParams metaParams = ReplicationParams.meta(prev);
ClusterMetadata.Transformer transformer = prev.transformer();
DataPlacement placement = prev.placements.get(metaParams);
// Reset any partially completed transition by removing the pending replica from the write group
if (reconfigureCMS.next.activeTransition != null)
{
InetAddressAndPort pendingEndpoint = prev.directory.endpoint(reconfigureCMS.next.activeTransition.nodeId);
Replica pendingReplica = new Replica(pendingEndpoint, entireRange, true);
placement = placement.unbuild()
.withoutWriteReplica(prev.nextEpoch(), pendingReplica)
.build();
// see what the placements for the meta keyspace will be after cancelling the active join
CMSMembership cms = prev.cmsMembership.cancelJoining(reconfigureCMS.next.activeTransition.nodeId);
if (!cms.joiningMembers().isEmpty())
return new Rejected(ExceptionCode.INVALID,
String.format("Placements will be inconsistent if this transformation is applied:" +
"\nFull members %s\nJoining members: %s",
cms.fullMembers(),
cms.joiningMembers()));
// if all is good, actually cancel the joining member
transformer = transformer.cancelJoiningCMS(reconfigureCMS.next.activeTransition.nodeId);
// Recalculate the replication params for the meta keyspace based on the actual placement as it will be
ReplicationParams recalculated = getAccurateReplication(prev.directory, cms);
// If they no longer match the replication params in schema, i.e. the transitions completed so far did not
// bring the membership/placements into line with configuration, update schema to match what we actually have
if (!recalculated.equals(ReplicationParams.meta(prev)))
{
KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites,
recalculated,
FastPathStrategy.simple()));
transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)));
}
}
if (!placement.reads.equivalentTo(placement.writes))
return new Rejected(ExceptionCode.INVALID, String.format("Placements will be inconsistent if this transformation is applied:\nReads %s\nWrites: %s",
placement.reads,
placement.writes));
// Reset the replication params for the meta keyspace based on the actual placement in case they no longer match
ReplicationParams fromPlacement = getAccurateReplication(prev.directory, placement);
// If they no longer match, i.e. the transitions completed so far did not bring the placements into line with
// the configuration, remove the entry keyed by the existing configured params.
DataPlacements.Builder builder = prev.placements.unbuild();
if (!metaParams.equals(fromPlacement))
{
builder = builder.without(metaParams);
// Also update schema with the corrected params
KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple()));
transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace)));
}
// finally, add the possibly corrected placement keyed by the possibly corrected params
builder = builder.with(fromPlacement, placement);
transformer = transformer.with(builder.build());
return Transformation.success(transformer.with(prev.inProgressSequences.without(ReconfigureCMS.SequenceKey.instance))
.with(prev.lockedRanges.unlock(reconfigureCMS.next.lockKey)),
MetaStrategy.affectedRanges(prev));
}
private ReplicationParams getAccurateReplication(Directory directory, DataPlacement placement)
private ReplicationParams getAccurateReplication(Directory directory, CMSMembership membership)
{
Map<String, Integer> replicasPerDc = new HashMap<>();
placement.writes.byEndpoint().keySet().forEach(i -> {
String dc = directory.location(directory.peerId(i)).datacenter;
membership.fullMembers().forEach(id -> {
String dc = directory.location(id).datacenter;
int count = replicasPerDc.getOrDefault(dc, 0);
replicasPerDc.put(dc, ++count);
});

4
src/java/org/apache/cassandra/tcm/sequences/Move.java
View File

@ -321,7 +321,7 @@ public class Move extends MultiStepOperation<Epoch>
StreamPlan streamPlan = new StreamPlan(StreamOperation.RELOCATION);
Keyspaces keyspaces = Schema.instance.getNonLocalStrategyKeyspaces();
Map<ReplicationParams, EndpointsByReplica> movementMap = movementMap(FailureDetector.instance,
metadata.placements,
metadata.placements(),
toSplitRanges,
startMove.delta(),
midMove.delta(),
@ -430,7 +430,7 @@ public class Move extends MultiStepOperation<Epoch>
@Override
public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
{
DataPlacements placements = metadata.placements;
DataPlacements placements = metadata.placements();
switch (next)
{

6
src/java/org/apache/cassandra/tcm/sequences/ProgressBarrier.java
View File

@ -58,6 +58,7 @@ import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.Retry;
import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
@ -170,8 +171,9 @@ public class ProgressBarrier
Set<Range<Token>> ranges = e.getValue();
for (Range<Token> range : ranges)
{
EndpointsForRange writes = metadata.placements.get(params).writes.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
EndpointsForRange reads = metadata.placements.get(params).reads.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
DataPlacement placement = metadata.placement(params);
EndpointsForRange writes = placement.writes.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
EndpointsForRange reads = placement.reads.matchRange(range).get().filter(r -> filter.test(r.endpoint()));
// Affected ranges can contain ranges which are the results of merging or splitting and may not exist
// as keys in the existing ReplicaGroups. As such, no replicas will be found for these ranges and so no
// WaitFor is necessary.

2
src/java/org/apache/cassandra/tcm/sequences/RemoveNodeStreams.java
View File

@ -121,7 +121,7 @@ public class RemoveNodeStreams implements LeaveStreams
RangesByEndpoint startWriteAdditions = startDelta.get(params).writes.additions;
RangesByEndpoint startWriteRemovals = startDelta.get(params).writes.removals;
// find current placements from the metadata, we need to stream from replicas that are not changed and are therefore not in the deltas
ReplicaGroups currentPlacements = metadata.placements.get(params).reads;
ReplicaGroups currentPlacements = metadata.placement(params).reads;
startWriteAdditions.flattenValues()
.forEach(newReplica -> {
EndpointsForRange.Builder candidateBuilder = new EndpointsForRange.Builder(newReplica.range());

2
src/java/org/apache/cassandra/tcm/sequences/ReplaceSameAddress.java
View File

@ -44,7 +44,7 @@ public class ReplaceSameAddress
{
MovementMap.Builder builder = MovementMap.builder();
InetAddressAndPort addr = metadata.directory.endpoint(nodeId);
metadata.placements.forEach((params, placement) -> {
metadata.placements().forEach((params, placement) -> {
EndpointsByReplica.Builder sources = new EndpointsByReplica.Builder();
placement.reads.byEndpoint().get(addr).forEach(destination -> {
placement.reads.forRange(destination.range()).forEach(potentialSource -> {

2
src/java/org/apache/cassandra/tcm/sequences/UnbootstrapAndLeave.java
View File

@ -253,7 +253,7 @@ public class UnbootstrapAndLeave extends MultiStepOperation<Epoch>
@Override
public ClusterMetadata.Transformer cancel(ClusterMetadata metadata)
{
DataPlacements placements = metadata.placements;
DataPlacements placements = metadata.placements();
switch (next)
{
// need to undo MID_LEAVE and START_LEAVE, but PrepareLeave doesn't affect placement

14
src/java/org/apache/cassandra/tcm/serialization/Version.java
View File

@ -38,13 +38,13 @@ public enum Version
*/
V0(0),
/**
* - Moved Partitioner in ClusterMetadata serializer to be the first field
* - Added a counter to Directory serializer to keep track of NodeIds
* - Moved Partitioner in ClusterMetadata serializer to be the first field
* - Added a counter to Directory serializer to keep track of NodeIds
*/
V1(1),
/**
* - Added version to PlacementForRange serializer
* - Serialize MemtableParams when serializing TableParams
* - Added version to PlacementForRange serializer
* - Serialize MemtableParams when serializing TableParams
*/
V2(2),
/**
@ -78,6 +78,11 @@ public enum Version
* - Comments and security labels for schema elements (keyspaces, tables, columns, UDTs, and UDT fields)
*/
V8(8),
/**
* - DataPlacements don't include MetaStrategy, replaced by ClusterMetadata.CMSMembership
* - Size of DataPlacements is encoded as vint
*/
V9(9),
UNKNOWN(Integer.MAX_VALUE);
@ -95,6 +100,7 @@ public enum Version
}
private final int version;
Version(int version)
{
this.version = version;

2
src/java/org/apache/cassandra/tcm/transformations/AccordMarkStale.java
View File

@ -78,7 +78,7 @@ public class AccordMarkStale implements Transformation
for (KeyspaceMetadata keyspace : prev.schema.getKeyspaces().without(SchemaConstants.REPLICATED_SYSTEM_KEYSPACE_NAMES))
{
List<AccordTopology.KeyspaceShard> shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements, prev.directory);
List<AccordTopology.KeyspaceShard> shards = AccordTopology.KeyspaceShard.forKeyspace(keyspace, prev.placements(), prev.directory);
for (AccordTopology.KeyspaceShard shard : shards)
{

2
src/java/org/apache/cassandra/tcm/transformations/AlterSchema.java
View File

@ -244,7 +244,7 @@ public class AlterSchema implements Transformation
DataPlacements.Builder newPlacementsBuilder = DataPlacements.builder(calculatedPlacements.size());
calculatedPlacements.forEach((params, newPlacement) -> {
DataPlacement previousPlacement = prev.placements.get(params);
DataPlacement previousPlacement = prev.placement(params);
// Preserve placement versioning that has resulted from natural application where possible
if (previousPlacement.equivalentTo(newPlacement))
newPlacementsBuilder.with(params, previousPlacement);

4
src/java/org/apache/cassandra/tcm/transformations/AlterTopology.java
View File

@ -130,11 +130,11 @@ public class AlterTopology implements Transformation
for (Map.Entry<NodeId, Location> update : updates.entrySet())
updated = updated.withUpdatedRackAndDc(update.getKey(), update.getValue());
ClusterMetadata proposed = prev.transformer().with(updated).build().metadata;
DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements.lastModified(),
DataPlacements proposedPlacements = placementProvider.calculatePlacements(prev.placements().lastModified(),
proposed.tokenMap.toRanges(),
proposed,
proposed.schema.getKeyspaces());
if (!proposedPlacements.equivalentTo(prev.placements))
if (!proposedPlacements.equivalentTo(prev.placements()))
{
logger.info("Rejecting topology modifications which would materially change data placements: {}", updates);
return new Rejected(INVALID, "Proposed updates modify data placements, violating consistency guarantees");

2
src/java/org/apache/cassandra/tcm/transformations/ApplyPlacementDeltas.java
View File

@ -82,7 +82,7 @@ public abstract class ApplyPlacementDeltas implements Transformation
ClusterMetadata.Transformer next = prev.transformer();
if (!delta.isEmpty())
next = next.with(delta.apply(prev.nextEpoch(), prev.placements));
next = next.with(delta.apply(prev.nextEpoch(), prev.placements()));
next = transform(prev, next);

6
src/java/org/apache/cassandra/tcm/transformations/PrepareJoin.java
View File

@ -89,7 +89,7 @@ import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
*/
public class PrepareJoin implements Transformation
{
public static final Serializer<PrepareJoin> serializer = new Serializer<PrepareJoin>()
public static final Serializer<PrepareJoin> serializer = new Serializer<>()
{
public PrepareJoin construct(NodeId nodeId, Set<Token> tokens, PlacementProvider placementProvider, boolean joinTokenRing, boolean streamData)
{
@ -168,10 +168,10 @@ public class PrepareJoin implements Transformation
startJoin, midJoin, finishJoin,
joinTokenRing, streamData);
if (!prev.tokenMap.isEmpty())
assertPreExistingWriteReplica(prev.placements, transitionPlan);
assertPreExistingWriteReplica(prev.placements(), transitionPlan);
LockedRanges newLockedRanges = prev.lockedRanges.lock(lockKey, rangesToLock);
DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements);
DataPlacements startingPlacements = transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements());
ClusterMetadata.Transformer proposed = prev.transformer()
.with(newLockedRanges)
.with(startingPlacements)

2
src/java/org/apache/cassandra/tcm/transformations/PrepareLeave.java
View File

@ -115,7 +115,7 @@ public class PrepareLeave implements Transformation
PlacementDeltas startDelta = transitionPlan.addToWrites();
PlacementDeltas midDelta = transitionPlan.moveReads();
PlacementDeltas finishDelta = transitionPlan.removeFromWrites();
transitionPlan.assertPreExistingWriteReplica(prev.placements);
transitionPlan.assertPreExistingWriteReplica(prev.placements());
LockedRanges.Key unlockKey = LockedRanges.keyFor(proposed.epoch);

4
src/java/org/apache/cassandra/tcm/transformations/PrepareMove.java
View File

@ -109,7 +109,7 @@ public class PrepareMove implements Transformation
StartMove startMove = new StartMove(nodeId, transitionPlan.addToWrites(), lockKey);
MidMove midMove = new MidMove(nodeId, transitionPlan.moveReads(), lockKey);
FinishMove finishMove = new FinishMove(nodeId, tokens, transitionPlan.removeFromWrites(), lockKey);
transitionPlan.assertPreExistingWriteReplica(prev.placements);
transitionPlan.assertPreExistingWriteReplica(prev.placements());
Move sequence = Move.newSequence(prev.nextEpoch(),
lockKey,
@ -123,7 +123,7 @@ public class PrepareMove implements Transformation
return Transformation.success(prev.transformer()
.withNodeState(nodeId, NodeState.MOVING)
.with(prev.lockedRanges.lock(lockKey, rangesToLock))
.with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements))
.with(transitionPlan.toSplit.apply(prev.nextEpoch(), prev.placements()))
.with(prev.inProgressSequences.with(nodeId, sequence)),
rangesToLock);
}

2
src/java/org/apache/cassandra/tcm/transformations/PrepareReplace.java
View File

@ -109,7 +109,7 @@ public class PrepareReplace implements Transformation
StartReplace start = new StartReplace(replaced, replacement, transitionPlan.addToWrites(), unlockKey);
MidReplace mid = new MidReplace(replaced, replacement, transitionPlan.moveReads(), unlockKey);
FinishReplace finish = new FinishReplace(replaced, replacement, transitionPlan.removeFromWrites(), unlockKey);
transitionPlan.assertPreExistingWriteReplica(prev.placements);
transitionPlan.assertPreExistingWriteReplica(prev.placements());
Set<Token> tokens = new HashSet<>(prev.tokenMap.tokens(replaced));
BootstrapAndReplace plan = BootstrapAndReplace.newSequence(prev.nextEpoch(),

2
src/java/org/apache/cassandra/tcm/transformations/Register.java
View File

@ -109,7 +109,7 @@ public class Register implements Transformation
if (isReplacingSameAddress())
{
NodeId self = ClusterMetadata.current().myNodeId();
if (self == null)
if (self == NodeId.UNREGISTERED)
throw new IllegalStateException("Tried to replace same address, but node does not seem to be registered");
return self;

27
src/java/org/apache/cassandra/tcm/transformations/Startup.java
View File

@ -24,10 +24,6 @@ import java.util.Objects;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Transformation;
@ -35,14 +31,12 @@ import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.MetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
import static org.apache.cassandra.locator.MetaStrategy.entireRange;
public class Startup implements Transformation
{
@ -96,32 +90,13 @@ public class Startup implements Transformation
if (!nodeId.equals(existingNodeId) && addresses.conflictsWith(existingAddresses))
return new Rejected(INVALID, String.format("New addresses %s conflicts with existing node %s with addresses %s", addresses, entry.getKey(), existingAddresses));
}
next = next.withNewAddresses(nodeId, addresses);
Keyspaces allKeyspaces = prev.schema.getKeyspaces().withAddedOrReplaced(prev.schema.getKeyspaces());
DataPlacements newPlacement = ClusterMetadataService.instance()
.placementProvider()
.calculatePlacements(prev.nextEpoch(),
prev.tokenMap.toRanges(),
next.build().metadata,
allKeyspaces);
if (prev.isCMSMember(prev.directory.endpoint(nodeId)))
{
ReplicationParams metaParams = ReplicationParams.meta(prev);
InetAddressAndPort endpoint = prev.directory.endpoint(nodeId);
Replica leavingReplica = new Replica(endpoint, entireRange, true);
Replica joiningReplica = new Replica(addresses.broadcastAddress, entireRange, true);
DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild();
builder.reads.withoutReplica(prev.nextEpoch(), leavingReplica);
builder.writes.withoutReplica(prev.nextEpoch(), leavingReplica);
builder.reads.withReplica(prev.nextEpoch(), joiningReplica);
builder.writes.withReplica(prev.nextEpoch(), joiningReplica);
newPlacement = newPlacement.unbuild().with(metaParams, builder.build()).build();
}
prev.schema.getKeyspaces());
next = next.with(newPlacement);
}

68
src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java
View File

@ -29,15 +29,12 @@ import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.RangesByEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.MultiStepOperation;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.sequences.InProgressSequences;
import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
@ -45,7 +42,6 @@ import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
import static org.apache.cassandra.locator.MetaStrategy.entireRange;
import static org.apache.cassandra.tcm.MultiStepOperation.Kind.RECONFIGURE_CMS;
/**
@ -154,38 +150,26 @@ public class AdvanceCMSReconfiguration implements Transformation
{
// Pop the next node to be added from the list diff.additions
NodeId addition = diff.additions.get(0);
InetAddressAndPort endpoint = prev.directory.endpoint(addition);
Replica replica = new Replica(endpoint, entireRange, true);
List<NodeId> newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size()));
// Check that the candidate is not already a CMS member
ReplicationParams metaParams = ReplicationParams.meta(prev);
RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint();
RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint();
if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint))
return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS");
CMSMembership cms = prev.cmsMembership;
if (cms.joiningMembers().contains(addition) || cms.fullMembers().contains(addition))
return new Transformation.Rejected(INVALID, "Endpoint is already a full or joining member of the CMS: " + prev.directory.endpoint(addition));
ClusterMetadata.Transformer transformer = prev.transformer();
// Add the candidate as a write replica
DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild()
.withWriteReplica(prev.nextEpoch(), replica);
transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build());
// Add the candidate as a joining member
ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(addition);
// Construct a set of sources for the new member to stream log tables from (essentially this is the existing members)
Set<InetAddressAndPort> streamCandidates = new HashSet<>();
for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues())
{
if (!replica.equals(r))
streamCandidates.add(r.endpoint());
}
Set<InetAddressAndPort> streamCandidates = prev.fullCMSMembers();
// Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence,
// including the remaining add/remove operations and the streaming that needs to be done by the joining node
List<NodeId> newAdditions = new ArrayList<>(diff.additions.subList(1, diff.additions.size()));
AdvanceCMSReconfiguration next = next(prev.nextEpoch(),
newAdditions,
diff.removals,
new ReconfigureCMS.ActiveTransition(addition, streamCandidates));
// Create a new sequence instance with the next step to reflect that the state has progressed.
ReconfigureCMS advanced = sequence.advance(next);
// Finally, replace the existing reconfiguration sequence with this updated one.
@ -206,21 +190,21 @@ public class AdvanceCMSReconfiguration implements Transformation
*/
private Transformation.Result finishAdd(ClusterMetadata prev, ReconfigureCMS sequence, NodeId addition)
{
// Check that the candidate is already a joining CMS member
CMSMembership cms = prev.cmsMembership;
if (!cms.joiningMembers().contains(addition))
return new Transformation.Rejected(INVALID, "Endpoint is not a in the process of joining the CMS: " + prev.directory.endpoint(addition));
// Add the new member as a full read replica, able to participate in quorums for log updates
ReplicationParams metaParams = ReplicationParams.meta(prev);
InetAddressAndPort endpoint = prev.directory.endpoint(addition);
Replica replica = new Replica(endpoint, entireRange, true);
ClusterMetadata.Transformer transformer = prev.transformer();
DataPlacement.Builder builder = prev.placements.get(metaParams)
.unbuild()
.withReadReplica(prev.nextEpoch(), replica);
transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build());
ClusterMetadata.Transformer transformer = prev.transformer().finishJoiningCMS(addition);
// Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence,
// which includes the remaining add/remove operations
AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, diff.removals, null);
// Create a new sequence instance with the next step to reflect that the state has progressed.
ReconfigureCMS advanced = sequence.advance(next);
// Finally, replace the existing reconfiguration sequence with this updated one.
transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced));
return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));
@ -238,29 +222,25 @@ public class AdvanceCMSReconfiguration implements Transformation
List<NodeId> newRemovals = new ArrayList<>(diff.removals.subList(1, diff.removals.size()));
// Check that the candidate is actually a CMS member
ClusterMetadata.Transformer transformer = prev.transformer();
Set<NodeId> cms = prev.fullCMSMemberIds();
InetAddressAndPort endpoint = prev.directory.endpoint(removal);
Replica replica = new Replica(endpoint, entireRange, true);
ReplicationParams metaParams = ReplicationParams.meta(prev);
if (!prev.fullCMSMembers().contains(endpoint))
if (!cms.contains(removal))
return new Transformation.Rejected(INVALID, String.format("%s is not currently a CMS member, cannot remove it", endpoint));
// Check that the candidate is not the only CMS member
DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild();
builder.reads.withoutReplica(prev.nextEpoch(), replica);
builder.writes.withoutReplica(prev.nextEpoch(), replica);
DataPlacement proposed = builder.build();
if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty())
if (cms.size() == 1)
return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint));
// Actually remove the candidate
transformer = transformer.with(prev.placements.unbuild().with(metaParams, proposed).build());
// Remove the CMS member
ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(removal);
// Set up the next step in the sequence. This encapsulates the entire state of the reconfiguration sequence,
// which includes the remaining add/remove operations
AdvanceCMSReconfiguration next = next(prev.nextEpoch(), diff.additions, newRemovals, null);
// Create a new sequence instance with the next step to reflect that the state has progressed.
ReconfigureCMS advanced = sequence.advance(next);
// Finally, replace the existing reconfiguration sequence with this updated one.
transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced));
return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));

10
src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java
View File

@ -26,7 +26,6 @@ import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.MultiStepOperation;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.sequences.AddToCMS;
import org.apache.cassandra.tcm.sequences.InProgressSequences;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
@ -84,12 +83,9 @@ public class FinishAddToCMS extends BaseMembershipTransformation
InetAddressAndPort endpoint = prev.directory.endpoint(targetNode);
Replica replica = new Replica(endpoint, entireRange, true);
ClusterMetadata.Transformer transformer = prev.transformer();
DataPlacement.Builder builder = prev.placements.get(metaParams)
.unbuild()
.withReadReplica(prev.nextEpoch(), replica);
transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build())
.with(prev.inProgressSequences.without(targetNode));
ClusterMetadata.Transformer transformer = prev.transformer()
.finishJoiningCMS(targetNode)
.with(prev.inProgressSequences.without(targetNode));
return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));
}

40
src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java
View File

@ -20,6 +20,8 @@ package org.apache.cassandra.tcm.transformations.cms;
import java.io.IOException;
import com.google.common.collect.ImmutableSet;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
@ -29,12 +31,10 @@ import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.DistributedMetadataLogKeyspace;
import org.apache.cassandra.schema.DistributedSchema;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.ownership.DataPlacements;
import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
@ -72,7 +72,6 @@ public class PreInitialize implements Transformation
assert metadata.epoch.isBefore(Epoch.FIRST);
ClusterMetadata.Transformer transformer = metadata.transformer();
// This null check is a leftover from previous implementations. In earlier versions the address and datacenter
// were not be included in the serialized form of this transform and so were not written to the local or
// distributed logs nor included in the log entries sent over the wire between peers.
@ -84,30 +83,33 @@ public class PreInitialize implements Transformation
// PRE_INITIALIZE_CMS becomes irrelevant.
if (addr != null)
{
DataPlacement.Builder dataPlacementBuilder = DataPlacement.builder();
Replica replica = new Replica(addr,
MetaStrategy.partitioner.getMinimumToken(),
MetaStrategy.partitioner.getMinimumToken(),
true);
dataPlacementBuilder.reads.withReplica(Epoch.FIRST, replica);
dataPlacementBuilder.writes.withReplica(Epoch.FIRST, replica);
DataPlacements initialPlacement = metadata.placements.unbuild()
.with(ReplicationParams.simpleMeta(1, datacenter),
dataPlacementBuilder.build()).build();
transformer.with(initialPlacement);
// create the distributed metadata keyspace in schema with replication settings based on the DC of the
// initial CMS node
Keyspaces updated = metadata.schema.getKeyspaces()
.withAddedOrReplaced(DistributedMetadataLogKeyspace.initialMetadata(datacenter));
transformer.with(new DistributedSchema(updated, Epoch.FIRST));
ClusterMetadata.Transformer.Transformed transformed = transformer.build();
// This is required to bootstrap the initialization process. Because committing to the metadata log uses the
// previous ClusterMetadata to identify CMS members who form the consensus group, the first CMS member must
// be routable at that point. After the INITIALIZE_CMS is committed, and on instances other than the first
// CMS member, this placement is derived from the CMS membership list of node ids.
DataPlacement.Builder initialPlacement = DataPlacement.builder();
Replica replica = new Replica(addr,
MetaStrategy.partitioner.getMinimumToken(),
MetaStrategy.partitioner.getMinimumToken(),
true);
initialPlacement.reads.withReplica(Epoch.FIRST, replica);
initialPlacement.writes.withReplica(Epoch.FIRST, replica);
metadata = transformed.metadata.forcePreInitializedState(initialPlacement.build());
}
else
{
metadata = metadata.forceEpoch(Epoch.FIRST);
}
ClusterMetadata.Transformer.Transformed transformed = transformer.build();
metadata = transformed.metadata.forceEpoch(Epoch.FIRST);
assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch;
return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, transformed.modifiedKeys);
return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, ImmutableSet.of());
}
@Override

13
src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java
View File

@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -83,7 +82,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
logger.info("Proposed CMS reconfiguration resulted in {}", diff);
LockedRanges.Key lockKey = LockedRanges.keyFor(prev.nextEpoch());
Set<NodeId> cms = prev.fullCMSMembers().stream().map(prev.directory::peerId).collect(Collectors.toSet());
Set<NodeId> cms = prev.fullCMSMemberIds();
Set<NodeId> tmp = new HashSet<>(cms);
tmp.addAll(diff.additions);
tmp.removeAll(diff.removals);
@ -267,9 +266,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME);
KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple()));
return executeInternal(prev,
transformer -> transformer.with(prev.placements.replaceParams(prev.nextEpoch(), ReplicationParams.meta(prev), replicationParams))
.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))));
return executeInternal(prev, transformer -> transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))));
}
public String toString()
@ -312,7 +309,6 @@ public abstract class PrepareCMSReconfiguration implements Transformation
public static Diff diff(Set<NodeId> currentCms, Set<NodeId> newCms)
{
assert !currentCms.contains(null) : "Current CMS contains a null value " + currentCms;
assert !newCms.contains(null) : "New CMS contains a null value " + newCms;
List<NodeId> additions = new ArrayList<>();
@ -355,10 +351,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation
public static boolean needsReconfiguration(ClusterMetadata metadata)
{
Map<String, Integer> dcRf = extractRf(ReplicationParams.meta(metadata));
Set<NodeId> currentCms = metadata.fullCMSMembers()
.stream()
.map(metadata.directory::peerId)
.collect(Collectors.toSet());
Set<NodeId> currentCms = metadata.fullCMSMemberIds();
int expectedSize = dcRf.values().stream().mapToInt(Integer::intValue).sum();
if (currentCms.size() != expectedSize)
return true;

25
src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java
View File

@ -29,20 +29,16 @@ import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.MultiStepOperation;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.sequences.InProgressSequences;
import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
import static org.apache.cassandra.locator.MetaStrategy.entireRange;
/**
* This class along with AddToCMS, StartAddToCMS & FinishAddToCMS, contain a high degree of duplication with their intended
@ -87,11 +83,7 @@ public class RemoveFromCMS extends BaseMembershipTransformation
if (sequence != null)
return new Transformation.Rejected(INVALID, String.format("Can't remove %s from CMS as there are ongoing range movements on it", endpoint));
ReplicationParams metaParams = ReplicationParams.meta(prev);
DataPlacement placements = prev.placements.get(metaParams);
int minProposedSize = (int) Math.min(placements.reads.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count(),
placements.writes.forRange(replica.range()).get().stream().filter(r -> !r.endpoint().equals(endpoint)).count());
int minProposedSize = prev.fullCMSMemberIds().size() - 1;
if (minProposedSize < MIN_SAFE_CMS_SIZE)
{
logger.warn("Removing {} from CMS members would reduce the service size to {} which is below the " +
@ -109,19 +101,8 @@ public class RemoveFromCMS extends BaseMembershipTransformation
if (minProposedSize == 0)
return new Transformation.Rejected(INVALID, String.format("Removing %s from the CMS would leave no members in CMS.", endpoint));
ClusterMetadata.Transformer transformer = prev.transformer();
Replica replica = new Replica(endpoint, entireRange, true);
DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild();
builder.reads.withoutReplica(prev.nextEpoch(), replica);
builder.writes.withoutReplica(prev.nextEpoch(), replica);
DataPlacement proposed = builder.build();
if (proposed.reads.byEndpoint().isEmpty() || proposed.writes.byEndpoint().isEmpty())
return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint));
return Transformation.success(transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()),
MetaStrategy.affectedRanges(prev));
ClusterMetadata.Transformer transformer = prev.transformer().leaveCMS(nodeId);
return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));
}
@Override

29
src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java
View File

@ -18,25 +18,20 @@
package org.apache.cassandra.tcm.transformations.cms;
import java.util.HashSet;
import java.util.Set;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.RangesByEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.MultiStepOperation;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.sequences.AddToCMS;
import org.apache.cassandra.tcm.sequences.ReconfigureCMS;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import static org.apache.cassandra.exceptions.ExceptionCode.INVALID;
import static org.apache.cassandra.locator.MetaStrategy.entireRange;
/**
* This class along with AddToCMS, FinishAddToCMS & RemoveFromCMS, contain a high degree of duplication with their intended
@ -76,27 +71,13 @@ public class StartAddToCMS extends BaseMembershipTransformation
if (prev.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance) != null)
return new Rejected(INVALID, String.format("Cannot add node to CMS as a CMS reconfiguration is currently active"));
Replica replica = new Replica(endpoint, entireRange, true);
ReplicationParams metaParams = ReplicationParams.meta(prev);
RangesByEndpoint readReplicas = prev.placements.get(metaParams).reads.byEndpoint();
RangesByEndpoint writeReplicas = prev.placements.get(metaParams).writes.byEndpoint();
if (readReplicas.containsKey(endpoint) || writeReplicas.containsKey(endpoint))
CMSMembership cms = prev.cmsMembership;
if (cms.joiningMembers().contains(nodeId) || cms.fullMembers().contains(nodeId))
return new Transformation.Rejected(INVALID, "Endpoint is already a member of CMS");
ClusterMetadata.Transformer transformer = prev.transformer();
DataPlacement.Builder builder = prev.placements.get(metaParams).unbuild()
.withWriteReplica(prev.nextEpoch(), replica);
transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build());
Set<InetAddressAndPort> streamCandidates = new HashSet<>();
for (Replica r : prev.placements.get(metaParams).reads.byEndpoint().flattenValues())
{
if (!replica.equals(r))
streamCandidates.add(r.endpoint());
}
ClusterMetadata.Transformer transformer = prev.transformer().startJoiningCMS(nodeId);
Set<InetAddressAndPort> streamCandidates = prev.fullCMSMembers();
AddToCMS joinSequence = new AddToCMS(prev.nextEpoch(), nodeId, streamCandidates, new FinishAddToCMS(endpoint));
transformer = transformer.with(prev.inProgressSequences.with(nodeId, joinSequence));
return Transformation.success(transformer, MetaStrategy.affectedRanges(prev));

32
src/java/org/apache/cassandra/tools/CMSOfflineTool.java
View File

@ -37,10 +37,9 @@ import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileInputStreamPlus;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.MultiStepOperation;
@ -441,24 +440,15 @@ public class CMSOfflineTool implements Runnable
nodeState + " state. Only a JOINED node can be set as CMS member.");
}
InetAddressAndPort endpoint = metadata.directory.getNodeAddresses(nodeId).broadcastAddress;
ReplicationParams metaParams = ReplicationParams.meta(metadata);
Iterable<Replica> currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues();
DataPlacement.Builder placementBuilder = metadata.placements.get(metaParams).unbuild();
for (Replica replica : currentReplicas)
{
placementBuilder.withoutReadReplica(metadata.epoch, replica)
.withoutWriteReplica(metadata.epoch, replica);
}
CMSMembership cms = metadata.cmsMembership;
ClusterMetadata.Transformer transformer = metadata.transformer();
for (NodeId id : cms.fullMembers())
transformer = transformer.leaveCMS(id);
for (NodeId id : cms.joiningMembers())
transformer = transformer.cancelJoiningCMS(id);
Replica newCMS = MetaStrategy.replica(endpoint);
placementBuilder.withReadReplica(metadata.epoch, newCMS)
.withWriteReplica(metadata.epoch, newCMS);
return metadata.transformer()
.with(metadata.placements.unbuild()
.with(metaParams, placementBuilder.build())
.build())
.build().metadata;
transformer = transformer.startJoiningCMS(nodeId).finishJoiningCMS(nodeId);
return transformer.build().metadata;
}
}
@ -619,7 +609,7 @@ public class CMSOfflineTool implements Runnable
output.out.printf("Cluster Metadata Service:%n");
output.out.printf("Members: %s%n", members);
output.out.printf("Needs reconfiguration: %s%n", needsReconfiguration(metadata));
output.out.printf("Service State: %s%n", ClusterMetadataService.state(metadata));
output.out.printf("Service State: %s%n", ClusterMetadataService.State.OFFLINE_TOOL);
output.out.printf("Epoch: %s%n", metadata.epoch.getEpoch());
output.out.printf("Replication factor: %s%n", ReplicationParams.meta(metadata).toString());
}
@ -782,7 +772,7 @@ public class CMSOfflineTool implements Runnable
throw new IllegalArgumentException("Keyspace " + keyspace + " not found in cluster metadata.");
}
DataPlacement placement = metadata.placements.get(keyspaceMetadata.params.replication);
DataPlacement placement = metadata.placement(keyspaceMetadata.params.replication);
List<Object[]> rows = new ArrayList<>();
rows.addAll(replicaGroupsToRows(placement.reads, "read"));
rows.addAll(replicaGroupsToRows(placement.writes, "write"));

27
src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java
View File

@ -27,13 +27,11 @@ import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.io.util.FileInputStreamPlus;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
@ -60,9 +58,9 @@ public class TransformClusterMetadataHelper
DatabaseDescriptor.setPartitionerUnsafe(partitioner);
ClusterMetadataService.initializeForTools(false);
ClusterMetadata metadata = ClusterMetadataService.deserializeClusterMetadata(sourceFile);
System.out.println("Old CMS: " + metadata.placements.get(ReplicationParams.meta(metadata)));
System.out.println("Old CMS: " + metadata.placement(ReplicationParams.meta(metadata)));
metadata = makeCMS(metadata, InetAddressAndPort.getByNameUnchecked(args[1]));
System.out.println("New CMS: " + metadata.placements.get(ReplicationParams.meta(metadata)));
System.out.println("New CMS: " + metadata.placement(ReplicationParams.meta(metadata)));
Path p = Files.createTempFile("clustermetadata", "dump");
try (FileOutputStreamPlus out = new FileOutputStreamPlus(p))
{
@ -73,20 +71,9 @@ public class TransformClusterMetadataHelper
public static ClusterMetadata makeCMS(ClusterMetadata metadata, InetAddressAndPort endpoint)
{
ReplicationParams metaParams = ReplicationParams.meta(metadata);
Iterable<Replica> currentReplicas = metadata.placements.get(metaParams).writes.byEndpoint().flattenValues();
DataPlacement.Builder builder = metadata.placements.get(metaParams).unbuild();
for (Replica replica : currentReplicas)
{
builder.withoutReadReplica(metadata.epoch, replica)
.withoutWriteReplica(metadata.epoch, replica);
}
Replica newCMS = MetaStrategy.replica(endpoint);
builder.withReadReplica(metadata.epoch, newCMS)
.withWriteReplica(metadata.epoch, newCMS);
return metadata.transformer().with(metadata.placements.unbuild().with(metaParams,
builder.build())
.build())
.build().metadata;
NodeId id = metadata.directory.peerId(endpoint);
if (id == null)
throw new IllegalStateException("No node id found for endpoint: " + endpoint);
return metadata.transformer().startJoiningCMS(id).finishJoiningCMS(id).build().metadata;
}
}

2
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -839,7 +839,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
{
CommitLog.instance.recoverSegmentsOnDisk();
NodeId self = ClusterMetadata.current().myNodeId();
if (self != null)
if (self != NodeId.UNREGISTERED)
AccordService.localStartup(self);
}
catch (IOException e)

8
test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java
View File

@ -444,8 +444,8 @@ public class ClusterUtils
for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces())
{
List[] placements = new List[2];
placements[0] = metadata.placements.get(keyspace.params.replication).reads.toReplicaStringList();
placements[1] = metadata.placements.get(keyspace.params.replication).writes.toReplicaStringList();
placements[0] = metadata.placement(keyspace.params.replication).reads.toReplicaStringList();
placements[1] = metadata.placement(keyspace.params.replication).writes.toReplicaStringList();
byKeyspace.put(keyspace.name, placements);
}
return byKeyspace;
@ -465,10 +465,10 @@ public class ClusterUtils
StringBuilder builder = new StringBuilder();
builder.append("'keyspace' { 'name':").append(keyspace.name).append("', ");
builder.append("'reads':['");
ReplicaGroups placement = metadata.placements.get(keyspace.params.replication).reads;
ReplicaGroups placement = metadata.placement(keyspace.params.replication).reads;
builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString());
builder.append("'], 'writes':['");
placement = metadata.placements.get(keyspace.params.replication).writes;
placement = metadata.placement(keyspace.params.replication).writes;
builder.append(byEndpoint ? placement.toStringByEndpoint() : placement.toString());
builder.append("']}");
keyspaces.add(builder.toString());

5
test/distributed/org/apache/cassandra/distributed/test/hostreplacement/AssassinateCMSNodeTest.java
View File

@ -27,7 +27,6 @@ import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.distributed.api.Feature.GOSSIP;
import static org.apache.cassandra.distributed.api.Feature.NETWORK;
@ -46,8 +45,8 @@ public class AssassinateCMSNodeTest extends TestBaseImpl
InetSocketAddress toAssassinate = cluster.get(2).broadcastAddress();
cluster.get(2).shutdown().get();
cluster.get(1).nodetoolResult("assassinate", toAssassinate.getHostString()).asserts().success();
cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())));
cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort())));
cluster.get(1).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember()));
cluster.get(3).runOnInstance(() -> assertTrue(ClusterMetadata.current().isCMSMember()));
cluster.get(1).nodetoolResult("cms").asserts().success();
}
}

16
test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java
View File

@ -32,14 +32,11 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.tcm.CMSOperations;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer;
import static org.apache.cassandra.distributed.shared.ClusterUtils.start;
@ -108,15 +105,10 @@ public class BootWithMetadataTest extends TestBaseImpl
try
{
ClusterMetadata metadata = ClusterMetadata.current();
Replica oldCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.1"));
Replica newCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.2"));
NodeId oldCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.1"));
NodeId newCMS = metadata.directory.peerId(InetAddressAndPort.getByNameUnchecked("127.0.0.2"));
ClusterMetadata.Transformer transformer = metadata.transformer();
DataPlacement.Builder builder = metadata.placements.get(ReplicationParams.meta(metadata)).unbuild()
.withoutReadReplica(metadata.nextEpoch(), oldCMS)
.withoutWriteReplica(metadata.nextEpoch(), oldCMS)
.withWriteReplica(metadata.nextEpoch(), newCMS)
.withReadReplica(metadata.nextEpoch(), newCMS);
transformer = transformer.with(metadata.placements.unbuild().with(ReplicationParams.meta(metadata), builder.build()).build());
transformer.leaveCMS(oldCMS).startJoiningCMS(newCMS).finishJoiningCMS(newCMS);
ClusterMetadata toDump = transformer.build().metadata.forceEpoch(Epoch.create(1000));
Path p = Files.createTempFile("clustermetadata", "dump");
try (FileOutputStreamPlus out = new FileOutputStreamPlus(p))

3
test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataDumpTest.java
View File

@ -66,7 +66,8 @@ public class ClusterMetadataDumpTest extends TestBaseImpl
epochsSeen++;
}
assertEquals(3, unsafeJoinSeen);
assertEquals(3, registerSeen);
// Only 2 REGISTER transforms are expected as the first CMS node is registered implicitly by INITIALIZE_CMS
assertEquals(2, registerSeen);
assertTrue(epochsSeen > 15);
res = cluster.get(1).nodetoolResult("cms", "dumplog", "--start", "10", "--end", "15");

36
test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java
View File

@ -64,6 +64,7 @@ import org.apache.cassandra.service.accord.topology.AccordFastPath;
import org.apache.cassandra.service.accord.topology.AccordStaleReplicas;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.tcm.AtomicLongBackedProcessor;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Commit;
@ -159,7 +160,8 @@ public class ClusterMetadataTestHelper
public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner)
{
return new ClusterMetadata(epoch, Murmur3Partitioner.instance,
return new ClusterMetadata(epoch,
Murmur3Partitioner.instance,
DistributedSchema.empty(),
Directory.EMPTY,
new TokenMap(partitioner),
@ -169,7 +171,8 @@ public class ClusterMetadataTestHelper
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
ImmutableMap.of(),
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
}
public static ClusterMetadata minimalForTesting(IPartitioner partitioner)
@ -177,6 +180,11 @@ public class ClusterMetadataTestHelper
return minimalForTesting(Epoch.EMPTY, partitioner);
}
public static ClusterMetadata minimalForTesting(Epoch e, Keyspaces keyspaces, CMSMembership cms)
{
return minimalForTesting(e, Murmur3Partitioner.instance, new DistributedSchema(keyspaces), cms);
}
public static ClusterMetadata minimalForTesting(Keyspaces keyspaces)
{
return minimalForTesting(Murmur3Partitioner.instance).transformer()
@ -185,6 +193,28 @@ public class ClusterMetadataTestHelper
.metadata.forceEpoch(Epoch.EMPTY);
}
public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema)
{
return minimalForTesting(epoch, partitioner, schema, CMSMembership.EMPTY);
}
public static ClusterMetadata minimalForTesting(Epoch epoch, IPartitioner partitioner, DistributedSchema schema, CMSMembership cms)
{
return new ClusterMetadata(epoch,
Murmur3Partitioner.instance,
schema,
Directory.EMPTY,
new TokenMap(partitioner),
DataPlacements.empty(),
AccordFastPath.EMPTY,
LockedRanges.EMPTY,
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
ImmutableMap.of(),
AccordStaleReplicas.EMPTY,
cms);
}
public static ClusterMetadataService syncInstanceForTest()
{
LocalLog log = LocalLog.logSpec()
@ -1057,7 +1087,7 @@ public class ClusterMetadataTestHelper
public static VersionedEndpoints.ForToken getNaturalReplicasForToken(ClusterMetadata metadata, String keyspace, Token searchPosition)
{
KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspace);
return metadata.placements.get(keyspaceMetadata.params.replication).reads.forToken(searchPosition);
return metadata.placement(keyspaceMetadata.params.replication).reads.forToken(searchPosition);
}
public static BootstrapAndJoin getBootstrapPlan(int idx)

12
test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
View File

@ -508,7 +508,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
Set<NodeId> bouncing = new HashSet<>();
Set<NodeId> replicasFromBouncedReplicaSets = new HashSet<>();
outer:
for (VersionedEndpoints.ForRange placements : sut.service.metadata().placements.get(rf.asKeyspaceParams().replication).writes.endpoints)
for (VersionedEndpoints.ForRange placements : sut.service.metadata().placement(rf.asKeyspaceParams().replication).writes.endpoints)
{
List<NodeId> replicas = new ArrayList<>(metadata.directory.toNodeIds(placements.get().endpoints()));
List<NodeId> bounceCandidates = new ArrayList<>();
@ -545,8 +545,8 @@ public class MetadataChangeSimulationTest extends CMSTestBase
ClusterMetadata actualMetadata = sut.service.metadata();
ReplicationParams replication = actualMetadata.schema.getKeyspaces().get("test").get().params.replication;
Assert.assertEquals(replication, sut.rf.asKeyspaceParams().replication);
match(actualMetadata.placements.get(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
match(actualMetadata.placements.get(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
match(actualMetadata.placement(replication).reads, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
match(actualMetadata.placement(replication).writes, sut.rf.replicate(modelState.simulatedPlacements.nodes).asMap());
}
public static void validatePlacements(CMSTestBase.CMSSut sut, ModelState modelState) throws Throwable
@ -559,7 +559,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
Assert.assertEquals(modelState.simulatedPlacements.nodes.stream().map(Node::token).collect(Collectors.toSet()),
actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).getLongValue()).collect(Collectors.toSet()));
for (Map.Entry<ReplicationParams, DataPlacement> e : actualMetadata.placements.asMap().entrySet())
for (Map.Entry<ReplicationParams, DataPlacement> e : actualMetadata.placements().asMap().entrySet())
{
if (!e.getKey().equals(replication))
continue;
@ -569,7 +569,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
match(placement.reads, modelState.simulatedPlacements.readPlacements);
}
validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements);
validatePlacements(sut.partitioner, sut.rf, modelState, actualMetadata.placements());
}
public static ModelChecker.Pair<ModelState, Node> registerNewNode(ModelState state, CMSSut sut, int dcIdx, int rackIdx)
@ -953,7 +953,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
validatePlacements(sut, state);
}
// Finally verify that the predicted placements match the actual ones
Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placements.get(ksm.params.replication)));
Assert.assertTrue(allSettled.equivalentTo(sut.service.metadata().placement(ksm.params.replication)));
}
}
}

3
test/distributed/org/apache/cassandra/distributed/test/log/OperationalEquivalenceTest.java
View File

@ -116,8 +116,7 @@ public class OperationalEquivalenceTest extends CMSTestBase
withMove = ClusterMetadata.current();
}
assertPlacements(simulateAndCompare(rf, equivalentNodes).placements,
withMove.placements);
assertPlacements(simulateAndCompare(rf, equivalentNodes).placements(), withMove.placements());
}
private static ClusterMetadata simulateAndCompare(ReplicationFactor rf, List<Node> nodes) throws Exception

18
test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java
View File

@ -41,7 +41,6 @@ import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.locator.MetaStrategy;
import org.apache.cassandra.schema.DistributedMetadataLogKeyspace;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.paxos.Ballot;
import org.apache.cassandra.service.paxos.PaxosRepairHistory;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -57,6 +56,7 @@ import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin
import static org.apache.cassandra.distributed.shared.ClusterUtils.replaceHostAndStart;
import static org.apache.cassandra.distributed.shared.NetworkTopology.dcAndRack;
import static org.apache.cassandra.distributed.shared.NetworkTopology.networkTopology;
import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.psjava.util.AssertStatus.assertTrue;
@ -85,10 +85,10 @@ public class ReconfigureCMSTest extends FuzzTestBase
ClusterMetadata metadata = ClusterMetadata.current();
assertEquals(5, metadata.fullCMSMembers().size());
assertEquals(ReplicationParams.simpleMeta(5, metadata.directory.knownDatacenters()),
metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get());
metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication);
});
cluster.stream().forEach(i -> {
Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0);
Assert.assertTrue(i.executeInternal(String.format("SELECT * FROM %s.%s", METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME)).length > 0);
});
cluster.get(nodeSelector.get()).nodetoolResult("cms", "reconfigure", "1").asserts().success();
@ -96,7 +96,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
ClusterMetadata metadata = ClusterMetadata.current();
assertEquals(1, metadata.fullCMSMembers().size());
assertEquals(ReplicationParams.simpleMeta(1, metadata.directory.knownDatacenters()),
metadata.placements.keys().stream().filter(ReplicationParams::isMeta).findFirst().get());
metadata.schema.getKeyspaceMetadata(METADATA_KEYSPACE_NAME).params.replication);
});
}
}
@ -136,7 +136,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance));
assertEquals(2, metadata.fullCMSMembers().size());
ReplicationParams params = ReplicationParams.meta(metadata);
DataPlacement placements = metadata.placements.get(params);
DataPlacement placements = metadata.placements().get(params);
assertTrue(placements.reads.equivalentTo(placements.writes));
assertEquals(metadata.fullCMSMembers().size(), Integer.parseInt(params.asMap().get("dc0")));
});
@ -161,7 +161,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance));
Assert.assertTrue(metadata.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort()));
assertEquals(3, metadata.fullCMSMembers().size());
DataPlacement placements = metadata.placements.get(ReplicationParams.meta(metadata));
DataPlacement placements = metadata.placements().get(ReplicationParams.meta(metadata));
Assert.assertTrue(placements.reads.equivalentTo(placements.writes));
});
}
@ -216,7 +216,7 @@ public class ReconfigureCMSTest extends FuzzTestBase
awaitRingJoin(replacingNode, cluster.get(1));
replacingNode.runOnInstance(() -> {
ClusterMetadata metadata = ClusterMetadata.current();
assertTrue(metadata.isCMSMember(FBUtilities.getBroadcastAddressAndPort()));
assertTrue(metadata.isCMSMember());
assertEquals(3, metadata.fullCMSMembers().size());
});
}
@ -326,11 +326,11 @@ public class ReconfigureCMSTest extends FuzzTestBase
Object[][] rows = instance.executeInternal("select points from system.paxos_repair_history " +
"where keyspace_name = ? " +
"and table_name = ?",
SchemaConstants.METADATA_KEYSPACE_NAME,
METADATA_KEYSPACE_NAME,
DistributedMetadataLogKeyspace.TABLE_NAME);
if (rows.length == 0)
return PaxosRepairHistory.empty(SchemaConstants.METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME);
return PaxosRepairHistory.empty(METADATA_KEYSPACE_NAME, DistributedMetadataLogKeyspace.TABLE_NAME);
assertEquals(1, rows.length);
//noinspection unchecked
List<ByteBuffer> points = (List<ByteBuffer>)rows[0][0];

156
test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java
View File

@ -19,12 +19,13 @@
package org.apache.cassandra.distributed.test.log;
import java.io.IOException;
import java.net.UnknownHostException;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.EnumSet;
import org.junit.Test;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInstanceConfig;
@ -33,9 +34,11 @@ import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.MetadataSnapshots;
import org.apache.cassandra.tcm.Transformation;
import org.apache.cassandra.tcm.membership.Location;
@ -45,15 +48,21 @@ import org.apache.cassandra.tcm.membership.NodeState;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.PlacementProvider;
import org.apache.cassandra.tcm.sequences.LeaveStreams;
import org.apache.cassandra.tcm.sequences.LockedRanges;
import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave;
import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.tcm.transformations.CustomTransformation;
import org.apache.cassandra.tcm.transformations.PrepareLeave;
import org.apache.cassandra.tcm.transformations.Register;
import org.apache.cassandra.tcm.transformations.TriggerSnapshot;
import org.apache.cassandra.tcm.transformations.Unregister;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.vint.VIntCoding;
import static org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper.addr;
import static org.apache.cassandra.tcm.membership.NodeVersion.CURRENT_METADATA_VERSION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public class RegisterTest extends TestBaseImpl
{
@ -106,57 +115,42 @@ public class RegisterTest extends TestBaseImpl
{
cluster.get(1).startup();
cluster.get(1).runOnInstance(() -> {
// Run a custom transformation to inject a fake node into the directory with a known id and an
// artificially lowered max supported serialization version
CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer);
CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME,
new RegisterNodeWithOldVersion());
ClusterMetadataService.instance().commit(injectOldNode);
// Doesn't matter which specific Transformation we use here, we're testing that the serializer uses
// the correct lower bound
Transformation t = new Register(NodeAddresses.current(), TEST_LOCATION, NodeVersion.CURRENT);
try
{
// Unregister to make directory empty
ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(),
EnumSet.allOf(NodeState.class),
ClusterMetadataService.instance().placementProvider()));
// Register a ghost node with V0 (bypasses version check because directory is now empty).
// In a real world cluster we will always be upgrading from a smaller version.
ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")),
TEST_LOCATION,
new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0)));
NodeId oldNode = ClusterMetadata.current().directory.peerId(InetAddressAndPort.getByName("127.0.0.100"));
// Register a node with upgraded version
CassandraVersion currentVersion = NodeVersion.CURRENT.cassandraVersion;
NodeVersion upgraded = new NodeVersion(new CassandraVersion(String.format("%d.%d.%d", currentVersion.major + 1, 0, 0)),
NodeVersion.CURRENT_METADATA_VERSION);
ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.200")), TEST_LOCATION, upgraded));
// Doesn't matter which specific Transformation we use here, we're testing that the serializer uses
// the correct lower bound
Transformation t = new Register(NodeAddresses.current(), new Location("DC", "RACK"), NodeVersion.CURRENT);
try
assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, RegisterNodeWithOldVersion.METADATA_VERSION);
ByteBuffer bytes = t.kind().toVersionedBytes(t);
try (DataInputBuffer buf = new DataInputBuffer(bytes, true))
{
assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion,
Version.V0.asInt());
ByteBuffer bytes = t.kind().toVersionedBytes(t);
try (DataInputBuffer buf = new DataInputBuffer(bytes, true))
{
// Because ClusterMetadata.current().directory still contains oldNode we must serialize at
// the version it supports
assertEquals(Version.V0, Version.fromInt(buf.readUnsignedVInt32()));
}
// If we unregister oldNode, then the ceiling for serialization version will rise
ClusterMetadataService.instance().commit(new Unregister(oldNode, EnumSet.allOf(NodeState.class), ClusterMetadataService.instance().placementProvider()));
assertEquals(ClusterMetadata.current().directory.clusterMinVersion.serializationVersion,
NodeVersion.CURRENT_METADATA_VERSION.asInt());
bytes = t.kind().toVersionedBytes(t);
try (DataInputBuffer buf = new DataInputBuffer(bytes, true))
{
assertEquals(NodeVersion.CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32()));
}
// Because ClusterMetadata.current().directory still contains the fake old node we must
// serialize at the version _it_ supports
assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32()));
}
catch (IOException e)
// If we unregister the fake node, then the ceiling for serialization version will rise
Unregister unregisterOldNode = new Unregister(RegisterNodeWithOldVersion.NODE_ID,
EnumSet.allOf(NodeState.class),
ClusterMetadataService.instance().placementProvider());
ClusterMetadataService.instance().commit(unregisterOldNode);
assertEquals(ClusterMetadata.current().directory.commonSerializationVersion, CURRENT_METADATA_VERSION);
bytes = t.kind().toVersionedBytes(t);
try (DataInputBuffer buf = new DataInputBuffer(bytes, true))
{
throw new RuntimeException(e);
assertEquals(CURRENT_METADATA_VERSION, Version.fromInt(buf.readUnsignedVInt32()));
}
}
catch (UnknownHostException e)
catch (IOException e)
{
throw new RuntimeException(e);
}
@ -172,28 +166,70 @@ public class RegisterTest extends TestBaseImpl
{
cluster.get(1).startup();
cluster.get(1).runOnInstance(() -> {
// Run a custom transformation to inject a fake node into the directory with a known id and an
// artificially lowered max supported serialization version
CustomTransformation.registerExtension(RegisterNodeWithOldVersion.NAME, RegisterNodeWithOldVersion.serializer);
CustomTransformation injectOldNode = new CustomTransformation(RegisterNodeWithOldVersion.NAME,
new RegisterNodeWithOldVersion());
ClusterMetadataService.instance().commit(injectOldNode);
// Now trigger a snapshot which must be written to the snapshot using the old serialization version
Epoch epoch = ClusterMetadataService.instance().commit(TriggerSnapshot.instance).epoch;
// fetch the raw bytes of the snapshot we just serialized
ByteBuffer bytes = SystemKeyspace.getSnapshot(epoch);
assertNotNull(bytes);
// assert the prepended version matches
Version writtenVersion = null;
try
{
// Unregister to make directory empty
ClusterMetadataService.instance().commit(new Unregister(ClusterMetadata.current().myNodeId(),
EnumSet.allOf(NodeState.class),
ClusterMetadataService.instance().placementProvider()));
// Register a ghost node with V0 (bypasses version check because directory is now empty).
// In a real world cluster we will always be upgrading from a smaller version.
ClusterMetadataService.instance().commit(new Register(new NodeAddresses(InetAddressAndPort.getByName("127.0.0.100")),
TEST_LOCATION,
new NodeVersion(NodeVersion.CURRENT.cassandraVersion, Version.V0)));
writtenVersion = Version.fromInt(VIntCoding.readUnsignedVInt32(new DataInputBuffer(bytes, false)));
}
catch (UnknownHostException e)
catch (IOException e)
{
throw new RuntimeException(e);
}
ClusterMetadataService.instance().commit(TriggerSnapshot.instance);
assertEquals(RegisterNodeWithOldVersion.METADATA_VERSION, writtenVersion);
// load the snapshot using the standard mechanism and assert it matches current cluster metadata
ClusterMetadata cm = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots().getSnapshot(ClusterMetadata.current().epoch);
cm.equals(ClusterMetadata.current());
});
}
}
// Custom transforms to lock/unlock an arbitrary set of ranges to
// avoid having to actually initiate some range movement
public static class RegisterNodeWithOldVersion implements Transformation, Serializable
{
public static final AsymmetricMetadataSerializer<Transformation, RegisterNodeWithOldVersion> serializer = new AsymmetricMetadataSerializer<Transformation, RegisterNodeWithOldVersion>()
{
@Override
public void serialize(Transformation t, DataOutputPlus out, Version version){}
@Override
public RegisterNodeWithOldVersion deserialize(DataInputPlus in, Version version) {return new RegisterNodeWithOldVersion();}
@Override
public long serializedSize(Transformation t, Version version) {return 0;}
};
public static final String NAME = "TestRegisterNodeWithOldVersion";
public static final NodeId NODE_ID = new NodeId(99);
public static final Version METADATA_VERSION = Version.V0;
@Override
public Kind kind()
{
return Kind.CUSTOM;
}
@Override
public Result execute(ClusterMetadata metadata)
{
ClusterMetadata.Transformer transformer = metadata.transformer()
.unsafeRegisterForTesting(NODE_ID,
new NodeAddresses(addr(99)),
TEST_LOCATION,
new NodeVersion(NodeVersion.CURRENT.cassandraVersion,
METADATA_VERSION));
return Transformation.success(transformer, LockedRanges.AffectedRanges.EMPTY);
}
}
}

4
test/distributed/org/apache/cassandra/distributed/test/log/ResumableStartupTest.java
View File

@ -146,12 +146,12 @@ public class ResumableStartupTest extends FuzzTestBase
KeyspaceMetadata ksm = metadata.schema.getKeyspaceMetadata(keyspace);
boolean isWriteReplica = false;
boolean isReadReplica = false;
for (InetAddressAndPort readReplica : metadata.placements.get(ksm.params.replication).reads.byEndpoint().keySet())
for (InetAddressAndPort readReplica : metadata.placement(ksm.params.replication).reads.byEndpoint().keySet())
{
if (readReplica.getHostAddressAndPort().equals(newAddress))
isReadReplica = true;
}
for (InetAddressAndPort writeReplica : metadata.placements.get(ksm.params.replication).writes.byEndpoint().keySet())
for (InetAddressAndPort writeReplica : metadata.placement(ksm.params.replication).writes.byEndpoint().keySet())
{
if (writeReplica.getHostAddressAndPort().equals(newAddress))
isWriteReplica = true;

4
test/distributed/org/apache/cassandra/distributed/test/log/SimulatedOperation.java
View File

@ -177,8 +177,8 @@ public abstract class SimulatedOperation
sutActions.next();
ClusterMetadata m2 = ClusterMetadata.current();
Map<Range<Token>, VersionedEndpoints.ForRange> after = m2.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.asMap();
m1.placements.get(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> {
Map<Range<Token>, VersionedEndpoints.ForRange> after = m2.placement(simulatedState.rf.asKeyspaceParams().replication).reads.asMap();
m1.placement(simulatedState.rf.asKeyspaceParams().replication).reads.forEach((k, beforePlacements) -> {
if (after.containsKey(k))
{
VersionedEndpoints.ForRange afterPlacements = after.get(k);

2
test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java
View File

@ -69,7 +69,7 @@ public class SnapshotTest extends TestBaseImpl
ClusterMetadata before = ClusterMetadata.current();
ClusterMetadata after = ClusterMetadataService.instance().triggerSnapshot();
ClusterMetadata serialized = ClusterMetadataService.instance().snapshotManager().getSnapshot(after.epoch);
assertEquals(before.placements, serialized.placements);
assertEquals(before.placements(), serialized.placements());
assertEquals(before.tokenMap, serialized.tokenMap);
assertEquals(before.directory, serialized.directory);
assertEquals(before.schema, serialized.schema);

2
test/distributed/org/apache/cassandra/distributed/test/ring/RangeVersioningTest.java
View File

@ -54,7 +54,7 @@ public class RangeVersioningTest extends FuzzTestBase
for (int i = 1; i <= 4; i++)
{
Epoch smallestSeen = null;
for (VersionedEndpoints.ForRange fr : metadata.placements.get(ReplicationParams.simple(i)).writes.endpoints)
for (VersionedEndpoints.ForRange fr : metadata.placement(ReplicationParams.simple(i)).writes.endpoints)
{
if (smallestSeen == null || fr.lastModified().isBefore(smallestSeen))
smallestSeen = fr.lastModified();

3
test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterReplacementTest.java
View File

@ -34,7 +34,6 @@ import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.distributed.shared.ClusterUtils.addInstance;
import static org.apache.cassandra.distributed.shared.ClusterUtils.awaitRingJoin;
@ -80,7 +79,7 @@ public class CMSPlacementAfterReplacementTest extends TestBaseImpl
IInvokableInstance nodeToRemove = cluster.get(2);
cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success();
cluster.get(2).runOnInstance(() -> {
assertTrue(ClusterMetadata.current().isCMSMember(FBUtilities.getBroadcastAddressAndPort()));
assertTrue(ClusterMetadata.current().isCMSMember());
});
nodeToRemove.shutdown().get();
IInvokableInstance replacingNode = addInstance(cluster, nodeToRemove.config(),

2
test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeAssassinateTest.java
View File

@ -60,7 +60,7 @@ public class ClusterMetadataUpgradeAssassinateTest extends UpgradeTestBase
((IInvokableInstance) i).runOnInstance(() -> {
ClusterMetadata metadata = ClusterMetadata.current();
InetAddressAndPort ep = InetAddressAndPort.getByNameUnchecked(host);
metadata.placements.asMap().forEach((key, value) -> {
metadata.placements().forEach((key, value) -> {
if (key.isMeta())
return;
boolean existsInPlacements = Streams.concat(value.reads.endpoints.stream(),

8
test/distributed/org/apache/cassandra/distributed/upgrade/ClusterMetadataUpgradeDelayedInitializeTest.java
View File

@ -181,7 +181,7 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase
try
{
new ByteBuddy().rebase(ClusterMetadata.class)
.method(named("initializeClusterIdentifier"))
.method(named("forceInitializedState"))
.intercept(MethodDelegation.to(ClusterMetadataUpgradeDelayedInitializeTest.BBInterceptor.class))
.make()
.load(classLoader, ClassLoadingStrategy.Default.INJECTION);
@ -201,13 +201,13 @@ public class ClusterMetadataUpgradeDelayedInitializeTest extends UpgradeTestBase
public static class BBInterceptor
{
@SuppressWarnings("unused")
public static ClusterMetadata initializeClusterIdentifier(@SuperCall Callable<ClusterMetadata> zuper)
public static ClusterMetadata forceInitializedState(@SuperCall Callable<ClusterMetadata> zuper)
{
try
{
logger.info("initializeClusterIdentifier waiting...");
logger.info("forceInitializedState waiting...");
BBState.latch.await(60, TimeUnit.SECONDS);
logger.info("initializeClusterIdentifier continuing...");
logger.info("forceInitializedState continuing...");
return zuper.call();
}
catch (Throwable e)

2
test/distributed/org/apache/cassandra/fuzz/topology/TopologyMixupTestBase.java
View File

@ -945,7 +945,7 @@ public abstract class TopologyMixupTestBase<S extends TopologyMixupTestBase.Sche
{
return inst.callOnInstance(() -> {
ClusterMetadata current = ClusterMetadata.current();
Set<InetAddressAndPort> members = current.placements.get(ReplicationParams.meta(current)).writes.byEndpoint().keySet();
Set<InetAddressAndPort> members = current.placement(ReplicationParams.meta(current)).writes.byEndpoint().keySet();
// Why not just use 'current.fullCMSMembers()'? That uses the "read" replicas, so "could" have less endpoints
// It would be more consistent to use fullCMSMembers but thought process is knowing the full set is better
// than the coordination set.

4
test/simulator/main/org/apache/cassandra/simulator/cluster/OnClusterReplace.java
View File

@ -78,7 +78,7 @@ class OnClusterReplace extends OnClusterChangeTopology
List<Map.Entry<String, String>> repairRanges = actions.cluster.get(leaving).unsafeApplyOnThisThread(
(String keyspaceName) -> {
ClusterMetadata metadata = ClusterMetadata.current();
return metadata.placements.get(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication)
return metadata.placement(metadata.schema.getKeyspace(keyspaceName).getMetadata().params.replication)
.writes.ranges()
.stream()
.map(OnClusterReplace::toStringEntry)
@ -93,7 +93,7 @@ class OnClusterReplace extends OnClusterChangeTopology
(String keyspaceName, String tk) -> {
ClusterMetadata metadata = ClusterMetadata.current();
KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaces().getNullable(keyspaceName);
return metadata.placements.get(keyspaceMetadata.params.replication).reads
return metadata.placement(keyspaceMetadata.params.replication).reads
.forToken(Utils.parseToken(tk))
.get()
.stream().map(Replica::endpoint)

8
test/unit/org/apache/cassandra/auth/GrantAndRevokeTest.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.auth;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@ -34,6 +35,8 @@ import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableMetadata;
@ -59,6 +62,11 @@ public class GrantAndRevokeTest extends CQLTester
ServerTestUtils.daemonInitialization();
DatabaseDescriptor.setPermissionsValidity(0);
DatabaseDescriptor.setRolesValidity(0);
// for the tables in the distributed metadata keyspace to be queryable, there needs to be a valid placement
// which is derived from the CMS membership. Most unit tests don't actually need to query those dist tables
// which is why this isn't done as a matter of routine
ClusterMetadataTestHelper.reconfigureCms(ReplicationParams.ntsMeta(Collections.singletonMap(DatabaseDescriptor.getLocalDataCenter(), 1)));
ServerTestUtils.markCMS();
requireAuthentication();
requireNetwork();
CassandraDaemon.getInstanceForTesting().setupVirtualKeyspaces();

3
test/unit/org/apache/cassandra/dht/BootStrapperTest.java
View File

@ -204,7 +204,7 @@ public class BootStrapperTest
ClusterMetadata metadata = ClusterMetadata.current();
Pair<MovementMap, MovementMap> movements = Pair.create(MovementMap.empty(), MovementMap.empty());
if (metadata.myNodeId() == null)
if (metadata.myNodeId() == NodeId.UNREGISTERED)
{
Token myToken = metadata.partitioner.getRandomToken();
InetAddressAndPort myEndpoint = InetAddressAndPort.getByName("127.0.0.1");
@ -265,5 +265,4 @@ public class BootStrapperTest
ClusterMetadataTestHelper.addEndpoint(addr, tokens);
}
}
}

4
test/unit/org/apache/cassandra/locator/MetaStrategyTest.java
View File

@ -36,6 +36,7 @@ import org.apache.cassandra.schema.DistributedSchema;
import org.apache.cassandra.service.accord.topology.AccordFastPath;
import org.apache.cassandra.service.accord.topology.AccordStaleReplicas;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationState;
import org.apache.cassandra.tcm.CMSMembership;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.Directory;
@ -96,7 +97,8 @@ public class MetaStrategyTest
InProgressSequences.EMPTY,
ConsensusMigrationState.EMPTY,
ImmutableMap.of(),
AccordStaleReplicas.EMPTY);
AccordStaleReplicas.EMPTY,
CMSMembership.EMPTY);
}
@Test

2
test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
View File

@ -372,6 +372,6 @@ public class SimpleStrategyTest extends CassandraTestBase
ReplicationParams replicationParams,
Token token)
{
return metadata.placements.get(replicationParams).writes.forToken(token).get();
return metadata.placement(replicationParams).writes.forToken(token).get();
}
}

2
test/unit/org/apache/cassandra/service/accord/AccordTopologyUtils.java
View File

@ -105,7 +105,7 @@ public class AccordTopologyUtils
{
ReplicationParams replication = keyspace.params.replication;
AbstractReplicationStrategy strategy = AbstractReplicationStrategy.createReplicationStrategy(keyspace.name, replication);
DataPlacements.Builder placements = metadata.placements.unbuild();
DataPlacements.Builder placements = metadata.placements().unbuild();
DataPlacement placement = strategy.calculateDataPlacement(Epoch.EMPTY, metadata.tokenMap.toRanges(), metadata);
placements.with(replication, placement);
metadata = transformer.with(placements.build()).build().metadata;

6
test/unit/org/apache/cassandra/service/accord/EpochSyncTest.java
View File

@ -344,7 +344,7 @@ public class EpochSyncTest
private static boolean joined(ClusterMetadata metadata, Node.Id id)
{
NodeAddresses address = metadata.directory.getNodeAddresses(new NodeId(id.id));
return metadata.placements.get(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress);
return metadata.placement(replication_params).reads.byEndpoint().keySet().contains(address.broadcastAddress);
}
public enum EpochTracker { topologyManager, accordSyncPropagator }
@ -615,7 +615,7 @@ public class EpochSyncTest
{
Topology t = AccordTopology.createAccordTopology(current);
Ranges ranges = t.ranges().mergeTouching();
if (!current.placements.get(replication_params).reads.isEmpty())
if (!current.placement(replication_params).reads.isEmpty())
Assertions.assertThat(ranges).hasSize(1);
cms.setMetadata(current);
for (Node.Id id : status(s -> s != Status.Removed))
@ -721,7 +721,7 @@ public class EpochSyncTest
case Registered:
Invariants.require(!t.nodes().contains(id), "Node was in Init state but present in the Topology!");
Invariants.require(current.directory.peerId(address(id)) != null, "Node exists but not in TCM");
if (current.placements.get(replication_params).writes.byEndpoint().keySet().contains(address(id)))
if (current.placement(replication_params).writes.byEndpoint().keySet().contains(address(id)))
status = Status.Joining;
break;
case Joining:

8
test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java
View File

@ -41,6 +41,7 @@ import org.apache.cassandra.io.util.FileOutputStreamPlus;
import org.apache.cassandra.tcm.membership.Directory;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.membership.MembershipUtils;
import org.apache.cassandra.tcm.membership.NodeAddresses;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.membership.NodeVersion;
import org.apache.cassandra.tcm.ownership.DataPlacements;
@ -110,7 +111,10 @@ public class BootWithMetadataTest
Directory directory = first.directory;
int nodeCount = 10;
int tokensPerNode = 5;
for (int i = 0; i < nodeCount; i++)
// Ensure that the "local" node is registered as it being member of the CMS is a precondition of booting from
// a ClusterMetadata.
directory = directory.with(NodeAddresses.current(), new Location("DC1", "RACK1"));
for (int i = 0; i < nodeCount-1; i++)
directory = directory.with(nodeAddresses(random), new Location("DC1", "RACK1"));
t = t.with(directory);
@ -156,7 +160,7 @@ public class BootWithMetadataTest
assertEquals(toWrite.schema, fromRead.schema);
assertEquals(toWrite.directory, fromRead.directory);
assertEquals(toWrite.tokenMap, fromRead.tokenMap);
assertEquals(toWrite.placements, fromRead.placements);
assertEquals(toWrite.placements(), fromRead.placements());
assertEquals(toWrite.lockedRanges, fromRead.lockedRanges);
assertEquals(toWrite.inProgressSequences, fromRead.inProgressSequences);
assertEquals(toWrite.extensions, fromRead.extensions);

Some files were not shown because too many files have changed in this diff Show More