diff --git a/CHANGES.txt b/CHANGES.txt index 5fe89cc440..c313028dad 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Remove period field from ClusterMetadata and metadata log tables (CASSANDRA-19482) * Enrich system_views.pending_hints vtable with hints sizes (CASSANDRA-19486) * Expose all dropwizard metrics in virtual tables (CASSANDRA-14572) * Ensured that PropertyFileSnitchTest do not overwrite cassandra-toploogy.properties (CASSANDRA-19502) diff --git a/src/java/org/apache/cassandra/db/PartitionPosition.java b/src/java/org/apache/cassandra/db/PartitionPosition.java index 5e1d6184e2..e8d29cbd94 100644 --- a/src/java/org/apache/cassandra/db/PartitionPosition.java +++ b/src/java/org/apache/cassandra/db/PartitionPosition.java @@ -17,11 +17,11 @@ */ package org.apache.cassandra.db; -import java.io.DataInput; import java.io.IOException; import java.nio.ByteBuffer; import org.apache.cassandra.dht.*; +import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.bytecomparable.ByteComparable; @@ -100,7 +100,7 @@ public interface PartitionPosition extends RingPosition, Byte Token.serializer.serialize(pos.getToken(), out, version); } - public PartitionPosition deserialize(DataInput in, IPartitioner p, int version) throws IOException + public PartitionPosition deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException { Kind kind = Kind.fromOrdinal(in.readByte()); if (kind == Kind.ROW_KEY) diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java index ff54d226b0..4e2f8aa933 100644 --- a/src/java/org/apache/cassandra/db/SystemKeyspace.java +++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java @@ -50,7 +50,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; -import com.google.common.io.ByteStreams; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +71,6 @@ import org.apache.cassandra.db.rows.Rows; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.LocalPartitioner; import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.ReversedLongLocalPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.gms.EndpointState; @@ -86,6 +84,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.RebufferingInputStream; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.metrics.RestorableMeter; import org.apache.cassandra.metrics.TopPartitionTracker; import org.apache.cassandra.net.MessagingService; @@ -487,16 +486,14 @@ public final class SystemKeyspace parse(METADATA_LOG, "Local Metadata Log", "CREATE TABLE %s (" - + "period bigint," - + "current_epoch bigint static," + "epoch bigint," + "entry_id bigint," + "transformation blob," + "kind text," - + "PRIMARY KEY (period, epoch))") + + "PRIMARY KEY (epoch))") + .partitioner(MetaStrategy.partitioner) .compaction(CompactionParams.twcs(ImmutableMap.of("compaction_window_unit","DAYS", "compaction_window_size","1"))) - .partitioner(new LocalPartitioner(LongType.instance)) .build(); public static final TableMetadata Snapshots = parse(SNAPSHOT_TABLE_NAME, @@ -505,7 +502,7 @@ public final class SystemKeyspace "epoch bigint PRIMARY KEY," + "period bigint," + "snapshot blob)") - .partitioner(ReversedLongLocalPartitioner.instance) + .partitioner(MetaStrategy.partitioner) .build(); @Deprecated(since = "4.0") @@ -1878,7 +1875,7 @@ public final class SystemKeyspace try { // See rangeToBytes above for why version is 0. - return (Range) Range.tokenSerializer.deserialize(ByteStreams.newDataInput(ByteBufferUtil.getArray(rawRange)), + return (Range) Range.tokenSerializer.deserialize(new DataInputBuffer(ByteBufferUtil.getArray(rawRange)), partitioner, 0); } @@ -1985,22 +1982,27 @@ public final class SystemKeyspace } } - public static void storeSnapshot(Epoch epoch, long period, ByteBuffer snapshot) + public static void storeSnapshot(Epoch epoch, ByteBuffer snapshot) { - logger.info("Storing snapshot of cluster metadata at epoch {} (period {})", epoch, period); - String query = String.format("INSERT INTO %s.%s (epoch, period, snapshot) VALUES (?, ?, ?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); - executeInternal(query, epoch.getEpoch(), period, snapshot); + Preconditions.checkArgument(epoch.isAfter(Epoch.FIRST), "Cannot store a snapshot for an epoch less than " + Epoch.FIRST.getEpoch()); + logger.info("Storing snapshot of cluster metadata at epoch {}", epoch); + String query = String.format("INSERT INTO %s.%s (epoch, snapshot) VALUES (?, ?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); + executeInternal(query, epoch.getEpoch(), snapshot); forceBlockingFlush(SNAPSHOT_TABLE_NAME); } public static ByteBuffer getSnapshot(Epoch epoch) { + Preconditions.checkArgument(epoch.isAfter(Epoch.FIRST), "Cannot retrieve a snapshot for an epoch less than " + Epoch.FIRST.getEpoch()); logger.info("Getting snapshot of epoch = {}", epoch); String query = String.format("SELECT snapshot FROM %s.%s WHERE epoch = ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); UntypedResultSet res = executeInternal(query, epoch.getEpoch()); if (res == null || res.isEmpty()) return null; - return res.one().getBytes("snapshot").duplicate(); + ByteBuffer bytes = res.one().getBytes("snapshot"); + if (bytes == null || !bytes.hasRemaining()) + return null; + return bytes.duplicate(); } /** @@ -2020,14 +2022,27 @@ public final class SystemKeyspace */ public static ByteBuffer findSnapshotBefore(Epoch search) { + // during gossip upgrade we have epoch = Long.MIN_VALUE + 1 (and the reverse partitioner doesn't support negative keys) + search = search.isBefore(Epoch.EMPTY) ? Epoch.EMPTY : search; String query = String.format("SELECT snapshot FROM %s.%s WHERE token(epoch) >= token(?) LIMIT 1", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); - UntypedResultSet res = executeInternal(query, search.getEpoch()); if (res != null && !res.isEmpty()) return res.one().getBytes("snapshot").duplicate(); return null; } + public static List listSnapshotsSince(Epoch search) + { + // during gossip upgrade we have epoch = Long.MIN_VALUE + 1 (and the reverse partitioner doesn't support negative keys) + search = search.isBefore(Epoch.EMPTY) ? Epoch.EMPTY : search; + String query = String.format("SELECT epoch FROM %s.%s WHERE token(epoch) < token(?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); + UntypedResultSet res = executeInternal(query, search.getEpoch()); + if (res == null) + return Collections.emptyList(); + + return res.stream().map(row -> Epoch.create(row.getLong("epoch"))).collect(Collectors.toList()); + } + /** * Find the latest snapshot we have in the log. */ diff --git a/src/java/org/apache/cassandra/db/virtual/ClusterMetadataLogTable.java b/src/java/org/apache/cassandra/db/virtual/ClusterMetadataLogTable.java index 4316a82cc0..1c345a19b4 100644 --- a/src/java/org/apache/cassandra/db/virtual/ClusterMetadataLogTable.java +++ b/src/java/org/apache/cassandra/db/virtual/ClusterMetadataLogTable.java @@ -25,7 +25,7 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.db.marshal.TimestampType; import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.dht.LocalPartitioner; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tcm.Transformation; @@ -36,7 +36,6 @@ import static org.apache.cassandra.schema.SchemaConstants.METADATA_KEYSPACE_NAME final class ClusterMetadataLogTable extends AbstractVirtualTable { - private static final String PERIOD = "period"; private static final String EPOCH = "epoch"; private static final String KIND = "kind"; private static final String TRANSFORMATION = "transformation"; @@ -48,9 +47,8 @@ final class ClusterMetadataLogTable extends AbstractVirtualTable super(TableMetadata.builder(keyspace, "cluster_metadata_log") .comment("cluster metadata log") .kind(TableMetadata.Kind.VIRTUAL) - .partitioner(new LocalPartitioner(LongType.instance)) - .addPartitionKeyColumn(PERIOD, LongType.instance) - .addClusteringColumn(EPOCH, LongType.instance) + .partitioner(MetaStrategy.partitioner) + .addPartitionKeyColumn(EPOCH, LongType.instance) .addRegularColumn(KIND, UTF8Type.instance) .addRegularColumn(TRANSFORMATION, UTF8Type.instance) .addRegularColumn(ENTRY_ID, LongType.instance) @@ -64,14 +62,14 @@ final class ClusterMetadataLogTable extends AbstractVirtualTable try { SimpleDataSet result = new SimpleDataSet(metadata()); - UntypedResultSet res = execute(format("SELECT period, epoch, kind, transformation, entry_id, writetime(kind) as wt " + + UntypedResultSet res = execute(format("SELECT epoch, kind, transformation, entry_id, writetime(kind) as wt " + "FROM %s.%s", METADATA_KEYSPACE_NAME, TABLE_NAME), ConsistencyLevel.QUORUM); for (UntypedResultSet.Row r : res) { Transformation.Kind kind = Transformation.Kind.valueOf(r.getString("kind")); Transformation transformation = kind.fromVersionedBytes(r.getBlob("transformation")); - result.row(r.getLong("period"), r.getLong("epoch")) + result.row(r.getLong("epoch")) .column(KIND, kind.toString()) .column(TRANSFORMATION, transformation.toString()) .column(ENTRY_ID, r.getLong("entry_id")) diff --git a/src/java/org/apache/cassandra/dht/AbstractBounds.java b/src/java/org/apache/cassandra/dht/AbstractBounds.java index 9faee77d20..aa3cfd1601 100644 --- a/src/java/org/apache/cassandra/dht/AbstractBounds.java +++ b/src/java/org/apache/cassandra/dht/AbstractBounds.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.dht; -import java.io.DataInput; import java.io.IOException; import java.io.Serializable; import java.util.Collection; @@ -27,6 +26,7 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.Pair; @@ -195,7 +195,7 @@ public abstract class AbstractBounds> implements Seria serializer.serialize(range.right, out, version); } - public AbstractBounds deserialize(DataInput in, IPartitioner p, int version) throws IOException + public AbstractBounds deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException { boolean isToken, startInclusive, endInclusive; // !WARNING! See serialize method above for why we still need to have that condition. diff --git a/src/java/org/apache/cassandra/dht/IPartitioner.java b/src/java/org/apache/cassandra/dht/IPartitioner.java index 7e63cb422e..9dba3e0d9d 100644 --- a/src/java/org/apache/cassandra/dht/IPartitioner.java +++ b/src/java/org/apache/cassandra/dht/IPartitioner.java @@ -18,7 +18,6 @@ package org.apache.cassandra.dht; import java.nio.ByteBuffer; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -37,20 +36,6 @@ public interface IPartitioner return DatabaseDescriptor.getPartitioner(); } - static void validate(Collection> allBounds) - { - for (AbstractBounds bounds : allBounds) - validate(bounds); - } - - static void validate(AbstractBounds bounds) - { - if (global() != bounds.left.getPartitioner()) - throw new AssertionError(String.format("Partitioner in bounds serialization. Expected %s, was %s.", - global().getClass().getName(), - bounds.left.getPartitioner().getClass().getName())); - } - /** * Transform key to object representation of the on-disk format. * diff --git a/src/java/org/apache/cassandra/dht/IPartitionerDependentSerializer.java b/src/java/org/apache/cassandra/dht/IPartitionerDependentSerializer.java index 3a9a76810b..5c75788c0b 100644 --- a/src/java/org/apache/cassandra/dht/IPartitionerDependentSerializer.java +++ b/src/java/org/apache/cassandra/dht/IPartitionerDependentSerializer.java @@ -17,9 +17,9 @@ */ package org.apache.cassandra.dht; -import java.io.DataInput; import java.io.IOException; +import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; /** @@ -49,7 +49,7 @@ public interface IPartitionerDependentSerializer * @return the type that was deserialized * @throws IOException if deserialization fails */ - public T deserialize(DataInput in, IPartitioner p, int version) throws IOException; + public T deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException; /** * Calculate serialized size of object without actually serializing. diff --git a/src/java/org/apache/cassandra/dht/ReversedLongLocalPartitioner.java b/src/java/org/apache/cassandra/dht/ReversedLongLocalPartitioner.java index c423ba7e13..c56f3e920e 100644 --- a/src/java/org/apache/cassandra/dht/ReversedLongLocalPartitioner.java +++ b/src/java/org/apache/cassandra/dht/ReversedLongLocalPartitioner.java @@ -17,6 +17,7 @@ */ package org.apache.cassandra.dht; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -30,7 +31,9 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.LongType; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.ObjectSizes; +import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.bytecomparable.ByteSourceInverse; @@ -53,9 +56,14 @@ public class ReversedLongLocalPartitioner implements IPartitioner return new CachedHashDecoratedKey(getToken(key), key); // CachedHashDecoratedKey is used for bloom filter hash calculation } - public Token midpoint(Token left, Token right) + public Token midpoint(Token ltoken, Token rtoken) { - throw new UnsupportedOperationException(); + // the symbolic MINIMUM token should act as ZERO: the empty bit array + BigInteger left = ltoken.equals(MIN_TOKEN) ? BigInteger.ZERO : BigInteger.valueOf(ltoken.getLongValue()); + BigInteger right = rtoken.equals(MIN_TOKEN) ? BigInteger.ZERO : BigInteger.valueOf(rtoken.getLongValue()); + Pair midpair = FBUtilities.midpoint(left, right, 63); + // discard the remainder + return new ReversedLongLocalToken(midpair.left.longValue()); } public Token split(Token left, Token right, double ratioToLeft) @@ -174,6 +182,12 @@ public class ReversedLongLocalPartitioner implements IPartitioner return token; } + @Override + public long getLongValue() + { + return token; + } + @Override public ByteSource asComparableBytes(ByteComparable.Version version) { diff --git a/src/java/org/apache/cassandra/dht/Token.java b/src/java/org/apache/cassandra/dht/Token.java index 41694277d8..3b78d7b84d 100644 --- a/src/java/org/apache/cassandra/dht/Token.java +++ b/src/java/org/apache/cassandra/dht/Token.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.dht; -import java.io.DataInput; import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; @@ -99,7 +98,13 @@ public abstract class Token implements RingPosition, Serializable { private static final int SERDE_VERSION = MessagingService.VERSION_40; + // Convenience method as Token has a reference to its Partitioner public void serialize(Token t, DataOutputPlus out, Version version) throws IOException + { + serialize(t, out, t.getPartitioner(), version); + } + + public void serialize(Token t, DataOutputPlus out, IPartitioner partitioner, Version version) throws IOException { serializer.serialize(t, out, SERDE_VERSION); } @@ -111,7 +116,13 @@ public abstract class Token implements RingPosition, Serializable return serializer.deserialize(in, partitioner, SERDE_VERSION); } + // Convenience method as Token has a reference to its Partitioner public long serializedSize(Token t, Version version) + { + return serializedSize(t, t.getPartitioner(), version); + } + + public long serializedSize(Token t, IPartitioner partitioner, Version version) { return serializer.serializedSize(t, SERDE_VERSION); } @@ -126,7 +137,7 @@ public abstract class Token implements RingPosition, Serializable p.getTokenFactory().serialize(token, out); } - public Token deserialize(DataInput in, IPartitioner p, int version) throws IOException + public Token deserialize(DataInputPlus in, IPartitioner p, int version) throws IOException { int size = deserializeSize(in); byte[] bytes = new byte[size]; @@ -134,7 +145,7 @@ public abstract class Token implements RingPosition, Serializable return p.getTokenFactory().fromByteArray(ByteBuffer.wrap(bytes)); } - public int deserializeSize(DataInput in) throws IOException + public int deserializeSize(DataInputPlus in) throws IOException { return in.readInt(); } diff --git a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java index 754687a199..6a90aa1770 100644 --- a/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java +++ b/src/java/org/apache/cassandra/locator/CMSPlacementStrategy.java @@ -28,13 +28,15 @@ import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.membership.NodeState; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.ownership.TokenMap; import static org.apache.cassandra.locator.SimpleStrategy.REPLICATION_FACTOR; @@ -115,8 +117,12 @@ public interface CMSPlacementStrategy } } - EndpointsForRange endpoints = NetworkTopologyStrategy.calculateNaturalReplicas(EntireRange.entireRange.left, - EntireRange.entireRange, + // Although MetaStrategy has its own entireRange, it uses a custom partitioner which isn't compatible with + // regular, non-CMS placements. For that reason, we select replicas here using tokens provided by the + // globally configured partitioner. + Token minToken = DatabaseDescriptor.getPartitioner().getMinimumToken(); + EndpointsForRange endpoints = NetworkTopologyStrategy.calculateNaturalReplicas(minToken, + new Range<>(minToken, minToken), directory, tokenMap, rf); diff --git a/src/java/org/apache/cassandra/locator/EndpointsByReplica.java b/src/java/org/apache/cassandra/locator/EndpointsByReplica.java index 9c4636082f..3f5bd6ec76 100644 --- a/src/java/org/apache/cassandra/locator/EndpointsByReplica.java +++ b/src/java/org/apache/cassandra/locator/EndpointsByReplica.java @@ -24,9 +24,9 @@ import com.google.common.collect.Maps; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.IPartitionerDependentSerializer; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; @@ -72,7 +72,7 @@ public class EndpointsByReplica extends ReplicaMultimap + public static class Serializer implements IPartitionerDependentSerializer { @Override public void serialize(EndpointsByReplica t, DataOutputPlus out, int version) throws IOException @@ -90,18 +90,18 @@ public class EndpointsByReplica extends ReplicaMultimap range = (Range) tokenSerializer.deserialize(in, IPartitioner.global(), version); + Replica replica = Replica.serializer.deserialize(in, partitioner, version); + Range range = (Range) tokenSerializer.deserialize(in, partitioner, version); int efrSize = in.readUnsignedVInt32(); EndpointsForRange.Builder efrBuilder = new EndpointsForRange.Builder(range, efrSize); for (int j = 0; j < efrSize; j++) - efrBuilder.add(Replica.serializer.deserialize(in, version), Conflict.NONE); + efrBuilder.add(Replica.serializer.deserialize(in, partitioner, version), Conflict.NONE); builder.putAll(replica, efrBuilder.build(), Conflict.NONE); } @@ -111,13 +111,13 @@ public class EndpointsByReplica extends ReplicaMultimap entry : t.map.entrySet()) { size += Replica.serializer.serializedSize(entry.getKey(), version); EndpointsForRange efr = entry.getValue(); size += tokenSerializer.serializedSize(efr.range(), version); - size += TypeSizes.sizeofVInt(efr.size()); + size += TypeSizes.sizeofUnsignedVInt(efr.size()); for (Replica replica : efr) size += Replica.serializer.serializedSize(replica, version); } diff --git a/src/java/org/apache/cassandra/locator/LocalStrategy.java b/src/java/org/apache/cassandra/locator/LocalStrategy.java index c98cbc2b94..69193090c4 100644 --- a/src/java/org/apache/cassandra/locator/LocalStrategy.java +++ b/src/java/org/apache/cassandra/locator/LocalStrategy.java @@ -20,17 +20,19 @@ package org.apache.cassandra.locator; import java.util.List; import java.util.Map; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; +import org.apache.cassandra.tcm.ownership.PlacementForRange; +import org.apache.cassandra.tcm.ownership.VersionedEndpoints; +import org.apache.cassandra.utils.FBUtilities; public class LocalStrategy extends SystemStrategy { private static final ReplicationFactor RF = ReplicationFactor.fullOnly(1); - public LocalStrategy(String keyspaceName, Map configOptions) { super(keyspaceName, configOptions); @@ -53,4 +55,17 @@ public class LocalStrategy extends SystemStrategy { return RF; } + + /** + * For lazy initialisation. In some circumstances, we may want to instantiate LocalStrategy without initialising + * DatabaseDescriptor; FQL replay is one such usage as we initialise the KeyspaceMetadata objects, which now eagerly + * creates the replication strategy. + */ + static class EntireRange + { + public static final Range entireRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), DatabaseDescriptor.getPartitioner().getMinimumToken()); + public static final EndpointsForRange localReplicas = EndpointsForRange.of(new Replica(FBUtilities.getBroadcastAddressAndPort(), entireRange, true)); + public static final DataPlacement placement = new DataPlacement(PlacementForRange.builder().withReplicaGroup(VersionedEndpoints.forRange(Epoch.FIRST, localReplicas)).build(), + PlacementForRange.builder().withReplicaGroup(VersionedEndpoints.forRange(Epoch.FIRST, localReplicas)).build()); + } } diff --git a/src/java/org/apache/cassandra/locator/MetaStrategy.java b/src/java/org/apache/cassandra/locator/MetaStrategy.java index a16d4e68bb..536ac2fa22 100644 --- a/src/java/org/apache/cassandra/locator/MetaStrategy.java +++ b/src/java/org/apache/cassandra/locator/MetaStrategy.java @@ -20,14 +20,15 @@ package org.apache.cassandra.locator; import java.util.List; import java.util.Map; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.ReversedLongLocalPartitioner; import org.apache.cassandra.dht.Token; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.ownership.DataPlacement; - -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import org.apache.cassandra.tcm.sequences.LockedRanges; /** * MetaStrategy is designed for distributed cluster metadata keyspace, and should not be used by @@ -40,6 +41,20 @@ import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; */ public class MetaStrategy extends SystemStrategy { + public static final IPartitioner partitioner = ReversedLongLocalPartitioner.instance; + public static final Range entireRange = new Range<>(partitioner.getMinimumToken(), + partitioner.getMinimumToken()); + + public static LockedRanges.AffectedRanges affectedRanges(ClusterMetadata metadata) + { + return LockedRanges.AffectedRanges.singleton(ReplicationParams.meta(metadata), entireRange); + } + + public static Replica replica(InetAddressAndPort addr) + { + return new Replica(addr, entireRange, true); + } + public MetaStrategy(String keyspaceName, Map configOptions) { super(keyspaceName, configOptions); @@ -67,6 +82,15 @@ public class MetaStrategy extends SystemStrategy return ReplicationFactor.fullOnly(rf); } + @Override + public RangesAtEndpoint getAddressReplicas(ClusterMetadata metadata, InetAddressAndPort endpoint) + { + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint); + if (metadata.fullCMSMembers().contains(endpoint)) + builder.add(replica(endpoint)); + return builder.build(); + } + @Override public boolean hasSameSettings(AbstractReplicationStrategy other) { diff --git a/src/java/org/apache/cassandra/locator/Replica.java b/src/java/org/apache/cassandra/locator/Replica.java index 86f190e42b..b1f68b2101 100644 --- a/src/java/org/apache/cassandra/locator/Replica.java +++ b/src/java/org/apache/cassandra/locator/Replica.java @@ -26,9 +26,9 @@ import com.google.common.base.Preconditions; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.IPartitionerDependentSerializer; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.utils.FBUtilities; @@ -53,7 +53,7 @@ import static org.apache.cassandra.dht.AbstractBounds.tokenSerializer; */ public final class Replica implements Comparable { - public static final IVersionedSerializer serializer = new Serializer(); + public static final IPartitionerDependentSerializer serializer = new Serializer(); private final Range range; private final InetAddressAndPort endpoint; @@ -202,7 +202,7 @@ public final class Replica implements Comparable return transientReplica(endpoint, new Range<>(start, end)); } - public static class Serializer implements IVersionedSerializer + public static class Serializer implements IPartitionerDependentSerializer { @Override public void serialize(Replica t, DataOutputPlus out, int version) throws IOException @@ -213,9 +213,9 @@ public final class Replica implements Comparable } @Override - public Replica deserialize(DataInputPlus in, int version) throws IOException + public Replica deserialize(DataInputPlus in, IPartitioner partitioner, int version) throws IOException { - Range range = (Range) tokenSerializer.deserialize(in, IPartitioner.global(), version); + Range range = (Range) tokenSerializer.deserialize(in, partitioner, version); InetAddressAndPort endpoint = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version); boolean isFull = in.readBoolean(); return new Replica(endpoint, range, isFull); diff --git a/src/java/org/apache/cassandra/repair/RepairJobDesc.java b/src/java/org/apache/cassandra/repair/RepairJobDesc.java index dba336b5e7..796b1c6f57 100644 --- a/src/java/org/apache/cassandra/repair/RepairJobDesc.java +++ b/src/java/org/apache/cassandra/repair/RepairJobDesc.java @@ -34,7 +34,11 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; @@ -121,7 +125,8 @@ public class RepairJobDesc desc.sessionId.serialize(out); out.writeUTF(desc.keyspace); out.writeUTF(desc.columnFamily); - IPartitioner.validate(desc.ranges); + if (version >= MessagingService.VERSION_51) + out.writeUTF(getPartitioner(desc).getClass().getCanonicalName()); out.writeInt(desc.ranges.size()); for (Range rt : desc.ranges) AbstractBounds.tokenSerializer.serialize(rt, out, version); @@ -135,15 +140,16 @@ public class RepairJobDesc TimeUUID sessionId = TimeUUID.deserialize(in); String keyspace = in.readUTF(); String columnFamily = in.readUTF(); + IPartitioner partitioner = version >= MessagingService.VERSION_51 + ? FBUtilities.newPartitioner(in.readUTF()) + : IPartitioner.global(); int nRanges = in.readInt(); Collection> ranges = new ArrayList<>(nRanges); Range range; - for (int i = 0; i < nRanges; i++) { - range = (Range) AbstractBounds.tokenSerializer.deserialize(in, - IPartitioner.global(), version); + range = (Range) AbstractBounds.tokenSerializer.deserialize(in, partitioner, version); ranges.add(range); } @@ -158,6 +164,11 @@ public class RepairJobDesc size += TimeUUID.sizeInBytes(); size += TypeSizes.sizeof(desc.keyspace); size += TypeSizes.sizeof(desc.columnFamily); + if (version >= MessagingService.VERSION_51) + { + String partitioner = getPartitioner(desc).getClass().getCanonicalName(); + size += TypeSizes.sizeof(partitioner); + } size += TypeSizes.sizeof(desc.ranges.size()); for (Range rt : desc.ranges) { @@ -165,5 +176,13 @@ public class RepairJobDesc } return size; } + + private IPartitioner getPartitioner(RepairJobDesc desc) + { + TableMetadata tm = ClusterMetadata.current().schema.getKeyspaceMetadata(desc.keyspace) + .getTableOrViewNullable(desc.columnFamily); + return tm != null ? tm.partitioner : IPartitioner.global(); + + } } } diff --git a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java index fc44d7d0c4..81ae098349 100644 --- a/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java +++ b/src/java/org/apache/cassandra/repair/messages/PrepareMessage.java @@ -35,12 +35,14 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.TimeUUID; public class PrepareMessage extends RepairMessage { public final List tableIds; + public final IPartitioner partitioner; public final Collection> ranges; public final TimeUUID parentRepairSession; @@ -49,11 +51,12 @@ public class PrepareMessage extends RepairMessage public final boolean isGlobal; public final PreviewKind previewKind; - public PrepareMessage(TimeUUID parentRepairSession, List tableIds, Collection> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind) + public PrepareMessage(TimeUUID parentRepairSession, List tableIds, IPartitioner partitioner, Collection> ranges, boolean isIncremental, long repairedAt, boolean isGlobal, PreviewKind previewKind) { super(null); this.parentRepairSession = parentRepairSession; this.tableIds = tableIds; + this.partitioner = partitioner; this.ranges = ranges; this.isIncremental = isIncremental; this.repairedAt = repairedAt; @@ -79,13 +82,14 @@ public class PrepareMessage extends RepairMessage previewKind == other.previewKind && repairedAt == other.repairedAt && tableIds.equals(other.tableIds) && + partitioner.getClass().equals(other.partitioner.getClass()) && ranges.equals(other.ranges); } @Override public int hashCode() { - return Objects.hash(parentRepairSession, isGlobal, previewKind, isIncremental, repairedAt, tableIds, ranges); + return Objects.hash(parentRepairSession, isGlobal, previewKind, isIncremental, repairedAt, tableIds, ranges, partitioner); } private static final String MIXED_MODE_ERROR = "Some nodes involved in repair are on an incompatible major version. " + @@ -104,12 +108,11 @@ public class PrepareMessage extends RepairMessage for (TableId tableId : message.tableIds) tableId.serialize(out); message.parentRepairSession.serialize(out); + if (version >= MessagingService.VERSION_51) + out.writeUTF(message.partitioner.getClass().getCanonicalName()); out.writeInt(message.ranges.size()); for (Range r : message.ranges) - { - IPartitioner.validate(r); Range.tokenSerializer.serialize(r, out, version); - } out.writeBoolean(message.isIncremental); out.writeLong(message.repairedAt); out.writeBoolean(message.isGlobal); @@ -120,21 +123,23 @@ public class PrepareMessage extends RepairMessage { Preconditions.checkArgument(version == MessagingService.current_version, String.format(MIXED_MODE_ERROR, version, MessagingService.current_version)); - int tableIdCount = in.readInt(); List tableIds = new ArrayList<>(tableIdCount); for (int i = 0; i < tableIdCount; i++) tableIds.add(TableId.deserialize(in)); TimeUUID parentRepairSession = TimeUUID.deserialize(in); + IPartitioner partitioner = version >= MessagingService.VERSION_51 + ? FBUtilities.newPartitioner(in.readUTF()) + : IPartitioner.global(); int rangeCount = in.readInt(); List> ranges = new ArrayList<>(rangeCount); for (int i = 0; i < rangeCount; i++) - ranges.add((Range) Range.tokenSerializer.deserialize(in, IPartitioner.global(), version)); + ranges.add((Range) Range.tokenSerializer.deserialize(in, partitioner, version)); boolean isIncremental = in.readBoolean(); long timestamp = in.readLong(); boolean isGlobal = in.readBoolean(); PreviewKind previewKind = PreviewKind.deserialize(in.readInt()); - return new PrepareMessage(parentRepairSession, tableIds, ranges, isIncremental, timestamp, isGlobal, previewKind); + return new PrepareMessage(parentRepairSession, tableIds, partitioner, ranges, isIncremental, timestamp, isGlobal, previewKind); } public long serializedSize(PrepareMessage message, int version) @@ -144,6 +149,8 @@ public class PrepareMessage extends RepairMessage for (TableId tableId : message.tableIds) size += tableId.serializedSize(); size += TimeUUID.sizeInBytes(); + if (version >= MessagingService.VERSION_51) + size += TypeSizes.sizeof(message.partitioner.getClass().getCanonicalName()); size += TypeSizes.sizeof(message.ranges.size()); for (Range r : message.ranges) size += Range.tokenSerializer.serializedSize(r, version); diff --git a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java index a5579add83..92a1ac4eb8 100644 --- a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java +++ b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java @@ -32,8 +32,10 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.tcm.ClusterMetadata; import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; @@ -100,10 +102,7 @@ public class SyncRequest extends RepairMessage inetAddressAndPortSerializer.serialize(message.dst, out, version); out.writeInt(message.ranges.size()); for (Range range : message.ranges) - { - IPartitioner.validate(range); AbstractBounds.tokenSerializer.serialize(range, out, version); - } out.writeInt(message.previewKind.getSerializationVal()); out.writeBoolean(message.asymmetric); } @@ -116,8 +115,11 @@ public class SyncRequest extends RepairMessage InetAddressAndPort dst = inetAddressAndPortSerializer.deserialize(in, version); int rangesCount = in.readInt(); List> ranges = new ArrayList<>(rangesCount); + IPartitioner partitioner = ClusterMetadata.current().schema.getKeyspaceMetadata(desc.keyspace).params.replication.isMeta() + ? MetaStrategy.partitioner + : IPartitioner.global(); for (int i = 0; i < rangesCount; ++i) - ranges.add((Range) AbstractBounds.tokenSerializer.deserialize(in, IPartitioner.global(), version)); + ranges.add((Range) AbstractBounds.tokenSerializer.deserialize(in, partitioner, version)); PreviewKind previewKind = PreviewKind.deserialize(in.readInt()); boolean asymmetric = in.readBoolean(); return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric); diff --git a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java index a2e162659c..896f2bf51b 100644 --- a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java +++ b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java @@ -24,6 +24,8 @@ import java.util.Set; import java.util.function.Supplier; import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.utils.JVMStabilityInspector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,11 +35,9 @@ import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.statements.schema.CreateTableStatement; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.exceptions.CasWriteTimeoutException; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LogReader; @@ -63,18 +63,17 @@ public final class DistributedMetadataLogKeyspace */ public static final long GENERATION = 0; + public static final TableId LOG_TABLE_ID = TableId.unsafeDeterministic(SchemaConstants.METADATA_KEYSPACE_NAME, TABLE_NAME); public static final String LOG_TABLE_CQL = "CREATE TABLE %s.%s (" - + "period bigint," - + "current_epoch bigint static," - + "sealed boolean static," + "epoch bigint," + "entry_id bigint," + "transformation blob," + "kind text," - + "PRIMARY KEY (period, epoch))"; + + "PRIMARY KEY (epoch))"; public static final TableMetadata Log = parse(LOG_TABLE_CQL, TABLE_NAME, "Log") + .partitioner(MetaStrategy.partitioner) .compaction(CompactionParams.twcs(ImmutableMap.of("compaction_window_unit","DAYS", "compaction_window_size","1"))) .build(); @@ -83,20 +82,20 @@ public final class DistributedMetadataLogKeyspace { try { - String init = String.format("INSERT INTO %s.%s (period, epoch, current_epoch, transformation, kind, entry_id, sealed) " + - "VALUES(?, ?, ?, ?, ?, ?, false) " + + String init = String.format("INSERT INTO %s.%s (epoch, transformation, kind, entry_id) " + + "VALUES(?, ?, ?, ?) " + "IF NOT EXISTS", SchemaConstants.METADATA_KEYSPACE_NAME, TABLE_NAME); UntypedResultSet result = QueryProcessor.execute(init, ConsistencyLevel.QUORUM, - Period.FIRST, FIRST.getEpoch(), FIRST.getEpoch(), - Transformation.Kind.PRE_INITIALIZE_CMS.toVersionedBytes(PreInitialize.blank()), Transformation.Kind.PRE_INITIALIZE_CMS.toString(), Entry.Id.NONE.entryId); + FIRST.getEpoch(), + Transformation.Kind.PRE_INITIALIZE_CMS.toVersionedBytes(PreInitialize.blank()), + Transformation.Kind.PRE_INITIALIZE_CMS.toString(), + Entry.Id.NONE.entryId); UntypedResultSet.Row row = result.one(); if (row.getBoolean("[applied]")) return true; if (row.getLong("epoch") == FIRST.getEpoch() && - row.getLong("period") == Period.FIRST && - row.getLong("current_epoch") == FIRST.getEpoch() && row.getLong("entry_id") == Entry.Id.NONE.entryId && Transformation.Kind.PRE_INITIALIZE_CMS.toString().equals(row.getString("kind"))) return true; @@ -119,10 +118,7 @@ public final class DistributedMetadataLogKeyspace public static boolean tryCommit(Entry.Id entryId, Transformation transform, Epoch previousEpoch, - Epoch nextEpoch, - long previousPeriod, - long nextPeriod, - boolean sealCurrentPeriod) + Epoch nextEpoch) { try { @@ -132,30 +128,16 @@ public final class DistributedMetadataLogKeyspace // TODO get lowest supported metadata version from ClusterMetadata ByteBuffer serializedEvent = transform.kind().toVersionedBytes(transform); - UntypedResultSet result; - if (previousPeriod + 1 == nextPeriod || ClusterMetadataService.state() == ClusterMetadataService.State.RESET) - { - String query = String.format("INSERT INTO %s.%s (period, epoch, current_epoch, entry_id, transformation, kind, sealed) " + - "VALUES (?, ?, ?, ?, ?, ?, false) " + - "IF NOT EXISTS;", - SchemaConstants.METADATA_KEYSPACE_NAME, TABLE_NAME); - result = QueryProcessor.execute(query, ConsistencyLevel.QUORUM, - nextPeriod, nextEpoch.getEpoch(), nextEpoch.getEpoch(), entryId.entryId, serializedEvent, transform.kind().toString()); - } - else - { - assert previousPeriod == nextPeriod; - String query = String.format("UPDATE %s.%s SET current_epoch = ?, sealed = ?, entry_id = ?, transformation = ?, kind = ? " + - "WHERE period = ? AND epoch = ? " + - "IF current_epoch = ? and sealed = false;", - SchemaConstants.METADATA_KEYSPACE_NAME, TABLE_NAME); - - result = QueryProcessor.execute(query, - ConsistencyLevel.QUORUM, - nextEpoch.getEpoch(), sealCurrentPeriod, - entryId.entryId, serializedEvent, transform.kind().toString(), - previousPeriod, nextEpoch.getEpoch(), previousEpoch.getEpoch()); - } + String query = String.format("INSERT INTO %s.%s (epoch, entry_id, transformation, kind) " + + "VALUES (?, ?, ?, ?) " + + "IF NOT EXISTS;", + SchemaConstants.METADATA_KEYSPACE_NAME, TABLE_NAME); + UntypedResultSet result = QueryProcessor.execute(query, + ConsistencyLevel.QUORUM, + nextEpoch.getEpoch(), + entryId.entryId, + serializedEvent, + transform.kind().toString()); return result.one().getBoolean("[applied]"); } @@ -176,7 +158,7 @@ public final class DistributedMetadataLogKeyspace public static LogState getLogState(Epoch since, boolean consistentFetch) { - return (consistentFetch ? serialLogReader : localLogReader).getLogState(ClusterMetadata.current().period, since); + return (consistentFetch ? serialLogReader : localLogReader).getLogState(since); } public static class DistributedTableLogReader implements LogReader @@ -195,18 +177,21 @@ public final class DistributedMetadataLogKeyspace this(consistencyLevel, () -> ClusterMetadataService.instance().snapshotManager()); } - public EntryHolder getEntries(long period, Epoch since) throws IOException + public EntryHolder getEntries(Epoch since) throws IOException { - UntypedResultSet resultSet = execute(String.format("SELECT epoch, kind, transformation, entry_id, sealed FROM %s.%s WHERE period = ? AND epoch >= ?", + // during gossip upgrade we have epoch = Long.MIN_VALUE + 1 (and the reverse partitioner doesn't support negative keys) + since = since.isBefore(Epoch.EMPTY) ? Epoch.EMPTY : since; + // note that we want all entries with epoch >= since - but since we use a reverse partitioner, we actually + // want all entries where the token is less than token(since) + UntypedResultSet resultSet = execute(String.format("SELECT epoch, kind, transformation, entry_id FROM %s.%s WHERE token(epoch) <= token(?)", SchemaConstants.METADATA_KEYSPACE_NAME, TABLE_NAME), - consistencyLevel, period, since.getEpoch()); + consistencyLevel, since.getEpoch()); EntryHolder entryHolder = new EntryHolder(since); for (UntypedResultSet.Row row : resultSet) { - long epochl = row.getLong("epoch"); - Epoch epoch = Epoch.create(epochl); - Transformation.Kind kind = Transformation.Kind.valueOf(row.getString("kind")); long entryId = row.getLong("entry_id"); + Epoch epoch = Epoch.create(row.getLong("epoch")); + Transformation.Kind kind = Transformation.Kind.valueOf(row.getString("kind")); Transformation transform = kind.fromVersionedBytes(row.getBlob("transformation")); entryHolder.add(new Entry(new Entry.Id(entryId), epoch, transform)); } @@ -230,7 +215,7 @@ public final class DistributedMetadataLogKeyspace private static TableMetadata.Builder parse(String cql, String table, String description) { return CreateTableStatement.parse(String.format(cql, SchemaConstants.METADATA_KEYSPACE_NAME, table), SchemaConstants.METADATA_KEYSPACE_NAME) - .id(TableId.unsafeDeterministic(SchemaConstants.METADATA_KEYSPACE_NAME, table)) + .id(LOG_TABLE_ID) .epoch(FIRST) .comment(description); } diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index a7252eca2b..9682c8e959 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -60,6 +60,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.config.DurationSpec; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; @@ -100,6 +101,7 @@ import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.repair.state.CoordinatorState; import org.apache.cassandra.repair.state.ParticipateState; import org.apache.cassandra.repair.state.ValidationState; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.PaxosRepair; @@ -663,11 +665,18 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai Set failedNodes = synchronizedSet(new HashSet<>()); AsyncPromise promise = new AsyncPromise<>(); + Set partitioners = new HashSet<>(1); List tableIds = new ArrayList<>(columnFamilyStores.size()); for (ColumnFamilyStore cfs : columnFamilyStores) + { tableIds.add(cfs.metadata.id); + partitioners.add(cfs.getPartitioner()); + } - PrepareMessage message = new PrepareMessage(parentRepairSession, tableIds, options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind()); + if (partitioners.size() > 1) + failRepair(parentRepairSession, "The tables involved in repair are configured with multiple partitioners."); + + PrepareMessage message = new PrepareMessage(parentRepairSession, tableIds, columnFamilyStores.get(0).getPartitioner(), options.getRanges(), options.isIncremental(), repairedAt, options.isGlobal(), options.getPreviewKind()); register(new ParticipateState(ctx.clock(), ctx.broadcastAddressAndPort(), message)); for (InetAddressAndPort neighbour : endpoints) { @@ -1129,21 +1138,18 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai { for (TableMetadata table : tables) { - Set endpoints = metadata.placements - .get(keyspace.getMetadata().params.replication) - .reads - .forRange(range) - .get() - .filter(FailureDetector.isReplicaAlive).endpoints(); - if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, endpoints)) + + ReplicationParams replication = keyspace.getMetadata().params.replication; + // Special case meta keyspace as it uses a custom partitioner/tokens, but the paxos table and repairs + // are based on the system partitioner + EndpointsForRange endpoints = replication.isMeta() + ? ClusterMetadata.current().fullCMSMembersAsReplicas() + : ClusterMetadata.current().placements.get(replication).reads.forRange(range).get(); + + Set liveEndpoints = endpoints.filter(FailureDetector.isReplicaAlive).endpoints(); + if (!PaxosRepair.hasSufficientLiveNodesForTopologyChange(keyspace, range, liveEndpoints)) { - Set downEndpoints = metadata.placements - .get(keyspace.getMetadata().params.replication) - .reads - .forRange(range) - .get() - .filter(e -> !endpoints.contains(e.endpoint())).endpoints(); - downEndpoints.removeAll(endpoints); + Set downEndpoints = endpoints.filter(e -> !liveEndpoints.contains(e.endpoint())).endpoints(); throw new RuntimeException(String.format("Insufficient live nodes to repair paxos for %s in %s for %s.\n" + "There must be enough live nodes to satisfy EACH_QUORUM, but the following nodes are down: %s\n" + @@ -1165,7 +1171,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai range, table.keyspace, table.name, ClusterMetadata.current(), PAXOS_REPAIR_ALLOW_MULTIPLE_PENDING_UNSAFE.getKey())); } - futures.add(() -> PaxosCleanup.cleanup(ctx, endpoints, table, Collections.singleton(range), false, repairCommandExecutor())); + futures.add(() -> PaxosCleanup.cleanup(ctx, liveEndpoints, table, Collections.singleton(range), false, repairCommandExecutor())); } } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 928be162a0..4dce59ef92 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -146,6 +146,7 @@ import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.LocalStrategy; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.RangesByEndpoint; import org.apache.cassandra.locator.Replica; @@ -1971,13 +1972,21 @@ public class StorageService extends NotificationBroadcasterSupport implements IE Map, EndpointsForRange> rangeToEndpointMap = new HashMap<>(ranges.size()); if (null != keyspaceMetadata) { - TokenMap tokenMap = metadata.tokenMap; - - for (Range range : ranges) + if (keyspaceMetadata.params.replication.isMeta()) { - Token token = tokenMap.nextToken(tokenMap.tokens(), range.right.getToken()); - rangeToEndpointMap.put(range, metadata.placements.get(keyspaceMetadata.params.replication) - .reads.forRange(token).get()); + rangeToEndpointMap.put(MetaStrategy.entireRange, + metadata.placements.get(keyspaceMetadata.params.replication) + .reads.forRange(MetaStrategy.entireRange).get()); + } + else + { + TokenMap tokenMap = metadata.tokenMap; + for (Range range : ranges) + { + Token token = tokenMap.nextToken(tokenMap.tokens(), range.right.getToken()); + rangeToEndpointMap.put(range, metadata.placements.get(keyspaceMetadata.params.replication) + .reads.forRange(token).get()); + } } } else @@ -3149,7 +3158,10 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public Pair> repair(String keyspace, Map repairSpec, List listeners) { - RepairOption option = RepairOption.parse(repairSpec, ClusterMetadata.current().partitioner); + IPartitioner partitioner = Keyspace.open(keyspace).getMetadata().params.replication.isMeta() + ? MetaStrategy.partitioner + : IPartitioner.global(); + RepairOption option = RepairOption.parse(repairSpec, partitioner); return repair(keyspace, option, listeners); } @@ -4469,6 +4481,8 @@ public class StorageService extends NotificationBroadcasterSupport implements IE List keys = new ArrayList<>(); for (Keyspace keyspace : Keyspace.nonLocalStrategy()) { + if (keyspace.getMetadata().params.replication.isMeta()) + continue; for (Range range : getPrimaryRangesForEndpoint(keyspace.getName(), getBroadcastAddressAndPort())) keys.addAll(keySamples(keyspace.getColumnFamilyStores(), range)); } diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java index f3938f1ccb..fed3cc0785 100644 --- a/src/java/org/apache/cassandra/service/paxos/Paxos.java +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java @@ -45,6 +45,7 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InOurDc; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.locator.ReplicaLayout.ForTokenWrite; @@ -253,7 +254,12 @@ public class Paxos static Electorate get(TableMetadata table, DecoratedKey key, ConsistencyLevel consistency) { - return get(consistency, forTokenWriteLiveAndDown(Keyspace.open(table.keyspace), key.getToken())); + // MetaStrategy distributes the entire keyspace to all replicas. In addition, its tables (currently only + // the dist log table) don't use the globally configured partitioner. For these reasons we don't lookup the + // replicas using the supplied token as this can actually be of the incorrect type (for example when + // performing Paxos repair). + final Token token = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : key.getToken(); + return get(consistency, forTokenWriteLiveAndDown(Keyspace.open(table.keyspace), token)); } static Electorate get(ConsistencyLevel consistency, ForTokenWrite all) @@ -425,13 +431,18 @@ public class Paxos static Participants get(ClusterMetadata metadata, TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus, Predicate isReplicaAlive) { KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaceMetadata(table.keyspace); - ReplicaLayout.ForTokenWrite all = forTokenWriteLiveAndDown(keyspaceMetadata, token); + // MetaStrategy distributes the entire keyspace to all replicas. In addition, its tables (currently only + // the dist log table) don't use the globally configured partitioner. For these reasons we don't lookup the + // replicas using the supplied token as this can actually be of the incorrect type (for example when + // performing Paxos repair). + final Token actualToken = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : token; + ReplicaLayout.ForTokenWrite all = forTokenWriteLiveAndDown(keyspaceMetadata, actualToken); ReplicaLayout.ForTokenWrite electorate = consistencyForConsensus.isDatacenterLocal() ? all.filter(InOurDc.replicas()) : all; EndpointsForToken live = all.all().filter(isReplicaAlive); return new Participants(metadata.epoch, Keyspace.open(table.keyspace), consistencyForConsensus, all, electorate, live, - (cm) -> get(cm, table, token, consistencyForConsensus)); + (cm) -> get(cm, table, actualToken, consistencyForConsensus)); } static Participants get(TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus) @@ -1125,8 +1136,13 @@ public class Paxos public static boolean isInRangeAndShouldProcess(InetAddressAndPort from, DecoratedKey key, TableMetadata table, boolean includesRead) { Keyspace keyspace = Keyspace.open(table.keyspace); - return (includesRead ? EndpointsForToken.natural(keyspace, key.getToken()).get() - : ReplicaLayout.forTokenWriteLiveAndDown(keyspace, key.getToken()).all() + // MetaStrategy distributes the entire keyspace to all replicas. In addition, its tables (currently only + // the dist log table) don't use the globally configured partitioner. For these reasons we don't lookup the + // replicas using the supplied token as this can actually be of the incorrect type (for example when + // performing Paxos repair). + Token token = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : key.getToken(); + return (includesRead ? EndpointsForToken.natural(keyspace, token).get() + : ReplicaLayout.forTokenWriteLiveAndDown(keyspace, token).all() ).contains(getBroadcastAddressAndPort()); } diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java index fd220f83b7..cc6bc72325 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosRepair.java @@ -52,6 +52,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.RequestCallbackWithFailure; import org.apache.cassandra.repair.SharedContext; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; @@ -545,7 +546,13 @@ public class PaxosRepair extends AbstractPaxosRepair */ public static boolean hasSufficientLiveNodesForTopologyChange(Keyspace keyspace, Range range, Collection liveEndpoints) { - return hasSufficientLiveNodesForTopologyChange(ClusterMetadata.current().placements.get(keyspace.getMetadata().params.replication).reads.forRange(range).endpoints(), + ReplicationParams replication = keyspace.getMetadata().params.replication; + // Special case meta keyspace as it uses a custom partitioner/tokens, but the paxos table and repairs + // are based on the system partitioner + Collection allEndpoints = replication.isMeta() + ? ClusterMetadata.current().fullCMSMembers() + : ClusterMetadata.current().placements.get(replication).reads.forRange(range).endpoints(); + return hasSufficientLiveNodesForTopologyChange(allEndpoints, liveEndpoints, DatabaseDescriptor.getEndpointSnitch()::getDatacenter, DatabaseDescriptor.paxoTopologyRepairNoDcChecks(), diff --git a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java index b341ef351e..3130333813 100644 --- a/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java +++ b/src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java @@ -51,6 +51,8 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.FSReadError; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.locator.MetaStrategy; +import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; @@ -126,14 +128,32 @@ public class UncommittedTableData Range range = rangeIterator.peek(); - Token token = peeking.peek().key.getToken(); - if (!range.contains(token)) + PaxosKeyState peeked = peeking.peek(); + Token token = peeked.key.getToken(); + + // If repairing the distributed metadata log table, we skip the filtering of paxos keys where the token + // is outside the range of the repair. This check would be complicated by the fact that the system.paxos + // table keys are tokenized with the global partitioner, but the log table uses its own specific + // partitioner. This means that the repair ranges will be a ReversedLongLocalToken pair, while the + // tokens obtained from the PaxosKeyState iterator will be whatever the global partitioner uses. + // However, as the replicas of the distributed log table (i.e. CMS members) always replicate the entire + // table, the range check is superfluous in this case anyway. + // For the purposes of the actual paxos repair (and for paxos reads/writes in general), this mismatch is + // also fine as the keys/tokens are opaque to the paxos implentation itself. + if (range.left.getPartitioner() == MetaStrategy.partitioner) { - if (range.right.compareTo(token) < 0) - rangeIterator.next(); - else - peeking.next(); - continue; + assert peeked.tableId.equals(DistributedMetadataLogKeyspace.LOG_TABLE_ID); + } + else + { + if (!range.contains(token)) + { + if (range.right.compareTo(token) < 0) + rangeIterator.next(); + else + peeking.next(); + continue; + } } PaxosKeyState next = peeking.next(); diff --git a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java index b159218ac4..969247b722 100644 --- a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java @@ -34,9 +34,9 @@ import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; -import org.apache.cassandra.locator.LocalStrategy; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.compatibility.TokenRingUtils; import org.apache.cassandra.utils.AbstractIterator; @@ -60,7 +60,8 @@ class ReplicaPlanIterator extends AbstractIterator this.keyspace = keyspace; this.consistency = consistency; - List> l = keyspace.getReplicationStrategy() instanceof LocalStrategy + ReplicationParams replication = keyspace.getMetadata().params.replication; + List> l = replication.isLocal() || replication.isMeta() ? keyRange.unwrap() : getRestrictedRanges(keyRange); this.ranges = l.iterator(); diff --git a/src/java/org/apache/cassandra/streaming/StreamRequest.java b/src/java/org/apache/cassandra/streaming/StreamRequest.java index 0c8542fcfa..8aaf17c17a 100644 --- a/src/java/org/apache/cassandra/streaming/StreamRequest.java +++ b/src/java/org/apache/cassandra/streaming/StreamRequest.java @@ -31,8 +31,10 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.SchemaConstants; import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; @@ -69,6 +71,7 @@ public class StreamRequest out.writeInt(request.columnFamilies.size()); inetAddressAndPortSerializer.serialize(request.full.endpoint(), out, version); + serializeReplicas(request.full, out, version); serializeReplicas(request.transientReplicas, out, version); for (String cf : request.columnFamilies) @@ -81,7 +84,6 @@ public class StreamRequest for (Replica replica : replicas) { - IPartitioner.validate(replica.range()); Token.serializer.serialize(replica.range().left, out, version); Token.serializer.serialize(replica.range().right, out, version); } @@ -93,15 +95,21 @@ public class StreamRequest int cfCount = in.readInt(); InetAddressAndPort endpoint = inetAddressAndPortSerializer.deserialize(in, version); - RangesAtEndpoint full = deserializeReplicas(in, version, endpoint, true); - RangesAtEndpoint transientReplicas = deserializeReplicas(in, version, endpoint, false); + // TODO: It would be nicer to actually serialize the partitioner rather than infer it from the keyspace + // but at the moment that would require us to get it from the tokens of the full/transient replicas or + // looking up the KeyspaceMetadata. This way is not pleasant but it is cheap and easy. + IPartitioner partitioner = keyspace.equals(SchemaConstants.METADATA_KEYSPACE_NAME) + ? MetaStrategy.partitioner + : IPartitioner.global(); + RangesAtEndpoint full = deserializeReplicas(in, version, endpoint, true, partitioner); + RangesAtEndpoint transientReplicas = deserializeReplicas(in, version, endpoint, false, partitioner); List columnFamilies = new ArrayList<>(cfCount); for (int i = 0; i < cfCount; i++) columnFamilies.add(in.readUTF()); return new StreamRequest(keyspace, full, transientReplicas, columnFamilies); } - RangesAtEndpoint deserializeReplicas(DataInputPlus in, int version, InetAddressAndPort endpoint, boolean isFull) throws IOException + RangesAtEndpoint deserializeReplicas(DataInputPlus in, int version, InetAddressAndPort endpoint, boolean isFull, IPartitioner partitioner) throws IOException { int replicaCount = in.readInt(); @@ -111,8 +119,8 @@ public class StreamRequest //TODO, super need to review the usage of streaming vs not streaming endpoint serialization helper //to make sure I'm not using the wrong one some of the time, like do repair messages use the //streaming version? - Token left = Token.serializer.deserialize(in, IPartitioner.global(), version); - Token right = Token.serializer.deserialize(in, IPartitioner.global(), version); + Token left = Token.serializer.deserialize(in, partitioner, version); + Token right = Token.serializer.deserialize(in, partitioner, version); replicas.add(new Replica(endpoint, new Range<>(left, right), isFull)); } return replicas.build(); diff --git a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java index a72b5b664f..247aa45670 100644 --- a/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AbstractLocalProcessor.java @@ -90,10 +90,7 @@ public abstract class AbstractLocalProcessor implements Processor { Epoch nextEpoch = result.success().metadata.epoch; // If metadata applies, try committing it to the log - boolean applied = tryCommitOne(entryId, transform, - previous.epoch, nextEpoch, - previous.period, previous.nextPeriod(), - result.success().metadata.lastInPeriod); + boolean applied = tryCommitOne(entryId, transform, previous.epoch, nextEpoch); // Application here semantially means "succeeded in committing to the distributed log". if (applied) @@ -195,7 +192,6 @@ public abstract class AbstractLocalProcessor implements Processor @Override public abstract ClusterMetadata fetchLogAndWait(Epoch waitFor, Retry.Deadline retryPolicy); - protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, - Epoch previousEpoch, Epoch nextEpoch, - long previousPeriod, long nextPeriod, boolean sealPeriod); + protected abstract boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch); + } \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java index 20b32cf4d1..df00253433 100644 --- a/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/AtomicLongBackedProcessor.java @@ -59,9 +59,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor } @Override - protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, - Epoch previousEpoch, Epoch nextEpoch, - long previousPeriod, long nextPeriod, boolean sealPeriod) + protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch) { if (epochHolder.get() == 0) { @@ -90,7 +88,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor } @Override - public synchronized void append(long period, Entry entry) + public synchronized void append(Entry entry) { boolean needsSorting = entries.isEmpty() ? false : entry.epoch.isDirectlyAfter(entries.get(entries.size() - 1).epoch); entries.add(entry); @@ -99,18 +97,19 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor } @Override - public synchronized LogState getLogState(long currentPeriod, Epoch since) + public synchronized LogState getLogState(Epoch startEpoch) { ImmutableList.Builder builder = ImmutableList.builder(); ClusterMetadata latest = metadataSnapshots.getLatestSnapshot(); - Epoch actualSince = latest != null && latest.epoch.isAfter(since) ? latest.epoch : since; + Epoch actualSince = latest != null && latest.epoch.isAfter(startEpoch) ? latest.epoch : startEpoch; entries.stream().filter(e -> e.epoch.isAfter(actualSince)).forEach(builder::add); return new LogState(latest, builder.build()); } + @Override public synchronized LogState getPersistedLogState() { - return getLogState(0, Epoch.EMPTY); + return getLogState(Epoch.EMPTY); } public synchronized LogState getLogStateBetween(ClusterMetadata base, Epoch end) @@ -127,7 +126,7 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor } @Override - public synchronized EntryHolder getEntries(long period, Epoch since) + public synchronized EntryHolder getEntries(Epoch since) { throw new IllegalStateException("We have overridden all callers of this method, it should never be called"); } @@ -163,6 +162,14 @@ public class AtomicLongBackedProcessor extends AbstractLocalProcessor return snapshots.lastEntry().getValue(); } + @Override + public List listSnapshotsSince(Epoch epoch) + { + List epochs = new ArrayList<>(); + snapshots.tailMap(epoch).forEach((e, s) -> epochs.add(e)); + return epochs; + } + @Override public void storeSnapshot(ClusterMetadata metadata) { diff --git a/src/java/org/apache/cassandra/tcm/CMSOperations.java b/src/java/org/apache/cassandra/tcm/CMSOperations.java index 8eacd53873..12d6c94186 100644 --- a/src/java/org/apache/cassandra/tcm/CMSOperations.java +++ b/src/java/org/apache/cassandra/tcm/CMSOperations.java @@ -139,11 +139,11 @@ public class CMSOperations implements CMSOperationsMBean } @Override - public void sealPeriod() + public void snapshotClusterMetadata() { - logger.info("Sealing current period in metadata log"); - long period = cms.sealPeriod().period; - logger.info("Current period {} is sealed", period); + logger.info("Triggering cluster metadata snapshot"); + Epoch epoch = cms.triggerSnapshot().epoch; + logger.info("Cluster metadata snapshot triggered at {}", epoch); } @Override diff --git a/src/java/org/apache/cassandra/tcm/CMSOperationsMBean.java b/src/java/org/apache/cassandra/tcm/CMSOperationsMBean.java index 926c09d1d3..4f25a5bcfa 100644 --- a/src/java/org/apache/cassandra/tcm/CMSOperationsMBean.java +++ b/src/java/org/apache/cassandra/tcm/CMSOperationsMBean.java @@ -32,7 +32,7 @@ public interface CMSOperationsMBean public void cancelReconfigureCms(); public Map describeCMS(); - public void sealPeriod(); + public void snapshotClusterMetadata(); public void unsafeRevertClusterMetadata(long epoch); public String dumpClusterMetadata(long epoch, long transformToEpoch, String version) throws IOException; diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java index cc1636bf14..51a6d1f2e6 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadata.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadata.java @@ -43,8 +43,10 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.net.CMSIdentifierMismatchException; import org.apache.cassandra.schema.DistributedSchema; @@ -71,7 +73,6 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -import org.apache.cassandra.utils.vint.VIntCoding; import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR; import static org.apache.cassandra.db.TypeSizes.sizeof; @@ -84,8 +85,6 @@ public class ClusterMetadata public final int metadataIdentifier; public final Epoch epoch; - public final long period; - public final boolean lastInPeriod; public final IPartitioner partitioner; // Set during (initial) construction and not modifiable via Transformer public final DistributedSchema schema; @@ -97,7 +96,7 @@ public class ClusterMetadata public final ImmutableMap, ExtensionValue> extensions; // These two fields are lazy but only for the test purposes, since their computation requires initialization of the log ks - private Set fullCMSReplicas; + private EndpointsForRange fullCMSReplicas; private Set fullCMSEndpoints; public ClusterMetadata(IPartitioner partitioner) @@ -116,8 +115,6 @@ public class ClusterMetadata { this(EMPTY_METADATA_IDENTIFIER, Epoch.EMPTY, - Period.EMPTY, - true, partitioner, schema, directory, @@ -129,8 +126,6 @@ public class ClusterMetadata } public ClusterMetadata(Epoch epoch, - long period, - boolean lastInPeriod, IPartitioner partitioner, DistributedSchema schema, Directory directory, @@ -142,8 +137,6 @@ public class ClusterMetadata { this(EMPTY_METADATA_IDENTIFIER, epoch, - period, - lastInPeriod, partitioner, schema, directory, @@ -156,8 +149,6 @@ public class ClusterMetadata private ClusterMetadata(int metadataIdentifier, Epoch epoch, - long period, - boolean lastInPeriod, IPartitioner partitioner, DistributedSchema schema, Directory directory, @@ -173,8 +164,6 @@ public class ClusterMetadata assert tokenMap == null || tokenMap.partitioner().getClass().equals(partitioner.getClass()) : "Partitioner for TokenMap doesn't match base partitioner"; this.metadataIdentifier = metadataIdentifier; this.epoch = epoch; - this.period = period; - this.lastInPeriod = lastInPeriod; this.partitioner = partitioner; this.schema = schema; this.directory = directory; @@ -192,10 +181,10 @@ public class ClusterMetadata return fullCMSEndpoints; } - public Set fullCMSMembersAsReplicas() + public EndpointsForRange fullCMSMembersAsReplicas() { if (fullCMSReplicas == null) - this.fullCMSReplicas = ImmutableSet.copyOf(placements.get(ReplicationParams.meta(this)).reads.byEndpoint().flattenValues()); + fullCMSReplicas = placements.get(ReplicationParams.meta(this)).reads.forRange(MetaStrategy.entireRange).get(); return fullCMSReplicas; } @@ -206,12 +195,7 @@ public class ClusterMetadata public Transformer transformer() { - return new Transformer(this, this.nextEpoch(), false); - } - - public Transformer transformer(boolean sealPeriod) - { - return new Transformer(this, this.nextEpoch(), sealPeriod); + return new Transformer(this, this.nextEpoch()); } public ClusterMetadata forceEpoch(Epoch epoch) @@ -227,8 +211,6 @@ public class ClusterMetadata // greater than the epoch we're forcing here. return new ClusterMetadata(metadataIdentifier, epoch, - period, - lastInPeriod, partitioner, capLastModified(schema, epoch), capLastModified(directory, epoch), @@ -249,24 +231,6 @@ public class ClusterMetadata return new ClusterMetadata(clusterIdentifier, epoch, - period, - lastInPeriod, - partitioner, - schema, - directory, - tokenMap, - placements, - lockedRanges, - inProgressSequences, - extensions); - } - - public ClusterMetadata forcePeriod(long period) - { - return new ClusterMetadata(metadataIdentifier, - epoch, - period, - false, partitioner, schema, directory, @@ -302,11 +266,6 @@ public class ClusterMetadata return epoch.nextEpoch(); } - public long nextPeriod() - { - return lastInPeriod ? period + 1 : period; - } - public DataPlacement writePlacementAllSettled(KeyspaceMetadata ksm) { ClusterMetadata metadata = this; @@ -325,6 +284,8 @@ public class ClusterMetadata { PlacementForRange writes = placements.get(ksm.params.replication).writes; PlacementForRange reads = placements.get(ksm.params.replication).reads; + if (ksm.params.replication.isMeta()) + return !reads.equals(writes); return !reads.forToken(token).equals(writes.forToken(token)); } @@ -391,8 +352,6 @@ public class ClusterMetadata { private final ClusterMetadata base; private final Epoch epoch; - private final long period; - private final boolean lastInPeriod; private final IPartitioner partitioner; private DistributedSchema schema; private Directory directory; @@ -403,12 +362,10 @@ public class ClusterMetadata private final Map, ExtensionValue> extensions; private final Set modifiedKeys; - private Transformer(ClusterMetadata metadata, Epoch epoch, boolean lastInPeriod) + private Transformer(ClusterMetadata metadata, Epoch epoch) { this.base = metadata; this.epoch = epoch; - this.period = metadata.nextPeriod(); - this.lastInPeriod = lastInPeriod; this.partitioner = metadata.partitioner; this.schema = metadata.schema; this.directory = metadata.directory; @@ -625,8 +582,6 @@ public class ClusterMetadata return new Transformed(new ClusterMetadata(base.metadataIdentifier, epoch, - period, - lastInPeriod, partitioner, schema, directory, @@ -642,8 +597,6 @@ public class ClusterMetadata { return new ClusterMetadata(base.metadataIdentifier, Epoch.UPGRADE_GOSSIP, - Period.EMPTY, - true, partitioner, schema, directory, @@ -660,7 +613,6 @@ public class ClusterMetadata return "Transformer{" + "baseEpoch=" + base.epoch + ", epoch=" + epoch + - ", lastInPeriod=" + lastInPeriod + ", partitioner=" + partitioner + ", schema=" + schema + ", directory=" + schema + @@ -772,7 +724,6 @@ public class ClusterMetadata if (!(o instanceof ClusterMetadata)) return false; ClusterMetadata that = (ClusterMetadata) o; return epoch.equals(that.epoch) && - lastInPeriod == that.lastInPeriod && schema.equals(that.schema) && directory.equals(that.directory) && tokenMap.equals(that.tokenMap) && @@ -790,10 +741,6 @@ public class ClusterMetadata { logger.warn("Epoch {} != {}", epoch, other.epoch); } - if (lastInPeriod != other.lastInPeriod) - { - logger.warn("lastInPeriod {} != {}", lastInPeriod, other.lastInPeriod); - } if (!schema.equals(other.schema)) { Keyspaces.KeyspacesDiff diff = Keyspaces.diff(schema.getKeyspaces(), other.schema.getKeyspaces()); @@ -831,7 +778,7 @@ public class ClusterMetadata @Override public int hashCode() { - return Objects.hash(epoch, lastInPeriod, schema, directory, tokenMap, placements, lockedRanges, inProgressSequences, extensions); + return Objects.hash(epoch, schema, directory, tokenMap, placements, lockedRanges, inProgressSequences, extensions); } public static ClusterMetadata current() @@ -900,8 +847,6 @@ public class ClusterMetadata out.writeUnsignedVInt32(metadata.metadataIdentifier); Epoch.serializer.serialize(metadata.epoch, out); - out.writeUnsignedVInt(metadata.period); - out.writeBoolean(metadata.lastInPeriod); if (version.isBefore(Version.V1)) out.writeUTF(metadata.partitioner.getClass().getCanonicalName()); @@ -938,8 +883,6 @@ public class ClusterMetadata } Epoch epoch = Epoch.serializer.deserialize(in); - long period = in.readUnsignedVInt(); - boolean lastInPeriod = in.readBoolean(); if (version.isBefore(Version.V1)) partitioner = FBUtilities.newPartitioner(in.readUTF()); @@ -961,8 +904,6 @@ public class ClusterMetadata } return new ClusterMetadata(clusterIdentifier, epoch, - period, - lastInPeriod, partitioner, schema, dir, @@ -985,8 +926,6 @@ public class ClusterMetadata size += TypeSizes.sizeofUnsignedVInt(metadata.metadataIdentifier); size += Epoch.serializer.serializedSize(metadata.epoch) + - VIntCoding.computeUnsignedVIntSize(metadata.period) + - TypeSizes.BOOL_SIZE + sizeof(metadata.partitioner.getClass().getCanonicalName()) + DistributedSchema.serializer.serializedSize(metadata.schema, version) + Directory.serializer.serializedSize(metadata.directory, version) + diff --git a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java index 50f293e1aa..7b022b04eb 100644 --- a/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java +++ b/src/java/org/apache/cassandra/tcm/ClusterMetadataService.java @@ -63,7 +63,7 @@ import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.tcm.transformations.ForceSnapshot; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.tcm.transformations.cms.PrepareCMSReconfiguration; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; @@ -160,7 +160,7 @@ public class ClusterMetadataService { log = logSpec.sync().createLog(); localProcessor = wrapProcessor.apply(new AtomicLongBackedProcessor(log, logSpec.isReset())); - fetchLogHandler = new FetchCMSLog.Handler((e, ignored) -> logSpec.storage().getLogState(log.metadata().period, e)); + fetchLogHandler = new FetchCMSLog.Handler((e, ignored) -> logSpec.storage().getLogState(e)); } else { @@ -354,7 +354,7 @@ public class ClusterMetadataService .collect(toImmutableSet()); Election.instance.nominateSelf(candidates, ignored, metadata::equals, metadata); - ClusterMetadataService.instance().sealPeriod(); + ClusterMetadataService.instance().triggerSnapshot(); } else { @@ -402,8 +402,7 @@ public class ClusterMetadataService logger.warn("Reverting to epoch {}", epoch); ClusterMetadata metadata = ClusterMetadata.current(); ClusterMetadata toApply = transformSnapshot(LogState.getForRecovery(epoch)) - .forceEpoch(metadata.epoch.nextEpoch()) - .forcePeriod(metadata.nextPeriod()); + .forceEpoch(metadata.epoch.nextEpoch()); forceSnapshot(toApply); } @@ -436,8 +435,7 @@ public class ClusterMetadataService logger.warn("Loading cluster metadata from {}", file); ClusterMetadata metadata = ClusterMetadata.current(); ClusterMetadata toApply = deserializeClusterMetadata(file) - .forceEpoch(metadata.epoch.nextEpoch()) - .forcePeriod(metadata.nextPeriod()); + .forceEpoch(metadata.epoch.nextEpoch()); forceSnapshot(toApply); } @@ -754,14 +752,9 @@ public class ClusterMetadataService return snapshots; } - public ClusterMetadata sealPeriod() + public ClusterMetadata triggerSnapshot() { - return ClusterMetadataService.instance.commit(SealPeriod.instance, - (ClusterMetadata metadata) -> metadata, - (code, reason) -> { - // If the transformation got rejected, someone else has beat us to seal this period - return ClusterMetadata.current(); - }); + return ClusterMetadataService.instance.commit(TriggerSnapshot.instance); } public boolean isMigrating() diff --git a/src/java/org/apache/cassandra/tcm/FetchPeerLog.java b/src/java/org/apache/cassandra/tcm/FetchPeerLog.java index 64b7381f8b..1347dcf049 100644 --- a/src/java/org/apache/cassandra/tcm/FetchPeerLog.java +++ b/src/java/org/apache/cassandra/tcm/FetchPeerLog.java @@ -82,7 +82,7 @@ public class FetchPeerLog ClusterMetadata metadata = ClusterMetadata.current(); logger.info("Received peer log fetch request {} from {}: start = {}, current = {}", request, message.from(), message.payload.start, metadata.epoch); - LogState delta = LogStorage.SystemKeyspace.getLogState(metadata.period, message.payload.start); + LogState delta = LogStorage.SystemKeyspace.getLogState(message.payload.start); TCMMetrics.instance.peerLogEntriesServed(message.payload.start, delta.latestEpoch()); logger.info("Responding with log delta: {}", delta); MessagingService.instance().send(message.responseWith(delta), message.from()); diff --git a/src/java/org/apache/cassandra/tcm/MetadataSnapshots.java b/src/java/org/apache/cassandra/tcm/MetadataSnapshots.java index 2c2d619e1b..f6b15eb278 100644 --- a/src/java/org/apache/cassandra/tcm/MetadataSnapshots.java +++ b/src/java/org/apache/cassandra/tcm/MetadataSnapshots.java @@ -20,6 +20,8 @@ package org.apache.cassandra.tcm; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,7 @@ public interface MetadataSnapshots ClusterMetadata getSnapshot(Epoch epoch); ClusterMetadata getSnapshotBefore(Epoch epoch); ClusterMetadata getLatestSnapshot(); + List listSnapshotsSince(Epoch epoch); void storeSnapshot(ClusterMetadata metadata); static ByteBuffer toBytes(ClusterMetadata metadata) throws IOException @@ -77,6 +80,13 @@ public interface MetadataSnapshots @Override public ClusterMetadata getLatestSnapshot() {return null;} + + @Override + public List listSnapshotsSince(Epoch epoch) + { + return Collections.emptyList(); + } + @Override public void storeSnapshot(ClusterMetadata metadata) {} } @@ -127,12 +137,18 @@ public interface MetadataSnapshots return null; } + @Override + public List listSnapshotsSince(Epoch epoch) + { + return SystemKeyspace.listSnapshotsSince(epoch); + } + @Override public void storeSnapshot(ClusterMetadata metadata) { try { - SystemKeyspace.storeSnapshot(metadata.epoch, metadata.period, toBytes(metadata)); + SystemKeyspace.storeSnapshot(metadata.epoch, toBytes(metadata)); } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java index 3c1bb3c306..19bdc7ae9b 100644 --- a/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java +++ b/src/java/org/apache/cassandra/tcm/PaxosBackedProcessor.java @@ -33,6 +33,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.metrics.TCMMetrics; @@ -63,11 +64,9 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor } @Override - protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, - Epoch previousEpoch, Epoch nextEpoch, - long previousPeriod, long nextPeriod, boolean sealPeriod) + protected boolean tryCommitOne(Entry.Id entryId, Transformation transform, Epoch previousEpoch, Epoch nextEpoch) { - return tryCommit(entryId, transform, previousEpoch, nextEpoch, previousPeriod, nextPeriod, sealPeriod); + return tryCommit(entryId, transform, previousEpoch, nextEpoch); } @Override @@ -91,7 +90,8 @@ public class PaxosBackedProcessor extends AbstractLocalProcessor logger.warn("Could not perform consistent fetch, downgrading to fetching from CMS peers.", t); } } - Set replicas = metadata.fullCMSMembersAsReplicas(); + + EndpointsForRange replicas = metadata.fullCMSMembersAsReplicas(); // We prefer to always perform a consistent fetch (i.e. Paxos read of the distributed log state table). // However, in some cases (specifically, during CMS membership changes) this may not be possible, as Paxos diff --git a/src/java/org/apache/cassandra/tcm/Period.java b/src/java/org/apache/cassandra/tcm/Period.java deleted file mode 100644 index 6e35aeb4ab..0000000000 --- a/src/java/org/apache/cassandra/tcm/Period.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.tcm; - -public class Period -{ - public static final long EMPTY = 0; - public static final long FIRST = 1; -} diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index 0d5e9c02c4..dfe4df8399 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -326,7 +326,7 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; ClusterMetadataService.instance().log().ready(); initMessaging.run(); ClusterMetadataService.instance().forceSnapshot(metadata.forceEpoch(metadata.nextEpoch())); - ClusterMetadataService.instance().sealPeriod(); + ClusterMetadataService.instance().triggerSnapshot(); CassandraRelevantProperties.TCM_UNSAFE_BOOT_WITH_CLUSTERMETADATA.reset(); assert ClusterMetadataService.state() == LOCAL; assert ClusterMetadataService.instance() != initial : "Aborting startup as temporary metadata service is still active"; diff --git a/src/java/org/apache/cassandra/tcm/Transformation.java b/src/java/org/apache/cassandra/tcm/Transformation.java index df86c0676b..8432d136b4 100644 --- a/src/java/org/apache/cassandra/tcm/Transformation.java +++ b/src/java/org/apache/cassandra/tcm/Transformation.java @@ -170,7 +170,7 @@ public interface Transformation PRE_INITIALIZE_CMS(0, () -> PreInitialize.serializer), INITIALIZE_CMS(1, () -> Initialize.serializer), FORCE_SNAPSHOT(2, () -> ForceSnapshot.serializer), - SEAL_PERIOD(3, () -> SealPeriod.serializer), + TRIGGER_SNAPSHOT(3, () -> TriggerSnapshot.serializer), SCHEMA_CHANGE(4, () -> AlterSchema.serializer), REGISTER(5, () -> Register.serializer), UNREGISTER(6, () -> Unregister.serializer), diff --git a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java index ab4ce55f11..f8df9fbc71 100644 --- a/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java +++ b/src/java/org/apache/cassandra/tcm/compatibility/GossipHelper.java @@ -56,7 +56,6 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MultiStepOperation; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.extensions.ExtensionKey; import org.apache.cassandra.tcm.extensions.ExtensionValue; import org.apache.cassandra.tcm.membership.Directory; @@ -277,8 +276,6 @@ public class GossipHelper public static ClusterMetadata emptyWithSchemaFromSystemTables(Set allKnownDatacenters) { return new ClusterMetadata(Epoch.UPGRADE_STARTUP, - Period.EMPTY, - true, DatabaseDescriptor.getPartitioner(), DistributedSchema.fromSystemTables(SchemaKeyspace.fetchNonSystemKeyspaces(), allKnownDatacenters), Directory.EMPTY, @@ -349,8 +346,6 @@ public class GossipHelper } ClusterMetadata forPlacementCalculation = new ClusterMetadata(Epoch.UPGRADE_GOSSIP, - Period.EMPTY, - true, partitioner, schema, directory, @@ -360,8 +355,6 @@ public class GossipHelper InProgressSequences.EMPTY, extensions); return new ClusterMetadata(Epoch.UPGRADE_GOSSIP, - Period.EMPTY, - true, partitioner, schema, directory, diff --git a/src/java/org/apache/cassandra/tcm/listeners/MetadataSnapshotListener.java b/src/java/org/apache/cassandra/tcm/listeners/MetadataSnapshotListener.java index 0157a12295..72e3fd5f83 100644 --- a/src/java/org/apache/cassandra/tcm/listeners/MetadataSnapshotListener.java +++ b/src/java/org/apache/cassandra/tcm/listeners/MetadataSnapshotListener.java @@ -33,16 +33,15 @@ public class MetadataSnapshotListener implements LogListener public void notify(Entry entry, Transformation.Result result) { ClusterMetadata next = result.success().metadata; - if (entry.transform.kind() == Transformation.Kind.SEAL_PERIOD) + if (entry.transform.kind() == Transformation.Kind.TRIGGER_SNAPSHOT) { - assert next.lastInPeriod; try { ClusterMetadataService.instance().snapshotManager().storeSnapshot(next); } catch (Throwable e) { - logger.warn("Unable to serialize metadata snapshot triggered by SealPeriod transformation", e); + logger.warn("Unable to serialize metadata snapshot triggered by TriggerSnapshot transformation", e); } } } diff --git a/src/java/org/apache/cassandra/tcm/log/LocalLog.java b/src/java/org/apache/cassandra/tcm/log/LocalLog.java index cf7e383877..7523fb2043 100644 --- a/src/java/org/apache/cassandra/tcm/log/LocalLog.java +++ b/src/java/org/apache/cassandra/tcm/log/LocalLog.java @@ -338,7 +338,7 @@ public abstract class LocalLog implements Closeable public LogState getCommittedEntries(Epoch since) { - return storage.getLogState(committed.get().period, since); + return storage.getLogState(since); } public ClusterMetadata waitForHighestConsecutive() @@ -481,7 +481,7 @@ public abstract class LocalLog implements Closeable next.epoch, pendingEntry.transform, prev.epoch); if (replayComplete.get()) - storage.append(transformed.success().metadata.period, pendingEntry.maybeUnwrapExecuted()); + storage.append(pendingEntry.maybeUnwrapExecuted()); notifyPreCommit(prev, next, isSnapshot); @@ -887,7 +887,7 @@ public abstract class LocalLog implements Closeable List list = new ArrayList<>(ClusterMetadata.current().fullCMSMembers()); list.sort(comparing(i -> i.addressBytes[i.addressBytes.length - 1])); if (list.get(0).equals(FBUtilities.getBroadcastAddressAndPort())) - ScheduledExecutors.nonPeriodicTasks.submit(() -> ClusterMetadataService.instance().sealPeriod()); + ScheduledExecutors.nonPeriodicTasks.submit(() -> ClusterMetadataService.instance().triggerSnapshot()); } }; } diff --git a/src/java/org/apache/cassandra/tcm/log/LogReader.java b/src/java/org/apache/cassandra/tcm/log/LogReader.java index b7c73dd315..b63ec4ecd6 100644 --- a/src/java/org/apache/cassandra/tcm/log/LogReader.java +++ b/src/java/org/apache/cassandra/tcm/log/LogReader.java @@ -19,93 +19,90 @@ package org.apache.cassandra.tcm.log; import java.io.IOException; -import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Ordering; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; public interface LogReader { /** - * Gets all entries where epoch >= since in the given period - could be empty if since is in a later epoch + * Gets all entries where epoch >= since - could be empty if since is a later epoch than the current highest seen */ - EntryHolder getEntries(long period, Epoch since) throws IOException; + EntryHolder getEntries(Epoch since) throws IOException; MetadataSnapshots snapshots(); /** - * Idea is to fill LogState "backwards" - we start querying the partition at currentPeriod, and if that doesn't - * include `since` we read currentPeriod - 1. Assuming we have something like - - * epoch | period | transformation - * 10 | 2 | SEAL_PERIOD - * 11 | 3 | SOMETHING - * 12 | 3 | SOMETHING - * 13 | 3 | SEAL_PERIOD - * 14 | 4 | SOMETHING - * 15 | 4 | SOMETHING - * 16 | 4 | SEAL_PERIOD - * 17 | 5 | SOMETHING + * Assuming we have something like: * - * and `since` is 14, we want to return epoch 15, 16, 17 - not the snapshot at 16 + entry at 17 since it is assumed that the full - * snapshot is much larger than the transformations. - * But, if `since` is 11, we want to return the most recent snapshot (at epoch 16) + entry at 17. + * epoch | transformation + * 10 | TRIGGER_SNAPSHOT + * 11 | SOMETHING + * 12 | SOMETHING + * 13 | TRIGGER_SNAPSHOT + * 14 | SOMETHING + * 15 | SOMETHING + * 16 | TRIGGER_SNAPSHOT + * 17 | SOMETHING * - * If a snapshot is missing we keep reading backwards until we find one, or we end up at period 0 and in that - * case we return all transformations in the log. + * and `startEpoch` is 14, we want to return epoch 15, 16, 17 - not the snapshot at 16 + entry at 17 since it is + * assumed that the full snapshot is much larger than the transformations. + * But, if `startEpoch` is 11, we want to return the most recent snapshot (at epoch 16) + entry at 17. + * + * If a snapshot is missing we keep reading backwards until we find one, or we end up at `startEpoch` and in that + * case we return all subsequent entries in the log. */ - default LogState getLogState(long currentPeriod, Epoch since) + default LogState getLogState(Epoch startEpoch) { try { - EntryHolder current = getEntries(currentPeriod, since); - if (current.done) - return new LogState(null, ImmutableList.sortedCopyOf(current.entries)); - List allEntries = new ArrayList<>(current.entries); - int i = 0; - while (true) - { - i++; - EntryHolder previous = getEntries(currentPeriod - i, since); - allEntries.addAll(previous.entries); - if (isContinuous(since, allEntries) && (previous.done || currentPeriod - i == Period.FIRST)) - { - return new LogState(null, ImmutableList.sortedCopyOf(allEntries)); - } - else - { - if (i == 1) - { - // we end up here if `since` is not in currentPeriod or currentPeriod - 1 - // so we should return a snapshot - we prefer returning the most recent snapshot + entries in `current` - // but if that doesn't exist we check previous. - if (current.min == null && previous.min == null) // we found no entries >= since in the tables -> since > current - return LogState.EMPTY; - ClusterMetadata snapshot = snapshots().getSnapshot(Epoch.create(current.min.getEpoch() - 1)); - if (snapshot != null) - return new LogState(snapshot, ImmutableList.sortedCopyOf(current.entries)); - } - else - { - // There were no entries in this period with epoch >= since (i.e. we have iterated back to before - // since) but we've also been unable to find a suitable snapshot. The best thing we can do is to - // include all the entries we have collected so far, even if they are non-contiguous. The caller - // will buffer them in its LocalLog and attempt to fill any gaps by requesting a LogState from - // other peers. - if (previous.min == null) - return new LogState(null, ImmutableList.sortedCopyOf(allEntries)); - } + EntryHolder entries = null; + // List of snapshots with an epoch > startEpoch + List snapshotEpochs = snapshots().listSnapshotsSince(startEpoch); - ClusterMetadata snapshot = snapshots().getSnapshot(Epoch.create(previous.min.getEpoch() - 1)); - if (snapshot != null) - return new LogState(snapshot, ImmutableList.sortedCopyOf(allEntries)); + // If there is at most 1 snapshot with an epoch > startEpoch, we prefer to skip that snapshot and just build a + // list of consecutive entries + if (snapshotEpochs.size() <= 1) + { + entries = getEntries(startEpoch); + if (entries.isContinuous()) + return new LogState(null, entries.immutable()); + // Gaps in a persisted log are never expected, but we have not been able to construct a continuous + // sequence of all entries between startEpoch and the current epoch, so fall back to the general case. + } + + assert Ordering.from(Comparator.reverseOrder()).isOrdered(snapshotEpochs) : "Epochs from snapshots().listSnapshotsSince(...) should be ordered by most recent epoch first"; + // From the list of snapshots which come after startEpoch, read the latest one available and create a + // LogState with that as the base plus any subsequent entries. If we have already read a list of entries, + // this must necessarily be a superset of entries startEpoch the available snapshot. + // This may include a non-continuous list of entries, which the caller will buffer in its LocalLog and + // attempt to fill any gaps by requesting additional LogStates from other peers. + for (Epoch snapshotAt : snapshotEpochs) + { + ClusterMetadata snapshot = snapshots().getSnapshot(snapshotAt); + if (null != snapshot) + { + ImmutableList sublist = entries != null + ? entries.immutable(snapshotAt) + : getEntries(snapshotAt).immutable(); + return new LogState(snapshot, sublist); } } + // We have been unable to find any suitable snapshot, so the best thing we can do is to include all the + // entries we have after startEpoch, even if that's a non-continuous list. If we have already read a list of + // entries subsequent to startEpoch, we can reuse that. + if (entries == null) + entries = getEntries(startEpoch); + return new LogState(null, entries.immutable()); + } catch (IOException e) { @@ -113,47 +110,47 @@ public interface LogReader } } - private static boolean isContinuous(Epoch since, List entries) - { - entries.sort(Entry::compareTo); - Epoch prev = since; - for (Entry e : entries) - { - if (!e.epoch.isDirectlyAfter(prev)) - return false; - prev = e.epoch; - } - return true; - } - class EntryHolder { - List entries; - Epoch min = null; + SortedSet entries; Epoch since; - boolean done = false; public EntryHolder(Epoch since) { - this.entries = new ArrayList<>(); + this.entries = new TreeSet<>(); this.since = since; } public void add(Entry entry) { if (entry.epoch.isAfter(since)) - { - if (entry.epoch.isDirectlyAfter(since)) - done = true; - if (min == null || min.isAfter(entry.epoch)) - min = entry.epoch; entries.add(entry); - } - else if (entry.epoch.is(since)) + } + + private boolean isContinuous() + { + Epoch prev = since; + for (Entry e : entries) { - // if `since` is the most recent epoch committed, this avoids an extra query - done = true; + if (!e.epoch.isDirectlyAfter(prev)) + return false; + prev = e.epoch; } + return true; + } + + private ImmutableList immutable() + { + return ImmutableList.copyOf(entries); + } + + private ImmutableList immutable(Epoch startExclusive) + { + ImmutableList.Builder list = ImmutableList.builder(); + for (Entry e : entries) + if (e.epoch.isAfter(startExclusive)) + list.add(e); + return list.build(); } } diff --git a/src/java/org/apache/cassandra/tcm/log/LogStorage.java b/src/java/org/apache/cassandra/tcm/log/LogStorage.java index e806efa08a..fbf60840c2 100644 --- a/src/java/org/apache/cassandra/tcm/log/LogStorage.java +++ b/src/java/org/apache/cassandra/tcm/log/LogStorage.java @@ -24,7 +24,7 @@ import org.apache.cassandra.tcm.MetadataSnapshots; public interface LogStorage extends LogReader { - void append(long period, Entry entry); + void append(Entry entry); LogState getPersistedLogState(); LogState getLogStateBetween(ClusterMetadata base, Epoch end); @@ -42,10 +42,10 @@ public interface LogStorage extends LogReader class NoOpLogStorage implements LogStorage { @Override - public void append(long period, Entry entry) {} + public void append(Entry entry) {} @Override - public LogState getLogState(long startPeriod, Epoch since) + public LogState getLogState(Epoch startEpoch) { return LogState.EMPTY; } @@ -57,7 +57,7 @@ public interface LogStorage extends LogReader } @Override - public EntryHolder getEntries(long period, Epoch since) + public EntryHolder getEntries(Epoch since) { return null; } diff --git a/src/java/org/apache/cassandra/tcm/log/SystemKeyspaceStorage.java b/src/java/org/apache/cassandra/tcm/log/SystemKeyspaceStorage.java index 631ead96db..ac419b28ff 100644 --- a/src/java/org/apache/cassandra/tcm/log/SystemKeyspaceStorage.java +++ b/src/java/org/apache/cassandra/tcm/log/SystemKeyspaceStorage.java @@ -35,7 +35,6 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.Transformation; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; @@ -68,16 +67,15 @@ public class SystemKeyspaceStorage implements LogStorage } // This method is always called from a single thread, so doesn't have to be synchonised. - public void append(long period, Entry entry) + public void append(Entry entry) { try { // TODO get lowest supported metadata version from ClusterMetadata ByteBuffer serializedTransformation = entry.transform.kind().toVersionedBytes(entry.transform); - String query = String.format("INSERT INTO %s.%s (period, epoch, current_epoch, entry_id, transformation, kind) VALUES (?,?,?,?,?,?)", + String query = String.format("INSERT INTO %s.%s (epoch, entry_id, transformation, kind) VALUES (?,?,?,?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, NAME); - executeInternal(query, period, entry.epoch.getEpoch(), entry.epoch.getEpoch(), - entry.id.entryId, serializedTransformation, entry.transform.kind().toString()); + executeInternal(query, entry.epoch.getEpoch(), entry.id.entryId, serializedTransformation, entry.transform.kind().toString()); // todo; should probably not flush every time, but it simplifies tests Keyspace.open(SchemaConstants.SYSTEM_KEYSPACE_NAME).getColumnFamilyStore(NAME).forceBlockingFlush(ColumnFamilyStore.FlushReason.INTERNALLY_FORCED); } @@ -123,19 +121,36 @@ public class SystemKeyspaceStorage implements LogStorage } @Override - public EntryHolder getEntries(long period, Epoch since) throws IOException + public EntryHolder getEntries(Epoch since) throws IOException + { + // during gossip upgrade we have epoch = Long.MIN_VALUE + 1 (and the reverse partitioner doesn't support negative keys) + since = since.isBefore(Epoch.EMPTY) ? Epoch.EMPTY : since; + UntypedResultSet resultSet = executeInternal(String.format("SELECT epoch, kind, transformation, entry_id FROM %s.%s WHERE token(epoch) <= token(?)", SchemaConstants.SYSTEM_KEYSPACE_NAME, NAME), + since.getEpoch()); + return toEntryHolder(since, resultSet); + } + + public EntryHolder getEntries(Epoch since, Epoch until) throws IOException + { + // during gossip upgrade we have epoch = Long.MIN_VALUE + 1 (and the reverse partitioner doesn't support negative keys) + since = since.isBefore(Epoch.EMPTY) ? Epoch.EMPTY : since; + UntypedResultSet resultSet = executeInternal(String.format("SELECT epoch, kind, transformation, entry_id " + + "FROM %s.%s " + + "WHERE token(epoch) <= token(?) AND token(epoch) >= token(?)", + SchemaConstants.SYSTEM_KEYSPACE_NAME, NAME), + since.getEpoch(), until.getEpoch()); + return toEntryHolder(since, resultSet); + } + + private static EntryHolder toEntryHolder(Epoch since, UntypedResultSet resultSet) throws IOException { - UntypedResultSet resultSet = executeInternal(String.format("SELECT epoch, kind, transformation, entry_id FROM %s.%s WHERE period = ? and epoch >= ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, NAME), - period, since.getEpoch()); EntryHolder holder = new EntryHolder(since); for (UntypedResultSet.Row row : resultSet) { long entryId = row.getLong("entry_id"); Epoch epoch = Epoch.create(row.getLong("epoch")); - ByteBuffer transformationBlob = row.getBlob("transformation"); Transformation.Kind kind = Transformation.Kind.valueOf(row.getString("kind")); - Transformation transform = kind.fromVersionedBytes(transformationBlob); - kind.fromVersionedBytes(transformationBlob); + Transformation transform = kind.fromVersionedBytes(row.getBlob("transformation")); holder.add(new Entry(new Entry.Id(entryId), epoch, transform)); } return holder; @@ -146,36 +161,18 @@ public class SystemKeyspaceStorage implements LogStorage { try { - ImmutableList.Builder allEntries = ImmutableList.builder(); - long period = (base == null ? Period.EMPTY : base.period) + 1; Epoch epoch = base == null ? Epoch.EMPTY : base.epoch; - while (true) - { - EntryHolder entryHolder = getEntries(period, epoch); - - if (entryHolder.entries.isEmpty()) - break; - boolean done = false; - for (Entry entry : entryHolder.entries) - { - if (entry.epoch.isEqualOrBefore(end)) - allEntries.add(entry); - else - done = true; - } - if (done) - break; - period++; - } - ImmutableList entries = allEntries.build(); + EntryHolder entryHolder = getEntries(epoch, end); + ImmutableList.Builder entries = ImmutableList.builder(); Epoch prevEpoch = epoch; - for (Entry e : entries) + for (Entry e : entryHolder.entries) { if (!prevEpoch.nextEpoch().is(e.epoch)) throw new IllegalStateException("Can't get replication between " + epoch + " and " + end + " - incomplete local log?"); prevEpoch = e.epoch; + entries.add(e); } - return new LogState(base, entries); + return new LogState(base, entries.build()); } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java b/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java index 367945f058..521084ffe5 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java +++ b/src/java/org/apache/cassandra/tcm/ownership/DataPlacement.java @@ -25,19 +25,27 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; public class DataPlacement { - public static final Serializer serializer = new Serializer(); + private static final Serializer globalSerializer = new Serializer(IPartitioner.global()); + private static final Serializer metaKeyspaceSerializer = new Serializer(MetaStrategy.partitioner); + public static Serializer serializerFor(ReplicationParams replication) + { + return replication.isMeta() ? metaKeyspaceSerializer : globalSerializer; + } private static final DataPlacement EMPTY = new DataPlacement(PlacementForRange.EMPTY, PlacementForRange.EMPTY); @@ -184,23 +192,29 @@ public class DataPlacement public static class Serializer implements MetadataSerializer { + private final IPartitioner partitioner; + private Serializer(IPartitioner partitioner) + { + this.partitioner = partitioner; + } + public void serialize(DataPlacement t, DataOutputPlus out, Version version) throws IOException { - PlacementForRange.serializer.serialize(t.reads, out, version); - PlacementForRange.serializer.serialize(t.writes, out, version); + PlacementForRange.serializer.serialize(t.reads, out, partitioner, version); + PlacementForRange.serializer.serialize(t.writes, out, partitioner, version); } public DataPlacement deserialize(DataInputPlus in, Version version) throws IOException { - PlacementForRange reads = PlacementForRange.serializer.deserialize(in, version); - PlacementForRange writes = PlacementForRange.serializer.deserialize(in, version); + PlacementForRange reads = PlacementForRange.serializer.deserialize(in, partitioner, version); + PlacementForRange writes = PlacementForRange.serializer.deserialize(in, partitioner, version); return new DataPlacement(reads, writes); } public long serializedSize(DataPlacement t, Version version) { - return PlacementForRange.serializer.serializedSize(t.reads, version) + - PlacementForRange.serializer.serializedSize(t.writes, version); + return PlacementForRange.serializer.serializedSize(t.reads, partitioner, version) + + PlacementForRange.serializer.serializedSize(t.writes, partitioner, version); } } } diff --git a/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java b/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java index a59d923591..73c83fb5fd 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java +++ b/src/java/org/apache/cassandra/tcm/ownership/DataPlacements.java @@ -19,7 +19,6 @@ package org.apache.cassandra.tcm.ownership; import java.io.IOException; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Map; @@ -32,6 +31,7 @@ import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.EndpointsForRange; @@ -44,7 +44,6 @@ import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.db.TypeSizes.sizeof; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; public class DataPlacements extends ReplicationMap implements MetadataValue { @@ -96,9 +95,14 @@ public class DataPlacements extends ReplicationMap implements Met // it's unlikely to happen, but perfectly safe to create multiple times, so no need to lock or statically init if (null == LOCAL_PLACEMENT) { - PlacementForRange placement = new PlacementForRange(Collections.singletonMap(entireRange, - VersionedEndpoints.forRange(Epoch.EMPTY, - EndpointsForRange.of(EntireRange.replica(FBUtilities.getBroadcastAddressAndPort()))))); + // todo remove this entirely + EndpointsForRange endpoints = EndpointsForRange.of(new Replica(FBUtilities.getBroadcastAddressAndPort(), + DatabaseDescriptor.getPartitioner().getMinimumToken(), + DatabaseDescriptor.getPartitioner().getMinimumToken(), + true)); + PlacementForRange placement = PlacementForRange.builder(1) + .withReplicaGroup(VersionedEndpoints.forRange(Epoch.EMPTY, endpoints)) + .build(); LOCAL_PLACEMENT = new DataPlacement(placement, placement); } return LOCAL_PLACEMENT; @@ -220,7 +224,7 @@ public class DataPlacements extends ReplicationMap implements Met for (Map.Entry entry : map.entrySet()) { ReplicationParams.serializer.serialize(entry.getKey(), out, version); - DataPlacement.serializer.serialize(entry.getValue(), out, version); + DataPlacement.serializerFor(entry.getKey()).serialize(entry.getValue(), out, version); } Epoch.serializer.serialize(t.lastModified, out, version); } @@ -232,8 +236,7 @@ public class DataPlacements extends ReplicationMap implements Met for (int i = 0; i < size; i++) { ReplicationParams params = ReplicationParams.serializer.deserialize(in, version); - DataPlacement dp = DataPlacement.serializer.deserialize(in, version); - map.put(params, dp); + map.put(params, DataPlacement.serializerFor(params).deserialize(in, version)); } Epoch lastModified = Epoch.serializer.deserialize(in, version); return new DataPlacements(lastModified, map); @@ -241,11 +244,11 @@ public class DataPlacements extends ReplicationMap implements Met public long serializedSize(DataPlacements t, Version version) { - int size = sizeof(t.size()); + long size = sizeof(t.size()); for (Map.Entry entry : t.asMap().entrySet()) { size += ReplicationParams.serializer.serializedSize(entry.getKey(), version); - size += DataPlacement.serializer.serializedSize(entry.getValue(), version); + size += DataPlacement.serializerFor(entry.getKey()).serializedSize(entry.getValue(), version); } size += Epoch.serializer.serializedSize(t.lastModified, version); return size; diff --git a/src/java/org/apache/cassandra/tcm/ownership/EntireRange.java b/src/java/org/apache/cassandra/tcm/ownership/EntireRange.java deleted file mode 100644 index 26bd3f0744..0000000000 --- a/src/java/org/apache/cassandra/tcm/ownership/EntireRange.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.tcm.ownership; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.locator.EndpointsForRange; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.schema.ReplicationParams; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.sequences.LockedRanges; -import org.apache.cassandra.utils.FBUtilities; - -public class EntireRange -{ - public static final Range entireRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), - DatabaseDescriptor.getPartitioner().getMinimumToken()); - public static LockedRanges.AffectedRanges affectedRanges(ClusterMetadata metadata) - { - return LockedRanges.AffectedRanges.singleton(ReplicationParams.meta(metadata), entireRange); - } - - public static Replica replica(InetAddressAndPort addr) - { - return new Replica(addr, entireRange, true); - } - - public static final EndpointsForRange localReplicas = EndpointsForRange.of(replica(FBUtilities.getBroadcastAddressAndPort())); - public static final DataPlacement placement = new DataPlacement(PlacementForRange.builder().withReplicaGroup(VersionedEndpoints.forRange(Epoch.FIRST, localReplicas)).build(), - PlacementForRange.builder().withReplicaGroup(VersionedEndpoints.forRange(Epoch.FIRST, localReplicas)).build()); -} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tcm/ownership/MovementMap.java b/src/java/org/apache/cassandra/tcm/ownership/MovementMap.java index 7337f83180..1f801e5bd0 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/MovementMap.java +++ b/src/java/org/apache/cassandra/tcm/ownership/MovementMap.java @@ -26,11 +26,13 @@ import java.util.Map; import com.google.common.collect.Maps; import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.EndpointsByReplica; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; @@ -144,7 +146,8 @@ public class MovementMap extends ReplicationMap for (int i = 0; i < size; i++) { ReplicationParams params = ReplicationParams.messageSerializer.deserialize(in, version); - EndpointsByReplica endpointsByReplica = EndpointsByReplica.serializer.deserialize(in, version); + IPartitioner partitioner = params.isMeta() ? MetaStrategy.partitioner : IPartitioner.global(); + EndpointsByReplica endpointsByReplica = EndpointsByReplica.serializer.deserialize(in, partitioner, version); builder.put(params, endpointsByReplica); } return builder.build(); @@ -153,7 +156,7 @@ public class MovementMap extends ReplicationMap @Override public long serializedSize(MovementMap t, int version) { - long size = TypeSizes.sizeofVInt(t.size()); + long size = TypeSizes.sizeofUnsignedVInt(t.size()); for (Map.Entry entry : t.asMap().entrySet()) { size += ReplicationParams.messageSerializer.serializedSize(entry.getKey(), version); diff --git a/src/java/org/apache/cassandra/tcm/ownership/PlacementForRange.java b/src/java/org/apache/cassandra/tcm/ownership/PlacementForRange.java index 4aaa1089f5..c72bd2a0d5 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/PlacementForRange.java +++ b/src/java/org/apache/cassandra/tcm/ownership/PlacementForRange.java @@ -39,7 +39,7 @@ import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.PartitionerAwareMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.db.TypeSizes.sizeof; @@ -330,9 +330,9 @@ public class PlacementForRange } } - public static class Serializer implements MetadataSerializer + public static class Serializer implements PartitionerAwareMetadataSerializer { - public void serialize(PlacementForRange t, DataOutputPlus out, Version version) throws IOException + public void serialize(PlacementForRange t, DataOutputPlus out, IPartitioner partitioner, Version version) throws IOException { out.writeInt(t.replicaGroups.size()); @@ -342,25 +342,24 @@ public class PlacementForRange VersionedEndpoints.ForRange efr = entry.getValue(); if (version.isAtLeast(Version.V2)) Epoch.serializer.serialize(efr.lastModified(), out, version); - Token.metadataSerializer.serialize(range.left, out, version); - Token.metadataSerializer.serialize(range.right, out, version); + Token.metadataSerializer.serialize(range.left, out, partitioner, version); + Token.metadataSerializer.serialize(range.right, out, partitioner, version); out.writeInt(efr.size()); for (int i = 0; i < efr.size(); i++) { Replica r = efr.get().get(i); - Token.metadataSerializer.serialize(r.range().left, out, version); - Token.metadataSerializer.serialize(r.range().right, out, version); + Token.metadataSerializer.serialize(r.range().left, out, partitioner, version); + Token.metadataSerializer.serialize(r.range().right, out, partitioner, version); InetAddressAndPort.MetadataSerializer.serializer.serialize(r.endpoint(), out, version); out.writeBoolean(r.isFull()); } } } - public PlacementForRange deserialize(DataInputPlus in, Version version) throws IOException + public PlacementForRange deserialize(DataInputPlus in, IPartitioner partitioner, Version version) throws IOException { int groupCount = in.readInt(); Map, VersionedEndpoints.ForRange> result = Maps.newHashMapWithExpectedSize(groupCount); - IPartitioner partitioner = ClusterMetadata.current().partitioner; for (int i = 0; i < groupCount; i++) { Epoch lastModified; @@ -394,9 +393,9 @@ public class PlacementForRange return new PlacementForRange(result); } - public long serializedSize(PlacementForRange t, Version version) + public long serializedSize(PlacementForRange t, IPartitioner partitioner, Version version) { - int size = sizeof(t.replicaGroups.size()); + long size = sizeof(t.replicaGroups.size()); for (Map.Entry, VersionedEndpoints.ForRange> entry : t.replicaGroups.entrySet()) { Range range = entry.getKey(); @@ -404,14 +403,14 @@ public class PlacementForRange if (version.isAtLeast(Version.V2)) size += Epoch.serializer.serializedSize(efr.lastModified(), version); - size += Token.metadataSerializer.serializedSize(range.left, version); - size += Token.metadataSerializer.serializedSize(range.right, version); + size += Token.metadataSerializer.serializedSize(range.left, partitioner, version); + size += Token.metadataSerializer.serializedSize(range.right, partitioner, version); size += sizeof(efr.size()); for (int i = 0; i < efr.size(); i++) { Replica r = efr.get().get(i); - size += Token.metadataSerializer.serializedSize(r.range().left, version); - size += Token.metadataSerializer.serializedSize(r.range().right, version); + size += Token.metadataSerializer.serializedSize(r.range().left, partitioner, version); + size += Token.metadataSerializer.serializedSize(r.range().right, partitioner, version); size += InetAddressAndPort.MetadataSerializer.serializer.serializedSize(r.endpoint(), version); size += sizeof(r.isFull()); } diff --git a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java index 723cd3d709..3d6499f61b 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java @@ -26,6 +26,7 @@ import org.apache.cassandra.exceptions.ExceptionCode; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; @@ -37,11 +38,10 @@ import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.ownership.DataPlacement; import org.apache.cassandra.tcm.ownership.DataPlacements; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; public class CancelCMSReconfiguration implements Transformation { @@ -104,7 +104,7 @@ public class CancelCMSReconfiguration implements Transformation return Transformation.success(transformer.with(prev.inProgressSequences.without(ReconfigureCMS.SequenceKey.instance)) .with(prev.lockedRanges.unlock(reconfigureCMS.next.lockKey)), - EntireRange.affectedRanges(prev)); + MetaStrategy.affectedRanges(prev)); } private ReplicationParams getAccurateReplication(Directory directory, DataPlacement placement) diff --git a/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java b/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java index e67f0b8f85..1a96833ffb 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java +++ b/src/java/org/apache/cassandra/tcm/sequences/ReconfigureCMS.java @@ -32,12 +32,16 @@ import com.google.common.collect.ImmutableSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.EndpointsByReplica; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.metrics.TCMMetrics; @@ -60,7 +64,6 @@ import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Retry; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.ownership.MovementMap; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.MetadataSerializer; @@ -71,7 +74,7 @@ import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.streaming.StreamOperation.RESTORE_REPLICA_COUNT; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; public class ReconfigureCMS extends MultiStepOperation { @@ -195,7 +198,7 @@ public class ReconfigureCMS extends MultiStepOperation entirePaxosRange = new Range<>(DatabaseDescriptor.getPartitioner().getMinimumToken(), + DatabaseDescriptor.getPartitioner().getMinimumToken()); List>> remaining = ActiveRepairService.instance().repairPaxosForTopologyChangeAsync(SchemaConstants.METADATA_KEYSPACE_NAME, - Collections.singletonList(entireRange), + Collections.singletonList(entirePaxosRange), "bootstrap"); while (!retry.reachedMax()) diff --git a/src/java/org/apache/cassandra/tcm/serialization/PartitionerAwareMetadataSerializer.java b/src/java/org/apache/cassandra/tcm/serialization/PartitionerAwareMetadataSerializer.java index e414d1b7b6..8f9c8c48d6 100644 --- a/src/java/org/apache/cassandra/tcm/serialization/PartitionerAwareMetadataSerializer.java +++ b/src/java/org/apache/cassandra/tcm/serialization/PartitionerAwareMetadataSerializer.java @@ -31,9 +31,11 @@ public interface PartitionerAwareMetadataSerializer * * @param t type that needs to be serialized * @param out DataOutput into which serialization needs to happen. + * @param partitioner The partitioner to use + * @param version protocol version * @throws IOException if serialization fails */ - void serialize(T t, DataOutputPlus out, Version version) throws IOException; + void serialize(T t, DataOutputPlus out, IPartitioner partitioner, Version version) throws IOException; /** * Deserialize into the specified DataInputStream instance. @@ -49,8 +51,9 @@ public interface PartitionerAwareMetadataSerializer /** * Calculate serialized size of object without actually serializing. * @param t object to calculate serialized size + * @param partitioner The partitioner to use * @param version protocol version * @return serialized size of object t */ - long serializedSize(T t, Version version); + long serializedSize(T t, IPartitioner partitioner, Version version); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/Startup.java b/src/java/org/apache/cassandra/tcm/transformations/Startup.java index b4d4007e43..7a2ecb34c5 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/Startup.java +++ b/src/java/org/apache/cassandra/tcm/transformations/Startup.java @@ -42,7 +42,7 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; public class Startup implements Transformation { diff --git a/src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java b/src/java/org/apache/cassandra/tcm/transformations/TriggerSnapshot.java similarity index 55% rename from src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java rename to src/java/org/apache/cassandra/tcm/transformations/TriggerSnapshot.java index 5e8222ebf4..6c53a6618b 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/SealPeriod.java +++ b/src/java/org/apache/cassandra/tcm/transformations/TriggerSnapshot.java @@ -28,35 +28,24 @@ import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; -import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; - /** - * Period is an entity that is completely transparent to the users of CMS, and is a consequence of a fact that - * LWTs only work on a single partition. It would be unwise to hold all epochs in a single partition, as it - * will eventually get extremely large. At the same time, we can not use Epoch as a primary key (even though - * IF NOT EXISTS queries would technically work for append puposes), since it would make log scans much more - * expensive. - * - * Another reason for having periods is that replaying an entire log for a freshly starting log in an old - * cluster can be very expensive, so in such cases the node can be caught up using a snapshot serving as a base - * state and a small number of entries instead. - * - * Transformation that seals the period and requests local state to take a snapshot. Snapshot taking is an - * asynchonous action, and we generally do not rely on the fact snapshot is, in fact going to be - * there all the time. Snapshots are used as a performance optimization. + * Snapshots are used during startup or catchup between peers to avoid having to replay or transmit the entire log. + * This transformation simply inserts a marker entry into the metadata log. Enacting the epoch on a peer triggers the + * snapshot action on that peer. By default, taking a snapshot taking is an asynchonous action, and we generally do not + * rely on the fact snapshot is, in fact going to be available immediately (or even durably). */ -public class SealPeriod implements Transformation +public class TriggerSnapshot implements Transformation { public static final Serializer serializer = new Serializer(); - public static SealPeriod instance = new SealPeriod(); + public static TriggerSnapshot instance = new TriggerSnapshot(); - private SealPeriod(){} + private TriggerSnapshot(){} @Override public Kind kind() { - return Kind.SEAL_PERIOD; + return Kind.TRIGGER_SNAPSHOT; } @Override @@ -68,20 +57,17 @@ public class SealPeriod implements Transformation @Override public Result execute(ClusterMetadata prev) { - if (prev.lastInPeriod) - return new Rejected(INVALID, "Have just sealed this period"); - - return Transformation.success(prev.transformer(true), LockedRanges.AffectedRanges.EMPTY); + return Transformation.success(prev.transformer(), LockedRanges.AffectedRanges.EMPTY); } - static class Serializer implements AsymmetricMetadataSerializer + static class Serializer implements AsymmetricMetadataSerializer { public void serialize(Transformation t, DataOutputPlus out, Version version) throws IOException { assert t == instance; } - public SealPeriod deserialize(DataInputPlus in, Version version) throws IOException + public TriggerSnapshot deserialize(DataInputPlus in, Version version) throws IOException { return instance; } @@ -95,6 +81,6 @@ public class SealPeriod implements Transformation @Override public String toString() { - return "SealPeriod{}"; + return "TriggerSnapshot{}"; } } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java index 13f009f8e3..614dd8a4ed 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/AdvanceCMSReconfiguration.java @@ -28,6 +28,7 @@ import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.RangesByEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; @@ -37,7 +38,6 @@ import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.sequences.LockedRanges; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; @@ -45,7 +45,7 @@ import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; import static org.apache.cassandra.tcm.MultiStepOperation.Kind.RECONFIGURE_CMS; /** @@ -129,7 +129,7 @@ public class AdvanceCMSReconfiguration implements Transformation return Transformation.success(prev.transformer() .with(prev.inProgressSequences.without(ReconfigureCMS.SequenceKey.instance)) .with(prev.lockedRanges.unlock(lockKey)), - EntireRange.affectedRanges(prev)); + MetaStrategy.affectedRanges(prev)); } } else @@ -190,7 +190,7 @@ public class AdvanceCMSReconfiguration implements Transformation ReconfigureCMS advanced = sequence.advance(next); // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } /** @@ -223,7 +223,7 @@ public class AdvanceCMSReconfiguration implements Transformation ReconfigureCMS advanced = sequence.advance(next); // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } /** @@ -263,7 +263,7 @@ public class AdvanceCMSReconfiguration implements Transformation ReconfigureCMS advanced = sequence.advance(next); // Finally, replace the existing reconfiguration sequence with this updated one. transformer.with(prev.inProgressSequences.with(ReconfigureCMS.SequenceKey.instance, (ReconfigureCMS old) -> advanced)); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } private AdvanceCMSReconfiguration next(Epoch latestModification, diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/BaseMembershipTransformation.java b/src/java/org/apache/cassandra/tcm/transformations/cms/BaseMembershipTransformation.java index ae7a3042d4..7ed5d62598 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/BaseMembershipTransformation.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/BaseMembershipTransformation.java @@ -24,9 +24,9 @@ import java.util.Objects; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.tcm.Transformation; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -38,7 +38,7 @@ public abstract class BaseMembershipTransformation implements Transformation protected BaseMembershipTransformation(InetAddressAndPort endpoint) { this.endpoint = endpoint; - this.replica = EntireRange.replica(endpoint); + this.replica = MetaStrategy.replica(endpoint); } // TODO: to node id diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java index eacf236a92..a1bed8ed2a 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/FinishAddToCMS.java @@ -19,6 +19,7 @@ package org.apache.cassandra.tcm.transformations.cms; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; @@ -26,13 +27,12 @@ import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.sequences.AddToCMS; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, StartAddToCMS & RemoveFromCMS, contain a high degree of duplication with their intended @@ -90,7 +90,7 @@ public class FinishAddToCMS extends BaseMembershipTransformation .withReadReplica(prev.nextEpoch(), replica); transformer = transformer.with(prev.placements.unbuild().with(metaParams, builder.build()).build()) .with(prev.inProgressSequences.without(targetNode)); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } public String toString() diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java index 07dbcc116a..8103e14f92 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/Initialize.java @@ -23,12 +23,12 @@ import java.io.IOException; import org.apache.cassandra.auth.AuthKeyspace; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.Keyspaces; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Transformation; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.tcm.transformations.ForceSnapshot; @@ -71,7 +71,7 @@ public class Initialize extends ForceSnapshot ClusterMetadata next = baseState; DistributedSchema initialSchema = new DistributedSchema(setUpDistributedSystemKeyspaces(next)); ClusterMetadata.Transformer transformer = next.transformer().with(initialSchema); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } public Keyspaces setUpDistributedSystemKeyspaces(ClusterMetadata next) diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java index ef6f8adb3a..dfc401e251 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PreInitialize.java @@ -20,14 +20,13 @@ package org.apache.cassandra.tcm.transformations.cms; import java.io.IOException; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -71,15 +70,14 @@ public class PreInitialize implements Transformation public Result execute(ClusterMetadata metadata) { assert metadata.epoch.isBefore(Epoch.FIRST); - assert metadata.period == Period.EMPTY; - ClusterMetadata.Transformer transformer = metadata.transformer(false); + ClusterMetadata.Transformer transformer = metadata.transformer(); if (addr != null) { DataPlacement.Builder dataPlacementBuilder = DataPlacement.builder(); Replica replica = new Replica(addr, - DatabaseDescriptor.getPartitioner().getMinimumToken(), - DatabaseDescriptor.getPartitioner().getMinimumToken(), + MetaStrategy.partitioner.getMinimumToken(), + MetaStrategy.partitioner.getMinimumToken(), true); dataPlacementBuilder.reads.withReplica(metadata.nextEpoch(), replica); dataPlacementBuilder.writes.withReplica(metadata.nextEpoch(), replica); @@ -90,7 +88,6 @@ public class PreInitialize implements Transformation ClusterMetadata.Transformer.Transformed transformed = transformer.build(); metadata = transformed.metadata; assert metadata.epoch.is(Epoch.FIRST) : metadata.epoch; - assert metadata.period == Period.FIRST : metadata.period; return new Success(metadata, LockedRanges.AffectedRanges.EMPTY, transformed.modifiedKeys); } diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java index 2480f37f65..87a4c870c5 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java @@ -45,7 +45,7 @@ import org.apache.cassandra.tcm.serialization.MetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; public class PrepareCMSReconfiguration { diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java index d60616ab70..8e37875e94 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/RemoveFromCMS.java @@ -28,6 +28,7 @@ import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; @@ -35,14 +36,13 @@ import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.sequences.InProgressSequences; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, StartAddToCMS & FinishAddToCMS, contain a high degree of duplication with their intended @@ -121,7 +121,7 @@ public class RemoveFromCMS extends BaseMembershipTransformation return new Transformation.Rejected(INVALID, String.format("Removing %s will leave no nodes in CMS", endpoint)); return Transformation.success(transformer.with(prev.placements.unbuild().with(metaParams, proposed).build()), - EntireRange.affectedRanges(prev)); + MetaStrategy.affectedRanges(prev)); } @Override diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java b/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java index af4f4f8e2a..90f3056f47 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/StartAddToCMS.java @@ -22,6 +22,7 @@ import java.util.HashSet; import java.util.Set; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.RangesByEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; @@ -30,13 +31,12 @@ import org.apache.cassandra.tcm.MultiStepOperation; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.sequences.AddToCMS; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.serialization.AsymmetricMetadataSerializer; import static org.apache.cassandra.exceptions.ExceptionCode.INVALID; -import static org.apache.cassandra.tcm.ownership.EntireRange.entireRange; +import static org.apache.cassandra.locator.MetaStrategy.entireRange; /** * This class along with AddToCMS, FinishAddToCMS & RemoveFromCMS, contain a high degree of duplication with their intended @@ -99,7 +99,7 @@ public class StartAddToCMS extends BaseMembershipTransformation AddToCMS joinSequence = new AddToCMS(prev.nextEpoch(), nodeId, streamCandidates, new FinishAddToCMS(endpoint)); transformer = transformer.with(prev.inProgressSequences.with(nodeId, joinSequence)); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } @Override diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java index e653e7e2b5..d31e69b1db 100644 --- a/src/java/org/apache/cassandra/tools/NodeTool.java +++ b/src/java/org/apache/cassandra/tools/NodeTool.java @@ -195,7 +195,6 @@ public class NodeTool ResumeHandoff.class, Ring.class, Scrub.class, - SealPeriod.class, SetAuthCacheConfig.class, SetBatchlogReplayThrottle.class, SetCacheCapacity.class, @@ -264,7 +263,8 @@ public class NodeTool .withDefaultCommand(CMSAdmin.DescribeCMS.class) .withCommand(CMSAdmin.DescribeCMS.class) .withCommand(CMSAdmin.InitializeCMS.class) - .withCommand(CMSAdmin.ReconfigureCMS.class); + .withCommand(CMSAdmin.ReconfigureCMS.class) + .withCommand(CMSAdmin.Snapshot.class); Cli parser = builder.build(); diff --git a/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java b/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java index cc97d6bfff..f5cf15d5b2 100644 --- a/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java +++ b/src/java/org/apache/cassandra/tools/TransformClusterMetadataHelper.java @@ -27,13 +27,13 @@ import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.util.FileInputStreamPlus; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import org.apache.cassandra.tcm.serialization.Version; @@ -84,7 +84,7 @@ public class TransformClusterMetadataHelper builder.withoutReadReplica(metadata.epoch, replica) .withoutWriteReplica(metadata.epoch, replica); } - Replica newCMS = EntireRange.replica(endpoint); + Replica newCMS = MetaStrategy.replica(endpoint); builder.withReadReplica(metadata.epoch, newCMS) .withWriteReplica(metadata.epoch, newCMS); return metadata.transformer().with(metadata.placements.unbuild().with(metaParams, diff --git a/src/java/org/apache/cassandra/tools/nodetool/CMSAdmin.java b/src/java/org/apache/cassandra/tools/nodetool/CMSAdmin.java index 0e07bdfa22..841da97fe7 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/CMSAdmin.java +++ b/src/java/org/apache/cassandra/tools/nodetool/CMSAdmin.java @@ -160,4 +160,15 @@ public abstract class CMSAdmin extends NodeTool.NodeToolCmd probe.getCMSOperationsProxy().reconfigureCMS(parsedRfs); } } + + @Command(name = "snapshot", description = "Request a checkpointing snapshot of cluster metadata") + public static class Snapshot extends NodeTool.NodeToolCmd + { + @Override + public void execute(NodeProbe probe) + { + probe.getCMSOperationsProxy().snapshotClusterMetadata(); + } + } + } diff --git a/src/java/org/apache/cassandra/tools/nodetool/SealPeriod.java b/src/java/org/apache/cassandra/tools/nodetool/SealPeriod.java deleted file mode 100644 index e7db33f453..0000000000 --- a/src/java/org/apache/cassandra/tools/nodetool/SealPeriod.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.tools.nodetool; - -import io.airlift.airline.Command; -import org.apache.cassandra.tools.NodeProbe; -import org.apache.cassandra.tools.NodeTool.NodeToolCmd; - -@Command(name = "sealperiod", description = "seal current period in the metadata log") -public class SealPeriod extends NodeToolCmd -{ - @Override - public void execute(NodeProbe probe) - { - probe.getCMSOperationsProxy().sealPeriod(); - } -} diff --git a/test/data/serialization/5.1/service.SyncComplete.bin b/test/data/serialization/5.1/service.SyncComplete.bin index 7c775cef66..69ddbdda04 100644 Binary files a/test/data/serialization/5.1/service.SyncComplete.bin and b/test/data/serialization/5.1/service.SyncComplete.bin differ diff --git a/test/data/serialization/5.1/service.SyncRequest.bin b/test/data/serialization/5.1/service.SyncRequest.bin index f853b20f9c..d1046c1435 100644 Binary files a/test/data/serialization/5.1/service.SyncRequest.bin and b/test/data/serialization/5.1/service.SyncRequest.bin differ diff --git a/test/data/serialization/5.1/service.ValidationComplete.bin b/test/data/serialization/5.1/service.ValidationComplete.bin index f0c502458a..85c0940b20 100644 Binary files a/test/data/serialization/5.1/service.ValidationComplete.bin and b/test/data/serialization/5.1/service.ValidationComplete.bin differ diff --git a/test/data/serialization/5.1/service.ValidationRequest.bin b/test/data/serialization/5.1/service.ValidationRequest.bin index 04c492a8a1..5f4cbf664b 100644 Binary files a/test/data/serialization/5.1/service.ValidationRequest.bin and b/test/data/serialization/5.1/service.ValidationRequest.bin differ diff --git a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java index 3e60a02523..b160b2600f 100644 --- a/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java +++ b/test/distributed/org/apache/cassandra/distributed/shared/ClusterUtils.java @@ -632,6 +632,14 @@ public class ClusterUtils return decode(inst.callOnInstance(() -> encode(ClusterMetadata.current().nextEpoch()))); } + public static Epoch snapshotClusterMetadata(IInvokableInstance inst) + { + return decode(inst.callOnInstance(() -> { + ClusterMetadata snapshotted = ClusterMetadataService.instance().triggerSnapshot(); + return encode(snapshotted.epoch); + })); + } + public static Map getPeerEpochs(IInvokableInstance requester) { Map map = requester.callOnInstance(() -> { diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java index fd554a189a..fc6dad5adf 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/BootWithMetadataTest.java @@ -32,6 +32,7 @@ import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.io.util.FileOutputStreamPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.tcm.CMSOperations; @@ -39,7 +40,6 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeVersion; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.serialization.VerboseMetadataSerializer; import static org.apache.cassandra.distributed.shared.ClusterUtils.start; @@ -103,8 +103,8 @@ public class BootWithMetadataTest extends TestBaseImpl try { ClusterMetadata metadata = ClusterMetadata.current(); - Replica oldCMS = EntireRange.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.1")); - Replica newCMS = EntireRange.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.2")); + Replica oldCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.1")); + Replica newCMS = MetaStrategy.replica(InetAddressAndPort.getByNameUnchecked("127.0.0.2")); ClusterMetadata.Transformer transformer = metadata.transformer(); DataPlacement.Builder builder = metadata.placements.get(ReplicationParams.meta(metadata)).unbuild() .withoutReadReplica(metadata.nextEpoch(), oldCMS) diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java index bd4ea3c7db..c1975cdbfe 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CMSTestBase.java @@ -27,6 +27,7 @@ import org.apache.cassandra.distributed.api.IIsolatedExecutor; import org.apache.cassandra.harry.sut.TokenPlacementModel; import org.apache.cassandra.locator.IEndpointSnitch; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.schema.DistributedSchema; @@ -40,7 +41,6 @@ import org.apache.cassandra.tcm.MetadataSnapshots; import org.apache.cassandra.tcm.Processor; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.log.LocalLog; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; import org.apache.cassandra.tcm.transformations.AlterSchema; import org.apache.cassandra.tcm.transformations.cms.Initialize; @@ -118,7 +118,7 @@ public class CMSTestBase ClusterMetadata next = baseState; DistributedSchema initialSchema = new DistributedSchema(prev.schema.getKeyspaces()); ClusterMetadata.Transformer transformer = next.transformer().with(initialSchema); - return Transformation.success(transformer, EntireRange.affectedRanges(prev)); + return Transformation.success(transformer, MetaStrategy.affectedRanges(prev)); } }); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java index 52126cd7ec..2e7dd85352 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ClusterMetadataTestHelper.java @@ -60,10 +60,8 @@ import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Commit; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.Transformation; import org.apache.cassandra.tcm.log.LocalLog; -import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeAddresses; import org.apache.cassandra.tcm.membership.NodeId; @@ -142,8 +140,6 @@ public class ClusterMetadataTestHelper public static ClusterMetadata minimalForTesting(IPartitioner partitioner) { return new ClusterMetadata(Epoch.EMPTY, - Period.EMPTY, - false, partitioner, null, null, @@ -157,8 +153,6 @@ public class ClusterMetadataTestHelper public static ClusterMetadata minimalForTesting(Keyspaces keyspaces) { return new ClusterMetadata(Epoch.EMPTY, - Period.EMPTY, - false, Murmur3Partitioner.instance, new DistributedSchema(keyspaces), null, @@ -169,25 +163,6 @@ public class ClusterMetadataTestHelper ImmutableMap.of()); } - public static void forceCurrentPeriodTo(long period) - { - ClusterMetadataService.unsetInstance(); - ClusterMetadataService.setInstance(instanceForTest()); - ClusterMetadata metadata = ClusterMetadata.currentNullable(); - metadata = new ClusterMetadata(metadata.epoch.nextEpoch(), - period, - metadata.lastInPeriod, - metadata.partitioner, - metadata.schema, - metadata.directory, - metadata.tokenMap, - metadata.placements, - metadata.lockedRanges, - metadata.inProgressSequences, - metadata.extensions); - ClusterMetadataService.instance().log().append(new LogState(metadata, LogState.EMPTY.entries)); - } - public static ClusterMetadataService syncInstanceForTest() { LocalLog log = LocalLog.logSpec() diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java index 219a06ee33..64c4e629e3 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java @@ -695,7 +695,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase case TCM_FETCH_CMS_LOG_REQ: { FetchCMSLog request = (FetchCMSLog) message.payload; - LogState logState = logStorage.getLogState(ClusterMetadata.current().period, request.lowerBound); + LogState logState = logStorage.getLogState(request.lowerBound); realCluster.deliverMessage(message.from(), Instance.serializeMessage(cms.addr(), message.from(), message.responseWith(logState))); return; diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/DistributedLogTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/DistributedLogTest.java index d5bef1961f..f4473d9c76 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/DistributedLogTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/DistributedLogTest.java @@ -29,7 +29,6 @@ import org.junit.Test; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.log.LogStorage; @@ -94,7 +93,7 @@ public class DistributedLogTest extends TestBaseImpl List actual = node.callOnInstance(() -> { List res = new ArrayList<>(); - for (Entry entry : LogStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, Epoch.FIRST).entries) + for (Entry entry : LogStorage.SystemKeyspace.getLogState(Epoch.FIRST).entries) { if (entry.transform instanceof CustomTransformation) { @@ -180,7 +179,7 @@ public class DistributedLogTest extends TestBaseImpl Set res = new HashSet<>(); // todo: add method to get the full log - for (Entry entry : LogStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, Epoch.FIRST).entries) + for (Entry entry : LogStorage.SystemKeyspace.getLogState(Epoch.FIRST).entries) { if (entry.transform instanceof CustomTransformation) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/FetchLogFromPeersTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/FetchLogFromPeersTest.java index 4a28128a2d..fbcd080a98 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/FetchLogFromPeersTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/FetchLogFromPeersTest.java @@ -46,7 +46,7 @@ import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.tcm.sequences.AddToCMS; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -315,14 +315,14 @@ public class FetchLogFromPeersTest extends TestBaseImpl LogState logState = (LogState) decoded.payload; // drop every other replication message to make sure pending buffer is non-consecutive if (decoded.epoch().getEpoch() % 2 == 0 && - logState.entries.stream().noneMatch((e) -> e.transform instanceof SealPeriod)) + logState.entries.stream().noneMatch((e) -> e.transform instanceof TriggerSnapshot)) return false; } return true; }) ).drop(); executeAlters(cluster); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); + cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().triggerSnapshot()); executeAlters(cluster); cluster.filters().reset(); @@ -353,7 +353,7 @@ public class FetchLogFromPeersTest extends TestBaseImpl filter = cluster.filters().inbound().to(2).verbs(Verb.TCM_FETCH_PEER_LOG_RSP.id, Verb.TCM_FETCH_CMS_LOG_REQ.id).drop(); executeAlters(cluster); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); + cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().triggerSnapshot()); executeAlters(cluster); filter.off(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java index 5a324fa70c..f8e88098d9 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java @@ -64,7 +64,7 @@ import org.apache.cassandra.tcm.ownership.DataPlacements; import org.apache.cassandra.tcm.ownership.PlacementForRange; import org.apache.cassandra.tcm.ownership.VersionedEndpoints; import org.apache.cassandra.tcm.transformations.Register; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import static org.apache.cassandra.distributed.test.log.PlacementSimulator.SimulatedPlacements; import static org.apache.cassandra.harry.sut.TokenPlacementModel.Node; @@ -449,7 +449,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase (state, sut, entropySource) -> { try { - sut.service.commit(SealPeriod.instance); + sut.service.commit(TriggerSnapshot.instance); } catch (IllegalStateException e) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java index 02a14a344e..e8713ae35f 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ReconfigureCMSTest.java @@ -28,13 +28,13 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.shared.NetworkTopology; +import org.apache.cassandra.locator.MetaStrategy; import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.tcm.ownership.EntireRange; import org.apache.cassandra.tcm.sequences.ProgressBarrier; import org.apache.cassandra.tcm.sequences.ReconfigureCMS; import org.apache.cassandra.tcm.transformations.cms.PrepareCMSReconfiguration; @@ -92,7 +92,7 @@ public class ReconfigureCMSTest extends FuzzTestBase ClusterMetadataService.instance().commit(new PrepareCMSReconfiguration.Complex(ReplicationParams.simple(3).asMeta())); ReconfigureCMS reconfigureCMS = (ReconfigureCMS) ClusterMetadata.current().inProgressSequences.get(ReconfigureCMS.SequenceKey.instance); ClusterMetadataService.instance().commit(reconfigureCMS.next); - ProgressBarrier.propagateLast(EntireRange.affectedRanges(ClusterMetadata.current())); + ProgressBarrier.propagateLast(MetaStrategy.affectedRanges(ClusterMetadata.current())); try { ClusterMetadataService.instance().commit(reconfigureCMS.next); @@ -107,7 +107,7 @@ public class ReconfigureCMSTest extends FuzzTestBase }); cluster.get(1).nodetoolResult("cms", "reconfigure", "--cancel").asserts().success(); cluster.get(1).runOnInstance(() -> { - ProgressBarrier.propagateLast(EntireRange.affectedRanges(ClusterMetadata.current())); + ProgressBarrier.propagateLast(MetaStrategy.affectedRanges(ClusterMetadata.current())); ClusterMetadata metadata = ClusterMetadata.current(); Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance)); Assert.assertEquals(2, metadata.fullCMSMembers().size()); @@ -119,20 +119,20 @@ public class ReconfigureCMSTest extends FuzzTestBase cluster.get(1).runOnInstance(() -> { ClusterMetadataService.instance().commit(new PrepareCMSReconfiguration.Complex(ReplicationParams.simple(4).asMeta())); - ProgressBarrier.propagateLast(EntireRange.affectedRanges(ClusterMetadata.current())); + ProgressBarrier.propagateLast(MetaStrategy.affectedRanges(ClusterMetadata.current())); ReconfigureCMS reconfigureCMS = (ReconfigureCMS) ClusterMetadata.current().inProgressSequences.get(ReconfigureCMS.SequenceKey.instance); ClusterMetadataService.instance().commit(reconfigureCMS.next); - ProgressBarrier.propagateLast(EntireRange.affectedRanges(ClusterMetadata.current())); + ProgressBarrier.propagateLast(MetaStrategy.affectedRanges(ClusterMetadata.current())); reconfigureCMS = (ReconfigureCMS) ClusterMetadata.current().inProgressSequences.get(ReconfigureCMS.SequenceKey.instance); ClusterMetadataService.instance().commit(reconfigureCMS.next); - ProgressBarrier.propagateLast(EntireRange.affectedRanges(ClusterMetadata.current())); + ProgressBarrier.propagateLast(MetaStrategy.affectedRanges(ClusterMetadata.current())); reconfigureCMS = (ReconfigureCMS) ClusterMetadata.current().inProgressSequences.get(ReconfigureCMS.SequenceKey.instance); Assert.assertNull(reconfigureCMS.next.activeTransition); }); cluster.get(1).nodetoolResult("cms", "reconfigure", "--cancel").asserts().success(); cluster.get(1).runOnInstance(() -> { - ProgressBarrier.propagateLast(EntireRange.affectedRanges(ClusterMetadata.current())); + ProgressBarrier.propagateLast(MetaStrategy.affectedRanges(ClusterMetadata.current())); ClusterMetadata metadata = ClusterMetadata.current(); Assert.assertNull(metadata.inProgressSequences.get(ReconfigureCMS.SequenceKey.instance)); Assert.assertTrue(metadata.fullCMSMembers().contains(FBUtilities.getBroadcastAddressAndPort())); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java index afcbef66e0..a32dfbc356 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/RegisterTest.java @@ -47,7 +47,7 @@ import org.apache.cassandra.tcm.sequences.UnbootstrapAndLeave; import org.apache.cassandra.tcm.serialization.Version; import org.apache.cassandra.tcm.transformations.PrepareLeave; import org.apache.cassandra.tcm.transformations.Register; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.tcm.transformations.Startup; import org.apache.cassandra.tcm.transformations.Unregister; import org.apache.cassandra.utils.CassandraVersion; @@ -85,7 +85,7 @@ public class RegisterTest extends TestBaseImpl }); cluster.get(1).runOnInstance(() -> { - ClusterMetadataService.instance().commit(SealPeriod.instance); + ClusterMetadataService.instance().commit(TriggerSnapshot.instance); }); IInstanceConfig config = cluster.newInstanceConfig(); @@ -181,7 +181,7 @@ public class RegisterTest extends TestBaseImpl { throw new RuntimeException(e); } - ClusterMetadataService.instance().commit(SealPeriod.instance); + ClusterMetadataService.instance().commit(TriggerSnapshot.instance); ClusterMetadata cm = new MetadataSnapshots.SystemKeyspaceMetadataSnapshots().getSnapshot(ClusterMetadata.current().epoch); cm.equals(ClusterMetadata.current()); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/ReplayPersistedTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/ReplayPersistedTest.java index 11b3520e42..0b0da8c445 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/ReplayPersistedTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/ReplayPersistedTest.java @@ -41,7 +41,7 @@ import org.apache.cassandra.tcm.log.Entry; import org.apache.cassandra.tcm.log.LogState; import org.apache.cassandra.tcm.log.SystemKeyspaceStorage; import org.apache.cassandra.tcm.transformations.CustomTransformation; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -134,7 +134,7 @@ public class ReplayPersistedTest extends TestBaseImpl { Entry last = entries.get(entries.size() - 1); // race, we might have got a SealPeriod since we grabbed ClusterMetadata.current (we don't block commit on that) - if (last.transform instanceof SealPeriod && + if (last.transform instanceof TriggerSnapshot && last.epoch.is(cur.epoch.nextEpoch())) { entries = state.entries.subList(0, state.entries.size() - 1); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java index ff3d047ef0..8e424e0bc5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SnapshotTest.java @@ -30,11 +30,13 @@ import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.test.TestBaseImpl; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.transformations.CustomTransformation; import static org.apache.cassandra.distributed.api.Feature.GOSSIP; import static org.apache.cassandra.distributed.api.Feature.NETWORK; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class SnapshotTest extends TestBaseImpl { @@ -61,20 +63,19 @@ public class SnapshotTest extends TestBaseImpl "WHERE id IS NOT NULL AND x IS NOT NULL\n" + "PRIMARY KEY (x, id)")); - cluster.get(1).runOnInstance(() -> { + long snapshotEpoch = cluster.get(1).callOnInstance(() -> { ClusterMetadata before = ClusterMetadata.current(); - ClusterMetadata after = ClusterMetadataService.instance().sealPeriod(); + ClusterMetadata after = ClusterMetadataService.instance().triggerSnapshot(); ClusterMetadata serialized = ClusterMetadataService.instance().snapshotManager().getSnapshot(after.epoch); assertEquals(before.placements, serialized.placements); assertEquals(before.tokenMap, serialized.tokenMap); assertEquals(before.directory, serialized.directory); assertEquals(before.schema, serialized.schema); + return serialized.epoch.getEpoch(); }); cluster.schemaChange(withKeyspace("create table %s.tbl2 (id int primary key, x int)")); - cluster.get(1).runOnInstance(() -> { - assertEquals(2, ClusterMetadata.current().period); - }); + assertEquals(snapshotEpoch + 1, ClusterUtils.getCurrentEpoch(cluster.get(1)).getEpoch()); } } @@ -91,15 +92,14 @@ public class SnapshotTest extends TestBaseImpl { cluster.schemaChange(withKeyspace("create table %s.tbl1 (id int primary key, x int)")); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); + Epoch snapshotEpoch = ClusterUtils.snapshotClusterMetadata(cluster.get(1)); cluster.schemaChange(withKeyspace("create table %s.tbl2 (id int primary key, x int)")); cluster.schemaChange(withKeyspace("create table %s.tbl3 (id int primary key, x int)")); - + Epoch expected = snapshotEpoch.nextEpoch().nextEpoch(); ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); for (int i = 1; i <= 2; i++) - cluster.get(i).runOnInstance(() -> { assertEquals(2, ClusterMetadata.current().period); }); - + assertTrue(expected.is(ClusterUtils.getCurrentEpoch(cluster.get(i)))); IInstanceConfig config = cluster.newInstanceConfig() .set("auto_bootstrap", true) @@ -110,18 +110,11 @@ public class SnapshotTest extends TestBaseImpl cluster.schemaChange(withKeyspace("create table %s.tbl4 (id int primary key, x int)")); cluster.schemaChange(withKeyspace("create table %s.tbl5 (id int primary key, x int)")); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); - + snapshotEpoch = ClusterUtils.snapshotClusterMetadata(cluster.get(1)); ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); + // no events executed after snapshot, epoch is unchanged for (int i = 1; i <= 3; i++) - { - cluster.get(i).runOnInstance(() -> { - // no events executed after NewPeriod, period is still 2 - assertEquals(2, ClusterMetadata.current().period); - // but the next one is 3 - assertEquals(true, ClusterMetadata.current().lastInPeriod); - }); - } + assertTrue(snapshotEpoch.is(ClusterUtils.getCurrentEpoch(cluster.get(i)))); config = cluster.newInstanceConfig() .set("auto_bootstrap", true) @@ -148,16 +141,15 @@ public class SnapshotTest extends TestBaseImpl { cluster.schemaChange(withKeyspace("create table %s.tbl1 (id int primary key, x int)")); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); - + Epoch snapshotEpoch = ClusterUtils.snapshotClusterMetadata(cluster.get(1)); cluster.schemaChange(withKeyspace("create table %s.tbl2 (id int primary key, x int)")); cluster.schemaChange(withKeyspace("create table %s.tbl3 (id int primary key, x int)")); - + Epoch expected = snapshotEpoch.nextEpoch().nextEpoch(); ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); for (int i = 1; i <= 3; i++) - cluster.get(i).runOnInstance(() -> { assertEquals(2, ClusterMetadata.current().period); }); + assertTrue(expected.is(ClusterUtils.getCurrentEpoch(cluster.get(i)))); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); + snapshotEpoch = ClusterUtils.snapshotClusterMetadata(cluster.get(1)); ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); long epochBefore = ClusterUtils.getCurrentEpoch(cluster.get(1)).getEpoch(); @@ -208,16 +200,15 @@ public class SnapshotTest extends TestBaseImpl { cluster.schemaChange(withKeyspace("create table %s.tbl1 (id int primary key, x int)")); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); - + Epoch snapshotEpoch = ClusterUtils.snapshotClusterMetadata(cluster.get(1)); cluster.schemaChange(withKeyspace("create table %s.tbl2 (id int primary key, x int)")); cluster.schemaChange(withKeyspace("create table %s.tbl3 (id int primary key, x int)")); - + Epoch expected = snapshotEpoch.nextEpoch().nextEpoch(); ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); for (int i = 1; i <= 3; i++) - cluster.get(i).runOnInstance(() -> { assertEquals(2, ClusterMetadata.current().period); }); + assertTrue(expected.is(ClusterUtils.getCurrentEpoch(cluster.get(i)))); - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); + cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().triggerSnapshot()); ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); long epochBefore = ClusterUtils.getCurrentEpoch(cluster.get(1)).getEpoch(); diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/SystemKeyspaceStorageTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/SystemKeyspaceStorageTest.java index b8e6dee0ae..e8bf239f56 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/log/SystemKeyspaceStorageTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/log/SystemKeyspaceStorageTest.java @@ -26,7 +26,6 @@ import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.QueryProcessor; -import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.distributed.test.ExecUtil; import org.apache.cassandra.harry.sut.TokenPlacementModel; import org.apache.cassandra.tcm.ClusterMetadataService; @@ -37,7 +36,6 @@ import org.apache.cassandra.tcm.FetchCMSLog; import org.apache.cassandra.tcm.transformations.CustomTransformation; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.utils.BiMultiValMap; import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.db.SystemKeyspace.SNAPSHOT_TABLE_NAME; @@ -50,34 +48,35 @@ public class SystemKeyspaceStorageTest extends CoordinatorPathTestBase public void testLogStateQuery() throws Throwable { cmsNodeTest((cluster, simulatedCluster) -> { - // Disable snapshots on CMS "client" nodes + // Disable periodic snapshotting DatabaseDescriptor.setMetadataSnapshotFrequency(Integer.MAX_VALUE); cluster.get(1).runOnInstance(() -> DatabaseDescriptor.setMetadataSnapshotFrequency(Integer.MAX_VALUE)); Random rng = new Random(1L); - // Map of epoch to period - BiMultiValMap epochToPeriod = new BiMultiValMap<>(); // Generate some epochs int cnt = 0; - int periodSize = rng.nextInt(10); - // set up a few epochs + int nextSnapshotIn = rng.nextInt(10); + List allEpochs = new ArrayList<>(); + List allSnapshots = new ArrayList<>(); + for (int i = 0; i < 500; i++) { try { - if (periodSize == 0) + if (nextSnapshotIn == 0) { - cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().sealPeriod()); + cluster.get(1).runOnInstance(() -> ClusterMetadataService.instance().triggerSnapshot()); ClusterMetadata metadata = ClusterMetadataService.instance().processor().fetchLogAndWait(); - epochToPeriod.put(metadata.epoch, metadata.period); - periodSize = rng.nextInt(10); + allEpochs.add(metadata.epoch); + allSnapshots.add(metadata.epoch); + nextSnapshotIn = rng.nextInt(10); } else { - periodSize--; + nextSnapshotIn--; } ClusterMetadata metadata = ClusterMetadataService.instance().commit(CustomTransformation.make(cnt++)); - epochToPeriod.put(metadata.epoch, metadata.period); + allEpochs.add(metadata.epoch); } catch (Throwable e) { @@ -87,23 +86,19 @@ public class SystemKeyspaceStorageTest extends CoordinatorPathTestBase ClusterMetadataService.instance().processor().fetchLogAndWait(); - List allEpochs = new ArrayList<>(epochToPeriod.keySet()); - List allPeriods = cluster.get(1).callOnInstance(() -> getAllSnapshots()); - List remainingPeriods = new ArrayList<>(allPeriods); + List remainingSnapshots = new ArrayList<>(allSnapshots); // Delete about a half (but potentially up to 100%) of all possible snapshots - for (int i = 0; i < allPeriods.size(); i++) + for (int i = 0; i < allSnapshots.size(); i++) { if (rng.nextBoolean()) { - // pick a sealed period for which we'll delete the snapshot - long toRemovePeriod = remainingPeriods.remove(rng.nextInt(remainingPeriods.size())); - // lookup the max epoch for the period, this is the key to the snapshots table - long toRemoveEpoch = maxEpochInPeriod(epochToPeriod, toRemovePeriod).getEpoch(); - cluster.get(1).runOnInstance(() -> deleteSnapshot(toRemoveEpoch)); + // pick a snapshot to delete + Epoch toRemoveSnapshot = remainingSnapshots.remove(rng.nextInt(remainingSnapshots.size())); + cluster.get(1).runOnInstance(() -> deleteSnapshot(toRemoveSnapshot.getEpoch())); } } - long latestSnapshot = remainingPeriods.get(remainingPeriods.size() - 1); + Epoch latestSnapshot = remainingSnapshots.get(remainingSnapshots.size() - 1); Epoch lastEpoch = allEpochs.stream().max(Comparator.naturalOrder()).get(); repeat(10, () -> { repeat(100, () -> { @@ -111,9 +106,8 @@ public class SystemKeyspaceStorageTest extends CoordinatorPathTestBase for (boolean consistentReplay : new boolean[]{ true, false }) { LogState logState = simulatedCluster.node(2).requestResponse(new FetchCMSLog(since, consistentReplay)); - // we now search backwards in the log, starting at the current period. // if we return a snapshot it is always the most recent one - // we don't return a snapshot if `since` is in the previous period + // we don't return a snapshot if there is only 1 snapshot after `since` Epoch start = since; if (logState.baseState == null) { @@ -125,7 +119,7 @@ public class SystemKeyspaceStorageTest extends CoordinatorPathTestBase } else { - assertEquals(latestSnapshot, logState.baseState.period); + assertEquals(latestSnapshot, logState.baseState.epoch); start = logState.baseState.epoch; if (logState.entries.isEmpty()) // no entries, snapshot should have the same epoch as since assertEquals(since, start); @@ -145,20 +139,11 @@ public class SystemKeyspaceStorageTest extends CoordinatorPathTestBase }); } - private Epoch maxEpochInPeriod(BiMultiValMap epochToPeriod, long period) - { - return epochToPeriod.inverse() - .get(period) - .stream() - .max(Comparator.naturalOrder()) - .get(); - } - @Test public void bounceNodeBootrappedFromSnapshot() throws Throwable { coordinatorPathTest(new TokenPlacementModel.SimpleReplicationFactor(3), (cluster, simulatedCluster) -> { - ClusterMetadataService.instance().sealPeriod(); + ClusterMetadataService.instance().triggerSnapshot(); ClusterMetadataService.instance().log().waitForHighestConsecutive(); ClusterMetadataService.instance().snapshotManager().storeSnapshot(ClusterMetadata.current()); @@ -194,16 +179,6 @@ public class SystemKeyspaceStorageTest extends CoordinatorPathTestBase } } - public static List getAllSnapshots() - { - List allPeriods = new ArrayList<>(); - String query = String.format("SELECT period FROM %s.%s", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); - for (UntypedResultSet.Row row : QueryProcessor.executeInternal(query)) - allPeriods.add(row.getLong("period")); - allPeriods.sort(Long::compare); - return allPeriods; - } - public static void deleteSnapshot(long epoch) { String query = String.format("DELETE FROM %s.%s WHERE epoch = ?", SchemaConstants.SYSTEM_KEYSPACE_NAME, SNAPSHOT_TABLE_NAME); diff --git a/test/distributed/org/apache/cassandra/distributed/test/tcm/RepairMetadataKeyspaceTest.java b/test/distributed/org/apache/cassandra/distributed/test/tcm/RepairMetadataKeyspaceTest.java new file mode 100644 index 0000000000..074a64913f --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tcm/RepairMetadataKeyspaceTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tcm; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.shared.ClusterUtils; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; +import org.apache.cassandra.tcm.Epoch; + +import static org.apache.cassandra.distributed.api.Feature.GOSSIP; +import static org.apache.cassandra.distributed.api.Feature.NETWORK; +import static org.apache.cassandra.distributed.shared.ClusterUtils.getDataDirectories; +import static org.apache.cassandra.distributed.shared.ClusterUtils.stopUnchecked; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class RepairMetadataKeyspaceTest extends TestBaseImpl +{ + @Test + public void testRepairMetadataKeyspace() throws Throwable + { + try (Cluster cluster = builder().withNodes(3) + .withConfig(config -> config.with(GOSSIP).with(NETWORK)) + .start()) + { + init(cluster); + cluster.get(1).nodetoolResult("cms", "reconfigure", "3").asserts().success(); + ClusterUtils.waitForCMSToQuiesce(cluster, cluster.get(1)); + Epoch currentEpoch = getConsistentEpoch(cluster); + + for (IInvokableInstance nodeToTest : cluster) + assertTrue(canReadCompleteLog(nodeToTest, currentEpoch)); + + IInvokableInstance toRepair = cluster.get(3); + stopUnchecked(toRepair); + String targetDir = DistributedMetadataLogKeyspace.TABLE_NAME + '-' + DistributedMetadataLogKeyspace.LOG_TABLE_ID.toHexString(); + for (File datadir : getDataDirectories(toRepair)) + { + List tabledirs = Files.find(datadir.toPath(), + Integer.MAX_VALUE, + (filePath, fileAttr) -> fileAttr.isDirectory() && filePath.getFileName().toString().equals(targetDir)) + .collect(Collectors.toList()); + for (Path tabledir : tabledirs) + { + Files.find(tabledir, + Integer.MAX_VALUE, + (path, attr) -> attr.isRegularFile()) + .map(File::new) + .forEach(File::delete); + } + } + + toRepair.startup(); + assertFalse(canReadCompleteLog(toRepair, currentEpoch)); + + toRepair.nodetoolResult("repair", "--full", "system_cluster_metadata").asserts().success(); + for (IInvokableInstance nodeToTest : cluster) + assertTrue(canReadCompleteLog(nodeToTest, currentEpoch)); + } + } + + private boolean canReadCompleteLog(IInvokableInstance instance, Epoch currentEpoch) + { + Object[][] res = instance.executeInternal("SELECT epoch FROM system_cluster_metadata.distributed_metadata_log"); + if (res.length != currentEpoch.getEpoch()) + return false; + + for (int i = res.length-1; i >= 0; i--) + { + long fromLog = (long)res[i][0]; + if (currentEpoch.getEpoch() - i != fromLog) + return false; + } + return true; + } + + private Epoch getConsistentEpoch(Cluster cluster) + { + Map epochs = getEpochsDirectly(cluster); + assertEquals(1, new HashSet<>(epochs.values()).size()); + return epochs.values().iterator().next(); + } + + private Map getEpochsDirectly(Cluster cluster) + { + Map epochs = new HashMap<>(); + cluster.forEach(inst -> epochs.put(inst.broadcastAddress().toString(), ClusterUtils.getClusterMetadataVersion(inst))); + return epochs; + } + +} diff --git a/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java b/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java index e706923fbd..97dea78484 100644 --- a/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java +++ b/test/unit/org/apache/cassandra/db/virtual/LocalRepairTablesTest.java @@ -322,7 +322,7 @@ public class LocalRepairTablesTest extends CQLTester private ParticipateState participate() { List> ranges = Arrays.asList(new Range<>(new Murmur3Partitioner.LongToken(0), new Murmur3Partitioner.LongToken(42))); - ParticipateState state = new ParticipateState(Clock.Global.clock(), FBUtilities.getBroadcastAddressAndPort(), new PrepareMessage(TimeUUID.Generator.nextTimeUUID(), Collections.emptyList(), ranges, true, 42, true, PreviewKind.ALL)); + ParticipateState state = new ParticipateState(Clock.Global.clock(), FBUtilities.getBroadcastAddressAndPort(), new PrepareMessage(TimeUUID.Generator.nextTimeUUID(), Collections.emptyList(), Murmur3Partitioner.instance, ranges, true, 42, true, PreviewKind.ALL)); ActiveRepairService.instance().register(state); return state; } diff --git a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java index 1e93b87df2..a357fc88a4 100644 --- a/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java +++ b/test/unit/org/apache/cassandra/locator/MetaStrategyTest.java @@ -34,7 +34,6 @@ import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.membership.Directory; import org.apache.cassandra.tcm.membership.Location; import org.apache.cassandra.tcm.membership.NodeAddresses; @@ -83,8 +82,6 @@ public class MetaStrategyTest } return new ClusterMetadata(Epoch.EMPTY, - Period.EMPTY, - true, Murmur3Partitioner.instance, DistributedSchema.empty(), directory, diff --git a/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java index 4a98435fbb..6b74d0febd 100644 --- a/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java @@ -244,7 +244,7 @@ public class RepairMessageVerbHandlerOutOfRangeTest } private static PrepareMessage prepareMsg(TimeUUID parentRepairSession, Collection> ranges) { - return new PrepareMessage(parentRepairSession, tableIds, ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, true, PreviewKind.NONE); + return new PrepareMessage(parentRepairSession, tableIds, Murmur3Partitioner.instance, ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, true, PreviewKind.NONE); } private static ValidationRequest validationMsg(Range range) diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java index 8e93641e91..9886076d70 100644 --- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java +++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java @@ -45,6 +45,10 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.SyncNodePair; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.SchemaTestUtil; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.schema.TableId; @@ -68,6 +72,9 @@ public class RepairMessageSerializationsTest DatabaseDescriptor.daemonInitialization(); originalPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); ClusterMetadataTestHelper.setInstanceForTest(); + SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create("serializationsTestKeyspace", + KeyspaceParams.simple(3))); + SchemaTestUtil.announceNewTable(TableMetadata.minimal("serializationsTestKeyspace", "repairMessages")); } @AfterClass @@ -175,6 +182,7 @@ public class RepairMessageSerializationsTest public void prepareMessage() throws IOException { PrepareMessage msg = new PrepareMessage(nextTimeUUID(), new ArrayList() {{add(TableId.generate());}}, + Murmur3Partitioner.instance, buildTokenRanges(), true, 100000L, false, PreviewKind.NONE); serializeRoundTrip(msg, PrepareMessage.serializer); diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java index b49c38b6f2..d10b65dd17 100644 --- a/test/unit/org/apache/cassandra/service/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java @@ -49,7 +49,11 @@ import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.Validator; import org.apache.cassandra.repair.messages.*; import org.apache.cassandra.repair.state.ValidationState; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.SchemaTestUtil; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.streaming.StreamSummary; @@ -73,6 +77,8 @@ public class SerializationsTest extends AbstractSerializationsTester DatabaseDescriptor.daemonInitialization(); partitionerSwitcher = Util.switchPartitioner(RandomPartitioner.instance); ClusterMetadataTestHelper.setInstanceForTest(); + SchemaTestUtil.addOrUpdateKeyspace(KeyspaceMetadata.create("Keyspace1", KeyspaceParams.simple(3))); + SchemaTestUtil.announceNewTable(TableMetadata.minimal("Keyspace1", "Standard1")); RANDOM_UUID = TimeUUID.fromString("743325d0-4c4b-11ec-8a88-2d67081686db"); FULL_RANGE = new Range<>(Util.testPartitioner().getMinimumToken(), Util.testPartitioner().getMinimumToken()); DESC = new RepairJobDesc(RANDOM_UUID, RANDOM_UUID, "Keyspace1", "Standard1", Arrays.asList(FULL_RANGE)); diff --git a/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java b/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java index ffb55e3048..2f77df4f60 100644 --- a/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java +++ b/test/unit/org/apache/cassandra/tcm/BootWithMetadataTest.java @@ -135,7 +135,7 @@ public class BootWithMetadataTest seq = addSequence(seq, move(partitioner, random, seq::contains)); seq = addSequence(seq, addToCMS(random, seq::contains)); t = t.with(seq); - ClusterMetadata toWrite = t.build().metadata.forceEpoch(epoch).forcePeriod(999); + ClusterMetadata toWrite = t.build().metadata.forceEpoch(epoch); // before exporting to file, make the local node the single CMS member in the CM instance // as CMS membership is a requirement for re-initialising from file diff --git a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java index da4df77b5e..3ec661f158 100644 --- a/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/GetLogStateTest.java @@ -19,6 +19,9 @@ package org.apache.cassandra.tcm; import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; import java.util.NavigableMap; import java.util.TreeMap; @@ -43,10 +46,12 @@ import org.apache.cassandra.tcm.transformations.CustomTransformation; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.psjava.util.AssertStatus.assertTrue; public class GetLogStateTest { - NavigableMap epochToPeriod = new TreeMap<>(); + Epoch maxEpoch = Epoch.EMPTY; NavigableMap epochToSnapshot = new TreeMap<>(); @BeforeClass public static void setup() throws IOException @@ -81,66 +86,301 @@ public class GetLogStateTest public void testGetLogState() { clearAndPopulate(); - // 1. `since` = max epoch - LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, epochToPeriod.lastKey()); + // Starting at the current epoch, iterate backwards. For each preceding epoch, fetch and verify a LogState + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(maxEpoch); assertEquals(logState, LogState.EMPTY); testGetLogStateHelper(); } @Test - public void testLostSnapshot() + public void testIncompleteLog1() { clearAndPopulate(); - QueryProcessor.executeInternal("DELETE FROM system.metadata_snapshots WHERE epoch = ?", epochToSnapshot.lastKey().getEpoch()); - epochToSnapshot.pollLastEntry(); - testGetLogStateHelper(); - } - - @Test - public void testIncompleteLog() - { - clearAndPopulate(); - // delete an entry in the log from the previous period + // delete an entry from the log which precedes the latest snapshot but is after toQuery. In this case we would + // prefer a simple list of entries, but will get the latest snapshot + subsequent entries instead. Epoch lastSnapshot = epochToSnapshot.lastKey(); Epoch toDelete = Epoch.create(lastSnapshot.getEpoch() - 2); Epoch toQuery = Epoch.create(lastSnapshot.getEpoch() - 3); - LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, toQuery); - assertCorrectLogState(logState, null, toQuery, epochToPeriod.lastKey()); - QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE period = ? and epoch = ?", epochToPeriod.get(toDelete), toDelete.getEpoch()); - logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, toQuery); - assertCorrectLogState(logState, epochToSnapshot.lastEntry().getValue(), null, epochToPeriod.lastKey()); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, epochToSnapshot.lastEntry().getValue(), null, maxEpoch); } @Test - public void testIncompleteLogAndLostSnapshot() + public void testIncompleteLog2() { clearAndPopulate(); - // delete an entry in the log from the previous period as well as the most recent snapshot. Both the deleted - // entry and snapshot are after 'since' so it's not possible to construct a contiguous sequence from since to - // current + // delete an entry from the log with multiple snapshots between toQuery and current. The deleted entry is one + // which comes after all available snapshots. Without losing a log entry, we should expect a LogState containing + // the most recent snapshot plus subsequent entries. + ClusterMetadata lastSnapshot = epochToSnapshot.lastEntry().getValue(); + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + for (int i=0; i<3; i++) + snapshots.next(); + Epoch toQuery = Epoch.create(snapshots.next().getEpoch() - 1); + Epoch toDelete = Epoch.create(maxEpoch.getEpoch() - 2); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, lastSnapshot, lastSnapshot.epoch, maxEpoch); + + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertEquals(lastSnapshot, logState.baseState); + boolean containsExpectedEntries = logState.entries.stream() + .map(entry -> entry.epoch.getEpoch()) + .allMatch(e -> e > lastSnapshot.epoch.getEpoch() + && e <= maxEpoch.getEpoch() + && e != toDelete.getEpoch()); + assertTrue(containsExpectedEntries); + } + + @Test + public void testIncompleteLog3() + { + clearAndPopulate(); + // delete an entry from the log where we have multiple snapshots after the deleted entry. In this case we should + // get the latest snapshot + subsequent entries and the deleted entry should not affect this. + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + ClusterMetadata lastSnapshot = epochToSnapshot.get(snapshots.next()); + // Jump back over a couple more snapshots to give us points to query for and delete + snapshots.next(); + Epoch e = snapshots.next(); + Epoch toQuery = Epoch.create(e.getEpoch() + 1); + Epoch toDelete = Epoch.create(toQuery.getEpoch() + 1); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, lastSnapshot, lastSnapshot.epoch, maxEpoch); + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, lastSnapshot, lastSnapshot.epoch, maxEpoch); + } + + @Test + public void testIncompleteLog4() + { + clearAndPopulate(); + // delete an entry from the log which comes after the most recent snapshot, which itself comes after toQuery. + // We would normally expect to skip over the intervening snapshot and return just a list of entries from toQuery + // to current. Because we cannot make that list continuous, instead we get the most recent snapshot and an + // incomplete list. + ClusterMetadata lastSnapshot = epochToSnapshot.lastEntry().getValue(); + Epoch toQuery = Epoch.create(lastSnapshot.epoch.getEpoch() - 2); + Epoch toDelete = Epoch.create(maxEpoch.getEpoch() - 1); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertEquals(lastSnapshot, logState.baseState); + boolean containsExpectedEntries = logState.entries.stream() + .map(entry -> entry.epoch.getEpoch()) + .allMatch(e -> e > lastSnapshot.epoch.getEpoch() + && e <= maxEpoch.getEpoch() + && e != toDelete.getEpoch()); + assertTrue(containsExpectedEntries); + } + + @Test + public void testCorruptSnapshot1() + { + clearAndPopulate(); + // with a single snapshot following toQuery, corrupt it so that it cannot be read. In this scenario, we already + // prefer to return a list of consecutive entries and no base snapshot, so the corruption should not matter. + Epoch lastSnapshot = epochToSnapshot.lastKey(); + + Epoch toQuery = Epoch.create(lastSnapshot.getEpoch() - 3); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + + QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", lastSnapshot.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + } + + @Test + public void testCorruptSnapshot2() + { + clearAndPopulate(); + // with a multiple snapshots following toQuery, corrupt the most recent. Ordinarily, we would expect the + // returned LogState to include the most recent snapshot plus subsequent entries. With the corruption, the base + // state should be the previous, valid snapshot plus all subsequent entries. + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + ClusterMetadata lastSnapshot = epochToSnapshot.get(snapshots.next()); + ClusterMetadata previousSnapshot = epochToSnapshot.get(snapshots.next()); + + Epoch toQuery = Epoch.create(previousSnapshot.epoch.getEpoch() - 3); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, lastSnapshot, lastSnapshot.epoch, maxEpoch); + + QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", lastSnapshot.epoch.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, previousSnapshot, previousSnapshot.epoch, maxEpoch); + } + + @Test + public void testCorruptSnapshot3() + { + clearAndPopulate(); + // with a multiple snapshots following toQuery, corrupt the n-most recent. Ordinarily, we would expect the + // returned LogState to include the most recent snapshot plus subsequent entries. With the corruption, the base + // state should be the first valid snapshot plus all subsequent entries. + List toCorrupt = new ArrayList<>(); + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + ClusterMetadata lastSnapshot = epochToSnapshot.get(snapshots.next()); + toCorrupt.add(lastSnapshot); + for ( int i=0; i < 3; i++) + toCorrupt.add(epochToSnapshot.get(snapshots.next())); + ClusterMetadata validSnapshot = epochToSnapshot.get(snapshots.next()); + + Epoch toQuery = Epoch.create(validSnapshot.epoch.getEpoch() - 3); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, lastSnapshot, lastSnapshot.epoch, maxEpoch); + + toCorrupt.forEach(cm -> QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", cm.epoch.getEpoch())); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, validSnapshot, validSnapshot.epoch, maxEpoch); + } + + @Test + public void testCorruptSnapshot4() + { + clearAndPopulate(); + // Corrupt every snapshot following toQuery, corrupt the n-most recent. Ordinarily, we would expect the + // returned LogState to include the most recent snapshot plus subsequent entries. With the corruption, the log + // state should contain just a list of all entries from toQuery to current. + List toCorrupt = new ArrayList<>(); + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + ClusterMetadata lastSnapshot = epochToSnapshot.get(snapshots.next()); + toCorrupt.add(lastSnapshot); + for ( int i=0; i < 3; i++) + toCorrupt.add(epochToSnapshot.get(snapshots.next())); + ClusterMetadata validSnapshot = epochToSnapshot.get(snapshots.next()); + + Epoch toQuery = Epoch.create(validSnapshot.epoch.getEpoch() + 1); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, lastSnapshot, lastSnapshot.epoch, maxEpoch); + + toCorrupt.forEach(cm -> QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", cm.epoch.getEpoch())); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + } + + @Test + public void testIncompleteLogAndCorruptSnapshot1() + { + clearAndPopulate(); + // delete an entry from the log which precedes the last snapshot and also corrupt that snapshot. Both the deleted + // entry and snapshot are after toQuery so it's not possible to construct a contiguous sequence from since to + // current, so instead we get just the entries which follow the deleted one Epoch lastSnapshot = epochToSnapshot.lastKey(); Epoch toDelete = Epoch.create(lastSnapshot.getEpoch() - 2); Epoch toQuery = Epoch.create(lastSnapshot.getEpoch() - 3); - LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, toQuery); - assertCorrectLogState(logState, null, toQuery, epochToPeriod.lastKey()); - QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE period = ? and epoch = ?", epochToPeriod.get(toDelete), toDelete.getEpoch()); - QueryProcessor.executeInternal("DELETE FROM system.metadata_snapshots WHERE epoch = ?", epochToSnapshot.lastKey().getEpoch()); - logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(ClusterMetadata.current().period, toQuery); - assertCorrectLogState(logState, null, toDelete, epochToPeriod.lastKey()); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", lastSnapshot.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toDelete, maxEpoch); } + @Test + public void testIncompleteLogAndCorruptSnapshot2() + { + clearAndPopulate(); + // delete an entry from the log which comes after the last snapshot and also corrupt that snapshot. + // We would normally expect to skip over the intervening snapshot and return just a list of entries from toQuery + // to current. Because we cannot make that list continuous, but we also cannot read the snapshot, instead we get + // just the entries which we are present and have an epoch > toQuery. + Epoch lastSnapshot = epochToSnapshot.lastKey(); + Epoch toDelete = Epoch.create(lastSnapshot.getEpoch() + 2); + Epoch toQuery = Epoch.create(lastSnapshot.getEpoch() - 3); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, null, toQuery, maxEpoch); + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", lastSnapshot.getEpoch()); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertNull(logState.baseState); + boolean containsExpectedEntries = logState.entries.stream() + .map(entry -> entry.epoch.getEpoch()) + .allMatch(e -> e > toQuery.getEpoch() + && e <= maxEpoch.getEpoch() + && e != toDelete.getEpoch()); + assertTrue(containsExpectedEntries); + } + + @Test + public void testIncompleteLogAndCorruptSnapshot3() + { + clearAndPopulate(); + // delete an entry from the log which is followed by multiple corrupt snapshots. Without corruption, querying + // from a point before the deleted entry would return a LogState with the most recent snapshot plus subsequent + // entries. Where all the relevant snapshots are corrupt, expect a LogState with only the non-continuous list + // of available entries between toQuery and current. + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + List toCorrupt = new ArrayList<>(); + for (int i=0; i<3; i++) + toCorrupt.add(snapshots.next()); + Epoch toQuery = Epoch.create(snapshots.next().getEpoch() + 1); + Epoch toDelete = Epoch.create(toQuery.getEpoch() + 2); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, epochToSnapshot.lastEntry().getValue(), epochToSnapshot.lastKey(), maxEpoch); + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + toCorrupt.forEach(e -> QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", e.getEpoch())); + + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertNull(logState.baseState); + boolean containsExpectedEntries = logState.entries.stream() + .map(entry -> entry.epoch.getEpoch()) + .allMatch(e -> e > toQuery.getEpoch() + && e <= maxEpoch.getEpoch() + && e != toDelete.getEpoch()); + assertTrue(containsExpectedEntries); + } + + @Test + public void testIncompleteLogAndCorruptSnapshot4() + { + clearAndPopulate(); + // Corrupt multiple snapshots and delete an entry which follows them in the log. Without corruption, querying + // from an epoch before those snapshots deleted entry would return a LogState with the most recent snapshot plus + // subsequent, non-consecutive entries. Where all the relevant snapshots are corrupt, expect a LogState with + // only the list of available, and still non-consecutive entries between toQuery and current. + Iterator snapshots = epochToSnapshot.descendingKeySet().iterator(); + List toCorrupt = new ArrayList<>(); + for (int i=0; i<3; i++) + toCorrupt.add(snapshots.next()); + Epoch toQuery = Epoch.create(snapshots.next().getEpoch() + 1); + Epoch toDelete = Epoch.create(maxEpoch.getEpoch() - 2); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertCorrectLogState(logState, epochToSnapshot.lastEntry().getValue(), epochToSnapshot.lastKey(), maxEpoch); + + QueryProcessor.executeInternal("DELETE FROM system.local_metadata_log WHERE epoch = ?", toDelete.getEpoch()); + toCorrupt.forEach(e -> QueryProcessor.executeInternal("DELETE snapshot FROM system.metadata_snapshots WHERE epoch = ?", e.getEpoch())); + logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(toQuery); + assertNull(logState.baseState); + boolean containsExpectedEntries = logState.entries.stream() + .map(entry -> entry.epoch.getEpoch()) + .allMatch(e -> e > toQuery.getEpoch() + && e <= maxEpoch.getEpoch() + && e != toDelete.getEpoch()); + assertTrue(containsExpectedEntries); + } + + private void testGetLogStateHelper() { + // If there is only a single snapshot between the starting point and the current epoch, we prefer a LogState + // with no base snapshot and just the consecutive entries. In all other cases, the LogState should include the + // most recent snapshot that can be successfully read and was taken after the starting epoch along with any + // subsequent entries. + Epoch lastSnapshotAt = epochToSnapshot.lastKey(); + Epoch lastEligibleSnapshotAt = epochToSnapshot.lowerKey(lastSnapshotAt); for (int i = 1; i < 20; i++) { - long currentPeriod = ClusterMetadata.current().period; - Epoch start = Epoch.create(epochToPeriod.lastKey().getEpoch() - i); - LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(currentPeriod, start); + Epoch start = Epoch.create(maxEpoch.getEpoch() - i); + LogState logState = SystemKeyspaceStorage.SystemKeyspace.getLogState(start); ClusterMetadata expectedBase; Epoch expectedStart; - // start.nextEpoch here because start is exclusive - so if we can fill the `since` without a snapshot within - // the previous two periods we do so without a snapshot. - if (epochToPeriod.get(start.nextEpoch()) == currentPeriod || epochToPeriod.get(start.nextEpoch()) == currentPeriod - 1) + if (start.isEqualOrAfter(lastEligibleSnapshotAt)) { expectedBase = null; expectedStart = start; @@ -150,7 +390,7 @@ public class GetLogStateTest expectedBase = epochToSnapshot.lastEntry().getValue(); expectedStart = null; } - assertCorrectLogState(logState, expectedBase, expectedStart, epochToPeriod.lastKey()); + assertCorrectLogState(logState, expectedBase, expectedStart, maxEpoch); } } @@ -179,19 +419,16 @@ public class GetLogStateTest private void clearAndPopulate() { - epochToPeriod.clear(); epochToSnapshot.clear(); for (int i = 0; i < 10; i++) { - ClusterMetadata metadata = ClusterMetadataService.instance().sealPeriod(); + ClusterMetadata metadata = ClusterMetadataService.instance().triggerSnapshot(); epochToSnapshot.put(metadata.epoch, metadata); - epochToPeriod.put(metadata.epoch, metadata.period); for (int j = 0; j < 4; j++) { - metadata = ClusterMetadataService.instance() - .commit(new CustomTransformation(CustomTransformation.PokeInt.NAME, - new CustomTransformation.PokeInt((int) ClusterMetadata.current().epoch.getEpoch()))); - epochToPeriod.put(metadata.epoch, metadata.period); + Transformation transform = new CustomTransformation(CustomTransformation.PokeInt.NAME, + new CustomTransformation.PokeInt((int) ClusterMetadata.current().epoch.getEpoch())); + maxEpoch = ClusterMetadataService.instance().commit(transform).epoch; } } } diff --git a/test/unit/org/apache/cassandra/tcm/LogStateTest.java b/test/unit/org/apache/cassandra/tcm/LogStateTest.java index 2258362fa7..9388ad6333 100644 --- a/test/unit/org/apache/cassandra/tcm/LogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/LogStateTest.java @@ -37,7 +37,7 @@ import org.apache.cassandra.tcm.log.LocalLog; import org.apache.cassandra.tcm.log.LogStorage; import org.apache.cassandra.tcm.ownership.UniformRangePlacement; import org.apache.cassandra.tcm.transformations.CustomTransformation; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -76,7 +76,7 @@ public class LogStateTest @Test public void testRevertEpoch() { - ClusterMetadataService.instance().sealPeriod(); + ClusterMetadataService.instance().triggerSnapshot(); List customEpochs = new ArrayList<>(40); for (int i=0; i < 10; i++) { @@ -87,18 +87,15 @@ public class LogStateTest new CustomTransformation.PokeInt((int) ClusterMetadata.current().epoch.getEpoch()))); customEpochs.add(ClusterMetadata.current().epoch); } - ClusterMetadataService.instance().commit(SealPeriod.instance); + ClusterMetadataService.instance().commit(TriggerSnapshot.instance); } for (Epoch epoch : customEpochs) { ClusterMetadataService.instance().revertToEpoch(epoch); ExtensionValue val = ClusterMetadata.current().extensions.get(CustomTransformation.PokeInt.METADATA_KEY); - if (val == null) - assertEquals(Epoch.create(2), epoch); // not yet any ints poked at epoch = 2 - else - // -1 since we poke the previous int to the extension: - assertEquals((int)(epoch.getEpoch() - 1), ClusterMetadata.current().extensions.get(CustomTransformation.PokeInt.METADATA_KEY).getValue()); + // -1 since we poke the previous int to the extension: + assertEquals((int)(epoch.getEpoch() - 1), ClusterMetadata.current().extensions.get(CustomTransformation.PokeInt.METADATA_KEY).getValue()); } } } diff --git a/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java b/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java index 56c7d4a031..37cee72373 100644 --- a/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/log/DistributedLogStateTest.java @@ -33,9 +33,8 @@ import org.apache.cassandra.schema.DistributedMetadataLogKeyspace; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.transformations.CustomTransformation; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.db.ColumnFamilyStore.FlushReason.UNIT_TESTS; @@ -66,8 +65,6 @@ public class DistributedLogStateTest extends LogStateTestBase // start test entries at FIRST + 1 as the pre-init transform is automatically inserted with Epoch.FIRST Epoch currentEpoch = Epoch.FIRST; Epoch nextEpoch; - long period = Period.FIRST; - long nextPeriod = period; boolean applied; final LogReader reader = new DistributedMetadataLogKeyspace.DistributedTableLogReader(ConsistencyLevel.SERIAL, () -> snapshots); @@ -85,29 +82,20 @@ public class DistributedLogStateTest extends LogStateTestBase boolean applied = DistributedMetadataLogKeyspace.tryCommit(new Entry.Id(currentEpoch.getEpoch()), CustomTransformation.make((int) currentEpoch.getEpoch()), currentEpoch, - nextEpoch, - period, - nextPeriod, - false); + nextEpoch); assertTrue(applied); currentEpoch = nextEpoch; - period = nextPeriod; } @Override - public void sealPeriod() + public void snapshotMetadata() { nextEpoch = currentEpoch.nextEpoch(); applied = DistributedMetadataLogKeyspace.tryCommit(new Entry.Id(currentEpoch.getEpoch()), - SealPeriod.instance, + TriggerSnapshot.instance, currentEpoch, - nextEpoch, - period, - period, - true); + nextEpoch); assertTrue(applied); - // after appending a SealPeriod, move to a new partition - nextPeriod++; currentEpoch = nextEpoch; // flush log table periodically so queries are served from disk ColumnFamilyStore.getIfExists(DistributedMetadataLogKeyspace.Log.id).forceBlockingFlush(UNIT_TESTS); @@ -116,19 +104,18 @@ public class DistributedLogStateTest extends LogStateTestBase @Override public LogState getLogState(Epoch since) { - return reader.getLogState(NUM_PERIODS + 1, since); + return reader.getLogState(since); } @Override public void dumpTables() throws IOException { - UntypedResultSet r = executeInternal("SELECT period, epoch, entry_id, kind FROM system_cluster_metadata.distributed_metadata_log"); + UntypedResultSet r = executeInternal("SELECT epoch, entry_id, kind FROM system_cluster_metadata.distributed_metadata_log"); r.forEach(row -> { - long p = row.getLong("period"); long e = row.getLong("epoch"); long i = row.getLong("entry_id"); String s = row.getString("kind"); - System.out.println(String.format("(%d, %d, %d, %s)", p, e, i, s)); + System.out.println(String.format("(%d, %d, %s)", e, i, s)); }); } }; diff --git a/test/unit/org/apache/cassandra/tcm/log/LocalLogTest.java b/test/unit/org/apache/cassandra/tcm/log/LocalLogTest.java index 816c491f1a..fac8d04d6f 100644 --- a/test/unit/org/apache/cassandra/tcm/log/LocalLogTest.java +++ b/test/unit/org/apache/cassandra/tcm/log/LocalLogTest.java @@ -42,7 +42,7 @@ import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.transformations.CustomTransformation; import org.apache.cassandra.tcm.transformations.ForceSnapshot; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import org.apache.cassandra.utils.concurrent.CountDownLatch; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; @@ -86,7 +86,7 @@ public class LocalLogTest } @Test - public void sealPeriodForceSnapshotCollisionWithGap() + public void forceSnapshotCollisionWithGap() { LocalLog log = LocalLog.logSpec() .sync() @@ -99,7 +99,7 @@ public class LocalLogTest entries.add(entry(i)); entries.add(new Entry(Entry.Id.NONE, Epoch.create(11), - SealPeriod.instance)); + TriggerSnapshot.instance)); entries.add(new Entry(Entry.Id.NONE, Epoch.create(11), @@ -127,7 +127,7 @@ public class LocalLogTest entries.add(entry(i)); entries.add(new Entry(Entry.Id.NONE, Epoch.create(11), - SealPeriod.instance)); + TriggerSnapshot.instance)); entries.add(new Entry(Entry.Id.NONE, Epoch.create(11), diff --git a/test/unit/org/apache/cassandra/tcm/log/LocalStorageLogStateTest.java b/test/unit/org/apache/cassandra/tcm/log/LocalStorageLogStateTest.java index 9b7d855aa6..5bc6ec0fa8 100644 --- a/test/unit/org/apache/cassandra/tcm/log/LocalStorageLogStateTest.java +++ b/test/unit/org/apache/cassandra/tcm/log/LocalStorageLogStateTest.java @@ -28,13 +28,11 @@ import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; import org.apache.cassandra.tcm.transformations.CustomTransformation; -import org.apache.cassandra.tcm.transformations.SealPeriod; +import org.apache.cassandra.tcm.transformations.TriggerSnapshot; import static org.apache.cassandra.cql3.QueryProcessor.executeInternal; import static org.apache.cassandra.db.SystemKeyspace.METADATA_LOG; @@ -62,7 +60,6 @@ public class LocalStorageLogStateTest extends LogStateTestBase { SystemKeyspaceStorage storage = new SystemKeyspaceStorage(() -> snapshots); Epoch epoch = Epoch.FIRST; - long period = Period.FIRST; @Override public void cleanup() @@ -78,40 +75,35 @@ public class LocalStorageLogStateTest extends LogStateTestBase // so fake an extra entry here to keep the test data in sync. if (epoch.is(Epoch.FIRST)) { - storage.append(period, new Entry(new Entry.Id(epoch.getEpoch()), epoch, CustomTransformation.make((int) epoch.getEpoch()))); + storage.append(new Entry(new Entry.Id(epoch.getEpoch()), epoch, CustomTransformation.make((int) epoch.getEpoch()))); epoch = epoch.nextEpoch(); } - storage.append(period, new Entry(new Entry.Id(epoch.getEpoch()), epoch, CustomTransformation.make((int) epoch.getEpoch()))); + storage.append(new Entry(new Entry.Id(epoch.getEpoch()), epoch, CustomTransformation.make((int) epoch.getEpoch()))); epoch = epoch.nextEpoch(); } @Override - public void sealPeriod() throws IOException + public void snapshotMetadata() throws IOException { - storage.append(period, new Entry(new Entry.Id(epoch.getEpoch()), epoch, SealPeriod.instance)); + storage.append(new Entry(new Entry.Id(epoch.getEpoch()), epoch, TriggerSnapshot.instance)); epoch = epoch.nextEpoch(); - period += 1; - // required so we have a starting point for finding the right period to build - // replication from _if_ the max_epoch -> period table is lost - ClusterMetadataTestHelper.forceCurrentPeriodTo(period); } @Override public LogState getLogState(Epoch since) { - return storage.getLogState(NUM_PERIODS + 1, since); + return storage.getLogState(since); } @Override public void dumpTables() throws IOException { - UntypedResultSet r = executeInternal("SELECT period, epoch, entry_id, kind FROM system.local_metadata_log"); + UntypedResultSet r = executeInternal("SELECT epoch, entry_id, kind FROM system.local_metadata_log"); r.forEach(row -> { - long p = row.getLong("period"); long e = row.getLong("epoch"); long i = row.getLong("entry_id"); String s = row.getString("kind"); - System.out.println(String.format("(%d, %d, %d, %s)", p, e, i, s)); + System.out.println(String.format("(%d, %d, %s)", e, i, s)); }); } }; diff --git a/test/unit/org/apache/cassandra/tcm/log/LogStateTestBase.java b/test/unit/org/apache/cassandra/tcm/log/LogStateTestBase.java index c073da8bb1..c7df0141fb 100644 --- a/test/unit/org/apache/cassandra/tcm/log/LogStateTestBase.java +++ b/test/unit/org/apache/cassandra/tcm/log/LogStateTestBase.java @@ -19,8 +19,8 @@ package org.apache.cassandra.tcm.log; import java.io.IOException; +import java.util.ArrayList; import java.util.List; -import java.util.Objects; import org.junit.Before; import org.junit.Test; @@ -30,7 +30,7 @@ import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.MetadataSnapshots; -import org.apache.cassandra.tcm.Period; +import org.apache.cassandra.tcm.sequences.SequencesUtils; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -39,17 +39,17 @@ import static org.junit.Assert.fail; public abstract class LogStateTestBase { - static int PERIOD_SIZE = 5; - static int NUM_PERIODS = 10; + static int SNAPSHOT_FREQUENCY = 5; + static int NUM_SNAPSHOTS = 10; static int EXTRA_ENTRIES = 2; - static int CURRENT_EPOCH = (NUM_PERIODS * PERIOD_SIZE) + EXTRA_ENTRIES; - static Sealed REAL_LAST_SEALED = new Sealed(NUM_PERIODS, Epoch.create(NUM_PERIODS * PERIOD_SIZE)); + static Epoch CURRENT_EPOCH = Epoch.create((NUM_SNAPSHOTS * SNAPSHOT_FREQUENCY) + EXTRA_ENTRIES); + static Epoch LATEST_SNAPSHOT_EPOCH = Epoch.create(NUM_SNAPSHOTS * SNAPSHOT_FREQUENCY); interface LogStateSUT { void cleanup() throws IOException; void insertRegularEntry() throws IOException; - void sealPeriod() throws IOException; + void snapshotMetadata() throws IOException; LogState getLogState(Epoch since); // just for manually checking the test data @@ -63,59 +63,71 @@ public abstract class LogStateTestBase { LogStateSUT sut = getSystemUnderTest(MetadataSnapshots.NO_OP); sut.cleanup(); - for (long i = 0; i < NUM_PERIODS; i++) + for (long i = 0; i < NUM_SNAPSHOTS; i++) { - // for the very first period (partition in the log table) we must write 1 fewer entries + // for the very first snapshot we must write 1 fewer entries // as the pre-init entry is automatically inserted with Epoch.FIRST when the table is empty - int entriesPerPeriod = PERIOD_SIZE - (i == 0 ? 2 : 1); - for (int j = 0; j < entriesPerPeriod; j++) + int entriesPerSnapshot = SNAPSHOT_FREQUENCY - (i == 0 ? 2 : 1); + for (int j = 0; j < entriesPerSnapshot; j++) sut.insertRegularEntry(); - sut.sealPeriod(); + sut.snapshotMetadata(); } - // Add 2 more Entries, which will be after the last sealed period. The point is to test what happens when - // we have to use the period to epoch reverse index, and the epochs are beyond the max indexed for (int i = 0; i < 2; i++) sut.insertRegularEntry(); sut.dumpTables(); } - static class TestSnapshots extends MetadataSnapshots.SystemKeyspaceMetadataSnapshots {}; - - static MetadataSnapshots withMissingSnapshot(Epoch ... expected) + static class TestSnapshots extends MetadataSnapshots.NoOp { - return new TestSnapshots() + + Epoch[] expected; + int idx; + boolean corrupt; + TestSnapshots(Epoch[] expected, boolean corrupt) { - int idx = 0; - @Override - public ClusterMetadata getSnapshot(Epoch since) - { - if (idx >= expected.length) - throw new AssertionError("Should not have gotten a query for "+since); - assertEquals(expected[idx++], since); - return null; - } - }; + this.expected = expected; + this.idx = 0; + this.corrupt = corrupt; + } + + @Override + public ClusterMetadata getSnapshot(Epoch since) + { + if (idx >= expected.length) + throw new AssertionError("Should not have gotten a query for "+since); + assertEquals(expected[idx++], since); + return corrupt ? null : ClusterMetadataTestHelper.minimalForTesting(Murmur3Partitioner.instance).forceEpoch(since); + } + + @Override + public List listSnapshotsSince(Epoch epoch) + { + List list = new ArrayList<>(); + for (Epoch e : expected) + if (e.isAfter(epoch)) + list.add(e); + + return list; + } + + }; + + static MetadataSnapshots withCorruptSnapshots(Epoch ... expected) + { + return new TestSnapshots(expected, true); } - static MetadataSnapshots withAvailableSnapshot(Epoch expected) + static MetadataSnapshots withAvailableSnapshots(Epoch ... expected) { - return new TestSnapshots() - { - @Override - public ClusterMetadata getSnapshot(Epoch since) - { - assertEquals(expected, since); - return ClusterMetadataTestHelper.minimalForTesting(Murmur3Partitioner.instance).forceEpoch(expected); - } - }; + return new TestSnapshots(expected, false); } static MetadataSnapshots throwing() { - return new TestSnapshots() + return new MetadataSnapshots.NoOp() { @Override public ClusterMetadata getSnapshot(Epoch epoch) @@ -127,115 +139,115 @@ public abstract class LogStateTestBase } @Test - public void sinceIsEmptyWithMissingSnapshot() + public void sinceIsEmptyWithCorruptSnapshots() { - Epoch [] queriedEpochs = new Epoch[NUM_PERIODS]; - for (int i = 0; i < NUM_PERIODS; i++) - queriedEpochs[i] = Epoch.create((REAL_LAST_SEALED.period - i) * PERIOD_SIZE); - MetadataSnapshots missingSnapshot = withMissingSnapshot(queriedEpochs); + Epoch [] queriedEpochs = new Epoch[NUM_SNAPSHOTS]; + for (int i = 0; i < NUM_SNAPSHOTS; i++) + queriedEpochs[i] = SequencesUtils.epoch((NUM_SNAPSHOTS - i) * SNAPSHOT_FREQUENCY); + MetadataSnapshots missingSnapshot = withCorruptSnapshots(queriedEpochs); LogState state = getSystemUnderTest(missingSnapshot).getLogState(Epoch.EMPTY); assertNull(state.baseState); - assertReplication(state.entries, 1, CURRENT_EPOCH); + assertEntries(state.entries, Epoch.FIRST, CURRENT_EPOCH); } @Test - public void sinceIsEmptyWithAvailableSnapshot() + public void sinceIsEmptyWithValidSnapshots() { - final Epoch expected = REAL_LAST_SEALED.epoch; - MetadataSnapshots withSnapshot = withAvailableSnapshot(expected); - LogState state = getSystemUnderTest(withSnapshot).getLogState(Epoch.EMPTY); - assertEquals(expected, state.baseState.epoch); - assertReplication(state.entries, expected.nextEpoch().getEpoch(), CURRENT_EPOCH); + MetadataSnapshots withSnapshots = withAvailableSnapshots(LATEST_SNAPSHOT_EPOCH, + SequencesUtils.epoch(((NUM_SNAPSHOTS - 1) * SNAPSHOT_FREQUENCY)), + SequencesUtils.epoch(((NUM_SNAPSHOTS - 2) * SNAPSHOT_FREQUENCY))); + LogState state = getSystemUnderTest(withSnapshots).getLogState(Epoch.EMPTY); + assertEquals(LATEST_SNAPSHOT_EPOCH, state.baseState.epoch); + assertEntries(state.entries, LATEST_SNAPSHOT_EPOCH.nextEpoch(), CURRENT_EPOCH); } @Test - public void sinceIsBeforeLastSealedButMissingSnapshot() + public void sinceIsBeforeLastSnapshotWithCorruptSnapshot() { - MetadataSnapshots missingSnapshot = withMissingSnapshot(REAL_LAST_SEALED.epoch, - Epoch.create(((REAL_LAST_SEALED.period - 1) * PERIOD_SIZE )), - Epoch.create(((REAL_LAST_SEALED.period - 2) * PERIOD_SIZE ))); - // an arbitrary epoch earlier than the last sealed - Epoch since = Epoch.create(((REAL_LAST_SEALED.period - 3) * PERIOD_SIZE ) + 2); + MetadataSnapshots missingSnapshot = withCorruptSnapshots(LATEST_SNAPSHOT_EPOCH, + SequencesUtils.epoch(((NUM_SNAPSHOTS - 1) * SNAPSHOT_FREQUENCY)), + SequencesUtils.epoch(((NUM_SNAPSHOTS - 2) * SNAPSHOT_FREQUENCY))); + // an arbitrary epoch earlier than the last snapshot + Epoch since = SequencesUtils.epoch(((NUM_SNAPSHOTS - 3) * SNAPSHOT_FREQUENCY) + 2); LogState state = getSystemUnderTest(missingSnapshot).getLogState(since); assertNull(state.baseState); - assertReplication(state.entries, since.nextEpoch().getEpoch(), CURRENT_EPOCH); + assertEntries(state.entries, since.nextEpoch(), CURRENT_EPOCH); } @Test - public void sinceIsBeforeLastSealedWithSnapshot() + public void sinceIsBeforeLastSnapshotWithValidSnapshot() { - final Epoch expected = REAL_LAST_SEALED.epoch; - MetadataSnapshots withSnapshot = withAvailableSnapshot(expected); - // an arbitrary epoch earlier than the last sealed - Epoch since = Epoch.create(((REAL_LAST_SEALED.period - 3) * PERIOD_SIZE ) + 2); - LogState state = getSystemUnderTest(withSnapshot).getLogState(since); - assertEquals(expected, state.baseState.epoch); - assertReplication(state.entries, expected.nextEpoch().getEpoch(), CURRENT_EPOCH); - } - - @Test - public void sinceIsMaxInLastSealedWithSnapshot() - { - // the max epoch in the last sealed period(but not the current highest epoch) - final Epoch since = REAL_LAST_SEALED.epoch; - MetadataSnapshots withSnapshot = withAvailableSnapshot(since); + MetadataSnapshots withSnapshot = withAvailableSnapshots(LATEST_SNAPSHOT_EPOCH); + // an arbitrary epoch earlier than the last snapshot + Epoch since = SequencesUtils.epoch(((NUM_SNAPSHOTS - 3) * SNAPSHOT_FREQUENCY) + 2); LogState state = getSystemUnderTest(withSnapshot).getLogState(since); assertNull(state.baseState); - assertReplication(state.entries, since.nextEpoch().getEpoch(), CURRENT_EPOCH); + assertEntries(state.entries, since.nextEpoch(), CURRENT_EPOCH); } @Test - public void sinceIsMaxInLastSealedButMissingSnapshot() + public void sinceIsEqualLastSnapshotWithValidSnapshot() { - // the max epoch in the last sealed period(but not the current highest epoch) - final Epoch since = REAL_LAST_SEALED.epoch; - MetadataSnapshots missingSnapshot = withMissingSnapshot(since); + // the max epoch in the last snapshot (but not the current highest epoch) + final Epoch since = LATEST_SNAPSHOT_EPOCH; + MetadataSnapshots withSnapshot = withAvailableSnapshots(since); + LogState state = getSystemUnderTest(withSnapshot).getLogState(since); + assertNull(state.baseState); + assertEntries(state.entries, since.nextEpoch(), CURRENT_EPOCH); + } + + @Test + public void sinceIsEqualLastSnapshotWithCorruptSnapshot() + { + // the max epoch in the last snapshot (but not the current highest epoch) + final Epoch since = LATEST_SNAPSHOT_EPOCH; + MetadataSnapshots missingSnapshot = withCorruptSnapshots(since); LogState state = getSystemUnderTest(missingSnapshot).getLogState(since); assertNull(state.baseState); - assertReplication(state.entries, since.nextEpoch().getEpoch(), CURRENT_EPOCH); + assertEntries(state.entries, since.nextEpoch(), CURRENT_EPOCH); } @Test - public void sinceIsAfterLastSealed() + public void sinceIsAfterLastSnapshot() { MetadataSnapshots snapshots = throwing(); - // an arbitrary epoch later than the last sealed (but not the current highest epoch) - Epoch since = Epoch.create(CURRENT_EPOCH - 1); + // an arbitrary epoch later than the last snapshot (but not the current highest epoch) + Epoch since = Epoch.create(CURRENT_EPOCH.getEpoch() - 1); LogState state = getSystemUnderTest(snapshots).getLogState(since); assertNull(state.baseState); - assertReplication(state.entries, since.nextEpoch().getEpoch(), CURRENT_EPOCH); + assertEntries(state.entries, since.nextEpoch(), CURRENT_EPOCH); } @Test - public void sinceIsMaxAfterLastSealed() + public void sinceIsMaxAfterLastSnapshot() { MetadataSnapshots snapshots = throwing(); - // the current highest epoch, which > the max epoch in the last sealed period - Epoch since = Epoch.create(CURRENT_EPOCH); + // the current highest epoch, which is > the epoch of the last snapshot + Epoch since = CURRENT_EPOCH; LogState state = getSystemUnderTest(snapshots).getLogState(since); assertNull(state.baseState); assertTrue(state.entries.isEmpty()); } @Test - public void sinceArbitraryEpochWithSealedButMissingSnapshot() + public void sinceArbitraryEpochWithMultipleCorruptSnapshots() { Epoch since = Epoch.create(35); - Epoch expected = REAL_LAST_SEALED.epoch; - MetadataSnapshots missingSnapshot = withMissingSnapshot(expected, // 50 - Epoch.create(expected.getEpoch() - PERIOD_SIZE), // 45 - Epoch.create(expected.getEpoch() - PERIOD_SIZE * 2L)); // 40 + Epoch expected = LATEST_SNAPSHOT_EPOCH; + MetadataSnapshots missingSnapshot = withCorruptSnapshots(expected, // 50 + Epoch.create(expected.getEpoch() - SNAPSHOT_FREQUENCY), // 45 + Epoch.create(expected.getEpoch() - SNAPSHOT_FREQUENCY * 2L)); // 40 LogState state = getSystemUnderTest(missingSnapshot).getLogState(since); assertNull(state.baseState); - assertReplication(state.entries, since.nextEpoch().getEpoch(), CURRENT_EPOCH); + assertEntries(state.entries, since.nextEpoch(), CURRENT_EPOCH); } - private void assertReplication(List entries, long min, long max) + private void assertEntries(List entries, Epoch min, Epoch max) { int idx = 0; - for (long i = min; i <= max; i++) + for (long i = min.getEpoch(); i <= max.getEpoch(); i++) { Entry e = entries.get(idx); assertEquals(e.epoch.getEpoch(), i); @@ -243,48 +255,4 @@ public abstract class LogStateTestBase } assertEquals(idx, entries.size()); } - - - public static class Sealed implements Comparable - { - public static final Sealed EMPTY = new Sealed(Period.EMPTY, Epoch.EMPTY); - public final long period; - public final Epoch epoch; - - public Sealed(long period, Epoch epoch) - { - this.period = period; - this.epoch = epoch; - } - - @Override - public String toString() - { - return "Sealed{" + - "period=" + period + - ", epoch=" + epoch + - '}'; - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (!(o instanceof Sealed)) return false; - Sealed sealed = (Sealed) o; - return period == sealed.period && epoch.equals(sealed.epoch); - } - - @Override - public int hashCode() - { - return Objects.hash(period, epoch); - } - - @Override - public int compareTo(Sealed o) - { - return Long.compare(this.epoch.getEpoch(), o.epoch.getEpoch()); - } - } } diff --git a/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java b/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java index 9ffac10c52..6c0580ea41 100644 --- a/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java +++ b/test/unit/org/apache/cassandra/tcm/sequences/SequencesUtils.java @@ -169,4 +169,9 @@ public class SequencesUtils new PrepareMove.FinishMove(node, tokens, deltas, key), true); } + + public static Epoch epoch(int epoch) + { + return Epoch.create(epoch); + } }