natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Initial witness non-serial single partition read/write path 

Patch by Ariel Weisberg; Reviewed by Abe Ratnosfky for CASSANDRA-20930
This commit is contained in:
Ariel Weisberg 2025-04-18 17:45:16 -04:00 committed by Blake Eggleston
parent 9d856f2c08
commit 1e036cf620
130 changed files with 1252 additions and 848 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/java/org/apache/cassandra/cql3/statements/schema/CreateTableStatement.java
View File

@ -73,7 +73,6 @@ import org.apache.cassandra.schema.TableParams;
import org.apache.cassandra.schema.Types;
import org.apache.cassandra.schema.UserFunctions;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.reads.repair.ReadRepairStrategy;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.serialization.Version;
import org.apache.cassandra.transport.Event.SchemaChange;
@ -176,11 +175,12 @@ public final class CreateTableStatement extends AlterSchemaStatement
TableMetadata table = builder.build();
table.validate();
if (keyspace.replicationStrategy.hasTransientReplicas()
&& table.params.readRepair != ReadRepairStrategy.NONE)
{
throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces");
}
// TODO (review): This can be removed right? ReadRepair is effectively not done anymore so the setting doesn't matter
// if (keyspace.replicationStrategy.hasTransientReplicas()
// && table.params.readRepair != ReadRepairStrategy.NONE)
// {
// throw ire("read_repair must be set to 'NONE' for transiently replicated keyspaces");
// }
if (!table.params.compression.isEnabled() && !SchemaConstants.isSystemKeyspace(table.keyspace))
Guardrails.uncompressedTablesEnabled.ensureEnabled(state);

10
src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java
View File

@ -29,6 +29,7 @@ import org.apache.cassandra.metrics.TCMMetrics;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -107,6 +108,7 @@ public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<
private ClusterMetadata checkTokenOwnership(ClusterMetadata metadata, Message<T> message)
{
boolean acceptsTransient = message.verb() == Verb.TRACKED_SUMMARY_REQ;
ReadCommand command = getCommand(message.payload);
if (command.metadata().isVirtual())
@ -136,11 +138,11 @@ public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<
throw InvalidRoutingException.forTokenRead(message.from(), token, metadata.epoch, command);
}
if (!command.acceptsTransient() && localReplica.isTransient())
if (!acceptsTransient && localReplica.isTransient())
{
MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s",
command.acceptsTransient() ? "transient" : "full",
acceptsTransient ? "transient" : "full",
localReplica.isTransient() ? "transient" : "full",
this));
}
@ -164,11 +166,11 @@ public abstract class AbstractReadCommandVerbHandler<T> implements IVerbHandler<
}
// TODO: preexisting issue: we should change the whole range for transient-ness, not just the right token
if (command.acceptsTransient() != maxTokenLocalReplica.isTransient())
if (!acceptsTransient && maxTokenLocalReplica.isTransient())
{
MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS);
throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s",
command.acceptsTransient() ? "transient" : "full",
acceptsTransient ? "transient" : "full",
maxTokenLocalReplica.isTransient() ? "transient" : "full",
this));
}

13
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -676,19 +676,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
return memtableFactory.streamFromMemtable();
}
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SerializationHeader header, ILifecycleTransaction txn)
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SerializationHeader header, ILifecycleTransaction txn)
{
return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, null, 0, header, txn);
return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, coordinatorLogOffsets, null, 0, header, txn);
}
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, IntervalSet<CommitLogPosition> commitLogPositions, SerializationHeader header, ILifecycleTransaction txn)
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, IntervalSet<CommitLogPosition> commitLogPositions, SerializationHeader header, ILifecycleTransaction txn)
{
return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, commitLogPositions, 0, header, txn);
return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, coordinatorLogOffsets, commitLogPositions, 0, header, txn);
}
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, IntervalSet<CommitLogPosition> commitLogPositions, int sstableLevel, SerializationHeader header, ILifecycleTransaction txn)
public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, IntervalSet<CommitLogPosition> commitLogPositions, int sstableLevel, SerializationHeader header, ILifecycleTransaction txn)
{
return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), txn);
return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, coordinatorLogOffsets, commitLogPositions, sstableLevel, header, indexManager.listIndexGroups(), txn);
}
public boolean supportsEarlyOpen()
@ -2441,7 +2441,6 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
keys,
0,
repairSessionID,
false,
logOffsetsBuilder.build(),
commitLogIntervals.build(),
new SerializationHeader(true,

2
src/java/org/apache/cassandra/db/DiskBoundaryManager.java
View File

@ -176,7 +176,7 @@ public class DiskBoundaryManager
weightedRanges.add(new Splitter.WeightedRange(1.0, r));
for (Range<Token> r : Range.sort(replicas.onlyTransient().ranges()))
weightedRanges.add(new Splitter.WeightedRange(0.1, r));
weightedRanges.add(new Splitter.WeightedRange(0.00001, r));
weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));

36
src/java/org/apache/cassandra/db/Keyspace.java
View File

@ -48,12 +48,15 @@ import org.apache.cassandra.db.repair.CassandraKeyspaceRepairManager;
import org.apache.cassandra.db.tracked.TrackedKeyspaceWriteHandler;
import org.apache.cassandra.db.view.ViewManager;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.index.Index;
import org.apache.cassandra.index.SecondaryIndexManager;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.metrics.KeyspaceMetrics;
import org.apache.cassandra.repair.KeyspaceRepairManager;
import org.apache.cassandra.replication.MutationTrackingService;
@ -64,6 +67,7 @@ import org.apache.cassandra.schema.SchemaProvider;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.consensus.migration.ConsensusMigrationMutationHelper;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.JVMStabilityInspector;
@ -611,6 +615,7 @@ public class Keyspace
private Future<?> applyInternalTracked(Mutation mutation, Promise<?> future)
{
Preconditions.checkState(getMetadata().useMutationTracking() && !mutation.id().isNone());
ClusterMetadata cm = ClusterMetadata.current();
if (TEST_FAIL_WRITES && getMetadata().name.equals(TEST_FAIL_WRITES_KS))
throw new RuntimeException("Testing write failures");
@ -631,8 +636,39 @@ public class Keyspace
continue;
}
// If this range is only witnessed then don't apply the update to the underlying column family store
// We still want the mutation tracking log to see the update so that it can witness it
// and participate in reconciliation of the mutation
AbstractReplicationStrategy replicationStrategy = cfs.keyspace.getReplicationStrategy();
if (replicationStrategy.hasTransientReplicas())
{
RangesAtEndpoint localRanges = replicationStrategy.getLocalRanges(cm);
Token token = upd.partitionKey().getToken();
boolean foundMatch = false;
for (Range<Token> r : localRanges.onlyFull().ranges())
{
if (r.contains(token))
{
foundMatch = true;
break;
}
}
if (!foundMatch)
{
// TODO checkState(!mutation.allowsPotentialTxnConflicts)
// Basically if a transaction system thinks it is writing to a non-witness but is writing to a
// witness then we are probably going to have issues.
// This is problematic/racy in general because schema changes aren't really synced with
// transaction systems yet when in reality they really should be completed by a transaction or some
// other similar solution.
continue;
}
}
cfs.getWriteHandler().write(mutation.id(), upd, ctx, true);
}
}
}

40
src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
View File

@ -78,7 +78,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
protected PartitionRangeReadCommand(Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -89,7 +88,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
Index.QueryPlan indexQueryPlan,
boolean trackWarnings)
{
super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
super(serializedAtEpoch, Kind.PARTITION_RANGE, isDigest, digestVersion, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata());
}
@ -101,7 +100,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
private static PartitionRangeReadCommand create(Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -116,7 +114,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
{
return new VirtualTablePartitionRangeReadCommand(isDigest,
digestVersion,
acceptsTransient,
metadata,
nowInSec,
columnFilter,
@ -129,7 +126,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return new PartitionRangeReadCommand(serializedAtEpoch,
isDigest,
digestVersion,
acceptsTransient,
potentialTxnConflicts,
metadata,
nowInSec,
@ -151,7 +147,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(metadata.epoch,
false,
0,
false,
PotentialTxnConflicts.DISALLOW,
metadata,
nowInSec,
@ -174,7 +169,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(metadata.epoch,
false,
0,
false,
potentialTxnConflicts,
metadata,
nowInSec,
@ -199,7 +193,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(metadata.epoch,
false,
0,
false,
PotentialTxnConflicts.DISALLOW,
metadata,
nowInSec,
@ -246,7 +239,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -268,7 +260,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
PotentialTxnConflicts.ALLOW,
metadata(),
nowInSec,
@ -285,7 +276,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
txnReadName,
@ -302,7 +292,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -320,25 +309,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
true,
digestVersion(),
false,
potentialTxnConflicts(),
metadata(),
nowInSec(),
columnFilter(),
rowFilter(),
limits(),
dataRange(),
indexQueryPlan(),
isTrackingWarnings());
}
@Override
protected PartitionRangeReadCommand copyAsTransientQuery()
{
return create(serializedAtEpoch(),
false,
0,
true,
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -356,7 +326,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -374,7 +343,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -635,7 +603,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -646,7 +613,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
throws IOException
{
DataRange range = DataRange.serializer.deserialize(in, version, metadata);
return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
return PartitionRangeReadCommand.create(serializedAtEpoch, isDigest, digestVersion, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, range, indexQueryPlan, false);
}
}
@ -654,7 +621,6 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
{
private VirtualTablePartitionRangeReadCommand(boolean isDigest,
int digestVersion,
boolean acceptsTransient,
TableMetadata metadata,
long nowInSec,
ColumnFilter columnFilter,
@ -664,7 +630,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
Index.QueryPlan indexQueryPlan,
boolean trackWarnings)
{
super(metadata.epoch, isDigest, digestVersion, acceptsTransient, PotentialTxnConflicts.ALLOW, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
super(metadata.epoch, isDigest, digestVersion, PotentialTxnConflicts.ALLOW, metadata, nowInSec, columnFilter, rowFilter, limits, dataRange, indexQueryPlan, trackWarnings);
}
@Override

48
src/java/org/apache/cassandra/db/ReadCommand.java
View File

@ -161,7 +161,7 @@ public abstract class ReadCommand extends AbstractReadQuery
this.allowed = allowed;
}
}
// Expose the active command running so transitive calls can lookup this command.
// This is useful for a few reasons, but mainly because the CQL query is here.
private static final FastThreadLocal<ReadCommand> COMMAND = new FastThreadLocal<>();
@ -169,7 +169,6 @@ public abstract class ReadCommand extends AbstractReadQuery
private final Kind kind;
private final boolean isDigestQuery;
private final boolean acceptsTransient;
private final Epoch serializedAtEpoch;
private final PotentialTxnConflicts potentialTxnConflicts;
@ -190,7 +189,6 @@ public abstract class ReadCommand extends AbstractReadQuery
Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -219,7 +217,6 @@ public abstract class ReadCommand extends AbstractReadQuery
Kind kind,
boolean isDigestQuery,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -231,13 +228,9 @@ public abstract class ReadCommand extends AbstractReadQuery
DataRange dataRange)
{
super(metadata, nowInSec, columnFilter, rowFilter, limits);
if (acceptsTransient && isDigestQuery)
throw new IllegalArgumentException("Attempted to issue a digest response to transient replica");
this.kind = kind;
this.isDigestQuery = isDigestQuery;
this.digestVersion = digestVersion;
this.acceptsTransient = acceptsTransient;
this.indexQueryPlan = indexQueryPlan;
this.potentialTxnConflicts = potentialTxnConflicts;
this.trackWarnings = trackWarnings;
@ -250,6 +243,11 @@ public abstract class ReadCommand extends AbstractReadQuery
return COMMAND.get();
}
public boolean acceptsTransient()
{
return false;
}
protected abstract void serializeSelection(DataOutputPlus out, int version) throws IOException;
protected abstract void serializeSelectionWithoutKey(DataOutputPlus out, int version) throws IOException;
protected abstract long selectionSerializedSize(int version);
@ -319,14 +317,6 @@ public abstract class ReadCommand extends AbstractReadQuery
return this;
}
/**
* @return Whether this query expects only a transient data response, or a full response
*/
public boolean acceptsTransient()
{
return acceptsTransient;
}
@Override
public void trackWarnings()
{
@ -388,28 +378,6 @@ public abstract class ReadCommand extends AbstractReadQuery
*/
public abstract ReadCommand copy();
/**
* Returns a copy of this command with acceptsTransient set to true.
*/
public ReadCommand copyAsTransientQuery(Replica replica)
{
checkArgument(replica.isTransient(),
"Can't make a transient request on a full replica: " + replica);
return copyAsTransientQuery();
}
/**
* Returns a copy of this command with acceptsTransient set to true.
*/
public ReadCommand copyAsTransientQuery(Iterable<Replica> replicas)
{
if (any(replicas, Replica::isFull))
throw new IllegalArgumentException("Can't make a transient request on full replicas: " + Iterables.toString(filter(replicas, Replica::isFull)));
return copyAsTransientQuery();
}
protected abstract ReadCommand copyAsTransientQuery();
/**
* Returns a copy of this command with isDigestQuery set to true.
*/
@ -1434,7 +1402,7 @@ public abstract class ReadCommand extends AbstractReadQuery
out.writeByte(
digestFlag(command.isDigestQuery())
| indexFlag(null != command.indexQueryPlan())
| acceptsTransientFlag(command.acceptsTransient())
// | acceptsTransientFlag(false) Deprecated flag, could be reused?
| needsReconciliationFlag(command.rowFilter().needsReconciliation())
| potentialTxnConflicts(command.potentialTxnConflicts)
);
@ -1494,7 +1462,7 @@ public abstract class ReadCommand extends AbstractReadQuery
indexQueryPlan = indexGroup.queryPlanFor(rowFilter);
}
return deserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
return deserializer.deserialize(in, version, schemaVersion, isDigest, digestVersion, potentialTxnConflicts, tableMetadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan);
}
public ReadCommand deserialize(DataInputPlus in, int version) throws IOException

3
src/java/org/apache/cassandra/db/SSTableImporter.java
View File

@ -468,8 +468,7 @@ public class SSTableImporter
if (options.clearRepaired)
{
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, ActiveRepairService.UNREPAIRED_SSTABLE,
null,
false);
null);
}
}
}

41
src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java
View File

@ -112,7 +112,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
protected SinglePartitionReadCommand(Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -125,7 +124,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
boolean trackWarnings,
DataRange dataRange)
{
super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
super(serializedAtEpoch, Kind.SINGLE_PARTITION, isDigest, digestVersion, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, indexQueryPlan, trackWarnings, dataRange);
assert partitionKey.getPartitioner() == metadata.partitioner;
this.partitionKey = partitionKey;
this.clusteringIndexFilter = clusteringIndexFilter;
@ -134,7 +133,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
private static SinglePartitionReadCommand create(Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -152,7 +150,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
{
return new VirtualTableSinglePartitionReadCommand(isDigest,
digestVersion,
acceptsTransient,
metadata,
nowInSec,
columnFilter,
@ -168,7 +165,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return new SinglePartitionReadCommand(serializedAtEpoch,
isDigest,
digestVersion,
acceptsTransient,
potentialTxnConflicts,
metadata,
nowInSec,
@ -187,7 +183,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(command.serializedAtEpoch(),
command.isDigestQuery(),
command.digestVersion(),
command.acceptsTransient(),
command.potentialTxnConflicts(),
command.metadata(),
command.nowInSec(),
@ -226,7 +221,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(metadata.epoch,
false,
0,
false,
PotentialTxnConflicts.DISALLOW,
metadata,
nowInSec,
@ -265,7 +259,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(metadata.epoch,
false,
0,
false,
potentialTxnConflicts,
metadata,
nowInSec,
@ -444,7 +437,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -463,26 +455,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(serializedAtEpoch(),
true,
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
columnFilter(),
rowFilter(),
limits(),
partitionKey(),
clusteringIndexFilter(),
indexQueryPlan(),
isTrackingWarnings());
}
@Override
protected SinglePartitionReadCommand copyAsTransientQuery()
{
return create(serializedAtEpoch(),
false,
0,
true,
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -501,7 +473,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
potentialTxnConflicts(),
metadata(),
nowInSec(),
@ -1399,7 +1370,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
return create(serializedAtEpoch(),
isDigestQuery(),
digestVersion(),
acceptsTransient(),
PotentialTxnConflicts.ALLOW,
metadata(),
nowInSeconds,
@ -1491,7 +1461,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -1503,7 +1472,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
{
DecoratedKey key = metadata.partitioner.decorateKey(metadata.partitionKeyType.readBuffer(in, DatabaseDescriptor.getMaxValueSize()));
ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
}
}
@ -1521,7 +1490,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
Epoch serializedAtEpoch,
boolean isDigest,
int digestVersion,
boolean acceptsTransient,
PotentialTxnConflicts potentialTxnConflicts,
TableMetadata metadata,
long nowInSec,
@ -1532,7 +1500,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
throws IOException
{
ClusteringIndexFilter filter = ClusteringIndexFilter.serializer.deserialize(in, version, metadata);
return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, acceptsTransient, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
return SinglePartitionReadCommand.create(serializedAtEpoch, isDigest, digestVersion, potentialTxnConflicts, metadata, nowInSec, columnFilter, rowFilter, limits, key, filter, indexQueryPlan, false);
}
}
@ -1568,7 +1536,6 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
{
protected VirtualTableSinglePartitionReadCommand(boolean isDigest,
int digestVersion,
boolean acceptsTransient,
TableMetadata metadata,
long nowInSec,
ColumnFilter columnFilter,
@ -1580,7 +1547,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar
boolean trackWarnings,
DataRange dataRange)
{
super(metadata.epoch, isDigest, digestVersion, acceptsTransient, PotentialTxnConflicts.ALLOW, metadata, nowInSec, columnFilter,
super(metadata.epoch, isDigest, digestVersion, PotentialTxnConflicts.ALLOW, metadata, nowInSec, columnFilter,
rowFilter, limits, partitionKey, clusteringIndexFilter, indexQueryPlan, trackWarnings, dataRange);
}

2
src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java
View File

@ -560,7 +560,6 @@ public abstract class AbstractCompactionStrategy
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
int sstableLevel,
@ -572,7 +571,6 @@ public abstract class AbstractCompactionStrategy
keyCount,
repairedAt,
pendingRepair,
isTransient,
coordinatorLogOffsets,
cfs.metadata,
commitLogPositions,

5
src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java
View File

@ -160,11 +160,11 @@ public abstract class AbstractStrategyHolder
* groups they deal with. IOW, if one holder returns true for a given isRepaired/isPendingRepair combo,
* none of the others should.
*/
public abstract boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient);
public abstract boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair);
public boolean managesSSTable(SSTableReader sstable)
{
return managesRepairedGroup(sstable.isRepaired(), sstable.isPendingRepair(), sstable.isTransient());
return managesRepairedGroup(sstable.isRepaired(), sstable.isPendingRepair());
}
public abstract AbstractCompactionStrategy getStrategyFor(SSTableReader sstable);
@ -196,7 +196,6 @@ public abstract class AbstractStrategyHolder
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
int sstableLevel,

32
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
View File

@ -1021,8 +1021,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
Iterator<SSTableReader> sstableIterator,
Collection<Range<Token>> ranges,
LifecycleTransaction txn,
TimeUUID sessionID,
boolean isTransient) throws IOException
TimeUUID sessionID) throws IOException
{
if (ranges.isEmpty())
return;
@ -1032,7 +1031,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
Set<SSTableReader> fullyContainedSSTables = findSSTablesToAnticompact(sstableIterator, normalizedRanges, sessionID);
cfs.metric.bytesMutatedAnticompaction.mark(SSTableReader.getTotalBytes(fullyContainedSSTables));
cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID, isTransient);
cfs.getCompactionStrategyManager().mutateRepaired(fullyContainedSSTables, UNREPAIRED_SSTABLE, sessionID);
// since we're just re-writing the sstable metdata for the fully contained sstables, we don't want
// them obsoleted when the anti-compaction is complete. So they're removed from the transaction here
txn.cancel(fullyContainedSSTables);
@ -1080,8 +1079,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
Set<SSTableReader> sstables = new HashSet<>(validatedForRepair);
validateSSTableBoundsForAnticompaction(sessionID, sstables, replicas);
mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), replicas.onlyFull().ranges(), txn, sessionID, false);
mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), replicas.onlyTransient().ranges(), txn, sessionID, true);
mutateFullyContainedSSTables(cfs, validatedForRepair, sstables.iterator(), replicas.ranges(), txn, sessionID);
assert txn.originals().equals(sstables);
if (!sstables.isEmpty())
@ -1652,7 +1650,7 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
{
StatsMetadata metadata = sstable.getSSTableMetadata();
// TODO(aratnofsky): filter coordinatorLogOffsets to exclude any CoordinatorLogIds we're no longer responsible for, after ownership change
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogOffsets, sstable, txn));
writer.switchWriter(createWriter(cfs, compactionFileLocation, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.coordinatorLogOffsets, sstable, txn));
long lastBytesScanned = 0;
while (ci.hasNext())
@ -1816,7 +1814,6 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
long expectedBloomFilterSize,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
SSTableReader sstable,
LifecycleTransaction txn)
@ -1828,7 +1825,6 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
.setKeyCount(expectedBloomFilterSize)
.setRepairedAt(repairedAt)
.setPendingRepair(pendingRepair)
.setTransientSSTable(isTransient)
.setCoordinatorLogOffsets(coordinatorLogOffsets)
.setTableMetadataRef(cfs.metadata)
.setMetadataCollector(new MetadataCollector(cfs.metadata().comparator).sstableLevel(sstable.getSSTableLevel()))
@ -1844,7 +1840,6 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
int expectedBloomFilterSize,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
Collection<SSTableReader> sstables,
ILifecycleTransaction txn)
{
@ -1873,7 +1868,6 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
.setRepairedAt(repairedAt)
.setPendingRepair(pendingRepair)
.setCoordinatorLogOffsets(logOffsetsBuilder.build())
.setTransientSSTable(isTransient)
.setTableMetadataRef(cfs.metadata)
.setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator).sstableLevel(minLevel))
.setSerializationHeader(SerializationHeader.make(cfs.metadata(), sstables))
@ -1995,7 +1989,6 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
CompactionStrategyManager strategy = cfs.getCompactionStrategyManager();
try (SharedTxn sharedTxn = new SharedTxn(txn);
SSTableRewriter fullWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
SSTableRewriter transWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
SSTableRewriter unrepairedWriter = SSTableRewriter.constructWithoutEarlyOpening(sharedTxn, false, groupMaxDataAge);
AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(txn.originals());
@ -2004,12 +1997,10 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
{
int expectedBloomFilterSize = Math.max(cfs.metadata().params.minIndexInterval, (int)(SSTableReader.getApproximateKeyCount(sstableAsSet)));
fullWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, false, sstableAsSet, txn));
transWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, true, sstableAsSet, txn));
unrepairedWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, false, sstableAsSet, txn));
fullWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, pendingRepair, sstableAsSet, txn));
unrepairedWriter.switchWriter(CompactionManager.createWriterForAntiCompaction(cfs, destination, expectedBloomFilterSize, UNREPAIRED_SSTABLE, NO_PENDING_REPAIR, sstableAsSet, txn));
Predicate<Token> fullChecker = !ranges.onlyFull().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyFull().ranges()) : t -> false;
Predicate<Token> transChecker = !ranges.onlyTransient().isEmpty() ? new Range.OrderedRangeContainmentChecker(ranges.onlyTransient().ranges()) : t -> false;
double compressionRatio = scanners.getCompressionRatio();
if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
compressionRatio = 1.0;
@ -2027,11 +2018,6 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
fullWriter.append(partition);
ci.setTargetDirectory(fullWriter.currentWriter().getFilename());
}
else if (transChecker.test(token))
{
transWriter.append(partition);
ci.setTargetDirectory(transWriter.currentWriter().getFilename());
}
else
{
// otherwise, append it to the unrepaired sstable
@ -2045,18 +2031,15 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
}
fullWriter.prepareToCommit();
transWriter.prepareToCommit();
unrepairedWriter.prepareToCommit();
txn.checkpoint();
txn.obsoleteOriginals();
txn.prepareToCommit();
List<SSTableReader> fullSSTables = new ArrayList<>(fullWriter.finished());
List<SSTableReader> transSSTables = new ArrayList<>(transWriter.finished());
List<SSTableReader> unrepairedSSTables = new ArrayList<>(unrepairedWriter.finished());
fullWriter.commit();
transWriter.commit();
unrepairedWriter.commit();
txn.commit();
logger.info("Anticompacted {} in {}.{} to full = {}, transient = {}, unrepaired = {} for {}",
@ -2064,10 +2047,9 @@ public class CompactionManager implements CompactionManagerMBean, ICompactionMan
cfs.getKeyspaceName(),
cfs.getTableName(),
fullSSTables,
transSSTables,
unrepairedSSTables,
pendingRepair);
return fullSSTables.size() + transSSTables.size() + unrepairedSSTables.size();
return fullSSTables.size() + unrepairedSSTables.size();
}
catch (Throwable e)
{

5
src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java
View File

@ -74,11 +74,10 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
}
@Override
public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair)
{
if (!isPendingRepair)
{
Preconditions.checkArgument(!isTransient, "isTransient can only be true for sstables pending repairs");
return this.isRepaired == isRepaired;
}
else
@ -226,7 +225,6 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
int sstableLevel,
@ -252,7 +250,6 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
keyCount,
repairedAt,
pendingRepair,
isTransient,
coordinatorLogOffsets,
commitLogPositions,
sstableLevel,

47
src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java
View File

@ -127,7 +127,6 @@ public class CompactionStrategyManager implements INotificationConsumer
/**
* Variables guarded by read and write lock above
*/
private final PendingRepairHolder transientRepairs;
private final PendingRepairHolder pendingRepairs;
private final CompactionStrategyHolder repaired;
private final CompactionStrategyHolder unrepaired;
@ -175,11 +174,10 @@ public class CompactionStrategyManager implements INotificationConsumer
return compactionStrategyIndexForDirectory(descriptor);
}
};
transientRepairs = new PendingRepairHolder(cfs, router, true);
pendingRepairs = new PendingRepairHolder(cfs, router, false);
pendingRepairs = new PendingRepairHolder(cfs, router);
repaired = new CompactionStrategyHolder(cfs, router, true);
unrepaired = new CompactionStrategyHolder(cfs, router, false);
holders = ImmutableList.of(transientRepairs, pendingRepairs, repaired, unrepaired);
holders = ImmutableList.of(pendingRepairs, repaired, unrepaired);
cfs.getTracker().subscribe(this);
logger.trace("Compaction manager for {}.{} subscribed to the data tracker.", cfs.keyspace.getName(), cfs.name);
@ -217,10 +215,6 @@ public class CompactionStrategyManager implements INotificationConsumer
if (repairFinishedTasks != null && !repairFinishedTasks.isEmpty())
return repairFinishedTasks;
repairFinishedTasks = transientRepairs.getNextRepairFinishedTasks();
if (repairFinishedTasks != null && !repairFinishedTasks.isEmpty())
return repairFinishedTasks;
// sort compaction task suppliers by remaining tasks descending
List<TaskSupplier> suppliers = new ArrayList<>(numPartitions * holders.size());
for (AbstractStrategyHolder holder : holders)
@ -430,18 +424,12 @@ public class CompactionStrategyManager implements INotificationConsumer
return pendingRepairs;
}
@VisibleForTesting
PendingRepairHolder getTransientRepairsUnsafe()
{
return transientRepairs;
}
public boolean hasDataForPendingRepair(TimeUUID sessionID)
{
readLock.lock();
try
{
return pendingRepairs.hasDataForSession(sessionID) || transientRepairs.hasDataForSession(sessionID);
return pendingRepairs.hasDataForSession(sessionID);
}
finally
{
@ -455,7 +443,7 @@ public class CompactionStrategyManager implements INotificationConsumer
readLock.lock();
try
{
return pendingRepairs.hasPendingRepairSSTable(sessionID, sstable) || transientRepairs.hasPendingRepairSSTable(sessionID, sstable);
return pendingRepairs.hasPendingRepairSSTable(sessionID, sstable);
}
finally
{
@ -948,19 +936,18 @@ public class CompactionStrategyManager implements INotificationConsumer
throw new IllegalStateException("No holder claimed " + sstable);
}
private AbstractStrategyHolder getHolder(long repairedAt, TimeUUID pendingRepair, boolean isTransient)
private AbstractStrategyHolder getHolder(long repairedAt, TimeUUID pendingRepair)
{
return getHolder(repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE,
pendingRepair != ActiveRepairService.NO_PENDING_REPAIR,
isTransient);
pendingRepair != ActiveRepairService.NO_PENDING_REPAIR);
}
@VisibleForTesting
AbstractStrategyHolder getHolder(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
AbstractStrategyHolder getHolder(boolean isRepaired, boolean isPendingRepair)
{
for (AbstractStrategyHolder holder : holders)
{
if (holder.managesRepairedGroup(isRepaired, isPendingRepair, isTransient))
if (holder.managesRepairedGroup(isRepaired, isPendingRepair))
return holder;
}
@ -1411,7 +1398,6 @@ public class CompactionStrategyManager implements INotificationConsumer
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
int sstableLevel,
@ -1419,16 +1405,15 @@ public class CompactionStrategyManager implements INotificationConsumer
Collection<Index.Group> indexGroups,
ILifecycleTransaction txn)
{
SSTable.validateRepairedMetadata(repairedAt, pendingRepair, isTransient);
SSTable.validateRepairedMetadata(repairedAt, pendingRepair);
maybeReloadDiskBoundaries();
readLock.lock();
try
{
return getHolder(repairedAt, pendingRepair, isTransient).createSSTableMultiWriter(descriptor,
return getHolder(repairedAt, pendingRepair).createSSTableMultiWriter(descriptor,
keyCount,
repairedAt,
pendingRepair,
isTransient,
coordinatorLogOffsets,
commitLogPositions,
sstableLevel,
@ -1499,7 +1484,7 @@ public class CompactionStrategyManager implements INotificationConsumer
* Mutates sstable repairedAt times and notifies listeners of the change with the writeLock held. Prevents races
* with other processes between when the metadata is changed and when sstables are moved between strategies.
*/
public void mutateRepaired(Collection<SSTableReader> sstables, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
public void mutateRepaired(Collection<SSTableReader> sstables, long repairedAt, TimeUUID pendingRepair) throws IOException
{
if (sstables.isEmpty())
return;
@ -1510,8 +1495,8 @@ public class CompactionStrategyManager implements INotificationConsumer
{
for (SSTableReader sstable: sstables)
{
sstable.mutateRepairedAndReload(repairedAt, pendingRepair, isTransient);
verifyMetadata(sstable, repairedAt, pendingRepair, isTransient);
sstable.mutateRepairedAndReload(repairedAt, pendingRepair);
verifyMetadata(sstable, repairedAt, pendingRepair);
changed.add(sstable);
}
}
@ -1530,14 +1515,12 @@ public class CompactionStrategyManager implements INotificationConsumer
}
}
private static void verifyMetadata(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient)
private static void verifyMetadata(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair)
{
if (!Objects.equals(pendingRepair, sstable.getPendingRepair()))
throw new IllegalStateException(String.format("Failed setting pending repair to %s on %s (pending repair is %s)", pendingRepair, sstable, sstable.getPendingRepair()));
if (repairedAt != sstable.getRepairedAt())
throw new IllegalStateException(String.format("Failed setting repairedAt to %d on %s (repairedAt is %d)", repairedAt, sstable, sstable.getRepairedAt()));
if (isTransient != sstable.isTransient())
throw new IllegalStateException(String.format("Failed setting isTransient to %b on %s (isTransient is %b)", isTransient, sstable, sstable.isTransient()));
}
public CleanupSummary releaseRepairData(Collection<TimeUUID> sessions)
@ -1546,7 +1529,7 @@ public class CompactionStrategyManager implements INotificationConsumer
readLock.lock();
try
{
for (PendingRepairManager prm : Iterables.concat(pendingRepairs.getManagers(), transientRepairs.getManagers()))
for (PendingRepairManager prm : pendingRepairs.getManagers())
cleanupTasks.add(prm.releaseSessionData(sessions));
}
finally

17
src/java/org/apache/cassandra/db/compaction/CompactionTask.java
View File

@ -442,23 +442,6 @@ public class CompactionTask extends AbstractCompactionTask
return ids.iterator().next();
}
public static boolean getIsTransient(Set<SSTableReader> sstables)
{
if (sstables.isEmpty())
{
return false;
}
boolean isTransient = sstables.iterator().next().isTransient();
if (!Iterables.all(sstables, sstable -> sstable.isTransient() == isTransient))
{
throw new RuntimeException("Attempting to compact transient sstables with non transient sstables");
}
return isTransient;
}
public static ImmutableCoordinatorLogOffsets getCoordinatorLogOffsets(Set<SSTableReader> sstables)
{
ImmutableCoordinatorLogOffsets.Builder builder = new ImmutableCoordinatorLogOffsets.Builder();

12
src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java
View File

@ -46,12 +46,10 @@ import org.apache.cassandra.utils.TimeUUID;
public class PendingRepairHolder extends AbstractStrategyHolder
{
private final List<PendingRepairManager> managers = new ArrayList<>();
private final boolean isTransient;
public PendingRepairHolder(ColumnFamilyStore cfs, DestinationRouter router, boolean isTransient)
public PendingRepairHolder(ColumnFamilyStore cfs, DestinationRouter router)
{
super(cfs, router);
this.isTransient = isTransient;
}
@Override
@ -71,15 +69,15 @@ public class PendingRepairHolder extends AbstractStrategyHolder
{
managers.clear();
for (int i = 0; i < numTokenPartitions; i++)
managers.add(new PendingRepairManager(cfs, params, isTransient));
managers.add(new PendingRepairManager(cfs, params));
}
@Override
public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
public boolean managesRepairedGroup(boolean isRepaired, boolean isPendingRepair)
{
Preconditions.checkArgument(!isPendingRepair || !isRepaired,
"SSTables cannot be both repaired and pending repair");
return isPendingRepair && (this.isTransient == isTransient);
return isPendingRepair;
}
@Override
@ -246,7 +244,6 @@ public class PendingRepairHolder extends AbstractStrategyHolder
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
int sstableLevel,
@ -264,7 +261,6 @@ public class PendingRepairHolder extends AbstractStrategyHolder
keyCount,
repairedAt,
pendingRepair,
isTransient,
coordinatorLogOffsets,
commitLogPositions,
sstableLevel,

36
src/java/org/apache/cassandra/db/compaction/PendingRepairManager.java
View File

@ -31,7 +31,6 @@ import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
@ -63,7 +62,6 @@ class PendingRepairManager
private final ColumnFamilyStore cfs;
private final CompactionParams params;
private final boolean isTransient;
private volatile ImmutableMap<TimeUUID, AbstractCompactionStrategy> strategies = ImmutableMap.of();
/**
@ -77,11 +75,10 @@ class PendingRepairManager
}
}
PendingRepairManager(ColumnFamilyStore cfs, CompactionParams params, boolean isTransient)
PendingRepairManager(ColumnFamilyStore cfs, CompactionParams params)
{
this.cfs = cfs;
this.params = params;
this.isTransient = isTransient;
}
private ImmutableMap.Builder<TimeUUID, AbstractCompactionStrategy> mapBuilder()
@ -162,7 +159,6 @@ class PendingRepairManager
synchronized void addSSTable(SSTableReader sstable)
{
Preconditions.checkArgument(sstable.isTransient() == isTransient);
getOrCreate(sstable).addSSTable(sstable);
}
@ -517,36 +513,18 @@ class PendingRepairManager
protected void runMayThrow() throws Exception
{
boolean completed = false;
boolean obsoleteSSTables = isTransient && repairedAt > 0;
try
{
if (obsoleteSSTables)
{
logger.info("Obsoleting transient repaired sstables for {}", sessionID);
Preconditions.checkState(Iterables.all(transaction.originals(), SSTableReader::isTransient));
transaction.obsoleteOriginals();
}
else
{
logger.info("Moving {} from pending to repaired with repaired at = {} and session id = {}", transaction.originals(), repairedAt, sessionID);
cfs.getCompactionStrategyManager().mutateRepaired(transaction.originals(), repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false);
}
logger.info("Moving {} from pending to repaired with repaired at = {} and session id = {}", transaction.originals(), repairedAt, sessionID);
cfs.getCompactionStrategyManager().mutateRepaired(transaction.originals(), repairedAt, ActiveRepairService.NO_PENDING_REPAIR);
completed = true;
}
finally
{
if (obsoleteSSTables)
{
transaction.prepareToCommit();
transaction.commit();
}
else
{
// we abort here because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll
// anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other
// compactions from marking these sstables compacting, and unmarking them when we're done
transaction.abort();
}
// we abort here because mutating metadata isn't guarded by LifecycleTransaction, so this won't roll
// anything back. Also, we don't want to obsolete the originals. We're only using it to prevent other
// compactions from marking these sstables compacting, and unmarking them when we're done
transaction.abort();
if (completed)
{
removeSessionIfEmpty(sessionID);

2
src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java
View File

@ -300,7 +300,6 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
int sstableLevel,
@ -318,7 +317,6 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
keyCount,
repairedAt,
pendingRepair,
isTransient,
coordinatorLogOffsets,
commitLogPositions,
header,

1
src/java/org/apache/cassandra/db/compaction/Upgrader.java
View File

@ -79,7 +79,6 @@ public class Upgrader
.setKeyCount(estimatedRows)
.setRepairedAt(metadata.repairedAt)
.setPendingRepair(metadata.pendingRepair)
.setTransientSSTable(metadata.isTransient)
.setCoordinatorLogOffsets(metadata.coordinatorLogOffsets)
.setTableMetadataRef(cfs.metadata)
.setMetadataCollector(sstableMetadataCollector)

4
src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java
View File

@ -61,7 +61,6 @@ public class ShardedMultiWriter implements SSTableMultiWriter
private final long keyCount;
private final long repairedAt;
private final TimeUUID pendingRepair;
private final boolean isTransient;
private final ImmutableCoordinatorLogOffsets coordinatorLogOffsets;
private final IntervalSet<CommitLogPosition> commitLogPositions;
private final SerializationHeader header;
@ -76,7 +75,6 @@ public class ShardedMultiWriter implements SSTableMultiWriter
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
IntervalSet<CommitLogPosition> commitLogPositions,
SerializationHeader header,
@ -89,7 +87,6 @@ public class ShardedMultiWriter implements SSTableMultiWriter
this.keyCount = keyCount;
this.repairedAt = repairedAt;
this.pendingRepair = pendingRepair;
this.isTransient = isTransient;
this.coordinatorLogOffsets = coordinatorLogOffsets;
this.commitLogPositions = commitLogPositions;
this.header = header;
@ -117,7 +114,6 @@ public class ShardedMultiWriter implements SSTableMultiWriter
.setRepairedAt(repairedAt)
.setPendingRepair(pendingRepair)
.setCoordinatorLogOffsets(coordinatorLogOffsets)
.setTransientSSTable(isTransient)
.setTableMetadataRef(cfs.metadata)
.setMetadataCollector(metadataCollector)
.setSerializationHeader(header)

3
src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java
View File

@ -61,7 +61,6 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
protected final long maxAge;
protected final long minRepairedAt;
protected final TimeUUID pendingRepair;
protected final boolean isTransient;
protected final ImmutableCoordinatorLogOffsets coordinatorLogOffsets;
protected final SSTableRewriter sstableWriter;
@ -97,7 +96,6 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
sstableWriter = SSTableRewriter.construct(cfs, txn, keepOriginals, maxAge, earlyOpenAllowed);
minRepairedAt = CompactionTask.getMinRepairedAt(nonExpiredSSTables);
pendingRepair = CompactionTask.getPendingRepair(nonExpiredSSTables);
isTransient = CompactionTask.getIsTransient(nonExpiredSSTables);
coordinatorLogOffsets = CompactionTask.getCoordinatorLogOffsets(nonExpiredSSTables);
DiskBoundaries db = cfs.getDiskBoundaries();
diskBoundaries = db.positions;
@ -328,7 +326,6 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
{
return descriptor.getFormat().getWriterFactory().builder(descriptor)
.setTableMetadataRef(cfs.metadata)
.setTransientSSTable(isTransient)
.setRepairedAt(minRepairedAt)
.setPendingRepair(pendingRepair)
.setCoordinatorLogOffsets(coordinatorLogOffsets)

1
src/java/org/apache/cassandra/db/memtable/Flushing.java
View File

@ -243,7 +243,6 @@ public class Flushing
partitionCount,
ActiveRepairService.UNREPAIRED_SSTABLE,
ActiveRepairService.NO_PENDING_REPAIR,
false,
flushSet.coordinatorLogOffsets(),
new IntervalSet<>(flushSet.commitLogLowerBound(),
flushSet.commitLogUpperBound()),

2
src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
View File

@ -138,7 +138,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader
}
UnaryOperator<StatsMetadata> transform = stats -> stats.mutateLevel(header.sstableLevel)
.mutateRepairedMetadata(messageHeader.repairedAt, messageHeader.pendingRepair, false);
.mutateRepairedMetadata(messageHeader.repairedAt, messageHeader.pendingRepair);
String description = String.format("level %s and repairedAt time %s and pendingRepair %s",
header.sstableLevel, messageHeader.repairedAt, messageHeader.pendingRepair);
writer.descriptor().getMetadataSerializer().mutate(writer.descriptor(), description, transform);

2
src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
View File

@ -188,7 +188,7 @@ public class CassandraStreamReader implements IStreamReader
StreamReceiver streamReceiver = session.getAggregator(tableId);
Preconditions.checkState(streamReceiver instanceof CassandraStreamReceiver);
ILifecycleTransaction txn = createTxn();
RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, false, coordinatorLogOffsets, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata()));
RangeAwareSSTableWriter writer = new RangeAwareSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, coordinatorLogOffsets, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata()));
return new SSTableTxnSingleStreamWriter(txn, writer);
}

3
src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java
View File

@ -42,6 +42,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Locator;
import org.apache.cassandra.locator.NetworkTopologyStrategy;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.membership.NodeAddresses;
@ -86,7 +87,7 @@ public class TokenAllocation
// We create a fake NTS replication strategy with the specified RF in the local DC
HashMap<String, String> options = new HashMap<>();
options.put(localDatacenter, Integer.toString(replicas));
NetworkTopologyStrategy fakeReplicationStrategy = new NetworkTopologyStrategy(null, options);
NetworkTopologyStrategy fakeReplicationStrategy = new NetworkTopologyStrategy(null, options, ReplicationType.untracked);
return new TokenAllocation(metadata, fakeReplicationStrategy, numTokens);
}

1
src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
View File

@ -151,7 +151,6 @@ public abstract class AbstractSSTableSimpleWriter implements Closeable
0,
ActiveRepairService.UNREPAIRED_SSTABLE,
ActiveRepairService.NO_PENDING_REPAIR,
false,
ImmutableCoordinatorLogOffsets.NONE,
header,
indexGroups,

8
src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java
View File

@ -45,7 +45,6 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
private final long estimatedKeys;
private final long repairedAt;
private final TimeUUID pendingRepair;
private final boolean isTransient;
private final ImmutableCoordinatorLogOffsets coordinatorLogOffsets;
private final SSTableFormat<?, ?> format;
private final SerializationHeader header;
@ -55,7 +54,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
private final List<SSTableMultiWriter> finishedWriters = new ArrayList<>();
private SSTableMultiWriter currentWriter = null;
public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, boolean isTransient, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SSTableFormat<?, ?> format, int sstableLevel, long totalSize, ILifecycleTransaction txn, SerializationHeader header) throws IOException
public RangeAwareSSTableWriter(ColumnFamilyStore cfs, long estimatedKeys, long repairedAt, TimeUUID pendingRepair, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SSTableFormat<?, ?> format, int sstableLevel, long totalSize, ILifecycleTransaction txn, SerializationHeader header) throws IOException
{
DiskBoundaries db = cfs.getDiskBoundaries();
directories = db.directories;
@ -64,7 +63,6 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
this.estimatedKeys = estimatedKeys / directories.size();
this.repairedAt = repairedAt;
this.pendingRepair = pendingRepair;
this.isTransient = isTransient;
this.coordinatorLogOffsets = coordinatorLogOffsets;
this.format = format;
this.txn = txn;
@ -77,7 +75,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
throw new IOException(String.format("Insufficient disk space to store %s",
FBUtilities.prettyPrintMemory(totalSize)));
Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, null, sstableLevel, header, txn);
currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, coordinatorLogOffsets, null, sstableLevel, header, txn);
}
}
@ -99,7 +97,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
finishedWriters.add(currentWriter);
Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format);
currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, null, sstableLevel, header, txn);
currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, coordinatorLogOffsets, null, sstableLevel, header, txn);
}
}

4
src/java/org/apache/cassandra/io/sstable/SSTable.java
View File

@ -309,12 +309,10 @@ public abstract class SSTable
return String.format("%s:%s(path='%s')", getClass().getSimpleName(), descriptor.version.format.name(), getFilename());
}
public static void validateRepairedMetadata(long repairedAt, TimeUUID pendingRepair, boolean isTransient)
public static void validateRepairedMetadata(long repairedAt, TimeUUID pendingRepair)
{
Preconditions.checkArgument((pendingRepair == NO_PENDING_REPAIR) || (repairedAt == UNREPAIRED_SSTABLE),
"pendingRepair cannot be set on a repaired sstable");
Preconditions.checkArgument(!isTransient || (pendingRepair != NO_PENDING_REPAIR),
"isTransient can only be true for sstables pending repair");
}
/**

9
src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java
View File

@ -114,10 +114,10 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
}
@SuppressWarnings({"resource", "RedundantSuppression"}) // log and writer closed during doPostCleanup
public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SerializationHeader header)
public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SerializationHeader header)
{
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, header, txn);
SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, coordinatorLogOffsets, header, txn);
return new SSTableTxnWriter(txn, writer);
}
@ -136,7 +136,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
SSTableMultiWriter writer;
try
{
writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, type, 0, 0, txn, header);
writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, coordinatorLogOffsets, type, 0, 0, txn, header);
}
catch (IOException e)
{
@ -154,7 +154,6 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
SerializationHeader header,
Collection<Index.Group> indexGroups,
@ -162,7 +161,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
{
// if the column family store does not exist, we create a new default SSTableMultiWriter to use:
LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, coordinatorLogOffsets, metadata, null, 0, header, indexGroups, txn, owner);
SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, coordinatorLogOffsets, metadata, null, 0, header, indexGroups, txn, owner);
return new SSTableTxnWriter(txn, writer);
}
}

2
src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java
View File

@ -117,7 +117,6 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
long keyCount,
long repairedAt,
TimeUUID pendingRepair,
boolean isTransient,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
TableMetadataRef metadata,
IntervalSet<CommitLogPosition> commitLogPositions,
@ -137,7 +136,6 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
.setKeyCount(keyCount)
.setRepairedAt(repairedAt)
.setPendingRepair(pendingRepair)
.setTransientSSTable(isTransient)
.setCoordinatorLogOffsets(coordinatorLogOffsets)
.setTableMetadataRef(metadata)
.setMetadataCollector(metadataCollector)

9
src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
View File

@ -1240,11 +1240,6 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
return sstableMetadata.repairedAt;
}
public boolean isTransient()
{
return sstableMetadata.isTransient;
}
public boolean intersects(Collection<Range<Token>> ranges)
{
Bounds<Token> range = new Bounds<>(first.getToken(), last.getToken());
@ -1397,11 +1392,11 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
/**
* Mutate sstable repair metadata with a lock to avoid racing with entire-sstable-streaming and then reload sstable metadata
*/
public void mutateRepairedAndReload(long newRepairedAt, TimeUUID newPendingRepair, boolean isTransient) throws IOException
public void mutateRepairedAndReload(long newRepairedAt, TimeUUID newPendingRepair) throws IOException
{
synchronized (tidy.global)
{
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, newRepairedAt, newPendingRepair, isTransient);
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, newRepairedAt, newPendingRepair);
reloadSSTableMetadata();
}
}

17
src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
View File

@ -82,7 +82,6 @@ public abstract class SSTableWriter extends SSTable implements Transactional
protected long repairedAt;
protected TimeUUID pendingRepair;
protected boolean isTransient;
protected ImmutableCoordinatorLogOffsets coordinatorLogOffsets;
protected long maxDataAge = -1;
protected final long keyCount;
@ -112,7 +111,6 @@ public abstract class SSTableWriter extends SSTable implements Transactional
this.keyCount = builder.getKeyCount();
this.repairedAt = builder.getRepairedAt();
this.pendingRepair = builder.getPendingRepair();
this.isTransient = builder.isTransientSSTable();
this.coordinatorLogOffsets = builder.getCoordinatorLogOffsets();
this.metadataCollector = builder.getMetadataCollector();
this.header = builder.getSerializationHeader();
@ -368,7 +366,6 @@ public abstract class SSTableWriter extends SSTable implements Transactional
metadata().params.bloomFilterFpChance,
repairedAt,
pendingRepair,
isTransient,
coordinatorLogOffsets,
header,
first.retainable().getKey(),
@ -489,7 +486,6 @@ public abstract class SSTableWriter extends SSTable implements Transactional
private long keyCount;
private long repairedAt;
private TimeUUID pendingRepair;
private boolean transientSSTable;
private SerializationHeader serializationHeader;
private List<Index.Group> indexGroups;
@Nullable
@ -526,12 +522,6 @@ public abstract class SSTableWriter extends SSTable implements Transactional
return (B) this;
}
public B setTransientSSTable(boolean transientSSTable)
{
this.transientSSTable = transientSSTable;
return (B) this;
}
public B setSerializationHeader(SerializationHeader serializationHeader)
{
this.serializationHeader = serializationHeader;
@ -611,11 +601,6 @@ public abstract class SSTableWriter extends SSTable implements Transactional
return pendingRepair;
}
public boolean isTransientSSTable()
{
return transientSSTable;
}
public ImmutableCoordinatorLogOffsets getCoordinatorLogOffsets()
{
return coordinatorLogOffsets;
@ -642,7 +627,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional
{
checkNotNull(getComponents());
validateRepairedMetadata(getRepairedAt(), getPendingRepair(), isTransientSSTable());
validateRepairedMetadata(getRepairedAt(), getPendingRepair());
return buildInternal(txn, owner);
}

4
src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
View File

@ -190,7 +190,7 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
Refs<SSTableReader> refs = Refs.ref(Collections.singleton(sstable)))
{
StatsMetadata metadata = sstable.getSSTableMetadata();
writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogOffsets, sstable, transaction));
writer.switchWriter(CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, metadata.repairedAt, metadata.pendingRepair, metadata.coordinatorLogOffsets, sstable, transaction));
scrubInternal(writer);
@ -240,7 +240,7 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
// out of order partitions/rows, but no bad partition found - we can keep our repairedAt time
long repairedAt = badPartitions > 0 ? ActiveRepairService.UNREPAIRED_SSTABLE : sstable.getSSTableMetadata().repairedAt;
SSTableReader newInOrderSstable;
try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.isTransient, metadata.coordinatorLogOffsets, sstable, transaction))
try (SSTableWriter inOrderWriter = CompactionManager.createWriter(cfs, destination, expectedBloomFilterSize, repairedAt, metadata.pendingRepair, metadata.coordinatorLogOffsets, sstable, transaction))
{
for (Partition partition : outOfOrder)
inOrderWriter.append(partition.unfilteredIterator());

2
src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
View File

@ -129,7 +129,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
{
try
{
sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair(), sstable.isTransient());
sstable.mutateRepairedAndReload(ActiveRepairService.UNREPAIRED_SSTABLE, sstable.getPendingRepair());
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
}
catch (IOException ioe)

2
src/java/org/apache/cassandra/io/sstable/metadata/IMetadataSerializer.java
View File

@ -94,7 +94,7 @@ public interface IMetadataSerializer
* NOTE: mutating stats metadata of a live sstable will race with entire-sstable-streaming, please use
* {@link SSTableReader#mutateLevelAndReload} instead on live sstable.
*/
public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, TimeUUID newPendingRepair, boolean isTransient) throws IOException;
public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, TimeUUID newPendingRepair) throws IOException;
/**
* Replace the sstable metadata file ({@code -Statistics.db}) with the given components.

3
src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
View File

@ -413,7 +413,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
return totalRows;
}
public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey)
public Map<MetadataType, MetadataComponent> finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, ImmutableCoordinatorLogOffsets coordinatorLogOffsets, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey)
{
assert minClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || minClustering.kind().isStart();
assert maxClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || maxClustering.kind().isEnd();
@ -448,7 +448,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
tokenSpaceCoverage,
originatingHostId,
pendingRepair,
isTransient,
hasPartitionLevelDeletions,
coordinatorLogOffsets,
firstKey,

4
src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java
View File

@ -240,13 +240,13 @@ public class MetadataSerializer implements IMetadataSerializer
}
@Override
public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, TimeUUID newPendingRepair, boolean isTransient) throws IOException
public void mutateRepairMetadata(Descriptor descriptor, long newRepairedAt, TimeUUID newPendingRepair) throws IOException
{
if (logger.isTraceEnabled())
logger.trace("Mutating {} to repairedAt time {} and pendingRepair {}",
descriptor.fileFor(Components.STATS), newRepairedAt, newPendingRepair);
mutate(descriptor, stats -> stats.mutateRepairedMetadata(newRepairedAt, newPendingRepair, isTransient));
mutate(descriptor, stats -> stats.mutateRepairedMetadata(newRepairedAt, newPendingRepair));
}
private void mutate(Descriptor descriptor, UnaryOperator<StatsMetadata> transform) throws IOException

12
src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java
View File

@ -79,7 +79,6 @@ public class StatsMetadata extends MetadataComponent
public final long totalRows;
public final UUID originatingHostId;
public final TimeUUID pendingRepair;
public final boolean isTransient;
public final ImmutableCoordinatorLogOffsets coordinatorLogOffsets;
// just holds the current encoding stats to avoid allocating - it is not serialized
public final EncodingStats encodingStats;
@ -122,7 +121,6 @@ public class StatsMetadata extends MetadataComponent
double tokenSpaceCoverage,
UUID originatingHostId,
TimeUUID pendingRepair,
boolean isTransient,
boolean hasPartitionLevelDeletions,
ImmutableCoordinatorLogOffsets coordinatorLogOffsets,
ByteBuffer firstKey,
@ -149,7 +147,6 @@ public class StatsMetadata extends MetadataComponent
this.tokenSpaceCoverage = tokenSpaceCoverage;
this.originatingHostId = originatingHostId;
this.pendingRepair = pendingRepair;
this.isTransient = isTransient;
this.coordinatorLogOffsets = coordinatorLogOffsets;
this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL);
this.hasPartitionLevelDeletions = hasPartitionLevelDeletions;
@ -209,14 +206,13 @@ public class StatsMetadata extends MetadataComponent
tokenSpaceCoverage,
originatingHostId,
pendingRepair,
isTransient,
hasPartitionLevelDeletions,
coordinatorLogOffsets,
firstKey,
lastKey);
}
public StatsMetadata mutateRepairedMetadata(long newRepairedAt, TimeUUID newPendingRepair, boolean newIsTransient)
public StatsMetadata mutateRepairedMetadata(long newRepairedAt, TimeUUID newPendingRepair)
{
return new StatsMetadata(estimatedPartitionSize,
estimatedCellPerPartitionCount,
@ -239,7 +235,6 @@ public class StatsMetadata extends MetadataComponent
tokenSpaceCoverage,
originatingHostId,
newPendingRepair,
newIsTransient,
hasPartitionLevelDeletions,
coordinatorLogOffsets,
firstKey,
@ -363,7 +358,7 @@ public class StatsMetadata extends MetadataComponent
if (version.hasIsTransient())
{
size += TypeSizes.sizeof(component.isTransient);
size += TypeSizes.sizeof(false); // deprecated field
}
if (version.hasOriginatingHostId())
@ -484,7 +479,7 @@ public class StatsMetadata extends MetadataComponent
if (version.hasIsTransient())
{
out.writeBoolean(component.isTransient);
out.writeBoolean(false); // deprecated field
}
if (version.hasOriginatingHostId())
@ -693,7 +688,6 @@ public class StatsMetadata extends MetadataComponent
tokenSpaceCoverage,
originatingHostId,
pendingRepair,
isTransient,
hasPartitionLevelDeletions,
coordinatorLogOffsets,
firstKey,

59
src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
View File

@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.function.Supplier;
import com.google.common.base.Preconditions;
@ -37,6 +38,7 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.service.AbstractWriteResponseHandler;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.DatacenterSyncWriteResponseHandler;
@ -48,20 +50,28 @@ import org.apache.cassandra.tcm.compatibility.TokenRingUtils;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
/**
* A abstract parent for all replication strategies.
*/
public abstract class AbstractReplicationStrategy
{
@SuppressWarnings("rawtypes")
private static final AtomicReferenceFieldUpdater<AbstractReplicationStrategy, Pair> LOCAL_RANGES_UPDATER = AtomicReferenceFieldUpdater.newUpdater(AbstractReplicationStrategy.class, Pair.class, "localRanges");
public final Map<String, String> configOptions;
public final ReplicationType replicationType;
// TODO: remove keyspace name; add a cache that allows going between replication params and replication strategy
protected final String keyspaceName;
protected AbstractReplicationStrategy(String keyspaceName, Map<String, String> configOptions)
private volatile Pair<Epoch, RangesAtEndpoint> localRanges;
protected AbstractReplicationStrategy(String keyspaceName, Map<String, String> configOptions, ReplicationType replicationType)
{
this.configOptions = configOptions == null ? Collections.<String, String>emptyMap() : configOptions;
this.keyspaceName = keyspaceName;
this.replicationType = replicationType;
}
/**
@ -248,15 +258,16 @@ public abstract class AbstractReplicationStrategy
private static AbstractReplicationStrategy createInternal(String keyspaceName,
Class<? extends AbstractReplicationStrategy> strategyClass,
Map<String, String> strategyOptions)
Map<String, String> strategyOptions,
ReplicationType replicationType)
throws ConfigurationException
{
AbstractReplicationStrategy strategy;
Class<?>[] parameterTypes = new Class[] {String.class, Map.class};
Class<?>[] parameterTypes = new Class[] {String.class, Map.class, ReplicationType.class};
try
{
Constructor<? extends AbstractReplicationStrategy> constructor = strategyClass.getConstructor(parameterTypes);
strategy = constructor.newInstance(keyspaceName, strategyOptions);
strategy = constructor.newInstance(keyspaceName, strategyOptions, replicationType);
}
catch (InvocationTargetException e)
{
@ -271,15 +282,17 @@ public abstract class AbstractReplicationStrategy
}
public static AbstractReplicationStrategy createReplicationStrategy(String keyspaceName,
ReplicationParams replicationParams)
ReplicationParams replicationParams,
ReplicationType replicationType)
{
return createReplicationStrategy(keyspaceName, replicationParams.klass, replicationParams.options);
return createReplicationStrategy(keyspaceName, replicationParams.klass, replicationParams.options, replicationType);
}
public static AbstractReplicationStrategy createReplicationStrategy(String keyspaceName,
Class<? extends AbstractReplicationStrategy> strategyClass,
Map<String, String> strategyOptions)
Map<String, String> strategyOptions,
ReplicationType replicationType)
{
AbstractReplicationStrategy strategy = createInternal(keyspaceName, strategyClass, strategyOptions);
AbstractReplicationStrategy strategy = createInternal(keyspaceName, strategyClass, strategyOptions, replicationType);
strategy.validateOptions();
return strategy;
}
@ -321,9 +334,10 @@ public abstract class AbstractReplicationStrategy
Class<? extends AbstractReplicationStrategy> strategyClass,
ClusterMetadata metadata,
Map<String, String> strategyOptions,
ReplicationType replicationType,
ClientState state) throws ConfigurationException
{
AbstractReplicationStrategy strategy = createInternal(keyspaceName, strategyClass, strategyOptions);
AbstractReplicationStrategy strategy = createInternal(keyspaceName, strategyClass, strategyOptions, replicationType);
strategy.validateExpectedOptions(metadata);
strategy.validateOptions();
strategy.maybeWarnOnOptions(state);
@ -363,6 +377,8 @@ public abstract class AbstractReplicationStrategy
{
if (DatabaseDescriptor.getNumTokens() > 1)
throw new ConfigurationException("Transient replication is not supported with vnodes yet");
if (!replicationType.isTracked())
throw new ConfigurationException("Transient replication requires mutation tracking");
}
}
catch (IllegalArgumentException e)
@ -394,4 +410,29 @@ public abstract class AbstractReplicationStrategy
throw new ConfigurationException(String.format("Unrecognized strategy option {%s} passed to %s for keyspace %s. Expected options: %s", key, getClass().getSimpleName(), keyspaceName, expectedOptions));
}
}
public boolean usesMutationTracking()
{
return replicationType.isTracked();
}
/**
* Returns local ranges for the epoch specified in the supplied cluster metadata or some later epoch. This caches
* the resulting RangesAtEndpoint so it should be a little more efficient.
*/
public RangesAtEndpoint getLocalRanges(ClusterMetadata cm)
{
while (true)
{
Pair<Epoch, RangesAtEndpoint> localRanges = this.localRanges;
if (localRanges != null && localRanges.left.isEqualOrAfter(cm.epoch))
return localRanges.right;
ClusterMetadata latestMetadata = ClusterMetadata.current();
RangesAtEndpoint newRanges = getAddressReplicas(latestMetadata, FBUtilities.getBroadcastAddressAndPort());
Pair<Epoch, RangesAtEndpoint> replacementLocalRanges = Pair.create(latestMetadata.epoch, newRanges);
if (LOCAL_RANGES_UPDATER.compareAndSet(this, localRanges, replacementLocalRanges))
return newRanges;
}
}
}

5
src/java/org/apache/cassandra/locator/LocalStrategy.java
View File

@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.ownership.DataPlacement;
@ -36,9 +37,9 @@ public class LocalStrategy extends SystemStrategy
private static final ReplicationFactor RF = ReplicationFactor.fullOnly(1);
private static final Map<IPartitioner, EntireRange> perPartitionerRanges = new IdentityHashMap<>();
public LocalStrategy(String keyspaceName, Map<String, String> configOptions)
public LocalStrategy(String keyspaceName, Map<String, String> configOptions, ReplicationType replicationType)
{
super(keyspaceName, configOptions);
super(keyspaceName, configOptions, replicationType);
}
@Override

5
src/java/org/apache/cassandra/locator/MetaStrategy.java
View File

@ -25,6 +25,7 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.ReversedLongLocalPartitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.ownership.DataPlacement;
@ -59,9 +60,9 @@ public class MetaStrategy extends SystemStrategy
private final ReplicationFactor rf;
public MetaStrategy(String keyspaceName, Map<String, String> configOptions)
public MetaStrategy(String keyspaceName, Map<String, String> configOptions, ReplicationType replicationType)
{
super(keyspaceName, configOptions);
super(keyspaceName, configOptions, replicationType);
int replicas = 0;
if (configOptions != null)
{

5
src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java
View File

@ -40,6 +40,7 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.ClientWarn;
@ -78,9 +79,9 @@ public class NetworkTopologyStrategy extends AbstractReplicationStrategy
private final ReplicationFactor aggregateRf;
private static final Logger logger = LoggerFactory.getLogger(NetworkTopologyStrategy.class);
public NetworkTopologyStrategy(String keyspaceName, Map<String, String> configOptions) throws ConfigurationException
public NetworkTopologyStrategy(String keyspaceName, Map<String, String> configOptions, ReplicationType replicationType) throws ConfigurationException
{
super(keyspaceName, configOptions);
super(keyspaceName, configOptions, replicationType);
int replicas = 0;
int trans = 0;

4
src/java/org/apache/cassandra/locator/RemoteStrategy.java
View File

@ -20,10 +20,12 @@ package org.apache.cassandra.locator;
import java.util.Map;
import org.apache.cassandra.schema.ReplicationType;
public class RemoteStrategy extends LocalStrategy
{
public RemoteStrategy(String keyspaceName, Map<String, String> configOptions)
{
super(keyspaceName, configOptions);
super(keyspaceName, configOptions, ReplicationType.untracked);
}
}

61
src/java/org/apache/cassandra/locator/ReplicaPlans.java
View File

@ -631,6 +631,9 @@ public class ReplicaPlans
}
};
// TODO (desired): Cheap quorums are not a goal for now, would really require speculation for writes
// and would trigger reconciliation work later, potentially quite a lot so I don't think it makes sense.
// So just remove selector?
/**
* Select all full nodes, live or down, as write targets. If there are insufficient nodes to complete the write,
* but there are live transient nodes, select a sufficient number of these to reach our consistency level.
@ -826,9 +829,67 @@ public class ReplicaPlans
return contactForEachQuorumRead(locator, (NetworkTopologyStrategy) replicationStrategy, candidates);
int count = consistencyLevel.blockFor(replicationStrategy) + (alwaysSpeculate ? 1 : 0);
// Fix for transient replica bug: ensure we always contact at least one full replica
// if the first replica is transient, reorder to put a full replica first
if (!candidates.isEmpty() && candidates.get(0).isTransient())
{
return contactWithFullReplicaFirst(candidates, count);
}
return candidates.subList(0, Math.min(count, candidates.size()));
}
/**
* Select replicas to contact ensuring that the first replica is a full replica
* followed by the required number of additional replicas in proximity order.
* This fixes the bug where CL.ONE could contact only a transient replica.
*/
private static <E extends Endpoints<E>> E contactWithFullReplicaFirst(E candidates, int count)
{
// Find the best full replica (first one in proximity-sorted candidates)
Replica fullReplica = null;
int fullReplicaIndex = -1;
for (int i = 0; i < candidates.size(); i++)
{
Replica replica = candidates.get(i);
if (replica.isFull())
{
fullReplica = replica;
fullReplicaIndex = i;
break;
}
}
if (fullReplica == null)
{
// No full replicas available - throw error similar to assureSufficientLiveReplicas
throw UnavailableException.create(ConsistencyLevel.ONE, count, 1, candidates.size(), 0);
}
// Build the contact list with full replica first
ReplicaCollection.Builder<E> contacts = candidates.newBuilder(count);
contacts.add(fullReplica);
int remaining = count - 1;
if (remaining > 0)
{
// Add the remaining replicas in proximity order, skipping the full replica we already added
int added = 0;
for (int i = 0; i < candidates.size() && added < remaining; i++)
{
if (i != fullReplicaIndex)
{
contacts.add(candidates.get(i));
added++;
}
}
}
return contacts.build();
}
/**
* Construct a plan for reading from a single node - this permits no speculation or read-repair

5
src/java/org/apache/cassandra/locator/SimpleStrategy.java
View File

@ -31,6 +31,7 @@ import org.apache.cassandra.db.guardrails.Guardrails;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.service.ClientState;
import org.apache.cassandra.service.ClientWarn;
@ -58,9 +59,9 @@ public class SimpleStrategy extends AbstractReplicationStrategy
private static final Logger logger = LoggerFactory.getLogger(SimpleStrategy.class);
private final ReplicationFactor rf;
public SimpleStrategy(String keyspaceName, Map<String, String> configOptions)
public SimpleStrategy(String keyspaceName, Map<String, String> configOptions, ReplicationType replicationType)
{
super(keyspaceName, configOptions);
super(keyspaceName, configOptions, replicationType);
validateOptionsInternal(configOptions);
this.rf = ReplicationFactor.fromString(this.configOptions.get(REPLICATION_FACTOR));
}

5
src/java/org/apache/cassandra/locator/SystemStrategy.java
View File

@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.Map;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.tcm.ClusterMetadata;
/**
@ -30,9 +31,9 @@ import org.apache.cassandra.tcm.ClusterMetadata;
*/
public abstract class SystemStrategy extends AbstractReplicationStrategy
{
public SystemStrategy(String keyspaceName, Map<String, String> configOptions)
public SystemStrategy(String keyspaceName, Map<String, String> configOptions, ReplicationType replicationType)
{
super(keyspaceName, configOptions);
super(keyspaceName, configOptions, replicationType);
}
public void validateOptions() throws ConfigurationException

2
src/java/org/apache/cassandra/replication/TrackedWriteRequest.java
View File

@ -87,7 +87,7 @@ public class TrackedWriteRequest
Keyspace keyspace = Keyspace.open(keyspaceName);
Token token = mutation.key().getToken();
ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(keyspace, consistencyLevel, token, ReplicaPlans.writeNormal);
ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(keyspace, consistencyLevel, token, ReplicaPlans.writeAll);
AbstractReplicationStrategy rs = plan.replicationStrategy();
if (plan.lookup(FBUtilities.getBroadcastAddressAndPort()) == null)

4
src/java/org/apache/cassandra/schema/KeyspaceMetadata.java
View File

@ -104,7 +104,7 @@ public final class KeyspaceMetadata implements SchemaElement
this.views = views;
this.types = types;
this.userFunctions = functions;
this.replicationStrategy = AbstractReplicationStrategy.createReplicationStrategy(keyspaceName, params.replication);
this.replicationStrategy = AbstractReplicationStrategy.createReplicationStrategy(keyspaceName, params.replication, params.replicationType);
}
@VisibleForTesting
@ -292,7 +292,7 @@ public final class KeyspaceMetadata implements SchemaElement
public boolean useMutationTracking()
{
return params.replicationType.isTracked();
return replicationStrategy.usesMutationTracking();
}
@Override

11
src/java/org/apache/cassandra/schema/KeyspaceParams.java
View File

@ -138,6 +138,11 @@ public final class KeyspaceParams
return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL, ReplicationType.untracked);
}
public static KeyspaceParams simpleWitness(String replicationFactor)
{
return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), ReplicationType.tracked);
}
public static KeyspaceParams simpleTransient(int replicationFactor)
{
return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL, ReplicationType.untracked);
@ -148,6 +153,10 @@ public final class KeyspaceParams
return new KeyspaceParams(true, ReplicationParams.nts(args), FastPathStrategy.simple(), replicationType);
}
public static KeyspaceParams ntsTracked(Object... args)
{
return nts(ReplicationType.tracked, args);
}
public static KeyspaceParams nts(Object... args)
{
@ -171,7 +180,7 @@ public final class KeyspaceParams
public void validate(String name, ClientState state, ClusterMetadata metadata)
{
replication.validate(name, state, metadata);
replication.validate(name, state, metadata, replicationType);
}
@Override

4
src/java/org/apache/cassandra/schema/ReplicationParams.java
View File

@ -191,10 +191,10 @@ public final class ReplicationParams
return new ReplicationParams(NetworkTopologyStrategy.class, options);
}
public void validate(String name, ClientState state, ClusterMetadata metadata)
public void validate(String name, ClientState state, ClusterMetadata metadata, ReplicationType replicationType)
{
// Attempt to instantiate the ARS, which will throw a ConfigurationException if the options aren't valid.
AbstractReplicationStrategy.validateReplicationStrategy(name, klass, metadata, options, state);
AbstractReplicationStrategy.validateReplicationStrategy(name, klass, metadata, options, replicationType, state);
}
public static ReplicationParams fromMap(Map<String, String> map) {

4
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -1557,7 +1557,7 @@ public class StorageProxy implements StorageProxyMBean
return;
// Always construct the replica plan to check availability
ReplicaPlan.ForWrite dataReplicaPlan = ReplicaPlans.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
ReplicaPlan.ForWrite dataReplicaPlan = ReplicaPlans.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeAll);
if (dataReplicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
writeMetrics.localRequests.mark();
@ -1819,7 +1819,7 @@ public class StorageProxy implements StorageProxyMBean
Keyspace keyspace = Keyspace.open(keyspaceName);
Token tk = mutation.key().getToken();
ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeNormal);
ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll);
if (replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null)
writeMetrics.localRequests.mark();

8
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -394,7 +394,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public RangesAtEndpoint getLocalReplicas(String keyspaceName)
{
return getReplicas(keyspaceName, FBUtilities.getBroadcastAddressAndPort());
return Keyspace.open(keyspaceName).getReplicationStrategy().getLocalRanges(ClusterMetadata.current());
}
public RangesAtEndpoint getReplicas(String keyspaceName, InetAddressAndPort endpoint)
@ -409,11 +409,9 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public List<Range<Token>> getLocalRanges(String ks)
{
InetAddressAndPort broadcastAddress = getBroadcastAddressAndPort();
Keyspace keyspace = Keyspace.open(ks);
List<Range<Token>> ranges = new ArrayList<>();
for (Replica r : keyspace.getReplicationStrategy().getAddressReplicas(ClusterMetadata.current(),
broadcastAddress))
for (Replica r : keyspace.getReplicationStrategy().getLocalRanges(ClusterMetadata.current()))
ranges.add(r.range());
return ranges;
}
@ -5829,7 +5827,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
Set<SSTableReader> result = table.runWithCompactionsDisabled(() -> {
Set<SSTableReader> sstables = table.getLiveSSTables().stream().filter(predicate).collect(Collectors.toSet());
if (!preview)
table.getCompactionStrategyManager().mutateRepaired(sstables, repairedAt, null, false);
table.getCompactionStrategyManager().mutateRepaired(sstables, repairedAt, null);
return sstables;
}, predicate, OperationType.ANTICOMPACTION, true, false, true);
sstablesTouched.addAll(result.stream().map(sst -> sst.descriptor.baseFile().name()).collect(Collectors.toList()));

2
src/java/org/apache/cassandra/service/accord/AccordSegmentCompactor.java
View File

@ -44,7 +44,7 @@ public class AccordSegmentCompactor<V> extends AbstractAccordSegmentCompactor<V>
SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
// TODO: Is ImmutableCoordinatorLogOffsets.NONE correct/reasonable here?
this.writer = SSTableTxnWriter.create(cfs, descriptor, estimatedKeyCount, 0, null, false, ImmutableCoordinatorLogOffsets.NONE, header);
this.writer = SSTableTxnWriter.create(cfs, descriptor, estimatedKeyCount, 0, null, ImmutableCoordinatorLogOffsets.NONE, header);
}
@Override

4
src/java/org/apache/cassandra/service/accord/interop/AccordInteropExecution.java
View File

@ -355,15 +355,13 @@ public class AccordInteropExecution implements ReadCoordinator
* Any nodes not contacted for read need to be sent commits
*/
@Override
public void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken transientRequests, EndpointsForToken digestRequests)
public void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken digestRequests)
{
if (readsCurrentlyUnderConstruction == null)
return;
for (int i = 0; i < fullDataRequests.size(); i++)
contacted.add(fullDataRequests.endpoint(i));
for (int i = 0; i < transientRequests.size(); i++)
contacted.add(transientRequests.endpoint(i));
for (int i = 0; i < digestRequests.size(); i++)
contacted.add(digestRequests.endpoint(i));
if (readsCurrentlyUnderConstruction.decrementAndGet() == 0)

18
src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java
View File

@ -123,14 +123,8 @@ public abstract class AbstractReadExecutor
makeRequests(command, replicas);
}
protected void makeTransientDataRequests(Iterable<Replica> replicas)
{
makeRequests(command.copyAsTransientQuery(replicas), replicas);
}
protected void makeDigestRequests(Iterable<Replica> replicas)
{
assert all(replicas, Replica::isFull);
// only send digest requests to full replicas, send data requests instead to the transient replicas
makeRequests(command.copyAsDigestQuery(replicas), replicas);
}
@ -143,7 +137,7 @@ public abstract class AbstractReadExecutor
for (Replica replica: replicas)
{
assert replica.isFull() || readCommand.acceptsTransient();
assert replica.isFull();
InetAddressAndPort endpoint = replica.endpoint();
if (replica.isSelf() && coordinator.localReadSupported())
@ -183,11 +177,9 @@ public abstract class AbstractReadExecutor
EndpointsForToken selected = replicaPlan().contacts();
EndpointsForToken fullDataRequests = selected.filter(Replica::isFull, initialDataRequestCount);
makeFullDataRequests(fullDataRequests);
EndpointsForToken transientRequests = selected.filter(Replica::isTransient);
makeTransientDataRequests(transientRequests);
EndpointsForToken digestRequests = selected.filter(r -> r.isFull() && !fullDataRequests.contains(r));
EndpointsForToken digestRequests = selected.filter(r -> !fullDataRequests.contains(r));
makeDigestRequests(digestRequests);
coordinator.notifyOfInitialContacts(fullDataRequests, transientRequests, digestRequests);
coordinator.notifyOfInitialContacts(fullDataRequests, digestRequests);
}
/**
@ -337,9 +329,7 @@ public abstract class AbstractReadExecutor
// we should only use a SpeculatingReadExecutor if we have an extra replica to speculate against
assert extraReplica != null;
retryCommand = extraReplica.isTransient()
? command.copyAsTransientQuery(extraReplica)
: command.copyAsDigestQuery(extraReplica);
retryCommand = command.copyAsDigestQuery(extraReplica);
}
else
{

2
src/java/org/apache/cassandra/service/reads/ReadCoordinator.java
View File

@ -71,7 +71,7 @@ public interface ReadCoordinator
return command;
}
void sendReadCommand(Message<ReadCommand> message, InetAddressAndPort to, RequestCallback<ReadResponse> callback);
default void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken transientRequests, EndpointsForToken digestRequests) {}
default void notifyOfInitialContacts(EndpointsForToken fullDataRequests, EndpointsForToken digestRequests) {}
void sendReadRepairMutation(Message<Mutation> message, InetAddressAndPort to, RequestCallback<Object> callback);
default PotentialTxnConflicts potentialTxnConflicts()
{

13
src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java
View File

@ -78,6 +78,8 @@ import org.apache.cassandra.transport.Dispatcher;
import org.apache.cassandra.utils.NoSpamLogger;
import org.apache.cassandra.utils.btree.BTreeSet;
import static com.google.common.base.Preconditions.checkState;
/**
* Helper in charge of collecting additional queries to be done on the coordinator to protect against invalid results
* being included due to replica-side filtering (secondary indexes or {@code ALLOW * FILTERING}).
@ -169,15 +171,14 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
new DataResolver<>(coordinator, cmd, replicaPlan, (NoopReadRepair<EndpointsForToken, ReplicaPlan.ForTokenRead>) NoopReadRepair.instance, requestTime);
ReadCallback<EndpointsForToken, ReplicaPlan.ForTokenRead> handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime);
// TODO No tracked path here yet so assert it doesn't handle transient replication correctly
checkState(!source.isTransient());
if (source.isSelf() && coordinator.localReadSupported())
{
Stage.READ.maybeExecuteImmediately(new StorageProxy.LocalReadRunnable(cmd, handler, requestTime));
}
else
{
if (source.isTransient())
cmd = cmd.copyAsTransientQuery(source);
cmd = coordinator.maybeAllowOutOfRangeReads(cmd, consistency);
MessagingService.instance().sendWithCallback(cmd.createMessage(false, requestTime), source.endpoint(), handler);
}
@ -196,7 +197,7 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
final EncodingStats stats;
final boolean[] silentRowAt;
final boolean[] silentColumnAt;
PartitionMergeListerner(DecoratedKey partitionKey, List<UnfilteredRowIterator> versions)
{
key = partitionKey;
@ -253,8 +254,8 @@ public class ReplicaFilteringProtection<E extends Endpoints<E>>
}
for (int i = 0; i < versions.length; i++)
// Mark the replica silent if it is silent about this column and there is actually
// divergence between the replicas. (i.e. If all replicas are silent for this
// Mark the replica silent if it is silent about this column and there is actually
// divergence between the replicas. (i.e. If all replicas are silent for this
// column, there is nothing to fetch to complete the row anyway.)
silentRowAt[i] |= silentColumnAt[i] && !allSilent;
}

5
src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java
View File

@ -47,6 +47,8 @@ import org.apache.cassandra.service.reads.repair.NoopReadRepair;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.Dispatcher;
import static com.google.common.base.Preconditions.checkState;
public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowIterator> implements MorePartitions<UnfilteredPartitionIterator>
{
private static final Logger logger = LoggerFactory.getLogger(ShortReadPartitionsProtection.class);
@ -192,8 +194,7 @@ public class ShortReadPartitionsProtection extends Transformation<UnfilteredRowI
}
else
{
if (source.isTransient())
cmd = cmd.copyAsTransientQuery(source);
checkState(!source.isTransient());
coordinator.sendReadCommand(cmd.createMessage(false, requestTime), source.endpoint(), handler);
}

6
src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java
View File

@ -68,9 +68,9 @@ import org.apache.cassandra.utils.AbstractIterator;
import org.apache.cassandra.utils.CloseableIterator;
import static com.google.common.base.Preconditions.checkState;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics;
import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetricsForLevel;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@VisibleForTesting
@ -219,6 +219,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
new DataResolver<>(readCoordinator, rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus);
ReadCallback<EndpointsForRange, ReplicaPlan.ForRangeRead> handler =
new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime);
checkState(!replicaPlan.contacts().anyMatch(Replica::isTransient), "Transient replication requires mutation tracking");
if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf() && readCoordinator.localReadSupported())
{
@ -229,8 +230,7 @@ public class RangeCommandIterator extends AbstractIterator<RowIterator> implemen
for (Replica replica : replicaPlan.contacts())
{
Tracing.trace("Enqueuing request to {}", replica);
ReadCommand command = replica.isFull() ? rangeCommand : rangeCommand.copyAsTransientQuery(replica);
Message<ReadCommand> message = command.createMessage(trackRepairedStatus && replica.isFull(), requestTime);
Message<ReadCommand> message = rangeCommand.createMessage(trackRepairedStatus && replica.isFull(), requestTime);
readCoordinator.sendReadCommand(message, replica.endpoint(), handler);
}
}

12
src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java
View File

@ -21,7 +21,6 @@ package org.apache.cassandra.service.reads.repair;
import java.util.function.Consumer;
import com.codahale.metrics.Meter;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -48,6 +47,7 @@ import org.apache.cassandra.service.reads.ReadCoordinator;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.transport.Dispatcher;
import static com.google.common.base.Preconditions.checkState;
import static java.util.concurrent.TimeUnit.MICROSECONDS;
public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends ReplicaPlan.ForRead<E, P>>
@ -103,13 +103,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
return;
}
if (to.isTransient())
{
// It's OK to send queries to transient nodes during RR, as we may have contacted them for their data request initially
// So long as we don't use these to generate repair mutations, we're fine, and this is enforced by requiring
// ReadOnlyReadRepair for transient keyspaces.
command = command.copyAsTransientQuery(to);
}
checkState(!to.isTransient(), "Transient replication requires mutation tracking so there should be no read repair");
if (Tracing.isTracing())
{
@ -187,7 +181,7 @@ public abstract class AbstractReadRepair<E extends Endpoints<E>, P extends Repli
public void maybeSendAdditionalReads()
{
Preconditions.checkState(command instanceof SinglePartitionReadCommand,
checkState(command instanceof SinglePartitionReadCommand,
"maybeSendAdditionalReads can only be called for SinglePartitionReadCommand");
DigestRepair<E, P> repair = digestRepair;
if (repair == null)

3
src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
View File

@ -423,8 +423,9 @@ public class SSTableMetadataViewer
field("ClusteringTypes", clusteringTypes.toString());
field("StaticColumns", FBUtilities.toString(statics));
field("RegularColumns", FBUtilities.toString(regulars));
// TODO (desired): This should be removed, but some tools might blow up expecting it
if (stats != null)
field("IsTransient", stats.isTransient);
field("IsTransient", false);
}
}

4
src/java/org/apache/cassandra/tools/SSTableRepairedAtSetter.java
View File

@ -79,11 +79,11 @@ public class SSTableRepairedAtSetter
if (setIsRepaired)
{
FileTime f = Files.getLastModifiedTime(descriptor.fileFor(Components.DATA).toPath());
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, f.toMillis(), null, false);
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, f.toMillis(), null);
}
else
{
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, 0, null, false);
descriptor.getMetadataSerializer().mutateRepairMetadata(descriptor, 0, null);
}
}
}

2
test/distributed/org/apache/cassandra/distributed/test/CountersTest.java
View File

@ -110,7 +110,7 @@ public class CountersTest extends TestBaseImpl
try
{
descriptor.getMetadataSerializer()
.mutateRepairMetadata(descriptor, System.currentTimeMillis(), null, false);
.mutateRepairMetadata(descriptor, System.currentTimeMillis(), null);
sstable.reloadSSTableMetadata();
}
catch (IOException e)

3
test/distributed/org/apache/cassandra/distributed/test/PreviewRepairSnapshotTest.java
View File

@ -177,8 +177,7 @@ public class PreviewRepairSnapshotTest extends TestBaseImpl
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor,
System.currentTimeMillis(),
null,
false);
null);
sstable.reloadSSTableMetadata();
}
catch (IOException e)

2
test/distributed/org/apache/cassandra/distributed/test/PreviewRepairTest.java
View File

@ -422,7 +422,7 @@ public class PreviewRepairTest extends TestBaseImpl
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table);
try
{
cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), ActiveRepairService.UNREPAIRED_SSTABLE, null, false);
cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), ActiveRepairService.UNREPAIRED_SSTABLE, null);
}
catch (IOException e)
{

2
test/distributed/org/apache/cassandra/distributed/test/RepairDigestTrackingTest.java
View File

@ -554,7 +554,7 @@ public class RepairDigestTrackingTest extends TestBaseImpl
SSTableReader sstable = sstables.next();
Descriptor descriptor = sstable.descriptor;
descriptor.getMetadataSerializer()
.mutateRepairMetadata(descriptor, currentTimeMillis(), null, false);
.mutateRepairMetadata(descriptor, currentTimeMillis(), null);
sstable.reloadSSTableMetadata();
}
} catch (IOException e) {

145
test/distributed/org/apache/cassandra/distributed/test/SharedClusterTestBase.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.io.IOException;
import java.util.function.Function;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.Cluster.Builder;
/**
* Base class for distributed tests that use a shared cluster across multiple test methods.
* This provides the same shared cluster lifecycle management as AccordTestBase but without
* the Accord-specific functionality.
*/
public abstract class SharedClusterTestBase extends TestBaseImpl
{
protected static final AtomicInteger COUNTER = new AtomicInteger(0);
protected static Cluster SHARED_CLUSTER;
protected String tableName;
protected String qualifiedTableName;
/**
* Sets up the shared cluster. Subclasses should call this from their @BeforeClass method.
*
* @param nodes the number of nodes in the cluster
* @param options function to customize the cluster builder
* @throws IOException if cluster creation fails
*/
protected static void setupCluster(int nodes, Function<Builder, Builder> options) throws IOException
{
SHARED_CLUSTER = options.apply(Cluster.build().withNodes(nodes)).start();
}
/**
* Sets up keyspace and tables for the test suite. Called once per test class.
* Subclasses should override this to customize keyspace and table creation.
*/
@BeforeClass
public static void setUpClass() throws Exception
{
// Subclasses should override to create keyspace and tables
}
/**
* Tears down keyspace and cluster. Called once per test class.
*/
@AfterClass
public static void tearDownClass()
{
if (SHARED_CLUSTER != null)
{
try
{
// Drop the test keyspace
SHARED_CLUSTER.schemaChange("DROP KEYSPACE IF EXISTS " + KEYSPACE);
}
catch (Exception e)
{
// Ignore errors during cleanup
}
SHARED_CLUSTER.close();
SHARED_CLUSTER = null;
}
}
/**
* Sets up test instance variables before each test method.
*/
@Before
public void setUp()
{
tableName = "tbl" + COUNTER.getAndIncrement();
qualifiedTableName = KEYSPACE + '.' + tableName;
}
/**
* Cleanup after each test method. Truncates tables created by the test.
*/
@After
public void tearDown()
{
if (SHARED_CLUSTER != null)
{
try
{
// Truncate tables to clean up data between tests
truncateTables();
}
catch (Exception e)
{
// Ignore errors during cleanup
}
}
}
/**
* Truncates tables after each test. Subclasses should override to specify which tables to truncate.
*/
protected void truncateTables()
{
// Default implementation does nothing, subclasses should override
}
/**
* Creates the keyspace with the specified replication strategy.
* Subclasses should call this from their setUpClass method.
*/
protected static void createKeyspace(String replicationStrategy)
{
SHARED_CLUSTER.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = " + replicationStrategy);
}
/**
* Creates a table with the specified DDL.
* Subclasses should call this from their setUpClass method.
*/
protected static void createTable(String tableDDL)
{
SHARED_CLUSTER.schemaChange(tableDDL);
}
}

3
test/distributed/org/apache/cassandra/distributed/test/TransientRangeMovement2Test.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import org.junit.Ignore;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
@ -29,6 +30,7 @@ import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.transformations.PrepareMove;
import org.apache.cassandra.utils.Pair;
@ -44,6 +46,7 @@ import static org.apache.cassandra.distributed.test.TransientRangeMovementTest.a
import static org.apache.cassandra.distributed.test.TransientRangeMovementTest.localStrs;
import static org.apache.cassandra.distributed.test.TransientRangeMovementTest.populate;
@Ignore(MutationTrackingUtils.IgnoreReasons.NO_RANGE_MOVEMENTS)
@SuppressWarnings("unchecked")
public class TransientRangeMovement2Test extends TestBaseImpl
{

3
test/distributed/org/apache/cassandra/distributed/test/TransientRangeMovementTest.java
View File

@ -31,6 +31,7 @@ import java.util.function.Function;
import com.google.common.collect.Sets;
import org.junit.Ignore;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
@ -40,6 +41,7 @@ import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.TokenSupplier;
import org.apache.cassandra.distributed.shared.NetworkTopology;
import org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.transformations.PrepareLeave;
@ -59,6 +61,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@Ignore(MutationTrackingUtils.IgnoreReasons.NO_RANGE_MOVEMENTS)
@SuppressWarnings("unchecked")
public class TransientRangeMovementTest extends TestBaseImpl
{

171
test/distributed/org/apache/cassandra/distributed/test/WitnessAlwaysReadsFullReplicaTest.java Normal file
View File

@ -0,0 +1,171 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.locator.BaseProximity;
import org.apache.cassandra.locator.Endpoint;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.locator.ReplicaCollection;
import org.apache.cassandra.utils.Sortable;
/**
* Make sure that if the fastest available replicas are transient that we don't hit an error
* by forgetting to contact a full replica.
*/
@RunWith(Parameterized.class)
public class WitnessAlwaysReadsFullReplicaTest extends SharedClusterTestBase
{
private static final Logger logger = LoggerFactory.getLogger(WitnessAlwaysReadsFullReplicaTest.class);
/**
* The consistency level to test.
*/
@Parameterized.Parameter
public ConsistencyLevel consistencyLevel;
private static String table = "test_tbl";
private static String qualifiedTable = KEYSPACE + "." + table;
@Parameterized.Parameters(name = "CL={0}")
public static Collection<Object[]> data()
{
List<Object[]> result = new ArrayList<>();
// Test all consistency levels except NODE_LOCAL as requested
for (ConsistencyLevel cl : ConsistencyLevel.values())
{
if (cl == ConsistencyLevel.ANY
|| cl == ConsistencyLevel.NODE_LOCAL
|| cl == ConsistencyLevel.SERIAL
|| cl == ConsistencyLevel.LOCAL_SERIAL)
continue;
result.add(new Object[]{ cl });
}
return result;
}
@BeforeClass
public static void setUpClass() throws Exception
{
// Set up shared cluster - for DC-aware consistency levels, we need multi-DC setup
setupCluster(6, builder -> builder.withRacks(2, 1, 3) // 2 DCs with 3 nodes each
.withConfig(cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", "true")
.set("transient_replication_enabled", "true")
.set("dynamic_snitch", false) // Disable dynamic snitch
.set("node_proximity", TransientFirstProximity.class.getName()))); // Use our custom proximity
// Create keyspace and table for the entire test suite
createKeyspace("{'class': 'NetworkTopologyStrategy', 'datacenter1': '3/1', 'datacenter2': '3/1'} AND replication_type='tracked'");
createTable("CREATE TABLE " + qualifiedTable + " (k int primary key, v int)");
}
@Override
protected void truncateTables()
{
// Truncate the test table after each test
SHARED_CLUSTER.schemaChange("TRUNCATE " + qualifiedTable);
}
/**
* Custom proximity implementation that prioritizes transient replicas first.
* This ensures that contactForRead will select transient replicas before full replicas,
* forcing the bug scenario where only transient replicas would be contacted.
*/
public static class TransientFirstProximity extends BaseProximity
{
@Override
public <C extends ReplicaCollection<? extends C>> C sortedByProximity(InetAddressAndPort address, C unsortedReplicas)
{
// Sort replicas to put transient replicas first, then full replicas
return unsortedReplicas.sorted((r1, r2) ->
{
// Transient replicas come first (lower value = higher priority)
if (r1.isTransient() && !r2.isTransient()) return -1; // r1 (transient) comes before r2 (full)
if (!r1.isTransient() && r2.isTransient()) return 1; // r2 (transient) comes before r1 (full)
return 0; // Same type, maintain stable order
});
}
@Override
public int compareEndpoints(InetAddressAndPort target, Replica r1, Replica r2)
{
// Transient replicas come first (lower value = higher priority)
if (r1.isTransient() && !r2.isTransient()) return -1; // r1 (transient) comes before r2 (full)
if (!r1.isTransient() && r2.isTransient()) return 1; // r2 (transient) comes before r1 (full)
return 0; // Same type, maintain stable order
}
@Override
public boolean supportCompareByEndpoint()
{
return true;
}
@Override
public <C extends Sortable<? extends Endpoint, ? extends C>> Comparator<Endpoint> endpointComparator(InetAddressAndPort address, C addresses)
{
return this::compareByEndpoint;
}
private int compareByEndpoint(Endpoint a, Endpoint b)
{
// For endpoints that are replicas, prioritize transient replicas first
if (a instanceof Replica && b instanceof Replica)
{
Replica r1 = (Replica) a;
Replica r2 = (Replica) b;
if (r1.isTransient() && !r2.isTransient()) return -1; // r1 (transient) comes before r2 (full)
if (!r1.isTransient() && r2.isTransient()) return 1; // r2 (transient) comes before r1 (full)
}
return 0; // Same type or not replicas, maintain stable order
}
}
@Test
public void testContactForReadBugCausesUnavailableException() throws Throwable
{
try
{
// This read should fail with UnavailableException because our custom proximity
// ensures transient replicas are sorted first, so contactForRead selects only transient replicas
SHARED_CLUSTER.coordinator(1).execute("SELECT * FROM " + qualifiedTable + " WHERE k = 1", consistencyLevel);
}
catch (Exception e)
{
throw e;
}
}
}

66
test/distributed/org/apache/cassandra/distributed/test/cql3/MultiNodeTableWalkWithWitnessesTest.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.cql3;
import java.util.List;
import accord.utils.RandomSource;
import org.apache.cassandra.cql3.ast.CreateIndexDDL;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.schema.TableMetadata;
public class MultiNodeTableWalkWithWitnessesTest extends MultiNodeTableWalkWithMutationTrackingTest
{
@Override
protected void clusterConfig(IInstanceConfig c)
{
super.clusterConfig(c);
// Enable transient replication replication
c.set("transient_replication_enabled", "true");
}
@Override
protected List<CreateIndexDDL.Indexer> supportedIndexers()
{
// TODO (expected): Implement supported indexers for witnesses
return List.of();
}
protected class MultiNodeState extends MutationTrackingState
{
public MultiNodeState(RandomSource rs, Cluster cluster)
{
super(rs, cluster);
}
@Override
protected String createKeyspaceCQL(TableMetadata metadata)
{
return createKeyspaceCQL(metadata, "3/1");
}
}
@Override
protected State createState(RandomSource rs, Cluster cluster)
{
return new MultiNodeState(rs, cluster);
}
}

11
test/distributed/org/apache/cassandra/distributed/test/cql3/StatefulASTBase.java
View File

@ -469,10 +469,15 @@ public class StatefulASTBase extends TestBaseImpl
cluster.schemaChange(metadata.toCqlString(false, false, false));
}
private String createKeyspaceCQL(TableMetadata metadata)
protected String createKeyspaceCQL(TableMetadata metadata)
{
return "CREATE KEYSPACE IF NOT EXISTS " + metadata.keyspace +
" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + Math.min(3, cluster.size()) + '}' +
return createKeyspaceCQL(metadata, String.valueOf(Math.min(3, cluster.size())));
}
protected String createKeyspaceCQL(TableMetadata metadata, String replicationFactor)
{
return "CREATE KEYSPACE IF NOT EXISTS " + metadata.keyspace +
" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '" + replicationFactor + "'}" +
" AND replication_type='" + metadata.keyspaceReplicationType + "';";
}

357
test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java
View File

@ -20,24 +20,28 @@ package org.apache.cassandra.distributed.test.tracking;
import java.util.UUID;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.replication.CoordinatorLogId;
import org.apache.cassandra.replication.MutationSummary;
import org.apache.cassandra.replication.Offsets;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.Util;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.gms.Gossiper;
import org.apache.cassandra.hints.HintsService;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.replication.CoordinatorLogId;
import org.apache.cassandra.replication.MutationSummary;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.replication.Offsets;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.schema.Schema;
@ -47,11 +51,22 @@ import org.apache.cassandra.utils.ByteBufferUtil;
import static org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils.getOnlyLogId;
import static org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils.summaryIdSpace;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
// TODO This test would be a lot faster if it had a shared cluster
public class MutationTrackingTest extends TestBaseImpl
{
private static final Logger logger = LoggerFactory.getLogger(MutationTrackingTest.class);
private static final String INSERT_FMT = "INSERT INTO " + KEYSPACE + ".tbl (k, v) VALUES (%d, %d)";
private static final String INSERT_CQL = String.format(INSERT_FMT, 1, 1);
private static final String CONDITIONAL_INSERT_CQL = INSERT_CQL + " IF NOT EXISTS";
private static final String BATCH_INSERT_FMT = "BEGIN %s BATCH%n"
+ " %s%n"
+ " %s%n"
+ "APPLY BATCH";
@Ignore
@Test
public void testBasicWritePath() throws Throwable
{
@ -72,7 +87,7 @@ public class MutationTrackingTest extends TestBaseImpl
cluster.get(1).runOnInstance(() -> {
KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
Assert.assertEquals(ReplicationType.tracked, keyspace.params.replicationType);
assertEquals(ReplicationType.tracked, keyspace.params.replicationType);
});
cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM);
@ -83,11 +98,331 @@ public class MutationTrackingTest extends TestBaseImpl
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
CoordinatorLogId logId = getOnlyLogId(summary);
Offsets summaryIds = summaryIdSpace(summary.get(logId));
Assert.assertEquals(1, summaryIds.offsetCount());
assertEquals(1, summaryIds.offsetCount());
});
}
}
@Ignore
@Test
public void testWitnessPaxosV1Reads() throws Throwable
{
testWitnessPaxosReads("v1");
}
@Ignore
@Test
public void testWitnessPaxosV2Reads() throws Throwable
{
testWitnessPaxosReads("v2");
}
private void testWitnessPaxosReads(String paxosVariant) throws Throwable
{
try (Cluster cluster = Cluster.build(3)
.withConfig(cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", "true")
.set("transient_replication_enabled", "true")
.set("paxos_variant", paxosVariant))
.start())
{
String keyspaceName = KEYSPACE;
cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': '3/1'} " +
"AND replication_type='tracked';"));
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);"));
// TODO shouldn't be necessary to mess with marking things in Gossip but there is no read speculation
// so the read fails because it routes to a node that is blocked
cluster.filters().allVerbs().to(3).drop().on();
cluster.filters().allVerbs().from(3).drop().on();
for (int i = 1; i < 3; i++)
cluster.get(i).runOnInstance(() -> Gossiper.instance.convict(InetAddressAndPort.getByNameUnchecked("127.0.0.3"), Double.MAX_VALUE));
cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM);
// Two nodes should know about the mutation
for (int i = 1; i <= 2; i++)
cluster.get(i).runOnInstance(() -> {
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(Util.dk(1), ColumnFamilyStore.getIfExists(keyspaceName, "tbl").metadata.id, true);
assertEquals(1, summary.size());
});
// Filter should stop the witness from getting the mutation so we can test pushing the mutation summary to the witness
cluster.get(3).runOnInstance(() -> {
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(Util.dk(1), ColumnFamilyStore.getIfExists(keyspaceName, "tbl").metadata.id, true);
assertEquals(0, summary.size());
});
int rowsFound = 0;
String singlePartitionSelectCQL = withKeyspace("SELECT * FROM %s.tbl WHERE k = 1");
for (IInvokableInstance instance : cluster)
{
Object[][] result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0-1 rows", result.length == 0 || result.length == 1);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 2, rowsFound);
cluster.filters().reset();
cluster.filters().allVerbs().to(2).drop().on();
cluster.filters().allVerbs().from(2).drop().on();
cluster.get(1).runOnInstance(() -> Gossiper.runInGossipStageBlocking(() -> {
InetAddressAndPort endpoint = InetAddressAndPort.getByNameUnchecked("127.0.0.3");
Gossiper.instance.realMarkAlive(endpoint, Gossiper.instance.getEndpointStateForEndpoint(endpoint));
}));
for (int i = 1; i < 4; i++)
if (i != 2)
cluster.get(i).runOnInstance(() -> Gossiper.instance.convict(InetAddressAndPort.getByNameUnchecked("127.0.0.2"), Double.MAX_VALUE));
Object[][] result = cluster.coordinator(1).execute(singlePartitionSelectCQL, ConsistencyLevel.SERIAL);
assertEquals(1, result.length);
assertEquals(1, result[0][0]);
assertEquals(1, result[0][1]);
// The read at SERIAL should propagate the mutation to the witness
cluster.get(3).runOnInstance(() -> {
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(Util.dk(1), ColumnFamilyStore.getIfExists(keyspaceName, "tbl").metadata.id, true);
assertEquals(1, summary.size());
});
}
}
@Ignore("Unlogged batches not supported with mutation tracking yet")
@Test
public void testWitnessUnloggedBatchSkippedPath() throws Throwable
{
testWitnessBatchWrites(false);
}
@Ignore("Logged batches not supported with mutation tracking yet")
@Test
public void testWitnessLoggedBatchSkippedPath() throws Throwable
{
testWitnessBatchWrites(true);
}
private void testWitnessBatchWrites(boolean logged) throws Throwable
{
try (Cluster cluster = Cluster.build(3)
.withConfig(cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", "true")
.set("transient_replication_enabled", "true"))
.start())
{
cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': '3/1'} " +
"AND replication_type='tracked';"));
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);"));
String keyspaceName = KEYSPACE;
cluster.get(1).runOnInstance(() -> {
KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
assertEquals(ReplicationType.tracked, keyspace.params.replicationType);
});
String insertCql = String.format(BATCH_INSERT_FMT, logged ? "" : "UNLOGGED", String.format(INSERT_FMT, KEYSPACE, 1, 1), String.format(INSERT_FMT, KEYSPACE, 2, 2));
cluster.coordinator(1).execute(insertCql, ConsistencyLevel.ALL);
// Only two instances should have the row
int rowsFound = 0;
String singlePartitionSelectCQL = withKeyspace("SELECT * FROM %s.tbl");
for (IInvokableInstance instance : cluster)
{
Object[][] result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0 or 2 rows", result.length == 0 || result.length == 2);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 4, rowsFound);
cluster.get(1).runOnInstance(() -> {
TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
CoordinatorLogId logId = getOnlyLogId(summary);
Offsets summaryIds = summaryIdSpace(summary.get(logId));
assertEquals(1, summaryIds.offsetCount());
});
Object[][] result = cluster.coordinator(1).execute(singlePartitionSelectCQL, ConsistencyLevel.ALL);
assertEquals(2, result.length);
String partitionRangeSelectCQL = withKeyspace("SELECT * FROM %s.tbl");
result = cluster.coordinator(1).execute(partitionRangeSelectCQL, ConsistencyLevel.ALL);
assertEquals(2, result.length);
// Read time reconciliation should not propagate the row to the witness node
rowsFound = 0;
for (IInvokableInstance instance : cluster)
{
result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0 or 2 rows", result.length == 0 || result.length == 2);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 4, rowsFound);
}
}
@Test
public void testWitnessHintSkippedPath() throws Throwable
{
}
@Ignore
@Test
public void testWitnessSerialPaxosV1WritesSkipped() throws Throwable
{
testWitnessWrites(CONDITIONAL_INSERT_CQL, ConsistencyLevel.SERIAL, "v1");
}
@Ignore
@Test
public void testWitnessSerialPaxosV2WritesSkipped() throws Throwable
{
testWitnessWrites(CONDITIONAL_INSERT_CQL, ConsistencyLevel.SERIAL, "v1");
}
@Test
public void testNonSerialWitnessWrites() throws Throwable
{
testWitnessWrites(INSERT_CQL, ConsistencyLevel.ALL, null);
}
private void testWitnessWrites(String insertCql, ConsistencyLevel cl, String paxosVariant) throws Throwable
{
String paxosVariantFinal = paxosVariant == null ? "v1" : paxosVariant;
try (Cluster cluster = Cluster.build(3)
.withConfig(cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", "true")
.set("transient_replication_enabled", "true")
.set("paxos_variant", paxosVariantFinal))
.start())
{
cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': '3/1'} " +
"AND replication_type='tracked';"));
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);"));
String keyspaceName = KEYSPACE;
cluster.get(1).runOnInstance(() -> {
KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
assertEquals(ReplicationType.tracked, keyspace.params.replicationType);
});
cluster.coordinator(1).execute(insertCql, cl, ConsistencyLevel.QUORUM);
// Only two instances should have the row
int rowsFound = 0;
String singlePartitionSelectCQL = withKeyspace("SELECT * FROM %s.tbl");
for (IInvokableInstance instance : cluster)
{
Object[][] result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0-1 rows", result.length == 0 || result.length == 1);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 2, rowsFound);
cluster.get(1).runOnInstance(() -> {
TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
CoordinatorLogId logId = getOnlyLogId(summary);
Offsets summaryIds = summaryIdSpace(summary.get(logId));
assertEquals(1, summaryIds.offsetCount());
});
Object[][] result = cluster.coordinator(1).execute(singlePartitionSelectCQL, ConsistencyLevel.ALL);
assertEquals(1, result.length);
String partitionRangeSelectCQL = withKeyspace("SELECT * FROM %s.tbl");
result = cluster.coordinator(1).execute(partitionRangeSelectCQL, ConsistencyLevel.ALL);
assertEquals(1, result.length);
// Read time reconciliation should not propagate the row to the witness node
rowsFound = 0;
for (IInvokableInstance instance : cluster)
{
result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0-1 rows", result.length == 0 || result.length == 1);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 2, rowsFound);
}
}
@Test
public void testWitnessWriteSkippedPath() throws Throwable
{
try (Cluster cluster = Cluster.build(3)
.withConfig(cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", "true")
.set("transient_replication_enabled", "true"))
.start())
{
cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': '3/1'} " +
"AND replication_type='tracked';"));
cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);"));
String keyspaceName = KEYSPACE;
cluster.get(1).runOnInstance(() -> {
KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName);
assertEquals(ReplicationType.tracked, keyspace.params.replicationType);
});
cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.ALL);
// Only two instances should have the row
int rowsFound = 0;
String singlePartitionSelectCQL = withKeyspace("SELECT * FROM %s.tbl WHERE k = 1");
for (IInvokableInstance instance : cluster)
{
Object[][] result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0-1 rows", result.length == 0 || result.length == 1);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 2, rowsFound);
cluster.get(1).runOnInstance(() -> {
TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl");
DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1));
MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false);
CoordinatorLogId logId = getOnlyLogId(summary);
Offsets summaryIds = summaryIdSpace(summary.get(logId));
assertEquals(1, summaryIds.offsetCount());
});
Object[][] result = cluster.coordinator(1).execute(singlePartitionSelectCQL, ConsistencyLevel.ALL);
assertEquals(1, result.length);
String partitionRangeSelectCQL = withKeyspace("SELECT * FROM %s.tbl");
result = cluster.coordinator(1).execute(partitionRangeSelectCQL, ConsistencyLevel.ALL);
assertEquals(1, result.length);
// Read time reconciliation should not propagate the row to the witness node
rowsFound = 0;
for (IInvokableInstance instance : cluster)
{
result = instance.executeInternal(singlePartitionSelectCQL);
assertTrue("Each node should have 0-1 rows", result.length == 0 || result.length == 1);
rowsFound += result.length;
}
assertEquals("Only two instances should have the row", 2, rowsFound);
}
}
@Test
public void testHintsNotWrittenOnFailedWrite() throws Throwable
{
@ -111,7 +446,7 @@ public class MutationTrackingTest extends TestBaseImpl
long hints = cluster.get(1).callOnInstance(() -> StorageMetrics.totalHints.getCount());
// confirm no hints for node 3
cluster.get(1).runOnInstance(() -> Assert.assertEquals(0, HintsService.instance.getTotalHintsSize(node3HostId)));
cluster.get(1).runOnInstance(() -> assertEquals(0, HintsService.instance.getTotalHintsSize(node3HostId)));
cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM);
// wait for write timeout
@ -119,7 +454,7 @@ public class MutationTrackingTest extends TestBaseImpl
// TODO: confirm hints aren't written
cluster.get(1).runOnInstance(() -> {
Assert.assertEquals(hints, StorageMetrics.totalHints.getCount());
assertEquals(hints, StorageMetrics.totalHints.getCount());
});
}
}

7
test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java
View File

@ -56,6 +56,13 @@ public class MutationTrackingUtils
private static final Logger logger = LoggerFactory.getLogger(MutationTrackingUtils.class);
private static final int VERSION = MessagingService.current_version;
public static class IgnoreReasons
{
public static final String NO_RANGE_MOVEMENTS = "NO_RANGE_MOVEMENTS";
}
public static byte[] encodeId(MutationId id)
{
int size = Ints.checkedCast(MutationId.serializer.serializedSize(id, VERSION));

10
test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
View File

@ -417,12 +417,6 @@ public abstract class ForwardingSSTableReader extends SSTableReader
return delegate.getRepairedAt();
}
@Override
public boolean isTransient()
{
return delegate.isTransient();
}
@Override
public boolean intersects(Collection<Range<Token>> ranges)
{
@ -532,9 +526,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader
}
@Override
public void mutateRepairedAndReload(long newRepairedAt, TimeUUID newPendingRepair, boolean isTransient) throws IOException
public void mutateRepairedAndReload(long newRepairedAt, TimeUUID newPendingRepair) throws IOException
{
delegate.mutateRepairedAndReload(newRepairedAt, newPendingRepair, isTransient);
delegate.mutateRepairedAndReload(newRepairedAt, newPendingRepair);
}
@Override

2
test/distributed/org/apache/cassandra/service/accord/NemesisAccordSegmentCompactor.java
View File

@ -71,7 +71,7 @@ public class NemesisAccordSegmentCompactor<V> extends AbstractAccordSegmentCompa
{
Descriptor descriptor = cfs.newSSTableDescriptor(cfs.getDirectories().getDirectoryForNewSSTables());
SerializationHeader header = new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS);
writers[i] = SSTableTxnWriter.create(cfs, descriptor, 0, 0, null, false, CoordinatorLogOffsets.NONE, header);
writers[i] = SSTableTxnWriter.create(cfs, descriptor, 0, 0, null, CoordinatorLogOffsets.NONE, header);
}
}

9
test/harry/main/org/apache/cassandra/harry/model/TokenPlacementModel.java
View File

@ -43,6 +43,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.SimpleStrategy;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.Location;
import org.apache.cassandra.tcm.membership.NodeId;
@ -399,14 +400,17 @@ public class TokenPlacementModel
{
Object[] args = new Object[replication.size() * 2];
int i = 0;
boolean hasTransient = false;
for (Map.Entry<String, DCReplicas> e : replication.entrySet())
{
args[i * 2] = e.getKey();
args[i * 2 + 1] = e.getValue().toString();
if (e.getValue().transientCount > 0)
hasTransient = true;
i++;
}
return KeyspaceParams.nts(args);
return hasTransient ? KeyspaceParams.ntsTracked(args) : KeyspaceParams.nts(args);
}
private static Function<Lookup, Map<String, DCReplicas>> mapFunction(int dcs, int nodesPerDc, int transientsPerDc)
@ -615,7 +619,8 @@ public class TokenPlacementModel
Map<String, String> options = new HashMap<>();
options.put(ReplicationParams.CLASS, SimpleStrategy.class.getName());
options.put(SimpleStrategy.REPLICATION_FACTOR, dcReplicas().toString());
return KeyspaceParams.create(true, options);
ReplicationType replicationType = dcReplicas().transientCount > 0 ? ReplicationType.tracked : ReplicationType.untracked;
return KeyspaceParams.create(true, options, replicationType);
}
public ReplicatedRanges replicate(Range[] ranges, List<Node> nodes)

5
test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java
View File

@ -233,7 +233,7 @@ public class LongLeveledCompactionStrategyTest
populateSSTables(store);
assertTrue(repaired.getSSTables().isEmpty());
assertFalse(unrepaired.getSSTables().isEmpty());
mgr.mutateRepaired(store.getLiveSSTables(), FBUtilities.nowInSeconds(), null, false);
mgr.mutateRepaired(store.getLiveSSTables(), FBUtilities.nowInSeconds(), null);
assertFalse(repaired.getSSTables().isEmpty());
assertTrue(unrepaired.getSSTables().isEmpty());
@ -250,8 +250,7 @@ public class LongLeveledCompactionStrategyTest
// mark unrepair
mgr.mutateRepaired(store.getLiveSSTables().stream().filter(s -> s.isRepaired()).collect(Collectors.toList()),
ActiveRepairService.UNREPAIRED_SSTABLE,
null,
false);
null);
assertTrue(repaired.getSSTables().isEmpty());
assertFalse(unrepaired.getSSTables().isEmpty());
}

3
test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeBench.java
View File

@ -34,6 +34,7 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.ISSTableScanner;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.SSTableWriter;
import org.apache.cassandra.replication.CoordinatorLogOffsets;
import org.apache.cassandra.schema.TableMetadataRef;
import org.apache.cassandra.tools.Util;
@ -54,7 +55,7 @@ public class SSTablePipeBench extends SSTableAbstractPipeBench
public void readAndWrite() throws Throwable
{
SSTableReader ssTableReader = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata));
try (SSTableWriter ssTableWriter = CompactionManager.createWriter(cfs, new org.apache.cassandra.io.util.File(tmpDir), -1, -1, null, false, ssTableReader, LifecycleTransaction.offline(OperationType.COMPACTION));)
try (SSTableWriter ssTableWriter = CompactionManager.createWriter(cfs, new org.apache.cassandra.io.util.File(tmpDir), -1, -1, null, CoordinatorLogOffsets.NONE, ssTableReader, LifecycleTransaction.offline(OperationType.COMPACTION));)
{
final ISSTableScanner currentScanner = ssTableReader.getScanner();
Stream<UnfilteredRowIterator> partitions = Util.iterToStream(currentScanner);

3
test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeCursorBench.java
View File

@ -32,6 +32,7 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.io.sstable.SSTableCursorReader;
import org.apache.cassandra.io.sstable.SSTableCursorWriter;
import org.apache.cassandra.io.sstable.format.SortedTableWriter;
import org.apache.cassandra.replication.CoordinatorLogOffsets;
@State(Scope.Benchmark)
@ -50,7 +51,7 @@ public class SSTablePipeCursorBench extends SSTableAbstractPipeBench
public void readAndWrite() throws Throwable
{
try(SSTableCursorReader cursorReader = SSTableCursorReader.fromDescriptor(desc);
SortedTableWriter ssTableWriter = (SortedTableWriter) CompactionManager.createWriter(cfs, new org.apache.cassandra.io.util.File(tmpDir), 0, 0, null, false, cursorReader.ssTableReader(), LifecycleTransaction.offline(OperationType.COMPACTION));
SortedTableWriter ssTableWriter = (SortedTableWriter) CompactionManager.createWriter(cfs, new org.apache.cassandra.io.util.File(tmpDir), 0, 0, null, CoordinatorLogOffsets.NONE, cursorReader.ssTableReader(), LifecycleTransaction.offline(OperationType.COMPACTION));
SSTableCursorWriter cursorWriter = new SSTableCursorWriter(ssTableWriter);){
SSTableCursorPipeUtil.copySSTable(cursorReader, cursorWriter);
}

5
test/unit/org/apache/cassandra/SchemaLoader.java
View File

@ -225,6 +225,8 @@ public class SchemaLoader
+ "v2 int"
+ ")";
// CQLKeyspace
// ReplicationType replicationType = DatabaseDescriptor.getMutationTrackingEnabled() ? ReplicationType.tracked : ReplicationType.untracked;
// schema.add(KeyspaceMetadata.create(ks_cql, KeyspaceParams.simple(1, replicationType), Tables.of(
schema.add(KeyspaceMetadata.create(ks_cql, KeyspaceParams.simple(1), Tables.of(
// Column Families
@ -241,7 +243,8 @@ public class SchemaLoader
schema.add(KeyspaceMetadata.create(ks_cql_replicated, KeyspaceParams.simple(3),
Tables.of(CreateTableStatement.parse(simpleTable, ks_cql_replicated).build())));
schema.add(KeyspaceMetadata.create(ks_with_transient, KeyspaceParams.simple("3/1"),
schema.add(KeyspaceMetadata.create(ks_with_transient,
KeyspaceParams.simpleWitness("3/1"),
Tables.of(CreateTableStatement.parse(simpleTable, ks_with_transient).build())));
if (DatabaseDescriptor.getPartitioner() instanceof Murmur3Partitioner)

2
test/unit/org/apache/cassandra/db/CleanupTest.java
View File

@ -281,7 +281,7 @@ public class CleanupTest
beforeFirstCleanup.forEach((sstable) -> {
try
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), null, false);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), null);
sstable.reloadSSTableMetadata();
}
catch (Exception e)

15
test/unit/org/apache/cassandra/db/CleanupTransientTest.java
View File

@ -41,6 +41,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -73,9 +74,10 @@ public class CleanupTransientTest extends CassandraTestBase
@BeforeClass
public static void setup() throws Exception
{
DatabaseDescriptor.setMutationTrackingEnabled(true);
DatabaseDescriptor.setTransientReplicationEnabledUnsafe(true);
SchemaLoader.createKeyspace(KEYSPACE1,
KeyspaceParams.simple("2/1"),
KeyspaceParams.simpleWitness("2/1"),
SchemaLoader.standardCFMD(KEYSPACE1, CF_STANDARD1),
SchemaLoader.compositeIndexCFMD(KEYSPACE1, CF_INDEXED1, true));
@ -132,7 +134,7 @@ public class CleanupTransientTest extends CassandraTestBase
}
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null, false);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null);
sstable.reloadSSTableMetadata();
// This should remove approximately 50% of the data, specifically whatever was transiently replicated
@ -153,10 +155,11 @@ public class CleanupTransientTest extends CassandraTestBase
{
String key = String.valueOf(i);
// create a row and update the birthdate value, test that the index query fetches the new version
new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBufferUtil.bytes(key))
.clustering(COLUMN)
.add(colName, VALUE)
.build()
Mutation mutation = new RowUpdateBuilder(cfs.metadata(), System.currentTimeMillis(), ByteBufferUtil.bytes(key))
.clustering(COLUMN)
.add(colName, VALUE)
.build();
mutation.withMutationId(MutationTrackingService.instance.nextMutationId(cfs.metadata().keyspace, mutation.key().getToken()))
.applyUnsafe();
}

2
test/unit/org/apache/cassandra/db/ImportTest.java
View File

@ -234,7 +234,7 @@ public class ImportTest extends CQLTester
Set<SSTableReader> sstables = getCurrentColumnFamilyStore().getLiveSSTables();
getCurrentColumnFamilyStore().clearUnsafe();
for (SSTableReader sstable : sstables)
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 111, null, false);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 111, null);
File backupdir = moveToBackupDir(sstables);
assertEquals(0, execute("select * from %s").size());

1
test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java
View File

@ -81,6 +81,7 @@ public class MutationVerbHandlerOutOfRangeTest
SchemaLoader.schemaDefinition(TEST_NAME);
ServerTestUtils.markCMS();
StorageService.instance.unsafeSetInitialized();
org.apache.cassandra.config.DatabaseDescriptor.setMutationTrackingEnabled(true);
}
@Before

48
test/unit/org/apache/cassandra/db/ReadCommandTest.java
View File

@ -76,9 +76,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.WrappedDataOutputStreamPlus;
import org.apache.cassandra.locator.EndpointsForToken;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.ReplicaUtils;
import org.apache.cassandra.metrics.ClearableHistogram;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
@ -1568,50 +1566,6 @@ public class ReadCommandTest
}
}
@Test (expected = IllegalArgumentException.class)
public void copyFullAsTransientTest()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
readCommand.copyAsTransientQuery(ReplicaUtils.full(FBUtilities.getBroadcastAddressAndPort()));
}
@Test (expected = IllegalArgumentException.class)
public void copyTransientAsDigestQuery()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
ReadCommand readCommand = Util.cmd(cfs, Util.dk("key")).build();
readCommand.copyAsDigestQuery(ReplicaUtils.trans(FBUtilities.getBroadcastAddressAndPort()));
}
@Test (expected = IllegalArgumentException.class)
public void copyMultipleFullAsTransientTest()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
DecoratedKey key = Util.dk("key");
Token token = key.getToken();
// Address is unimportant for this test
InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
ReadCommand readCommand = Util.cmd(cfs, key).build();
readCommand.copyAsTransientQuery(EndpointsForToken.of(token,
ReplicaUtils.trans(addr, token),
ReplicaUtils.full(addr, token)));
}
@Test (expected = IllegalArgumentException.class)
public void copyMultipleTransientAsDigestQuery()
{
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(CF6);
DecoratedKey key = Util.dk("key");
Token token = key.getToken();
// Address is unimportant for this test
InetAddressAndPort addr = FBUtilities.getBroadcastAddressAndPort();
ReadCommand readCommand = Util.cmd(cfs, key).build();
readCommand.copyAsDigestQuery(EndpointsForToken.of(token,
ReplicaUtils.trans(addr, token),
ReplicaUtils.full(addr, token)));
}
@Test
public void testToCQLString()
{
@ -1711,7 +1665,7 @@ public class ReadCommandTest
{
try
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingSession, false);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingSession);
sstable.reloadSSTableMetadata();
}
catch (IOException e)

2
test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java
View File

@ -220,7 +220,6 @@ public class ReadCommandVerbHandlerOutOfRangeTest
super(tmd.epoch,
false,
0,
false,
PotentialTxnConflicts.DISALLOW,
tmd,
FBUtilities.nowInSeconds(),
@ -256,7 +255,6 @@ public class ReadCommandVerbHandlerOutOfRangeTest
super(tmd.epoch,
false,
0,
false,
PotentialTxnConflicts.DISALLOW,
tmd,
FBUtilities.nowInSeconds(),

1
test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java
View File

@ -174,7 +174,6 @@ public class ReadCommandVerbHandlerTest
super(metadata.epoch,
false,
0,
false,
PotentialTxnConflicts.DISALLOW,
metadata,
FBUtilities.nowInSeconds(),

1
test/unit/org/apache/cassandra/db/ReadResponseTest.java
View File

@ -254,7 +254,6 @@ public class ReadResponseTest
super(metadata.epoch,
isDigest,
0,
false,
PotentialTxnConflicts.DISALLOW,
metadata,
FBUtilities.nowInSeconds(),

2
test/unit/org/apache/cassandra/db/RepairedDataTombstonesTest.java
View File

@ -314,7 +314,7 @@ public class RepairedDataTombstonesTest extends CQLTester
public static void repair(ColumnFamilyStore cfs, SSTableReader sstable) throws IOException
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null, false);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, 1, null);
sstable.reloadSSTableMetadata();
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
}

14
test/unit/org/apache/cassandra/db/compaction/AbstractPendingRepairTest.java
View File

@ -101,11 +101,11 @@ public class AbstractPendingRepairTest extends AbstractRepairTest
return sstable;
}
public static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient)
public static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair)
{
try
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair);
sstable.reloadSSTableMetadata();
}
catch (IOException e)
@ -116,17 +116,17 @@ public class AbstractPendingRepairTest extends AbstractRepairTest
public static void mutateRepaired(SSTableReader sstable, long repairedAt)
{
mutateRepaired(sstable, repairedAt, ActiveRepairService.NO_PENDING_REPAIR, false);
mutateRepaired(sstable, repairedAt, ActiveRepairService.NO_PENDING_REPAIR);
}
public static void mutateRepaired(SSTableReader sstable, TimeUUID pendingRepair, boolean isTransient)
public static void mutateRepaired(SSTableReader sstable, TimeUUID pendingRepair)
{
mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair, isTransient);
mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair);
}
public static void mutateRepaired(List<SSTableReader> sstables, TimeUUID pendingRepair, boolean isTransient)
public static void mutateRepaired(List<SSTableReader> sstables, TimeUUID pendingRepair)
{
for (SSTableReader sstable : sstables)
mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair, isTransient);
mutateRepaired(sstable, ActiveRepairService.UNREPAIRED_SSTABLE, pendingRepair);
}
}

77
test/unit/org/apache/cassandra/db/compaction/AntiCompactionTest.java
View File

@ -178,19 +178,12 @@ public class AntiCompactionTest
try (UnfilteredRowIterator row = scanner.next())
{
Token token = row.partitionKey().getToken();
if (sstable.isPendingRepair() && !sstable.isTransient())
if (sstable.isPendingRepair())
{
assertTrue(fullContains.test(token));
assertFalse(transContains.test(token));
stats.pendingKeys++;
}
else if (sstable.isPendingRepair() && sstable.isTransient())
{
assertTrue(transContains.test(token));
assertFalse(fullContains.test(token));
stats.transKeys++;
}
else
{
assertFalse(fullContains.test(token));
@ -222,30 +215,6 @@ public class AntiCompactionTest
assertOnDiskState(store, 2);
}
@Test
public void antiCompactOneMixed() throws Exception
{
ColumnFamilyStore store = prepareColumnFamilyStore();
SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), range(4, 8)));
assertEquals(3, stats.numLiveSSTables);
assertEquals(stats.pendingKeys, 4);
assertEquals(stats.transKeys, 4);
assertEquals(stats.unrepairedKeys, 2);
assertOnDiskState(store, 3);
}
@Test
public void antiCompactOneTransOnly() throws Exception
{
ColumnFamilyStore store = prepareColumnFamilyStore();
SSTableStats stats = antiCompactRanges(store, atEndpoint(NO_RANGES, range(0, 4)));
assertEquals(2, stats.numLiveSSTables);
assertEquals(stats.pendingKeys, 0);
assertEquals(stats.transKeys, 4);
assertEquals(stats.unrepairedKeys, 6);
assertOnDiskState(store, 2);
}
@Test
public void antiCompactionSizeTest() throws InterruptedException, IOException, NoSuchRepairSessionException
{
@ -282,7 +251,7 @@ public class AntiCompactionTest
File dir = cfs.getDirectories().getDirectoryForNewSSTables();
Descriptor desc = cfs.newSSTableDescriptor(dir);
try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, false, ImmutableCoordinatorLogOffsets.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS)))
try (SSTableTxnWriter writer = SSTableTxnWriter.create(cfs, desc, 0, 0, NO_PENDING_REPAIR, ImmutableCoordinatorLogOffsets.NONE, new SerializationHeader(true, cfs.metadata(), cfs.metadata().regularAndStaticColumns(), EncodingStats.NO_STATS)))
{
for (int i = 0; i < count; i++)
{
@ -336,48 +305,6 @@ public class AntiCompactionTest
assertOnDiskState(store, 10);
}
@Test
public void antiCompactTenTrans() throws IOException, NoSuchRepairSessionException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
store.disableAutoCompaction();
for (int table = 0; table < 10; table++)
{
generateSStable(store,Integer.toString(table));
}
SSTableStats stats = antiCompactRanges(store, atEndpoint(NO_RANGES, range(0, 4)));
/*
Anticompaction will be anti-compacting 10 SSTables but will be doing this two at a time
so there will be no net change in the number of sstables
*/
assertEquals(10, stats.numLiveSSTables);
assertEquals(stats.pendingKeys, 0);
assertEquals(stats.transKeys, 40);
assertEquals(stats.unrepairedKeys, 60);
assertOnDiskState(store, 10);
}
@Test
public void antiCompactTenMixed() throws IOException, NoSuchRepairSessionException
{
Keyspace keyspace = Keyspace.open(KEYSPACE1);
ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
store.disableAutoCompaction();
for (int table = 0; table < 10; table++)
{
generateSStable(store,Integer.toString(table));
}
SSTableStats stats = antiCompactRanges(store, atEndpoint(range(0, 4), range(4, 8)));
assertEquals(15, stats.numLiveSSTables);
assertEquals(stats.pendingKeys, 40);
assertEquals(stats.transKeys, 40);
assertEquals(stats.unrepairedKeys, 20);
assertOnDiskState(store, 15);
}
@Test
public void shouldMutatePendingRepair() throws InterruptedException, IOException, NoSuchRepairSessionException
{

2
test/unit/org/apache/cassandra/db/compaction/CompactionGarbageCollectOnlyPurgeRepairedTest.java
View File

@ -361,7 +361,7 @@ public class CompactionGarbageCollectOnlyPurgeRepairedTest extends CQLTester
*/
private static void repair(ColumnFamilyStore cfs, SSTableReader sstable) throws IOException
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), null, false);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, System.currentTimeMillis(), null);
sstable.reloadSSTableMetadata();
cfs.getTracker().notifySSTableRepairedStatusChanged(Collections.singleton(sstable));
}

106
test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerPendingRepairTest.java
View File

@ -45,12 +45,6 @@ import org.apache.cassandra.utils.TimeUUID;
*/
public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingRepairTest
{
private boolean transientContains(SSTableReader sstable)
{
return csm.getTransientRepairsUnsafe().containsSSTable(sstable);
}
private boolean pendingContains(SSTableReader sstable)
{
return csm.getPendingRepairsUnsafe().containsSSTable(sstable);
@ -71,11 +65,6 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
return !Iterables.isEmpty(csm.getPendingRepairsUnsafe().getStrategiesFor(sessionID));
}
private boolean hasTransientStrategiesFor(TimeUUID sessionID)
{
return !Iterables.isEmpty(csm.getTransientRepairsUnsafe().getStrategiesFor(sessionID));
}
/**
* Pending repair strategy should be created when we encounter a new pending id
*/
@ -90,11 +79,10 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertFalse(sstable.isRepaired());
Assert.assertFalse(sstable.isPendingRepair());
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
Assert.assertFalse(sstable.isRepaired());
Assert.assertTrue(sstable.isPendingRepair());
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
// add the sstable
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
@ -102,7 +90,6 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertFalse(unrepairedContains(sstable));
Assert.assertTrue(pendingContains(sstable));
Assert.assertTrue(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
}
@Test
@ -112,17 +99,16 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable1 = makeSSTable(true);
mutateRepaired(sstable1, repairID, false);
mutateRepaired(sstable1, repairID);
SSTableReader sstable2 = makeSSTable(true);
mutateRepaired(sstable2, repairID, false);
mutateRepaired(sstable2, repairID);
Assert.assertFalse(repairedContains(sstable1));
Assert.assertFalse(unrepairedContains(sstable1));
Assert.assertFalse(repairedContains(sstable2));
Assert.assertFalse(unrepairedContains(sstable2));
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
// add only
SSTableListChangedNotification notification;
@ -138,7 +124,6 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertFalse(unrepairedContains(sstable2));
Assert.assertFalse(pendingContains(sstable2));
Assert.assertTrue(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
// remove and add
notification = new SSTableListChangedNotification(Collections.singleton(sstable2),
@ -165,18 +150,16 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertTrue(unrepairedContains(sstable));
Assert.assertFalse(repairedContains(sstable));
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
SSTableRepairStatusChanged notification;
// change to pending repaired
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
notification = new SSTableRepairStatusChanged(Collections.singleton(sstable));
csm.handleNotification(notification, cfs.getTracker());
Assert.assertFalse(unrepairedContains(sstable));
Assert.assertFalse(repairedContains(sstable));
Assert.assertTrue(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
Assert.assertTrue(pendingContains(sstable));
// change to repaired
@ -195,7 +178,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
Assert.assertTrue(pendingContains(sstable));
@ -224,7 +207,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertTrue(strategies.get(2).isEmpty());
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
strategies = csm.getStrategies();
@ -242,11 +225,10 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
LocalSessionAccessor.finalizeUnsafe(repairID);
Assert.assertTrue(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
Assert.assertTrue(pendingContains(sstable));
Assert.assertTrue(sstable.isPendingRepair());
Assert.assertFalse(sstable.isRepaired());
@ -263,7 +245,6 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertFalse(unrepairedContains(sstable));
Assert.assertFalse(pendingContains(sstable));
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
// sstable should have pendingRepair cleared, and repairedAt set correctly
long expectedRepairedAt = ActiveRepairService.instance().getParentRepairSession(repairID).repairedAt;
@ -282,12 +263,11 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
LocalSessionAccessor.failUnsafe(repairID);
Assert.assertTrue(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
Assert.assertTrue(pendingContains(sstable));
Assert.assertTrue(sstable.isPendingRepair());
Assert.assertFalse(sstable.isRepaired());
@ -303,7 +283,6 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertFalse(repairedContains(sstable));
Assert.assertTrue(unrepairedContains(sstable));
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
// sstable should have pendingRepair cleared, and repairedAt set correctly
Assert.assertFalse(sstable.isPendingRepair());
@ -334,7 +313,7 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
}
// change to pending repair
mutateRepaired(sstables, repairID, false);
mutateRepaired(sstables, repairID);
csm.handleNotification(new SSTableAddedNotification(sstables, null), cfs.getTracker());
for (SSTableReader sstable : sstables)
{
@ -403,7 +382,6 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
System.out.println("*********************************************************************************************");
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
Assert.assertTrue(repairedContains(compactedSSTable));
Assert.assertFalse(unrepairedContains(compactedSSTable));
Assert.assertFalse(pendingContains(compactedSSTable));
@ -413,70 +391,4 @@ public class CompactionStrategyManagerPendingRepairTest extends AbstractPendingR
Assert.assertTrue(compactedSSTable.isRepaired());
Assert.assertEquals(expectedRepairedAt, compactedSSTable.getSSTableMetadata().repairedAt);
}
@Test
public void finalizedSessionTransientCleanup()
{
Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, true);
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
LocalSessionAccessor.finalizeUnsafe(repairID);
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertTrue(hasTransientStrategiesFor(repairID));
Assert.assertTrue(transientContains(sstable));
Assert.assertFalse(pendingContains(sstable));
Assert.assertFalse(repairedContains(sstable));
Assert.assertFalse(unrepairedContains(sstable));
cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
AbstractCompactionTask compactionTask = Iterables.getOnlyElement(csm.getNextBackgroundTasks(FBUtilities.nowInSeconds()), null);
Assert.assertNotNull(compactionTask);
Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
// run the compaction
compactionTask.execute(ActiveCompactionsTracker.NOOP);
Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
}
@Test
public void failedSessionTransientCleanup()
{
Assert.assertTrue(cfs.getLiveSSTables().isEmpty());
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, true);
csm.handleNotification(new SSTableAddedNotification(Collections.singleton(sstable), null), cfs.getTracker());
LocalSessionAccessor.failUnsafe(repairID);
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertTrue(hasTransientStrategiesFor(repairID));
Assert.assertTrue(transientContains(sstable));
Assert.assertFalse(pendingContains(sstable));
Assert.assertFalse(repairedContains(sstable));
Assert.assertFalse(unrepairedContains(sstable));
cfs.getCompactionStrategyManager().enable(); // enable compaction to fetch next background task
AbstractCompactionTask compactionTask = Iterables.getOnlyElement(csm.getNextBackgroundTasks(FBUtilities.nowInSeconds()), null);
Assert.assertNotNull(compactionTask);
Assert.assertSame(PendingRepairManager.RepairFinishedCompactionTask.class, compactionTask.getClass());
// run the compaction
compactionTask.execute(ActiveCompactionsTracker.NOOP);
Assert.assertFalse(cfs.getLiveSSTables().isEmpty());
Assert.assertFalse(hasPendingStrategiesFor(repairID));
Assert.assertFalse(hasTransientStrategiesFor(repairID));
Assert.assertFalse(transientContains(sstable));
Assert.assertFalse(pendingContains(sstable));
Assert.assertFalse(repairedContains(sstable));
Assert.assertTrue(unrepairedContains(sstable));
}
}

45
test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java
View File

@ -132,12 +132,12 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
if (i % 3 == 0)
{
//make 1 third of sstables repaired
cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, System.currentTimeMillis(), null, false);
cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, System.currentTimeMillis(), null);
}
else if (i % 3 == 1)
{
//make 1 third of sstables pending repair
cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, 0, nextTimeUUID(), false);
cfs.getCompactionStrategyManager().mutateRepaired(newSSTables, 0, nextTimeUUID());
}
previousSSTables = currentSSTables;
}
@ -275,19 +275,19 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
DatabaseDescriptor.setAutomaticSSTableUpgradeEnabled(false);
}
private static void assertHolderExclusivity(boolean isRepaired, boolean isPendingRepair, boolean isTransient, Class<? extends AbstractStrategyHolder> expectedType)
private static void assertHolderExclusivity(boolean isRepaired, boolean isPendingRepair, Class<? extends AbstractStrategyHolder> expectedType)
{
ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
AbstractStrategyHolder holder = csm.getHolder(isRepaired, isPendingRepair, isTransient);
AbstractStrategyHolder holder = csm.getHolder(isRepaired, isPendingRepair);
assertNotNull(holder);
assertSame(expectedType, holder.getClass());
int matches = 0;
for (AbstractStrategyHolder other : csm.getHolders())
{
if (other.managesRepairedGroup(isRepaired, isPendingRepair, isTransient))
if (other.managesRepairedGroup(isRepaired, isPendingRepair))
{
assertSame("holder assignment should be mutually exclusive", holder, other);
matches++;
@ -296,13 +296,13 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
assertEquals(1, matches);
}
private static void assertInvalieHolderConfig(boolean isRepaired, boolean isPendingRepair, boolean isTransient)
private static void assertInvalidHolderConfig(boolean isRepaired, boolean isPendingRepair)
{
ColumnFamilyStore cfs = Keyspace.open(KS_PREFIX).getColumnFamilyStore(TABLE_PREFIX);
CompactionStrategyManager csm = cfs.getCompactionStrategyManager();
try
{
csm.getHolder(isRepaired, isPendingRepair, isTransient);
csm.getHolder(isRepaired, isPendingRepair);
fail("Expected IllegalArgumentException");
}
catch (IllegalArgumentException e)
@ -318,14 +318,11 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
@Test
public void testMutualExclusiveHolderClassification() throws Exception
{
assertHolderExclusivity(false, false, false, CompactionStrategyHolder.class);
assertHolderExclusivity(true, false, false, CompactionStrategyHolder.class);
assertHolderExclusivity(false, true, false, PendingRepairHolder.class);
assertHolderExclusivity(false, true, true, PendingRepairHolder.class);
assertInvalieHolderConfig(true, true, false);
assertInvalieHolderConfig(true, true, true);
assertInvalieHolderConfig(false, false, true);
assertInvalieHolderConfig(true, false, true);
assertHolderExclusivity(false, false, CompactionStrategyHolder.class);
assertHolderExclusivity(true, false, CompactionStrategyHolder.class);
assertHolderExclusivity(false, true, PendingRepairHolder.class);
assertHolderExclusivity(false, true, PendingRepairHolder.class);
assertInvalidHolderConfig(true, true);
}
PartitionPosition forKey(int key)
@ -344,7 +341,6 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
ColumnFamilyStore cfs = createJBODMockCFS(numDir);
Keyspace.open(cfs.getKeyspaceName()).getColumnFamilyStore(cfs.name).disableAutoCompaction();
assertTrue(cfs.getLiveSSTables().isEmpty());
List<SSTableReader> transientRepairs = new ArrayList<>();
List<SSTableReader> pendingRepair = new ArrayList<>();
List<SSTableReader> unrepaired = new ArrayList<>();
List<SSTableReader> repaired = new ArrayList<>();
@ -352,15 +348,13 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
for (int i = 0; i < numDir; i++)
{
int key = 100 * i;
transientRepairs.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
pendingRepair.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
unrepaired.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
repaired.add(createSSTableWithKey(cfs.getKeyspaceName(), cfs.name, key++));
}
cfs.getCompactionStrategyManager().mutateRepaired(transientRepairs, 0, nextTimeUUID(), true);
cfs.getCompactionStrategyManager().mutateRepaired(pendingRepair, 0, nextTimeUUID(), false);
cfs.getCompactionStrategyManager().mutateRepaired(repaired, 1000, null, false);
cfs.getCompactionStrategyManager().mutateRepaired(pendingRepair, 0, nextTimeUUID());
cfs.getCompactionStrategyManager().mutateRepaired(repaired, 1000, null);
DiskBoundaries boundaries = new DiskBoundaries(cfs, cfs.getDirectories().getWriteableLocations(),
Lists.newArrayList(forKey(100), forKey(200), forKey(300)),
@ -368,7 +362,7 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
CompactionStrategyManager csm = new CompactionStrategyManager(cfs, () -> boundaries, true);
List<GroupedSSTableContainer> grouped = csm.groupSSTables(Iterables.concat( transientRepairs, pendingRepair, repaired, unrepaired));
List<GroupedSSTableContainer> grouped = csm.groupSSTables(Iterables.concat( pendingRepair, repaired, unrepaired));
for (int x=0; x<grouped.size(); x++)
{
@ -383,14 +377,7 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
expected = repaired.get(y);
else if (sstable.isPendingRepair())
{
if (sstable.isTransient())
{
expected = transientRepairs.get(y);
}
else
{
expected = pendingRepair.get(y);
}
expected = pendingRepair.get(y);
}
else
expected = unrepaired.get(y);

10
test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
View File

@ -328,9 +328,9 @@ public class CompactionTaskTest
Assert.assertTrue(checkpointCalledInSSTableRewriter);
}
private static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair, boolean isTransient) throws IOException
private static void mutateRepaired(SSTableReader sstable, long repairedAt, TimeUUID pendingRepair) throws IOException
{
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair, isTransient);
sstable.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable.descriptor, repairedAt, pendingRepair);
sstable.reloadSSTableMetadata();
}
@ -359,9 +359,9 @@ public class CompactionTaskTest
SSTableReader pending1 = sstables.get(2);
SSTableReader pending2 = sstables.get(3);
mutateRepaired(repaired, FBUtilities.nowInSeconds(), ActiveRepairService.NO_PENDING_REPAIR, false);
mutateRepaired(pending1, UNREPAIRED_SSTABLE, nextTimeUUID(), false);
mutateRepaired(pending2, UNREPAIRED_SSTABLE, nextTimeUUID(), false);
mutateRepaired(repaired, FBUtilities.nowInSeconds(), ActiveRepairService.NO_PENDING_REPAIR);
mutateRepaired(pending1, UNREPAIRED_SSTABLE, nextTimeUUID());
mutateRepaired(pending2, UNREPAIRED_SSTABLE, nextTimeUUID());
LifecycleTransaction txn = null;
List<SSTableReader> toCompact = new ArrayList<>(sstables);

2
test/unit/org/apache/cassandra/db/compaction/CompactionsBytemanTest.java
View File

@ -197,7 +197,7 @@ public class CompactionsBytemanTest extends CQLTester
}
Util.flush(cfs);
}
cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), System.currentTimeMillis(), null, false);
cfs.getCompactionStrategyManager().mutateRepaired(cfs.getLiveSSTables(), System.currentTimeMillis(), null);
for (int i = 0; i < 5; i++)
{
for (int j = 0; j < 10; j++)

2
test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java
View File

@ -384,7 +384,7 @@ public class LeveledCompactionStrategyTest
SSTableReader sstable1 = unrepaired.manifest.getLevel(2).iterator().next();
SSTableReader sstable2 = unrepaired.manifest.getLevel(1).iterator().next();
sstable1.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable1.descriptor, System.currentTimeMillis(), null, false);
sstable1.descriptor.getMetadataSerializer().mutateRepairMetadata(sstable1.descriptor, System.currentTimeMillis(), null);
sstable1.reloadSSTableMetadata();
assertTrue(sstable1.isRepaired());

30
test/unit/org/apache/cassandra/db/compaction/PendingRepairManagerTest.java
View File

@ -48,7 +48,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
@ -66,7 +66,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
LocalSessionAccessor.finalizeUnsafe(repairID);
@ -85,7 +85,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
LocalSessionAccessor.failUnsafe(repairID);
@ -97,7 +97,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
public void needsCleanupNoSession()
{
TimeUUID fakeID = nextTimeUUID();
PendingRepairManager prm = new PendingRepairManager(cfs, null, false);
PendingRepairManager prm = new PendingRepairManager(cfs, null);
Assert.assertTrue(prm.canCleanup(fakeID));
}
@ -109,7 +109,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
@ -125,7 +125,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
Assert.assertNotNull(prm.get(repairID));
@ -143,13 +143,13 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
LocalSessionAccessor.finalizeUnsafe(repairID);
@ -187,7 +187,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
Assert.assertNotNull(prm.get(repairID));
@ -205,7 +205,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
TimeUUID repairID = registerSession(cfs, true, true);
LocalSessionAccessor.prepareUnsafe(repairID, COORDINATOR, PARTICIPANTS);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertNotNull(prm.get(repairID));
Assert.assertNotNull(prm.get(repairID));
@ -228,7 +228,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
TimeUUID repairId = registerSession(cfs, true, true);
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairId, false);
mutateRepaired(sstable, repairId);
prm.addSSTable(sstable);
try (CompactionTasks tasks = csm.getUserDefinedTasks(Collections.singleton(sstable), 100))
@ -246,8 +246,8 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
SSTableReader sstable = makeSSTable(true);
SSTableReader sstable2 = makeSSTable(true);
mutateRepaired(sstable, repairId, false);
mutateRepaired(sstable2, repairId2, false);
mutateRepaired(sstable, repairId);
mutateRepaired(sstable2, repairId2);
prm.addSSTable(sstable);
prm.addSSTable(sstable2);
try (CompactionTasks tasks = csm.getUserDefinedTasks(Lists.newArrayList(sstable, sstable2), 100))
@ -290,7 +290,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
Assert.assertFalse(prm.hasDataForSession(repairID));
SSTableReader sstable = makeSSTable(true);
mutateRepaired(sstable, repairID, false);
mutateRepaired(sstable, repairID);
prm.addSSTable(sstable);
Assert.assertTrue(prm.hasDataForSession(repairID));
}
@ -301,7 +301,7 @@ public class PendingRepairManagerTest extends AbstractPendingRepairTest
PendingRepairManager prm = csm.getPendingRepairManagers().get(0);
SSTableReader sstable = makeSSTable(false);
TimeUUID id = nextTimeUUID();
mutateRepaired(sstable, id, false);
mutateRepaired(sstable, id);
prm.getOrCreate(sstable);
cfs.truncateBlocking();
Assert.assertFalse(cfs.getSSTables(SSTableSet.LIVE).iterator().hasNext());

4
test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java
View File

@ -1445,7 +1445,7 @@ public class LogTransactionTest extends AbstractTransactionalTest
DecoratedKey key = MockSchema.readerBounds(generation);
SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, ImmutableCoordinatorLogOffsets.NONE, header, key.getKey().slice(), key.getKey().slice())
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, ImmutableCoordinatorLogOffsets.NONE, header, key.getKey().slice(), key.getKey().slice())
.get(MetadataType.STATS);
SSTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components)
.setTableMetadataRef(cfs.metadata)
@ -1481,7 +1481,7 @@ public class LogTransactionTest extends AbstractTransactionalTest
DecoratedKey key = MockSchema.readerBounds(generation);
SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList());
StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator)
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, ImmutableCoordinatorLogOffsets.NONE, header, key.getKey().slice(), key.getKey().slice())
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, ImmutableCoordinatorLogOffsets.NONE, header, key.getKey().slice(), key.getKey().slice())
.get(MetadataType.STATS);
SSTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
.setTableMetadataRef(cfs.metadata)

Some files were not shown because too many files have changed in this diff Show More