natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CEP-45: Full Repair Support for Tracked Keyspaces 

Tracked keyspaces cannot accept new data without first registering it in
the log. Any unreconciled data that isn't present in the log will break
read monotonicity, since mutation tracking uses a single data read and
can only read reconcile mutation IDs that are present in the log.

Full repair sync tasks also deliver data to replicas, and require
integration with the log just like imports do. The general design of this
integration is to give repair SyncTasks the same two-phase commit as
import transfers, where we stream SSTables to a pending directory, then
once sufficient streams complete successfully, we "activate" those
streams and move them out of the pending directory and into the live set.

patch by Caleb Rackliffe; reviewed by Abe Ratnofsky for CASSANDRA-21066

Co-authored-by: Caleb Rackliffe <calebrackliffe@gmail.com>
Co-authored-by: Abe Ratnofsky <abe@aber.io>
This commit is contained in:
Abe Ratnofsky 2025-12-12 12:32:44 -05:00 committed by Caleb Rackliffe
parent 3d8f81c4e2
commit d4826900b9
69 changed files with 4912 additions and 2188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -1690,6 +1690,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
*
* param @ filename - filename just flushed to disk
*/
@VisibleForTesting
public void addSSTable(SSTableReader sstable)
{
assert sstable.getColumnFamilyName().equals(name);

13
src/java/org/apache/cassandra/db/SSTableImporter.java
View File

@ -235,7 +235,7 @@ public class SSTableImporter
cfs.indexManager.buildSSTableAttachedIndexesBlocking(newSSTables);
if (isTracked)
TrackedBulkTransfer.execute(cfs.keyspace.getName(), newSSTables);
MutationTrackingService.instance.executeTransfers(cfs.keyspace.getName(), newSSTables, ConsistencyLevel.ALL);
else
cfs.getTracker().addSSTables(newSSTables);
@ -253,17 +253,6 @@ public class SSTableImporter
return failedDirectories;
}
/**
* TODO: Support user-defined consistency level for import, for import with replicas down
*/
private static class TrackedBulkTransfer
{
private static void execute(String keyspace, Set<SSTableReader> sstables)
{
MutationTrackingService.instance.executeTransfers(keyspace, sstables, ConsistencyLevel.ALL);
}
}
/**
* Check the state of this node and throws an {@link InterruptedException} if it is currently draining
*

2
src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java
View File

@ -160,7 +160,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader
private File getDataDir(ColumnFamilyStore cfs, long totalSize) throws IOException
{
boolean isTracked = cfs.metadata().replicationType().isTracked();
boolean isTracked = cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable();
Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize);
if (localDir == null)

3
src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java
View File

@ -130,9 +130,10 @@ public class CassandraStreamManager implements TableStreamManager
Set<SSTableReader> sstables = Sets.newHashSet();
SSTableIntervalTree intervalTree = buildSSTableIntervalTree(ImmutableList.copyOf(view.select(SSTableSet.CANONICAL)));
Predicate<SSTableReader> predicate;
// reconciledKeyspaceOffsets are only included when mutation logs are streamed, since we include logs
// for all unreconciled mutations, and SSTables for all reconciled mutations
if (reconciledKeyspaceOffsets != null)
{
// TODO: relax these restrictions as repair support is add
Preconditions.checkArgument(previewKind == PreviewKind.NONE);
Preconditions.checkArgument(pendingRepair == ActiveRepairService.NO_PENDING_REPAIR);
predicate = getSSTablePredicateForKeyspaceRanges(reconciledKeyspaceOffsets);

4
src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
View File

@ -62,7 +62,6 @@ import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException;
import org.apache.cassandra.streaming.StreamReceiver;
import org.apache.cassandra.streaming.StreamSession;
@ -191,9 +190,8 @@ public class CassandraStreamReader implements IStreamReader
ILifecycleTransaction txn = createTxn();
RangeAwareSSTableWriter writer;
if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER)
if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable())
{
Preconditions.checkState(cfs.metadata().replicationType().isTracked());
writer = new RangeAwarePendingSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, coordinatorLogOffsets, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata()), session.planId());
}
else

8
src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java
View File

@ -54,7 +54,6 @@ import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyReq
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.replication.PendingLocalTransfer;
import org.apache.cassandra.streaming.IncomingStream;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamReceiver;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.tcm.ClusterMetadata;
@ -135,9 +134,8 @@ public class CassandraStreamReceiver implements StreamReceiver
sstables.addAll(finished);
receivedEntireSSTable = file.isEntireSSTable();
if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER)
if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable())
{
Preconditions.checkState(cfs.metadata().replicationType().isTracked());
PendingLocalTransfer transfer = new PendingLocalTransfer(cfs.metadata().id, session.planId(), sstables);
MutationTrackingService.instance.received(transfer);
}
@ -266,8 +264,8 @@ public class CassandraStreamReceiver implements StreamReceiver
// add sstables (this will build non-SSTable-attached secondary indexes too, see CASSANDRA-10130)
logger.debug("[Stream #{}] Received {} sstables from {} ({})", session.planId(), readers.size(), session.peer, readers);
// Don't mark as live until activated by the stream coordinator
if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER)
// SSTables involved in a coordinated transfer become live when the transfer is activated
if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable())
return;
cfs.addSSTables(readers);

16
src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
View File

@ -35,15 +35,12 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.Clock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.AbstractBounds;
@ -62,6 +59,10 @@ import org.apache.cassandra.io.sstable.metadata.MetadataComponent;
import org.apache.cassandra.io.sstable.metadata.MetadataType;
import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
import org.apache.cassandra.io.util.MmappedRegionsCache;
import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.utils.Clock;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Transactional;
@ -336,9 +337,10 @@ public abstract class SSTableWriter extends SSTable implements Transactional
protected Map<MetadataType, MetadataComponent> finalizeMetadata()
{
// Migration from incremental repair to mutation tracking will be supported, but support for mixing
// incremental repair and mutation tracking is not planned
if (metadata().replicationType().isTracked() && repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
// Reconciliation should not occur before activation for coordinated transfer streams for tracked keyspaces.
boolean reconcile = txn.opType() != OperationType.STREAM;
if (metadata().replicationType().isTracked() && repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE && reconcile)
{
Preconditions.checkState(Objects.equals(pendingRepair, ActiveRepairService.NO_PENDING_REPAIR));
if (MutationTrackingService.instance.isDurablyReconciled(coordinatorLogOffsets))

13
src/java/org/apache/cassandra/net/Verb.java
View File

@ -74,12 +74,13 @@ import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.repair.messages.ValidationRequest;
import org.apache.cassandra.repair.messages.ValidationResponse;
import org.apache.cassandra.replication.ActivationResponse;
import org.apache.cassandra.replication.BroadcastLogOffsets;
import org.apache.cassandra.replication.ForwardedWrite;
import org.apache.cassandra.replication.LocalTransfers;
import org.apache.cassandra.replication.TransferTrackingService;
import org.apache.cassandra.replication.PullMutationsRequest;
import org.apache.cassandra.replication.PushMutationRequest;
import org.apache.cassandra.replication.TransferActivation;
import org.apache.cassandra.replication.ActivationRequest;
import org.apache.cassandra.replication.TransferFailed;
import org.apache.cassandra.schema.SchemaMutationsSerializer;
import org.apache.cassandra.schema.SchemaPullVerbHandler;
@ -339,10 +340,10 @@ public enum Verb
TRACKED_SUMMARY_RSP (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler ),
TRACKED_SUMMARY_REQ (911, P3, readTimeout, READ, () -> TrackedRead.SummaryRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_SUMMARY_RSP ),
TRACKED_TRANSFER_ACTIVATE_RSP (912, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER),
TRACKED_TRANSFER_ACTIVATE_REQ (913, P1, repairTimeout, ANTI_ENTROPY, () -> TransferActivation.serializer, () -> TransferActivation.verbHandler, TRACKED_TRANSFER_ACTIVATE_RSP),
TRACKED_TRANSFER_FAILED_RSP (914, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER),
TRACKED_TRANSFER_FAILED_REQ (915, P1, repairTimeout, ANTI_ENTROPY, () -> TransferFailed.serializer, () -> LocalTransfers.verbHandler, TRACKED_TRANSFER_FAILED_RSP),
TRACKED_TRANSFER_ACTIVATE_RSP (912, P1, repairTimeout, REQUEST_RESPONSE, () -> ActivationResponse.serializer, RESPONSE_HANDLER),
TRACKED_TRANSFER_ACTIVATE_REQ (913, P1, repairTimeout, ANTI_ENTROPY, () -> ActivationRequest.serializer, () -> ActivationRequest.verbHandler, TRACKED_TRANSFER_ACTIVATE_RSP),
TRACKED_TRANSFER_FAILED_RSP (914, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER),
TRACKED_TRANSFER_FAILED_REQ (915, P1, repairTimeout, ANTI_ENTROPY, () -> TransferFailed.serializer, () -> TransferTrackingService.verbHandler, TRACKED_TRANSFER_FAILED_RSP),
// accord
ACCORD_SIMPLE_RSP (119, P2, writeTimeout, IMMEDIATE, () -> accordEmbedded(EnumSerializer.simpleReply), AccordService::responseHandlerOrNoop ),

39
src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java
View File

@ -18,16 +18,21 @@
package org.apache.cassandra.repair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import com.google.common.base.Preconditions;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
/**
@ -38,29 +43,46 @@ import org.apache.cassandra.tracing.Tracing;
*/
public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask
{
public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind)
public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List<Range<Token>> differences, PreviewKind previewKind, ShortMutationId transferId)
{
super(ctx, desc, to, from, differences, previewKind);
super(ctx, desc, to, from, differences, previewKind, transferId);
}
@Override
public SyncTask withRanges(Collection<Range<Token>> newRanges)
{
List<Range<Token>> rangeList = newRanges instanceof List ? (List<Range<Token>>) newRanges : new ArrayList<>(newRanges);
return new AsymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, previewKind, transferId);
}
@Override
public SyncTask withTransferId(ShortMutationId transferId)
{
Preconditions.checkState(this.transferId == null);
return new AsymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, transferId);
}
public void startSync()
{
InetAddressAndPort local = ctx.broadcastAddressAndPort();
SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true);
SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true, transferId);
String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
Tracing.traceRepair(message);
sendRequest(request, request.src);
}
public void syncComplete(boolean success, List<SessionSummary> summaries)
public void syncComplete(SyncResponse response)
{
if (success)
if (response.success)
{
trySuccess(stat.withSummaries(summaries));
trySuccess(stat.withSummaries(response.summaries, response.planId, response.transferId));
}
else
{
tryFailure(RepairException.warn(desc, previewKind, String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer)));
String message = transferId == null
? String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer)
: String.format("Sync failed between %s and %s for transfer %s", nodePair.coordinator, nodePair.peer, transferId);
tryFailure(RepairException.warn(desc, previewKind, message));
}
}
@ -70,6 +92,7 @@ public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRem
return "AsymmetricRemoteSyncTask{" +
"rangesToSync=" + rangesToSync +
", nodePair=" + nodePair +
", transfer ID=" + transferId +
'}';
}
}

6
src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java
View File

@ -18,11 +18,9 @@
package org.apache.cassandra.repair;
import java.util.List;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.repair.messages.SyncResponse;
public interface CompletableRemoteSyncTask
{
void syncComplete(boolean success, List<SessionSummary> summaries);
void syncComplete(SyncResponse response);
}

31
src/java/org/apache/cassandra/repair/LocalSyncTask.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.cassandra.repair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
@ -30,6 +32,7 @@ import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamEvent;
@ -66,9 +69,9 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
List<Range<Token>> diff, TimeUUID pendingRepair,
boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
boolean requestRanges, boolean transferRanges, PreviewKind previewKind, ShortMutationId transferId)
{
super(ctx, desc, local, remote, diff, previewKind);
super(ctx, desc, local, remote, diff, previewKind, transferId);
Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job");
Preconditions.checkArgument(local.equals(ctx.broadcastAddressAndPort()));
@ -77,6 +80,27 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
this.transferRanges = transferRanges;
}
public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote,
List<Range<Token>> diff, TimeUUID pendingRepair,
boolean requestRanges, boolean transferRanges, PreviewKind previewKind)
{
this(ctx, desc, local, remote, diff, pendingRepair, requestRanges, transferRanges, previewKind, null);
}
@Override
public SyncTask withRanges(Collection<Range<Token>> newRanges)
{
List<Range<Token>> rangeList = newRanges instanceof List ? (List<Range<Token>>) newRanges : new ArrayList<>(newRanges);
return new LocalSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, pendingRepair, requestRanges, transferRanges, previewKind, transferId);
}
@Override
public SyncTask withTransferId(ShortMutationId transferId)
{
Preconditions.checkState(this.transferId == null);
return new LocalSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, pendingRepair, requestRanges, transferRanges, previewKind, transferId);
}
@VisibleForTesting
StreamPlan createStreamPlan()
{
@ -167,7 +191,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
status, desc.sessionId, nodePair.coordinator, nodePair.peer, desc.columnFamily);
logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
Tracing.traceRepair(message);
trySuccess(result.hasAbortedSession() ? stat : stat.withSummaries(result.createSummaries()));
trySuccess(result.hasAbortedSession() ? stat : stat.withSummaries(result.createSummaries(), result.planId, transferId));
finished();
}
}
@ -190,6 +214,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler
", transferRanges=" + transferRanges +
", rangesToSync=" + rangesToSync +
", nodePair=" + nodePair +
", transfer ID=" + transferId +
'}';
}

45
src/java/org/apache/cassandra/repair/RepairJob.java
View File

@ -50,6 +50,7 @@ import org.apache.cassandra.repair.asymmetric.HostDifferences;
import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter;
import org.apache.cassandra.repair.asymmetric.ReduceHelper;
import org.apache.cassandra.repair.state.JobState;
import org.apache.cassandra.replication.TransferTrackingService;
import org.apache.cassandra.schema.SystemDistributedKeyspace;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.accord.IAccordService;
@ -120,7 +121,6 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if ((!session.repairData && !session.repairPaxos) && !metadata.requiresAccordSupport())
throw new IllegalArgumentException(String.format("Cannot run accord only repair on %s.%s, which isn't configured for accord operations", cfs.keyspace.getName(), cfs.name));
}
public long getNowInSeconds()
@ -136,6 +136,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
}
public Collection<SyncTask> getSyncTasks()
{
return syncTasks;
}
@Override
public void run()
{
@ -254,6 +259,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
// that there are in memory at once. When all validations complete, submit sync tasks out of the scheduler.
syncResults = session.validationScheduler.schedule(() -> createSyncTasks(accordRepair, allSnapshotTasks, allEndpoints), taskExecutor)
.flatMap(this::executeTasks, taskExecutor);
// For tracked keyspaces, we need to ensure sync'd data is present in the log
boolean isTracked = cfs.metadata().replicationType().isTracked();
if (isTracked)
syncResults = TransferTrackingService.instance().onRepairSyncCompletion(this, syncResults, taskExecutor);
}
else
{
@ -306,7 +316,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}, taskExecutor);
}
private Future<List<SyncTask>> createSyncTasks(Future<AccordRepairResult> accordRepair, Future<?> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
private Future<SyncTasks> createSyncTasks(Future<AccordRepairResult> accordRepair, Future<?> allSnapshotTasks, List<InetAddressAndPort> allEndpoints)
{
Future<List<TreeResponse>> treeResponses;
if (allSnapshotTasks != null)
@ -330,9 +340,17 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
return a;
});
return treeResponses.map(session.optimiseStreams && !session.pullRepair
? this::createOptimisedSyncingSyncTasks
: this::createStandardSyncTasks, taskExecutor);
return treeResponses.map(trees -> {
List<SyncTask> syncTasks;
if (session.optimiseStreams && !session.pullRepair)
syncTasks = createOptimisedSyncingSyncTasks(trees);
else
syncTasks = createStandardSyncTasks(trees);
return ks.getMetadata().params.replicationType.isTracked()
? SyncTasks.tracked(ks, syncTasks)
: SyncTasks.untracked(syncTasks);
}, taskExecutor);
}
public synchronized void abort(@Nullable Throwable reason)
@ -413,18 +431,18 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
continue;
task = new LocalSyncTask(ctx, desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null,
requestRanges, transferRanges, previewKind);
requestRanges, transferRanges, previewKind, null);
}
else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint))
{
// Stream only from transient replica
TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2;
TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1;
task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind);
task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind, null);
}
else
{
task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind);
task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind, null);
}
syncTasks.add(task);
}
@ -437,7 +455,7 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
}
@VisibleForTesting
Future<List<SyncStat>> executeTasks(List<SyncTask> tasks)
Future<List<SyncStat>> executeTasks(SyncTasks tasks)
{
try
{
@ -447,10 +465,13 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if (!tasks.isEmpty())
state.phase.streamSubmitted();
if (cfs.metadata().replicationType().isTracked())
TransferTrackingService.instance().onRepairSyncExecution(tasks);
for (SyncTask task : tasks)
{
if (!task.isLocal())
session.trackSyncCompletion(Pair.create(desc, task.nodePair()), (CompletableRemoteSyncTask) task);
session.trackSyncCompletion(new SyncTask.SyncTaskId(desc, task.nodePair(), task.transferId), (CompletableRemoteSyncTask) task);
taskExecutor.execute(task);
}
@ -535,11 +556,11 @@ public class RepairJob extends AsyncFuture<RepairResult> implements Runnable
if (address.equals(local))
{
task = new LocalSyncTask(ctx, desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null,
true, false, previewKind);
true, false, previewKind, null);
}
else
{
task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind);
task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind, null);
}
syncTasks.add(task);

5
src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
View File

@ -311,7 +311,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message);
return;
}
SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst);
SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst, request.transferId);
if (!register(message, participate, state,
participate::register,
participate::sync))
@ -324,7 +324,8 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
request.ranges,
isIncremental(desc.parentSessionId) ? desc.parentSessionId : null,
request.previewKind,
request.asymmetric);
request.asymmetric,
request.transferId);
task.run();
sendAck(message);
}

10
src/java/org/apache/cassandra/repair/RepairSession.java
View File

@ -134,7 +134,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
// Each validation task waits response from replica in validating ConcurrentMap (keyed by CF name and endpoint address)
private final ConcurrentMap<Pair<RepairJobDesc, InetAddressAndPort>, ValidationTask> validating = new ConcurrentHashMap<>();
// Remote syncing jobs wait response in syncingTasks map
private final ConcurrentMap<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
private final ConcurrentMap<SyncTask.SyncTaskId, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>();
// Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor
public final SafeExecutor taskExecutor;
@ -224,7 +224,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
validating.put(key, task);
}
public synchronized void trackSyncCompletion(Pair<RepairJobDesc, SyncNodePair> key, CompletableRemoteSyncTask task)
public synchronized void trackSyncCompletion(SyncTask.SyncTaskId key, CompletableRemoteSyncTask task)
{
if (terminated)
return;
@ -269,7 +269,7 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
public void syncComplete(RepairJobDesc desc, Message<SyncResponse> message)
{
SyncNodePair nodes = message.payload.nodes;
CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes));
CompletableRemoteSyncTask task = syncingTasks.remove(new SyncTask.SyncTaskId(desc, nodes, message.payload.transferId));
// replies without a callback get dropped, so if in mixed mode this should be ignored
ctx.messaging().send(message.emptyResponse(), message.from());
if (task == null)
@ -277,11 +277,11 @@ public class RepairSession extends AsyncFuture<RepairSessionResult> implements I
if (logger.isDebugEnabled())
logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily);
task.syncComplete(message.payload.success, message.payload.summaries);
task.syncComplete(message.payload);
}
@VisibleForTesting
Map<Pair<RepairJobDesc, SyncNodePair>, CompletableRemoteSyncTask> getSyncingTasks()
Map<SyncTask.SyncTaskId, CompletableRemoteSyncTask> getSyncingTasks()
{
return Collections.unmodifiableMap(syncingTasks);
}

23
src/java/org/apache/cassandra/repair/StreamingRepairTask.java
View File

@ -20,6 +20,8 @@ package org.apache.cassandra.repair;
import java.util.Collections;
import java.util.Collection;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.locator.RangesAtEndpoint;
@ -32,6 +34,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.repair.state.SyncState;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.StreamEvent;
import org.apache.cassandra.streaming.StreamEventHandler;
@ -65,7 +68,20 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
private final TimeUUID pendingRepair;
private final PreviewKind previewKind;
public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection<Range<Token>> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric)
@Nullable
private final ShortMutationId transferId;
public StreamingRepairTask(SharedContext ctx,
SyncState state,
RepairJobDesc desc,
InetAddressAndPort initiator,
InetAddressAndPort src,
InetAddressAndPort dst,
Collection<Range<Token>> ranges,
TimeUUID pendingRepair,
PreviewKind previewKind,
boolean asymmetric,
ShortMutationId transferId)
{
this.ctx = ctx;
this.state = state;
@ -77,6 +93,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
this.asymmetric = asymmetric;
this.pendingRepair = pendingRepair;
this.previewKind = previewKind;
this.transferId = transferId;
}
public void run()
@ -120,7 +137,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
{
logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator);
this.state.phase.success();
RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries()), SYNC_RSP, initiator);
RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries(), state.planId, transferId), SYNC_RSP, initiator);
}
/**
@ -130,6 +147,6 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler
public void onFailure(Throwable t)
{
this.state.phase.fail(t);
RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList()), SYNC_RSP, initiator);
RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList(), null, transferId), SYNC_RSP, initiator);
}
}

33
src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.cassandra.repair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import com.google.common.base.Preconditions;
@ -28,8 +30,9 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RepairException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.repair.messages.SyncResponse;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.tracing.Tracing;
/**
@ -42,16 +45,30 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
{
private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class);
public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind)
public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List<Range<Token>> differences, PreviewKind previewKind, ShortMutationId transferId)
{
super(ctx, desc, r1, r2, differences, previewKind);
super(ctx, desc, r1, r2, differences, previewKind, transferId);
}
@Override
public SyncTask withRanges(Collection<Range<Token>> newRanges)
{
List<Range<Token>> rangeList = newRanges instanceof List ? (List<Range<Token>>) newRanges : new ArrayList<>(newRanges);
return new SymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, previewKind, transferId);
}
@Override
public SyncTask withTransferId(ShortMutationId transferId)
{
Preconditions.checkState(this.transferId == null);
return new SymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, transferId);
}
@Override
protected void startSync()
{
InetAddressAndPort local = ctx.broadcastAddressAndPort();
SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false);
SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false, transferId);
Preconditions.checkArgument(nodePair.coordinator.equals(request.src));
String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst);
logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message);
@ -59,11 +76,12 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
sendRequest(request, request.src);
}
public void syncComplete(boolean success, List<SessionSummary> summaries)
@Override
public void syncComplete(SyncResponse response)
{
if (success)
if (response.success)
{
trySuccess(stat.withSummaries(summaries));
trySuccess(stat.withSummaries(response.summaries, response.planId, response.transferId));
}
else
{
@ -78,6 +96,7 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo
return "SymmetricRemoteSyncTask{" +
"rangesToSync=" + rangesToSync +
", nodePair=" + nodePair +
", transfer ID=" + transferId +
'}';
}
}

37
src/java/org/apache/cassandra/repair/SyncStat.java
View File

@ -20,9 +20,13 @@ package org.apache.cassandra.repair;
import java.util.Collection;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.utils.TimeUUID;
/**
* Statistics about synchronizing two replica
@ -32,21 +36,48 @@ public class SyncStat
public final SyncNodePair nodes;
public final Collection<Range<Token>> differences;
public final List<SessionSummary> summaries;
public final TimeUUID planId;
@Nullable
public final ShortMutationId transferId;
public SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences)
{
this(nodes, differences, null);
}
public SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries)
private SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries)
{
this.nodes = nodes;
this.summaries = summaries;
this.differences = differences;
this.planId = null;
this.transferId = null;
}
public SyncStat withSummaries(List<SessionSummary> summaries)
private SyncStat(SyncNodePair nodes, Collection<Range<Token>> differences, List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
{
return new SyncStat(nodes, differences, summaries);
this.nodes = nodes;
this.summaries = summaries;
this.differences = differences;
this.planId = planId;
this.transferId = transferId;
}
public SyncStat withSummaries(List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
{
return new SyncStat(nodes, differences, summaries, planId, transferId);
}
@Override
public String toString()
{
return "SyncStat{" +
"nodes=" + nodes +
", differences=" + differences +
", summaries=" + summaries +
", planId=" + planId +
", transfer ID=" + transferId +
'}';
}
}

73
src/java/org/apache/cassandra/repair/SyncTask.java
View File

@ -18,13 +18,16 @@
package org.apache.cassandra.repair;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,8 +37,11 @@ import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import static org.apache.cassandra.net.Verb.SYNC_REQ;
import static org.apache.cassandra.repair.messages.RepairMessage.notDone;
@ -50,11 +56,13 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
public final List<Range<Token>> rangesToSync;
protected final PreviewKind previewKind;
protected final SyncNodePair nodePair;
protected final ShortMutationId transferId;
protected volatile TimeUUID planId;
protected volatile long startTime = Long.MIN_VALUE;
protected final SyncStat stat;
protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind)
protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List<Range<Token>> rangesToSync, PreviewKind previewKind, ShortMutationId transferId)
{
Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer);
this.ctx = ctx;
@ -62,16 +70,47 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
this.rangesToSync = rangesToSync;
this.nodePair = new SyncNodePair(primaryEndpoint, peer);
this.previewKind = previewKind;
this.transferId = transferId;
this.stat = new SyncStat(nodePair, rangesToSync);
addCallback((syncStat, failure) -> {
if (syncStat != null && syncStat.planId != null)
this.planId = syncStat.planId;
else if (failure instanceof org.apache.cassandra.streaming.StreamException)
this.planId = ((org.apache.cassandra.streaming.StreamException) failure).finalState.planId;
});
}
protected abstract void startSync();
/**
* Creates a new SyncTask with the same parameters but different ranges.
* Used for splitting sync tasks on shard boundaries.
*/
public abstract SyncTask withRanges(Collection<Range<Token>> newRanges);
public abstract SyncTask withTransferId(ShortMutationId transferId);
public SyncNodePair nodePair()
{
return nodePair;
}
public ShortMutationId getTransferId()
{
return transferId;
}
/**
* Returns the planId associated with this sync task's streaming operation.
* The planId is captured when the task completes (successfully or with a StreamException).
* @return the planId if streaming has completed, null otherwise
*/
public TimeUUID getPlanId()
{
return planId;
}
/**
* Compares trees, and triggers repairs for any ranges that mismatch.
*/
@ -118,4 +157,34 @@ public abstract class SyncTask extends AsyncFuture<SyncStat> implements Runnable
to,
this::tryFailure);
}
public static class SyncTaskId
{
RepairJobDesc desc;
SyncNodePair pair;
@Nullable
ShortMutationId transferId;
public SyncTaskId(RepairJobDesc desc, SyncNodePair pair, ShortMutationId transferId)
{
this.desc = desc;
this.pair = pair;
this.transferId = transferId;
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
SyncTaskId that = (SyncTaskId) o;
return Objects.equals(desc, that.desc) && Objects.equals(pair, that.pair) && Objects.equals(transferId, that.transferId);
}
@Override
public int hashCode()
{
return Objects.hash(desc, pair, transferId);
}
}
}

106
src/java/org/apache/cassandra/repair/SyncTasks.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.repair;
import java.util.AbstractCollection;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.replication.Participants;
import org.apache.cassandra.replication.Shard;
public class SyncTasks extends AbstractCollection<SyncTask>
{
private final List<ShardedSyncTask> shardedTasks = new ArrayList<>();
public static class ShardedSyncTask
{
public final String keyspace;
public final Participants participants;
public final SyncTask task;
public final Range<Token> range;
private ShardedSyncTask(String keyspace, Participants participants, SyncTask task, Range<Token> range)
{
this.keyspace = keyspace;
this.participants = participants;
this.task = task;
this.range = range;
}
}
static SyncTasks untracked(Collection<SyncTask> tasks)
{
SyncTasks syncTasks = new SyncTasks();
tasks.forEach(t -> syncTasks.shardedTasks.add(new ShardedSyncTask(null, null, t, null)));
return syncTasks;
}
/**
* Mutation Tracking manages tracking metadata within shards that are each responsible for a piece of the owned
* token space. Executing a full repair across an entire node's ownership will span multiple shards, so repair sync
* tasks need to be split to each align within a single tracking shard.
*/
static SyncTasks tracked(Keyspace keyspace, List<SyncTask> tasks)
{
return MutationTrackingService.instance.alignToShardBoundaries(keyspace, tasks);
}
public void add(Shard shard, SyncTask task)
{
// Narrow the ultimate scope of activation to the ranges in the sync tasks rather than the entire shard.
Set<Range<Token>> ranges = new HashSet<>(task.rangesToSync);
Range<Token> span = span(ranges);
shardedTasks.add(new ShardedSyncTask(shard.keyspace, shard.participants, task, span));
}
public static Range<Token> span(Set<Range<Token>> ranges)
{
List<Range<Token>> normalized = Range.normalize(ranges);
Token min = normalized.get(0).left;
Token max = normalized.get(normalized.size() - 1).right;
return new Range<>(min, max);
}
public void apply(Consumer<ShardedSyncTask> consumer)
{
shardedTasks.forEach(consumer);
}
@Override
public Iterator<SyncTask> iterator()
{
return shardedTasks.stream().map(shardedSyncTask -> shardedSyncTask.task).iterator();
}
@Override
public int size()
{
return shardedTasks.size();
}
}

39
src/java/org/apache/cassandra/repair/messages/SyncRequest.java
View File

@ -23,6 +23,8 @@ import java.util.Collection;
import java.util.List;
import java.util.Objects;
import javax.annotation.Nullable;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.IPartitioner;
@ -32,7 +34,9 @@ import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.PreviewKind;
import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer;
@ -51,6 +55,9 @@ public class SyncRequest extends RepairMessage
public final Collection<Range<Token>> ranges;
public final PreviewKind previewKind;
public final boolean asymmetric;
@Nullable
public final ShortMutationId transferId;
public SyncRequest(RepairJobDesc desc,
InetAddressAndPort initiator,
@ -58,7 +65,8 @@ public class SyncRequest extends RepairMessage
InetAddressAndPort dst,
Collection<Range<Token>> ranges,
PreviewKind previewKind,
boolean asymmetric)
boolean asymmetric,
ShortMutationId transferId)
{
super(desc);
this.initiator = initiator;
@ -67,6 +75,7 @@ public class SyncRequest extends RepairMessage
this.ranges = ranges;
this.previewKind = previewKind;
this.asymmetric = asymmetric;
this.transferId = transferId;
}
@Override
@ -81,13 +90,14 @@ public class SyncRequest extends RepairMessage
dst.equals(req.dst) &&
ranges.equals(req.ranges) &&
previewKind == req.previewKind &&
asymmetric == req.asymmetric;
asymmetric == req.asymmetric &&
transferId == req.transferId;
}
@Override
public int hashCode()
{
return Objects.hash(desc, initiator, src, dst, ranges, previewKind);
return Objects.hash(desc, initiator, src, dst, ranges, previewKind, transferId);
}
public static final IVersionedSerializer<SyncRequest> serializer = new IVersionedSerializer<SyncRequest>()
@ -103,6 +113,13 @@ public class SyncRequest extends RepairMessage
AbstractBounds.tokenSerializer.serialize(range, out, version);
out.writeInt(message.previewKind.getSerializationVal());
out.writeBoolean(message.asymmetric);
if (version >= MessagingService.Version.VERSION_52.value)
{
out.writeBoolean(message.transferId != null);
if (message.transferId != null)
ShortMutationId.serializer.serialize(message.transferId, out, version);
}
}
public SyncRequest deserialize(DataInputPlus in, int version) throws IOException
@ -118,7 +135,12 @@ public class SyncRequest extends RepairMessage
ranges.add((Range<Token>) AbstractBounds.tokenSerializer.deserialize(in, partitioner, version));
PreviewKind previewKind = PreviewKind.deserialize(in.readInt());
boolean asymmetric = in.readBoolean();
return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric);
ShortMutationId transferId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean()
? ShortMutationId.serializer.deserialize(in, version)
: null;
return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric, transferId);
}
public long serializedSize(SyncRequest message, int version)
@ -132,6 +154,14 @@ public class SyncRequest extends RepairMessage
size += AbstractBounds.tokenSerializer.serializedSize(range, version);
size += TypeSizes.sizeof(message.previewKind.getSerializationVal());
size += TypeSizes.sizeof(message.asymmetric);
if (version >= MessagingService.Version.VERSION_52.value)
{
size += TypeSizes.sizeof(false);
if (message.transferId != null)
size += ShortMutationId.serializer.serializedSize(message.transferId, version);
}
return size;
}
};
@ -146,6 +176,7 @@ public class SyncRequest extends RepairMessage
", ranges=" + ranges +
", previewKind=" + previewKind +
", asymmetric=" + asymmetric +
", transfer ID=" + transferId +
"} " + super.toString();
}
}

54
src/java/org/apache/cassandra/repair/messages/SyncResponse.java
View File

@ -22,15 +22,20 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import javax.annotation.Nullable;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.IPartitionerDependentSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.SyncNodePair;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.streaming.SessionSummary;
import org.apache.cassandra.utils.TimeUUID;
/**
*
@ -45,20 +50,29 @@ public class SyncResponse extends RepairMessage
public final List<SessionSummary> summaries;
public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries)
@Nullable
public final TimeUUID planId;
@Nullable
public final ShortMutationId transferId;
public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
{
super(desc);
this.nodes = nodes;
this.success = success;
this.summaries = summaries;
this.planId = planId;
this.transferId = transferId;
}
public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List<SessionSummary> summaries)
public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List<SessionSummary> summaries, TimeUUID planId, ShortMutationId transferId)
{
super(desc);
this.summaries = summaries;
this.nodes = new SyncNodePair(endpoint1, endpoint2);
this.success = success;
this.planId = planId;
this.transferId = transferId;
}
@Override
@ -70,13 +84,15 @@ public class SyncResponse extends RepairMessage
return desc.equals(other.desc) &&
success == other.success &&
nodes.equals(other.nodes) &&
summaries.equals(other.summaries);
summaries.equals(other.summaries) &&
Objects.equals(planId, other.planId) &&
Objects.equals(transferId, other.transferId);
}
@Override
public int hashCode()
{
return Objects.hash(desc, success, nodes, summaries);
return Objects.hash(desc, success, nodes, summaries, planId, transferId);
}
public static final IPartitionerDependentSerializer<SyncResponse> serializer = new IPartitionerDependentSerializer<SyncResponse>()
@ -92,6 +108,17 @@ public class SyncResponse extends RepairMessage
{
SessionSummary.serializer.serialize(summary, out, version);
}
if (version >= MessagingService.Version.VERSION_52.value)
{
out.writeBoolean(message.planId != null);
if (message.planId != null)
TimeUUID.Serializer.instance.serialize(message.planId, out);
out.writeBoolean(message.transferId != null);
if (message.transferId != null)
ShortMutationId.serializer.serialize(message.transferId, out, version);
}
}
@Override
@ -108,7 +135,13 @@ public class SyncResponse extends RepairMessage
summaries.add(SessionSummary.serializer.deserialize(in, partitioner, version));
}
return new SyncResponse(desc, nodes, success, summaries);
TimeUUID planId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean()
? TimeUUID.Serializer.instance.deserialize(in) : null;
ShortMutationId transferId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean()
? ShortMutationId.serializer.deserialize(in, version) : null;
return new SyncResponse(desc, nodes, success, summaries, planId, transferId);
}
public long serializedSize(SyncResponse message, int version)
@ -123,6 +156,17 @@ public class SyncResponse extends RepairMessage
size += SessionSummary.serializer.serializedSize(summary, version);
}
if (version >= MessagingService.Version.VERSION_52.value)
{
size += TypeSizes.sizeof(false);
if (message.planId != null)
size += TimeUUID.Serializer.instance.serializedSize(message.planId);
size += TypeSizes.sizeof(false);
if (message.transferId != null)
size += ShortMutationId.serializer.serializedSize(message.transferId, version);
}
return size;
}
};

17
src/java/org/apache/cassandra/repair/state/SyncState.java
View File

@ -20,8 +20,11 @@ package org.apache.cassandra.repair.state;
import java.util.Objects;
import javax.annotation.Nullable;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.utils.Clock;
public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
@ -31,9 +34,9 @@ public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
public final Phase phase = new Phase();
public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, ShortMutationId transferId)
{
super(clock, new Id(desc, initiator, src, dst), State.class);
super(clock, new Id(desc, initiator, src, dst, transferId), State.class);
}
public final class Phase extends BaseSkipPhase
@ -58,13 +61,17 @@ public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
{
public final RepairJobDesc desc;
public final InetAddressAndPort initiator, src, dst;
@Nullable
public final ShortMutationId transferId;
public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst)
public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, ShortMutationId transferId)
{
this.desc = desc;
this.initiator = initiator;
this.src = src;
this.dst = dst;
this.transferId = transferId;
}
@Override
@ -73,13 +80,13 @@ public class SyncState extends AbstractState<SyncState.State, SyncState.Id>
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Id id = (Id) o;
return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst);
return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst) && Objects.equals(transferId, id.transferId);
}
@Override
public int hashCode()
{
return Objects.hash(desc, initiator, src, dst);
return Objects.hash(desc, initiator, src, dst, transferId);
}
}
}

10
src/java/org/apache/cassandra/replication/ActivatedTransfers.java
View File

@ -87,11 +87,6 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
this.bounds = bounds;
}
private ActivatedTransfer(ShortMutationId id, Collection<SSTableReader> sstables)
{
this(id, covering(sstables));
}
public static final IVersionedSerializer<ActivatedTransfer> serializer = new IVersionedSerializer<>()
{
@Override
@ -151,7 +146,6 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
transfers.removeIf(transfer -> transfer.id.offset() == offset);
}
@VisibleForTesting
public void add(ShortMutationId transferId, Bounds<Token> bounds)
{
transfers.add(new ActivatedTransfer(transferId, bounds));
@ -159,7 +153,7 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
public void add(ShortMutationId transferId, Collection<SSTableReader> sstables)
{
transfers.add(new ActivatedTransfer(transferId, sstables));
transfers.add(new ActivatedTransfer(transferId, covering(sstables)));
}
public void addAll(ActivatedTransfers other)
@ -192,7 +186,7 @@ public class ActivatedTransfers implements Iterable<ShortMutationId>
return transfers.isEmpty();
}
private static Bounds<Token> covering(Collection<SSTableReader> sstables)
public static Bounds<Token> covering(Collection<SSTableReader> sstables)
{
Preconditions.checkArgument(!sstables.isEmpty());
Iterator<SSTableReader> iter = sstables.iterator();

260
src/java/org/apache/cassandra/replication/ActivationRequest.java Normal file
View File

@ -0,0 +1,260 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.io.IOException;
import java.util.Objects;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
/**
* Activation transitions a {@link PendingLocalTransfer} from being pending (durable on disk but not yet visible to
* reads) to live (visible to reads and compactions), by associating the streaming plan ID with a mutation ID, referred
* to as the transfer ID.
* <p>
* For tracked repairs, activation may occur without a plan ID and therefore without a {@link PendingLocalTransfer}. In
* this case, activation skips this step (as the data is already present) and simply updates log offsets.
* <p>
* See {@link CoordinatedTransfer} for the lifecycle of a transfer and when a {@link ActivationRequest} is sent.
*/
public class ActivationRequest
{
public final StreamOperation operation;
public final Pair<InetAddressAndPort, InetAddressAndPort> pair;
public final Phase phase;
public final ShortMutationId transferId;
public final NodeId coordinatorId;
public final String keyspace;
public final Range<Token> range;
@Nullable
public final TimeUUID planId;
public enum Phase
{
PREPARE(0),
COMMIT(1);
private final int id;
private static final Phase[] ids;
static
{
ids = new Phase[values().length];
for (Phase phase : values())
ids[phase.id] = phase;
}
Phase(int id)
{
this.id = id;
}
static Phase from(int id)
{
Phase phase = ids[id];
Preconditions.checkState(phase.id == id);
return phase;
}
}
public ActivationRequest(StreamOperation operation,
Pair<InetAddressAndPort, InetAddressAndPort> pair,
Phase phase,
ShortMutationId transferId,
NodeId coordinatorId,
Range<Token> range,
String keyspace,
TimeUUID planId)
{
Preconditions.checkArgument(operation.isTrackable(), "Operation " + operation.getDescription() + " is not trackable");
Preconditions.checkNotNull(pair, "Activations require an address pair");
Preconditions.checkArgument(!transferId.isNone(), "Activations require a transfer ID");
Preconditions.checkNotNull(coordinatorId, "Activations require a coordinator node ID");
this.operation = operation;
this.pair = pair;
this.transferId = transferId;
this.coordinatorId = coordinatorId;
this.phase = phase;
this.keyspace = keyspace;
this.range = range;
this.planId = planId;
}
ShortMutationId id()
{
return transferId;
}
public boolean isPrepare()
{
return phase == Phase.PREPARE;
}
public boolean isCommit()
{
return phase == Phase.COMMIT;
}
public void apply()
{
MutationTrackingService.instance.activateLocal(this);
}
public static final Serializer serializer = new Serializer();
public static class Serializer implements IVersionedSerializer<ActivationRequest>
{
@Override
public void serialize(ActivationRequest request, DataOutputPlus out, int version) throws IOException
{
out.writeUTF(request.operation.getDescription());
InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(request.pair.left, out, version);
InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(request.pair.right, out, version);
ShortMutationId.serializer.serialize(request.id(), out, version);
NodeId.messagingSerializer.serialize(request.coordinatorId, out, version);
out.writeByte(request.phase.id);
out.writeUTF(request.keyspace);
Range.serializer.serialize(request.range, out, null);
out.writeBoolean(request.planId != null);
if (request.planId != null)
TimeUUID.Serializer.instance.serialize(request.planId, out, version);
}
@Override
public ActivationRequest deserialize(DataInputPlus in, int version) throws IOException
{
StreamOperation operation = StreamOperation.fromString(in.readUTF());
InetAddressAndPort sender = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
InetAddressAndPort receiver = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
ShortMutationId id = ShortMutationId.serializer.deserialize(in, version);
NodeId coordinatorId = NodeId.messagingSerializer.deserialize(in, version);
Phase phase = Phase.from(in.readByte());
String keyspace = in.readUTF();
Range<Token> range = Range.serializer.deserialize(in, null);
TimeUUID planId = in.readBoolean() ? TimeUUID.Serializer.instance.deserialize(in, version) : null;
return new ActivationRequest(operation, Pair.create(sender, receiver), phase, id, coordinatorId, range, keyspace, planId);
}
@Override
public long serializedSize(ActivationRequest request, int version)
{
long size = 0;
size += TypeSizes.sizeof(request.operation.getDescription());
size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(request.pair.left, version);
size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(request.pair.right, version);
size += ShortMutationId.serializer.serializedSize(request.id(), version);
size += NodeId.messagingSerializer.serializedSize(request.coordinatorId, version);
size += TypeSizes.BYTE_SIZE; // Enum ordinal
size += TypeSizes.sizeof(request.keyspace);
size += Range.serializer.serializedSize(request.range, null);
size += TypeSizes.BOOL_SIZE;
if (request.planId != null)
size += TimeUUID.Serializer.instance.serializedSize(request.planId, version);
return size;
}
}
public static class VerbHandler implements IVerbHandler<ActivationRequest>
{
private static final Logger logger = LoggerFactory.getLogger(VerbHandler.class);
@Override
public void doVerb(Message<ActivationRequest> msg) throws IOException
{
TransferTrackingService.instance().executor.submit(() -> {
try
{
msg.payload.apply();
MessagingService.instance().respond(new ActivationResponse(msg.payload.pair), msg);
}
catch (Throwable t)
{
logger.error("Local activation of {} failed due to error", msg.payload, t);
MessagingService.instance().respondWithFailure(RequestFailureReason.forException(t), msg);
}
});
}
}
public static final VerbHandler verbHandler = new VerbHandler();
@Override
public String toString()
{
return "ActivationRequest{" +
"operation=" + operation +
", pair=" + pair +
", phase=" + phase +
", transferId=" + transferId +
", coordinatorId=" + coordinatorId +
", keyspace='" + keyspace + '\'' +
", range=" + range +
", planId=" + planId +
'}';
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
ActivationRequest that = (ActivationRequest) o;
return operation == that.operation && Objects.equals(pair, that.pair) && phase == that.phase && Objects.equals(transferId, that.transferId) && Objects.equals(coordinatorId, that.coordinatorId) && Objects.equals(keyspace, that.keyspace) && Objects.equals(range, that.range) && Objects.equals(planId, that.planId);
}
@Override
public int hashCode()
{
return Objects.hash(operation, pair, phase, transferId, coordinatorId, keyspace, range, planId);
}
}

94
src/java/org/apache/cassandra/replication/ActivationResponse.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.io.IOException;
import java.util.Objects;
import com.google.common.base.Preconditions;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.Pair;
public class ActivationResponse
{
public final Pair<InetAddressAndPort, InetAddressAndPort> syncPair;
public ActivationResponse(Pair<InetAddressAndPort, InetAddressAndPort> syncPair)
{
Preconditions.checkNotNull(syncPair, "Activations require a sync node address pair");
this.syncPair = syncPair;
}
public static final Serializer serializer = new Serializer();
@Override
public String toString()
{
return "ActivationResponse{" +
"syncPair=" + syncPair +
'}';
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
ActivationResponse that = (ActivationResponse) o;
return Objects.equals(syncPair, that.syncPair);
}
@Override
public int hashCode()
{
return Objects.hash(syncPair);
}
public static class Serializer implements IVersionedSerializer<ActivationResponse>
{
@Override
public void serialize(ActivationResponse activate, DataOutputPlus out, int version) throws IOException
{
InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(activate.syncPair.left, out, version);
InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(activate.syncPair.right, out, version);
}
@SuppressWarnings("SuspiciousNameCombination")
@Override
public ActivationResponse deserialize(DataInputPlus in, int version) throws IOException
{
InetAddressAndPort left = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
InetAddressAndPort right = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version);
return new ActivationResponse(Pair.create(left, right));
}
@Override
public long serializedSize(ActivationResponse activate, int version)
{
long size = 0;
size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(activate.syncPair.left, version);
size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(activate.syncPair.right, version);
return size;
}
}
}

7
src/java/org/apache/cassandra/replication/ActiveLogReconciler.java
View File

@ -17,7 +17,6 @@
*/
package org.apache.cassandra.replication;
import java.util.Collections;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
@ -132,7 +131,7 @@ public final class ActiveLogReconciler implements Shutdownable
{
private static Task from(ShortMutationId id, InetAddressAndPort toHost)
{
CoordinatedTransfer transfer = LocalTransfers.instance().getActivatedTransfer(id);
CoordinatedTransfer transfer = TransferTrackingService.instance().getActivatedTransfer(id);
if (transfer != null)
return new TransferTask(transfer, toHost);
else
@ -246,10 +245,10 @@ public final class ActiveLogReconciler implements Shutdownable
void send()
{
logger.debug("Sending activation to {}", toHost);
LocalTransfers.instance().executor.submit(() -> {
TransferTrackingService.instance().executor.submit(() -> {
try
{
transfer.activateOn(Collections.singleton(toHost));
transfer.activate(toHost);
onResponse(null);
}
catch (Throwable t)

566
src/java/org/apache/cassandra/replication/CoordinatedTransfer.java
View File

@ -20,66 +20,40 @@ package org.apache.cassandra.replication;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.annotation.CheckReturnValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.NoPayload;
import org.apache.cassandra.net.RequestCallbackWithFailure;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.streaming.OutgoingStream;
import org.apache.cassandra.streaming.StreamException;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.INIT;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP;
import static org.apache.cassandra.replication.TransferActivation.Phase;
import static org.apache.cassandra.replication.ActivationRequest.Phase;
/**
* Orchestrates the lifecycle of a tracked bulk data transfer for a single replica set, where the current instance is
@ -96,8 +70,8 @@ import static org.apache.cassandra.replication.TransferActivation.Phase;
* </li>
* <li>
* <b>Activation {@link Phase#PREPARE}</b>
* The coordinator sends PREPARE messages to verify replicas have the data persisted on disk and are ready for
* activation.
* The coordinator optinally sends PREPARE messages to verify replicas have the data persisted on disk and are
* ready for activation. Implementations may customize this via {@link #prepare}
* </li>
* <li>
* <b>Activation {@link Phase#COMMIT}</b>
@ -107,57 +81,41 @@ import static org.apache.cassandra.replication.TransferActivation.Phase;
* reconciliation processes (read reconciliation and background reconciliation).
* </li>
* </ol>
*
* For simplicity, the coordinator streams to itself rather than using direct file copy. This ensures we can use the
* same lifecycle management for crash-safety and atomic add.
* <p>
* <p>
* If a tracked data read is executed on a replica that's missing an activation, the read reconciliation process will
* apply the missing activation during reconciliation and a subsequent read will succeed. To minimize the gap between
* activations across replicas, avoid expensive operations like file copies or index builds during
* {@link TransferActivation#apply()}.
* {@link ActivationRequest#apply()}.
*/
public class CoordinatedTransfer
public abstract class CoordinatedTransfer
{
private static final Logger logger = LoggerFactory.getLogger(CoordinatedTransfer.class);
protected final String keyspace;
protected final Range<Token> range;
String logPrefix()
{
return String.format("[CoordinatedTransfer #%s]", id);
return String.format("[%s #%s]", getClass().getSimpleName(), id);
}
private final ShortMutationId id;
private final String keyspace;
private final Range<Token> range;
private final ConsistencyLevel cl;
final Collection<SSTableReader> sstables;
final ConcurrentMap<InetAddressAndPort, SingleTransferResult> streamResults;
final ConcurrentMap<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> streamResults;
@VisibleForTesting
CoordinatedTransfer(Range<Token> range, MutationId id)
public CoordinatedTransfer(ShortMutationId id, String keyspace, Range<Token> range)
{
this.keyspace = null;
this.range = range;
this.sstables = Collections.emptyList();
this.cl = null;
this.id = id;
this.streamResults = new ConcurrentHashMap<>();
}
CoordinatedTransfer(String keyspace, Range<Token> range, Participants participants, Collection<SSTableReader> sstables, ConsistencyLevel cl, Supplier<MutationId> nextId)
{
this.keyspace = keyspace;
this.range = range;
this.sstables = sstables;
this.cl = cl;
this.id = nextId.get();
}
ClusterMetadata cm = ClusterMetadata.current();
public CoordinatedTransfer(ShortMutationId id, Participants participants, String keyspace, Range<Token> range)
{
this.id = id;
this.streamResults = new ConcurrentHashMap<>(participants.size());
for (int i = 0; i < participants.size(); i++)
{
InetAddressAndPort addr = cm.directory.getNodeAddresses(new NodeId(participants.get(i))).broadcastAddress;
this.streamResults.put(addr, SingleTransferResult.Init());
}
this.keyspace = keyspace;
this.range = range;
}
ShortMutationId id()
@ -165,298 +123,76 @@ public class CoordinatedTransfer
return id;
}
void execute()
Bounds<Token> bounds()
{
logger.debug("{} Executing tracked bulk transfer {}", logPrefix(), this);
LocalTransfers.instance().save(this);
stream();
return new Bounds<>(range.left.nextValidToken(), range.right);
}
private void stream()
public boolean isCommitted()
{
// TODO: Don't stream multiple copies over the WAN, send one copy and indicate forwarding
List<Future<Void>> streaming = new ArrayList<>(streamResults.size());
for (InetAddressAndPort to : streamResults.keySet())
for (SingleTransferResult result : streamResults.values())
{
Future<Void> stream = LocalTransfers.instance().executor.submit(() -> {
stream(to);
return null;
});
streaming.add(stream);
if (result.state != SingleTransferResult.State.COMMITTED)
return false;
}
// Wait for all streams to complete, so we can clean up after failures. If we exit at the first failure, a
// future stream can complete.
LinkedList<Throwable> failures = null;
for (Future<Void> stream : streaming)
{
try
{
stream.get();
}
catch (InterruptedException | ExecutionException e)
{
if (failures == null)
failures = new LinkedList<>();
failures.add(e);
logger.error("{} Failed transfer due to", logPrefix(), e);
}
}
if (failures != null && !failures.isEmpty())
{
Throwable failure = failures.element();
Throwable cause = failure instanceof ExecutionException ? failure.getCause() : failure;
maybeCleanupFailedStreams(cause);
String msg = String.format("Failed streaming on %s instance(s): %s", failures.size(), failures);
throw new RuntimeException(msg, Throwables.unchecked(cause));
}
logger.info("{} All streaming completed successfully", logPrefix());
return true;
}
private boolean sufficient()
protected abstract ActivationRequest createActivation(Pair<InetAddressAndPort, InetAddressAndPort> pair, ActivationRequest.Phase phase);
final void activate(InetAddressAndPort peer)
{
AbstractReplicationStrategy ars = Keyspace.open(keyspace).getReplicationStrategy();
int blockFor = cl.blockFor(ars);
int responses = 0;
for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
{
if (entry.getValue().state == STREAM_COMPLETE)
responses++;
}
return responses >= blockFor;
activate(streamResults.keySet().stream().filter(pair -> pair.right.equals(peer)).collect(Collectors.toList()));
}
void stream(InetAddressAndPort to)
final void activate(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> pairs)
{
SingleTransferResult result;
try
// There's no reason to try to re-activate already COMMITTED peers...
List<Pair<InetAddressAndPort, InetAddressAndPort>> uncommittedPairs = new ArrayList<>(pairs.size());
for (Pair<InetAddressAndPort, InetAddressAndPort> pair : pairs)
if (streamResults.get(pair).state != COMMITTED)
uncommittedPairs.add(pair);
if (uncommittedPairs.isEmpty())
{
result = streamTask(to);
}
catch (StreamException | ExecutionException | InterruptedException | TimeoutException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
markStreamFailure(to, cause);
throw Throwables.unchecked(cause);
}
try
{
streamComplete(to, result);
}
catch (ExecutionException | InterruptedException | TimeoutException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
throw Throwables.unchecked(cause);
}
}
private void notifyFailure() throws ExecutionException, InterruptedException
{
class NotifyFailure extends AsyncFuture<Void> implements RequestCallbackWithFailure<NoPayload>
{
final Set<InetAddressAndPort> responses = ConcurrentHashMap.newKeySet(streamResults.size());
@Override
public void onResponse(Message<NoPayload> msg)
{
responses.remove(msg.from());
if (responses.isEmpty())
trySuccess(null);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
tryFailure(failure.failure);
}
}
NotifyFailure notifyFailure = new NotifyFailure();
for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
{
InetAddressAndPort to = entry.getKey();
// Coordinator cleans up CoordinatedTransfer and PendingLocalTransfer separately, does not need to notify
if (FBUtilities.getBroadcastAddressAndPort().equals(to))
continue;
SingleTransferResult result = entry.getValue();
if (result.planId == null)
{
logger.warn("{} Skipping notification of transfer failure to {} due to unknown planId", logPrefix(), to);
continue;
}
logger.debug("{}, Notifying {} of transfer failure for plan {}", logPrefix(), to, result.planId);
notifyFailure.responses.add(to);
Message<TransferFailed> msg = Message.out(Verb.TRACKED_TRANSFER_FAILED_REQ, new TransferFailed(result.planId));
MessagingService.instance().sendWithCallback(msg, to, notifyFailure);
}
notifyFailure.get();
}
private void markStreamFailure(InetAddressAndPort to, Throwable cause)
{
TimeUUID planId;
if (cause instanceof StreamException)
planId = ((StreamException) cause).finalState.planId;
else
planId = null;
streamResults.computeIfPresent(to, (peer, result) -> result.streamFailed(planId));
}
/**
* This shouldn't throw an exception, even if we fail to notify peers of the streaming failure.
*/
private void maybeCleanupFailedStreams(Throwable cause)
{
try
{
boolean purgeable = LocalTransfers.instance().purger.test(this);
if (!purgeable)
return;
notifyFailure();
LocalTransfers.instance().scheduleCleanup();
}
catch (Throwable t)
{
if (cause != null)
t.addSuppressed(cause);
logger.error("{} Failed to notify peers of stream failure", logPrefix(), t);
}
}
private void streamComplete(InetAddressAndPort to, SingleTransferResult result) throws ExecutionException, InterruptedException, TimeoutException
{
streamResults.put(to, result);
logger.info("{} Completed streaming to {}, {}", logPrefix(), to, this);
maybeActivate();
}
synchronized void maybeActivate()
{
// If any activations have already been sent out, send new activations to any received plans that have not yet
// been activated
boolean anyActivated = false;
Set<InetAddressAndPort> awaitingActivation = new HashSet<>();
for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
{
InetAddressAndPort peer = entry.getKey();
SingleTransferResult result = entry.getValue();
if (result.state == COMMITTING || result.state == COMMITTED)
{
anyActivated = true;
}
else if (result.state == STREAM_COMPLETE)
awaitingActivation.add(peer);
}
if (anyActivated && !awaitingActivation.isEmpty())
{
logger.debug("{} Transfer already activated on some peers, sending activations to remaining: {}", logPrefix(), awaitingActivation);
activateOn(awaitingActivation);
return;
}
// If no activations have been sent out, check whether we have enough planIds back to meet the required CL
else if (sufficient())
{
Set<InetAddressAndPort> peers = new HashSet<>();
for (Map.Entry<InetAddressAndPort, SingleTransferResult> entry : streamResults.entrySet())
{
InetAddressAndPort peer = entry.getKey();
SingleTransferResult result = entry.getValue();
if (result.state == STREAM_COMPLETE)
peers.add(peer);
}
logger.debug("{} Transfer meets consistency level {}, sending activations to {}", logPrefix(), cl, peers);
activateOn(peers);
logAlreadyCommitted(pairs);
return;
}
logger.debug("{} Nothing to activate", logPrefix());
activateInternal(uncommittedPairs);
}
void activateOn(Collection<InetAddressAndPort> peers)
private synchronized void activateInternal(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets)
{
Preconditions.checkState(!peers.isEmpty());
logger.debug("{} Activating {} on {}", logPrefix(), this, peers);
LocalTransfers.instance().activating(this);
logger.debug("{} Activating {} for {}", logPrefix(), this, targets);
// First phase ensures data is present on disk, then second phase does the actual import. This ensures that if
// something goes wrong (like a topology change during import), we don't have divergence.
class Prepare extends AsyncFuture<Void> implements RequestCallbackWithFailure<NoPayload>
{
final Set<InetAddressAndPort> responses = ConcurrentHashMap.newKeySet();
public Prepare()
{
responses.addAll(peers);
}
@Override
public void onResponse(Message<NoPayload> msg)
{
logger.debug("{} Got response from: {}", logPrefix(), msg.from());
responses.remove(msg.from());
if (responses.isEmpty())
trySuccess(null);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
logger.debug("{} Got failure {} from {}", logPrefix(), failure, from);
CoordinatedTransfer.this.streamResults.computeIfPresent(from, (peer, result) -> result.prepareFailed());
tryFailure(new RuntimeException("Tracked import failed during PREPARE on " + from + " due to " + failure.reason));
}
}
Prepare prepare = new Prepare();
for (InetAddressAndPort peer : peers)
{
TransferActivation activation = new TransferActivation(this, peer, Phase.PREPARE);
Message<TransferActivation> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
logger.debug("{} Sending {} to peer {}", logPrefix(), activation, peer);
MessagingService.instance().sendWithCallback(msg, peer, prepare);
CoordinatedTransfer.this.streamResults.computeIfPresent(peer, (peer0, result) -> result.preparing());
}
try
{
prepare.get();
}
catch (InterruptedException | ExecutionException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
throw Throwables.unchecked(cause);
}
logger.debug("{} Activation prepare complete for {}", logPrefix(), peers);
prepare(targets);
logger.debug("{} Activation prepare complete for {}", logPrefix(), targets);
// Acknowledgement of activation is equivalent to a remote write acknowledgement. The imported SSTables
// are now part of the live set, visible to reads.
class Commit extends AsyncFuture<Void> implements RequestCallbackWithFailure<Void>
class Commit extends AsyncFuture<Void> implements RequestCallbackWithFailure<ActivationResponse>
{
final Set<InetAddressAndPort> responses = ConcurrentHashMap.newKeySet();
final AtomicInteger responses = new AtomicInteger(0);
private Commit(Collection<InetAddressAndPort> peers)
private Commit(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> pairs)
{
responses.addAll(peers);
responses.addAndGet(pairs.size());
}
@Override
public void onResponse(Message<Void> msg)
public void onResponse(Message<ActivationResponse> msg)
{
logger.debug("{} Activation successfully applied on {}", logPrefix(), msg.from());
CoordinatedTransfer.this.streamResults.computeIfPresent(msg.from(), (peer, result) -> result.committed());
streamResults.computeIfPresent(msg.payload.syncPair, (peer, result) -> result.committed());
MutationTrackingService.instance.receivedActivationResponse(CoordinatedTransfer.this, msg.from());
responses.remove(msg.from());
if (responses.isEmpty())
if (responses.decrementAndGet() == 0)
{
// All activations complete, schedule cleanup to purge pending SSTables
LocalTransfers.instance().scheduleCleanup();
TransferTrackingService.instance().scheduleCleanup();
trySuccess(null);
}
}
@ -464,22 +200,21 @@ public class CoordinatedTransfer
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
logger.error("{} Failed activation on {} due to {}", logPrefix(), from, failure);
MutationTrackingService.instance.retryFailedTransfer(CoordinatedTransfer.this, from, failure.failure);
logger.error("{} Failed activation commit on {} due to {}", logPrefix(), from, failure);
// TODO(expected): should only fail if we don't meet requested CL
tryFailure(new RuntimeException("Tracked import failed during COMMIT on " + from + " due to " + failure.reason));
tryFailure(new RuntimeException("Tracked transfer failed during COMMIT on " + from + " due to " + failure.reason));
}
}
Commit commit = new Commit(peers);
for (InetAddressAndPort peer : peers)
Commit commit = new Commit(targets);
for (Pair<InetAddressAndPort, InetAddressAndPort> target : targets)
{
TransferActivation activation = new TransferActivation(this, peer, Phase.COMMIT);
Message<TransferActivation> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
ActivationRequest activation = createActivation(target, Phase.COMMIT);
Message<ActivationRequest> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
logger.debug("{} Sending {} to peer {}", logPrefix(), activation, peer);
MessagingService.instance().sendWithCallback(msg, peer, commit);
CoordinatedTransfer.this.streamResults.computeIfPresent(peer, (peer0, result) -> result.committing());
logger.debug("{} Sending commit {} to peer {}", logPrefix(), activation, target.right);
MessagingService.instance().sendWithCallback(msg, target.right, commit);
streamResults.computeIfPresent(target, (peer0, result) -> result.committing());
}
try
@ -491,17 +226,70 @@ public class CoordinatedTransfer
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
throw Throwables.unchecked(cause);
}
logger.debug("{} Activation commit complete for {}", logPrefix(), peers);
logger.debug("{} Activation commit complete for {}", logPrefix(), targets);
}
public boolean isCommitted()
protected abstract void prepare(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets);
private void logAlreadyCommitted(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> pairs)
{
for (SingleTransferResult result : streamResults.values())
logger.debug("Transfer {} for {} is already committed. Skipping activation...", this, pairs);
}
/**
* Notify all replicas that this transfer failed, triggering cleanup of pending SSTables.
* This is used by both {@link TrackedRepairTransfer} and {@link TrackedImportTransfer}.
*/
protected void notifyFailure() throws ExecutionException, InterruptedException
{
class NotifyFailure extends AsyncFuture<Void> implements RequestCallbackWithFailure<NoPayload>
{
if (result.state != COMMITTED)
return false;
// TODO: Does this actually work? What if there is a race between notification and callbacks where we decrement to zero before submitting a second request?
// It seems like we should add the pair up-front that actally have a plan ID...?
final AtomicInteger responses = new AtomicInteger(0);
@Override
public void onResponse(Message<NoPayload> msg)
{
if (responses.decrementAndGet() == 0)
trySuccess(null);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
// Log but don't fail - best effort cleanup
logger.warn("{} Failed to notify {} of transfer failure: {}", logPrefix(), from, failure);
if (responses.decrementAndGet() == 0)
trySuccess(null);
}
}
return true;
NotifyFailure notifyFailure = new NotifyFailure();
for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> entry : streamResults.entrySet())
{
InetAddressAndPort to = entry.getKey().right;
// Coordinator cleans up CoordinatedTransfer and PendingLocalTransfer separately, does not need to notify
if (FBUtilities.getBroadcastAddressAndPort().equals(to))
continue;
SingleTransferResult result = entry.getValue();
if (result.planId() == null)
{
// No planId means streaming never completed, so there's nothing to clean up on the replica
logger.debug("{} Skipping notification of transfer failure to {} - no planId", logPrefix(), to);
continue;
}
logger.debug("{} Notifying {} of transfer failure for plan {}", logPrefix(), to, result.planId());
notifyFailure.responses.incrementAndGet();
Message<TransferFailed> msg = Message.out(Verb.TRACKED_TRANSFER_FAILED_REQ, new TransferFailed(result.planId()));
MessagingService.instance().sendWithCallback(msg, to, notifyFailure);
}
// Only wait if we actually sent notifications
if (notifyFailure.responses.get() > 0)
notifyFailure.get();
}
/**
@ -511,6 +299,7 @@ public class CoordinatedTransfer
* <li>{@link State#INIT}: Transfer created, not yet streaming.</li>
* <li>{@link State#STREAM_COMPLETE}: Streaming successful, SSTables received on replica in pending directory.</li>
* <li>{@link State#STREAM_NOOP}: No data streamed (e.g., SSTable contains no rows in target range).</li>
* <li>{@link State#EMPTY_SYNC}: No repair sync (e.g., No Merkle tree disagreement).</li>
* <li>{@link State#STREAM_FAILED}: Streaming failed, may not have a streaming plan ID yet.</li>
* <li>{@link State#PREPARING}: Preparing for activation (first phase).</li>
* <li>{@link State#PREPARE_FAILED}: Prepare failed, aborting transfer.</li>
@ -523,8 +312,10 @@ public class CoordinatedTransfer
* ┌────────────────┐
* ↓ │
* INIT ──┬──→ STREAM_COMPLETE ──→ PREPARING ──┬──→ COMMITTING ──→ COMMITTED
* │ │
* ├──→ STREAM_NOOP └──→ PREPARE_FAILED
* │ ↑ │
* ├──→ EMPTY_SYNC ─────────────┘ │
* │ └──→ PREPARE_FAILED
* ├──→ STREAM_NOOP
* │
* └──→ STREAM_FAILED
* </pre>
@ -543,17 +334,19 @@ public class CoordinatedTransfer
PREPARING,
PREPARE_FAILED,
COMMITTING,
COMMITTED;
COMMITTED,
EMPTY_SYNC;
EnumSet<State> transitionFrom;
static
{
INIT.transitionFrom = EnumSet.noneOf(State.class);
EMPTY_SYNC.transitionFrom = EnumSet.noneOf(State.class);
STREAM_NOOP.transitionFrom = EnumSet.of(INIT);
STREAM_FAILED.transitionFrom = EnumSet.of(INIT);
STREAM_COMPLETE.transitionFrom = EnumSet.of(INIT);
PREPARING.transitionFrom = EnumSet.of(STREAM_COMPLETE, COMMITTING);
PREPARING.transitionFrom = EnumSet.of(STREAM_COMPLETE, COMMITTING, EMPTY_SYNC);
PREPARE_FAILED.transitionFrom = EnumSet.of(PREPARING);
COMMITTING.transitionFrom = EnumSet.of(PREPARING);
COMMITTED.transitionFrom = EnumSet.of(COMMITTING);
@ -570,14 +363,14 @@ public class CoordinatedTransfer
this.planId = planId;
}
private boolean canTransition(SingleTransferResult.State to)
boolean canTransition(State to)
{
return to.transitionFrom.contains(state);
}
public static SingleTransferResult Init()
{
return new SingleTransferResult(INIT, null);
return new SingleTransferResult(State.INIT, null);
}
@VisibleForTesting
@ -589,7 +382,13 @@ public class CoordinatedTransfer
@VisibleForTesting
static SingleTransferResult Noop()
{
return new SingleTransferResult(STREAM_NOOP, null);
return new SingleTransferResult(State.STREAM_NOOP, null);
}
@VisibleForTesting
static SingleTransferResult EmptySync()
{
return new SingleTransferResult(State.EMPTY_SYNC, null);
}
@CheckReturnValue
@ -607,31 +406,31 @@ public class CoordinatedTransfer
@CheckReturnValue
public SingleTransferResult streamFailed(TimeUUID planId)
{
return transition(STREAM_FAILED, planId);
return transition(State.STREAM_FAILED, planId);
}
@CheckReturnValue
public SingleTransferResult preparing()
{
return transition(PREPARING, this.planId);
return transition(State.PREPARING, this.planId);
}
@CheckReturnValue
public SingleTransferResult prepareFailed()
{
return transition(PREPARE_FAILED, this.planId);
return transition(State.PREPARE_FAILED, this.planId);
}
@CheckReturnValue
public SingleTransferResult committing()
{
return transition(COMMITTING, this.planId);
return transition(State.COMMITTING, this.planId);
}
@CheckReturnValue
public SingleTransferResult committed()
{
return transition(COMMITTED, this.planId);
return transition(State.COMMITTED, this.planId);
}
public TimeUUID planId()
@ -648,73 +447,4 @@ public class CoordinatedTransfer
'}';
}
}
private SingleTransferResult streamTask(InetAddressAndPort to) throws StreamException, ExecutionException, InterruptedException, TimeoutException
{
StreamPlan plan = new StreamPlan(StreamOperation.TRACKED_TRANSFER);
// No need to flush, only using non-live SSTables already on disk
plan.flushBeforeTransfer(false);
for (SSTableReader sstable : sstables)
{
List<Range<Token>> ranges = Collections.singletonList(range);
List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(ranges);
long estimatedKeys = sstable.estimatedKeysForRanges(ranges);
OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.TRACKED_TRANSFER, sstable.ref(), positions, ranges, estimatedKeys);
plan.transferStreams(to, Collections.singleton(stream));
}
long timeout = DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds();
logger.info("{} Starting streaming transfer {} to peer {}", logPrefix(), this, to);
StreamResultFuture execute = plan.execute();
StreamState state;
try
{
state = execute.get(timeout, TimeUnit.MILLISECONDS);
logger.debug("{} Completed streaming transfer {} to peer {}", logPrefix(), this, to);
}
catch (InterruptedException | ExecutionException | TimeoutException e)
{
logger.error("Stream session failed with error", e);
throw e;
}
if (state.hasFailedSession() || state.hasAbortedSession())
throw new StreamException(state, "Stream failed due to failed or aborted sessions");
// If the SSTable doesn't contain any rows in the provided range, no streams delivered, nothing to activate
if (state.sessions().isEmpty())
return SingleTransferResult.Noop();
return SingleTransferResult.StreamComplete(plan.planId());
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
CoordinatedTransfer transfer = (CoordinatedTransfer) o;
return Objects.equals(keyspace, transfer.keyspace) && Objects.equals(range, transfer.range) && Objects.equals(streamResults, transfer.streamResults) && Objects.equals(sstables, transfer.sstables) && cl == transfer.cl && Objects.equals(id, transfer.id);
}
@Override
public int hashCode()
{
return Objects.hash(keyspace, range, streamResults, sstables, cl, id);
}
@Override
public String toString()
{
return "CoordinatedTransfer{" +
"id=" + id +
", keyspace='" + keyspace + '\'' +
", range=" + range +
", cl=" + cl +
", sstables=" + sstables +
", streamResults=" + streamResults +
'}';
}
}

59
src/java/org/apache/cassandra/replication/CoordinatorLog.java
View File

@ -44,6 +44,7 @@ import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.metrics.MutationTrackingMetrics;
@ -329,12 +330,31 @@ public abstract class CoordinatorLog
MutationTrackingMetrics.instance.writeTimeOffsetsDiscovered.inc();
unreconciledMutations.finishWriting(mutation);
maybeMoveOffset(offset);
}
finally
{
lock.writeLock().unlock();
}
}
if (remoteReplicasWitnessed(offset))
{
reconciledOffsets.add(offset);
unreconciledMutations.remove(offset);
}
/*
- On local replicas after they've completed activation (onHostId == me)
*/
void finishActivation(Bounds<Token> bounds, ActivationRequest activation)
{
logger.trace("witnessed local transfer {}", activation.id());
lock.writeLock().lock();
try
{
int offset = activation.id().offset();
// we've raced with another write, no need to do anything else
if (!witnessedOffsets.get(localNodeId).add(offset))
return;
unreconciledMutations.activatedTransfer(activation.id(), bounds);
maybeMoveOffset(offset);
}
finally
{
@ -358,33 +378,12 @@ public abstract class CoordinatorLog
return othersWitnessed(offset, localNodeId);
}
/*
- On local replicas after they've completed activation (onHostId == me)
*/
void finishActivation(PendingLocalTransfer transfer, TransferActivation activation)
private void maybeMoveOffset(int offset)
{
logger.trace("witnessed local transfer {}", activation.id());
lock.writeLock().lock();
try
if (remoteReplicasWitnessed(offset))
{
int offset = activation.id().offset();
// we've raced with another write, no need to do anything else
if (!witnessedOffsets.get(localNodeId).add(offset))
return;
// This is the only difference with finishWriting - can we consolidate these methods?
unreconciledMutations.activatedTransfer(activation.id(), transfer.sstables);
if (remoteReplicasWitnessed(offset))
{
reconciledOffsets.add(offset);
unreconciledMutations.remove(offset);
}
}
finally
{
lock.writeLock().unlock();
reconciledOffsets.add(offset);
unreconciledMutations.remove(offset);
}
}

14
src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java
View File

@ -19,14 +19,12 @@
package org.apache.cassandra.replication;
import java.io.IOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;
import javax.annotation.concurrent.NotThreadSafe;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Iterators;
import org.slf4j.Logger;
@ -36,7 +34,6 @@ import org.agrona.collections.Long2ObjectHashMap;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.net.MessagingService;
@ -161,7 +158,6 @@ public class ImmutableCoordinatorLogOffsets implements CoordinatorLogOffsets<Off
return this;
}
@VisibleForTesting
public Builder addTransfer(ShortMutationId transferId, Bounds<Token> bounds)
{
if (transferId.isNone())
@ -172,16 +168,6 @@ public class ImmutableCoordinatorLogOffsets implements CoordinatorLogOffsets<Off
return this;
}
public Builder addTransfer(ShortMutationId transferId, Collection<SSTableReader> sstables)
{
if (transferId.isNone())
return this;
if (transfers == null)
transfers = new ActivatedTransfers();
transfers.add(transferId, sstables);
return this;
}
public Builder addTransfers(ActivatedTransfers other)
{
if (other.isEmpty())

198
src/java/org/apache/cassandra/replication/MutationTrackingService.java
View File

@ -18,6 +18,7 @@
package org.apache.cassandra.replication;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -28,6 +29,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
@ -46,28 +48,35 @@ import org.apache.cassandra.concurrent.ScheduledExecutorPlus;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.cql3.UntypedResultSet;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Splitter;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.gms.FailureDetector;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.EndpointsForRange;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.MutationTrackingMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.SyncTask;
import org.apache.cassandra.repair.SyncTasks;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.reads.tracked.TrackedLocalReads;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.Epoch;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tcm.listeners.ChangeListener;
@ -234,6 +243,35 @@ public class MutationTrackingService
}
}
// Requires that ranges is aligned to a single shard
public MutationId nextMutationId(String keyspace, Collection<Range<Token>> ranges)
{
shardLock.readLock().lock();
try
{
KeyspaceShards shards = getOrCreateShards(keyspace);
Shard shard = null;
for (Range<Token> range : ranges)
{
Shard curShard = shards.lookUp(range);
if (curShard == null)
throw new UnknownShardException(range, shards.groups);
if (shard == null)
shard = curShard;
else if (shard != curShard)
throw new IllegalStateException(String.format("Cannot generate a mutation ID for ranges (%s) that span across more than one shard (%s, %s)", ranges, shard, curShard));
}
Preconditions.checkNotNull(shard);
MutationId id = shard.nextId();
logger.trace("Created new mutation id {}", id);
return id;
}
finally
{
shardLock.readLock().unlock();
}
}
public void sentWriteRequest(Mutation mutation, IntHashSet toHostIds)
{
Preconditions.checkArgument(!mutation.id().isNone());
@ -375,10 +413,10 @@ public class MutationTrackingService
logger.info("Creating tracked bulk transfers for keyspace '{}' SSTables {}...", keyspace, sstables);
KeyspaceShards shards = checkNotNull(keyspaceShards.get(keyspace));
CoordinatedTransfers transfers = CoordinatedTransfers.create(keyspace, shards, sstables, cl);
TrackedImportTransfers transfers = TrackedImportTransfers.create(keyspace, shards, sstables, cl);
logger.info("Split input SSTables into transfers {}", transfers);
for (CoordinatedTransfer transfer : transfers)
for (TrackedImportTransfer transfer : transfers)
transfer.execute();
}
finally
@ -390,23 +428,52 @@ public class MutationTrackingService
public void received(PendingLocalTransfer transfer)
{
logger.debug("Received pending transfer for tracked table {}", transfer);
LocalTransfers.instance().received(transfer);
TransferTrackingService.instance().received(transfer);
}
void activateLocal(TransferActivation activation)
void activateLocal(ActivationRequest request)
{
PendingLocalTransfer pending = LocalTransfers.instance().getPendingTransfer(activation.planId);
if (pending == null)
throw new IllegalStateException(String.format("Cannot activate unknown local pending transfer %s", activation));
pending.activate(activation);
boolean committed = false;
String keyspace = request.keyspace;
Bounds<Token> bounds;
PendingLocalTransfer pending = null;
if (request.operation == StreamOperation.REPAIR)
{
bounds = new Bounds<>(request.range.left.nextValidToken(), request.range.right);
// A sync task does not necessarily stream to both replicas, which means there may be a plan ID without a
// pending local transfer. In this case, we simply treat this as an already committed transfer and update
// the required offsets in the log (unless we're just preparing).
committed = request.isCommit();
// If we have no plan ID, it means this replica did not participate in a sync.
if (request.planId != null)
pending = TransferTrackingService.instance().getPendingTransfer(request.planId);
}
else if (request.operation == StreamOperation.IMPORT)
{
pending = TransferTrackingService.instance().getPendingTransfer(request.planId);
if (pending == null)
throw new IllegalStateException(String.format("Cannot activate unknown local pending transfer %s", request));
bounds = ActivatedTransfers.covering(pending.sstables);
}
else
{
throw new IllegalArgumentException("Cannot activate transfer for stream operation " + request.operation);
}
if (pending != null)
committed = pending.activate(request, bounds);
shardLock.readLock().lock();
try
{
if (activation.isCommit())
if (committed)
{
keyspaceShards.get(pending.keyspace).lookUp(pending.range).finishActivation(pending, activation);
incomingMutations.invokeListeners(activation.transferId);
keyspaceShards.get(keyspace).lookUp(request.range).finishActivation(bounds, request);
incomingMutations.invokeListeners(request.transferId);
}
}
finally
@ -780,6 +847,66 @@ public class MutationTrackingService
forEachKeyspace(keyspace -> keyspace.collectDurablyReconciledOffsets(into));
}
public SyncTasks alignToShardBoundaries(Keyspace keyspace, List<SyncTask> tasks)
{
Preconditions.checkArgument(keyspace.getMetadata().replicationStrategy.replicationType.isTracked(), "Keyspace " + keyspace.getName() + " is not tracked");
KeyspaceShards shards = keyspaceShards.get(keyspace.getName());
Map<Shard, List<SyncTask>> tasksByShard = new HashMap<>();
// Shard ranges do not wrap, so unwrap the task ranges before we start comparing them.
for (SyncTask task : unwrapped(tasks))
{
Set<Shard> intersectingShards = new HashSet<>();
shards.forEachIntersectingShard(task.rangesToSync, intersectingShards::add);
for (Shard shard : intersectingShards)
{
// Ensure that we don't expand outside the ranges of the original sync tasks.
Set<Range<Token>> intersectingSyncRanges = new HashSet<>();
for (Range<Token> syncRange : task.rangesToSync)
intersectingSyncRanges.addAll(syncRange.intersectionWith(shard.range));
if (!intersectingSyncRanges.isEmpty())
tasksByShard.computeIfAbsent(shard, key -> new ArrayList<>()).add(task.withRanges(intersectingSyncRanges));
}
}
SyncTasks into = new SyncTasks();
for (Map.Entry<Shard, List<SyncTask>> entry : tasksByShard.entrySet())
{
Shard shard = entry.getKey();
Collection<SyncTask> syncTasks = entry.getValue();
// Assign a new transfer ID to each sync task and add to the tasks container
for (SyncTask task : syncTasks)
into.add(shard, task.withTransferId(shard.nextId()));
}
return into;
}
private static List<SyncTask> unwrapped(Collection<SyncTask> tasks)
{
List<SyncTask> unwrapped = new ArrayList<>();
for (SyncTask task : tasks)
{
List<Range<Token>> unwrappedRanges = new ArrayList<>();
for (Range<Token> range : task.rangesToSync)
{
if (range.isTrulyWrapAround())
unwrappedRanges.addAll(range.unwrap());
else
unwrappedRanges.add(range);
}
unwrapped.add(task.withRanges(unwrappedRanges));
}
return unwrapped;
}
public static class KeyspaceShards
{
private enum UpdateDecision
@ -993,6 +1120,14 @@ public class MutationTrackingService
});
}
private void forEachIntersectingShard(Collection<Range<Token>> ranges, Consumer<Shard> consumer)
{
shards.forEach((range0, shard) -> {
if (shard.range.intersects(ranges))
consumer.accept(shard);
});
}
void collectShardReconciledOffsetsToBuilder(ReconciledLogSnapshot.Builder builder)
{
ReconciledKeyspaceOffsets.Builder keyspaceBuilder = builder.getKeyspaceBuilder(keyspace);
@ -1098,8 +1233,9 @@ public class MutationTrackingService
private static class ReplicatedOffsetsBroadcaster
{
// TODO (later): a more intelligent heuristic for scheduling broadcasts
private static final long TRANSIENT_BROADCAST_INTERVAL_MILLIS = 200;
private static final long DURABLE_BROADCAST_INTERVAL_MILLIS = 60_000;
// TODO: Revert before merge, just increased frequency for test
private static final long TRANSIENT_BROADCAST_INTERVAL_MILLIS = 1_000;
private static final long DURABLE_BROADCAST_INTERVAL_MILLIS = 1_000;
private volatile boolean isPaused = false;
@ -1148,11 +1284,13 @@ public class MutationTrackingService
private static class LogStatePersister implements Runnable
{
// TODO (expected): consider a different interval
private static final long PERSIST_INTERVAL_MINUTES = 1;
// TODO: Revert before merge, just increased frequency for test
// private static final long PERSIST_INTERVAL_MILLIS = 60_000;
private static final long PERSIST_INTERVAL_MILLIS = 1_000;
void start()
{
executor.scheduleWithFixedDelay(this, PERSIST_INTERVAL_MINUTES, PERSIST_INTERVAL_MINUTES, TimeUnit.MINUTES);
executor.scheduleWithFixedDelay(this, PERSIST_INTERVAL_MILLIS, PERSIST_INTERVAL_MILLIS, TimeUnit.MILLISECONDS);
}
@Override
@ -1222,5 +1360,35 @@ public class MutationTrackingService
{
return service.keyspaceShards.get(keyspace);
}
/**
* Creates a test KeyspaceShards with the given shard ranges.
* The shards are created with minimal configuration suitable for testing.
*/
public static KeyspaceShards createTestKeyspaceShards(String keyspace, Set<Range<Token>> shardRanges)
{
Map<Range<Token>, Shard> shards = new HashMap<>();
Map<Range<Token>, VersionedEndpoints.ForRange> groups = new HashMap<>();
int localNodeId = 1;
AtomicInteger hostLogId = new AtomicInteger(0);
LongSupplier logId = () -> CoordinatorLogId.asLong(localNodeId, hostLogId.getAndIncrement());
Participants participants = new Participants(List.of(localNodeId));
for (Range<Token> range : shardRanges)
{
shards.put(range, new Shard(localNodeId, keyspace, range, participants, logId, (s, l) -> {}));
groups.put(range, VersionedEndpoints.forRange(Epoch.EMPTY, EndpointsForRange.empty(range)));
}
return new KeyspaceShards(keyspace, shards, new ReplicaGroups(groups));
}
/**
* Sets the keyspace shards for testing purposes.
*/
public static void setKeyspaceShards(MutationTrackingService service, String keyspace, KeyspaceShards shards)
{
service.keyspaceShards.put(keyspace, shards);
}
}
}

26
src/java/org/apache/cassandra/replication/PendingLocalTransfer.java
View File

@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
@ -48,7 +49,7 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
* Represents a bulk data transfer received on a replica, from completion of streaming into the pending location,
* through activation when it's made visible to reads. Pending transfers are identified by their streaming plan ID,
* and made live by {@link TransferActivation} which associates the streaming plan with a transfer ID that can be
* and made live by {@link ActivationRequest} which associates the streaming plan with a transfer ID that can be
* represented in mutation summaries.
*/
public class PendingLocalTransfer
@ -133,16 +134,16 @@ public class PendingLocalTransfer
* preserve is a transfer is only added to the live set iff the transfer ID is present in its mutation summaries.
* <p>
* We don't validate checksums here, mostly because a transfer can be activated during a read, if one replica
* missed the TransferActivation. Transfers should not be pending for very long, and should be protected by
* missed the {@link ActivationRequest}. Transfers should not be pending for very long, and should be protected by
* internode integrity checks provided by TLS.
* <p>
* Synchronized to prevent a single activation from running multiple times if requested during read reconciliation
* and in the background via {@link ActiveLogReconciler}.
*/
public synchronized void activate(TransferActivation activation)
public synchronized boolean activate(ActivationRequest request, Bounds<Token> bounds)
{
if (activated)
return;
return false;
Preconditions.checkState(isFullReplica());
@ -152,21 +153,19 @@ public class PendingLocalTransfer
Preconditions.checkNotNull(cfs);
Preconditions.checkState(!sstables.isEmpty());
if (activation.isPrepare())
if (request.isPrepare())
{
logger.info("{} Not adding SSTables to live set for dryRun {}", logPrefix(), activation);
return;
logger.info("{} Not adding SSTables to live set for dryRun {}", logPrefix(), request);
return false;
}
// Modify SSTables metadata to durably set transfer ID before importing
ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder()
.addTransfer(activation.transferId, sstables)
.build();
ImmutableCoordinatorLogOffsets logOffsets =
new ImmutableCoordinatorLogOffsets.Builder().addTransfer(request.transferId, bounds).build();
// Ensure no lingering mutation IDs, only activation IDs
for (SSTableReader sstable : sstables)
{
Preconditions.checkState(sstable.getCoordinatorLogOffsets().mutations().isEmpty());
try
{
sstable.mutateCoordinatorLogOffsetsAndReload(logOffsets);
@ -185,7 +184,7 @@ public class PendingLocalTransfer
// Retain the original SSTables in pending/ dir on the coordinator, so future streams can get the originals, and
// we don't need to isolate activated SSTables during compaction
boolean isCoordinator = activation.transferId.hostId == ClusterMetadata.current().myNodeId().id();
boolean isCoordinator = request.transferId.hostId == ClusterMetadata.current().myNodeId().id();
logger.debug("{} {} pending SSTables for activation to {}", isCoordinator ? "Copying" : "Moving", logPrefix(), dst);
dst.createFileIfNotExists();
@ -210,7 +209,8 @@ public class PendingLocalTransfer
long finishedActivation = currentTimeMillis();
logger.info("{} Finished activating transfer {} in {} ms", logPrefix(), this, finishedActivation - startedActivation);
LocalTransfers.instance().scheduleCleanup();
TransferTrackingService.instance().scheduleCleanup();
return true;
}
@Override

22
src/java/org/apache/cassandra/replication/Shard.java
View File

@ -41,6 +41,7 @@ import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
@ -58,9 +59,9 @@ public class Shard
private static final Logger logger = LoggerFactory.getLogger(Shard.class);
final int localNodeId;
final String keyspace;
final Range<Token> range;
final Participants participants;
public final String keyspace;
public final Range<Token> range;
public final Participants participants;
private final LongSupplier logIdProvider;
private final BiConsumer<Shard, CoordinatorLog> onNewLog;
private final NonBlockingHashMapLong<CoordinatorLog> logs;
@ -180,9 +181,9 @@ public class Shard
getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId);
}
void finishActivation(PendingLocalTransfer transfer, TransferActivation activation)
void finishActivation(Bounds<Token> bounds, ActivationRequest activation)
{
getOrCreate(activation.transferId).finishActivation(transfer, activation);
getOrCreate(activation.transferId).finishActivation(bounds, activation);
}
void receivedActivationResponse(CoordinatedTransfer transfer, InetAddressAndPort onHost)
@ -406,6 +407,17 @@ public class Shard
logs.values().forEach(log -> keyspaceBuilder.put(log.logId, log.collectReconciledOffsets(), range));
}
@Override
public String toString()
{
return "Shard{" +
"participants=" + participants +
", range=" + range +
", keyspace='" + keyspace + '\'' +
", localNodeId=" + localNodeId +
'}';
}
public DebugInfo getDebugInfo()
{
SortedMap<CoordinatorLogId, CoordinatorLog.DebugInfo> logDebugState = new TreeMap<>(Comparator.comparing(CoordinatorLogId::asLong));

421
src/java/org/apache/cassandra/replication/TrackedImportTransfer.java Normal file
View File

@ -0,0 +1,421 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.streaming.CassandraOutgoingFile;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailure;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.AbstractReplicationStrategy;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.RequestCallbackWithFailure;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.streaming.OutgoingStream;
import org.apache.cassandra.streaming.StreamException;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.streaming.StreamPlan;
import org.apache.cassandra.streaming.StreamResultFuture;
import org.apache.cassandra.streaming.StreamState;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.Throwables;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncFuture;
import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
import static org.apache.cassandra.replication.ActivationRequest.Phase;
/**
* Tracked imports mostly follow the default coordination process. They are notable in two ways. The first is that
* {@link #prepare} strictly verifies that streaming has completed on replicas. The second is that activation is based
* on the {@link #bounds} generated from {@link #sstables} rather than a sync range (as is the case with repair).
* <p>
* For simplicity, the coordinator streams to itself rather than using direct file copy. This ensures we can use the
* same lifecycle management for crash-safety and atomic add.
*/
public class TrackedImportTransfer extends CoordinatedTransfer
{
private static final Logger logger = LoggerFactory.getLogger(TrackedImportTransfer.class);
final Collection<SSTableReader> sstables;
private final ConsistencyLevel cl;
@VisibleForTesting
TrackedImportTransfer(Range<Token> range, MutationId id)
{
super(id, null, range);
this.sstables = Collections.emptyList();
this.cl = null;
}
TrackedImportTransfer(String keyspace, Range<Token> range, Participants participants, Collection<SSTableReader> sstables, ConsistencyLevel cl, Supplier<MutationId> nextId)
{
super(nextId.get(), participants, keyspace, range);
this.sstables = sstables;
this.cl = cl;
ClusterMetadata cm = ClusterMetadata.current();
for (int i = 0; i < participants.size(); i++)
{
InetAddressAndPort addr = cm.directory.getNodeAddresses(new NodeId(participants.get(i))).broadcastAddress;
this.streamResults.put(Pair.create(FBUtilities.getBroadcastAddressAndPort(), addr), SingleTransferResult.Init());
}
}
@Override
Bounds<Token> bounds()
{
return ActivatedTransfers.covering(sstables);
}
void execute()
{
logger.debug("{} Executing tracked import transfer {}", logPrefix(), this);
TransferTrackingService.instance().save(this);
stream();
}
private void stream()
{
// TODO: Don't stream multiple copies over the WAN, send one copy and indicate forwarding
List<Future<Void>> streaming = new ArrayList<>(streamResults.size());
for (Pair<InetAddressAndPort, InetAddressAndPort> pair : streamResults.keySet())
{
Future<Void> stream = TransferTrackingService.instance().executor.submit(() -> {
stream(pair);
return null;
});
streaming.add(stream);
}
// Wait for all streams to complete, so we can clean up after failures. If we exit at the first failure, a
// future stream can complete.
LinkedList<Throwable> failures = null;
for (Future<Void> stream : streaming)
{
try
{
stream.get();
}
catch (InterruptedException | ExecutionException e)
{
if (failures == null)
failures = new LinkedList<>();
failures.add(e);
logger.error("{} Failed transfer due to", logPrefix(), e);
}
}
if (failures != null && !failures.isEmpty())
{
Throwable failure = failures.element();
Throwable cause = failure instanceof ExecutionException ? failure.getCause() : failure;
maybeCleanupFailedStreams(cause);
String msg = String.format("Failed streaming on %s instance(s): %s", failures.size(), failures);
throw new RuntimeException(msg, Throwables.unchecked(cause));
}
logger.info("{} All streaming completed successfully", logPrefix());
}
private boolean sufficient()
{
AbstractReplicationStrategy ars = Keyspace.open(keyspace).getReplicationStrategy();
int blockFor = cl.blockFor(ars);
int responses = 0;
for (SingleTransferResult result : streamResults.values())
{
if (result.state == STREAM_COMPLETE)
responses++;
}
return responses >= blockFor;
}
void stream(Pair<InetAddressAndPort, InetAddressAndPort> pair)
{
SingleTransferResult result;
try
{
result = streamTask(pair.right);
}
catch (StreamException | ExecutionException | InterruptedException | TimeoutException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
markStreamFailure(pair, cause);
throw Throwables.unchecked(cause);
}
try
{
streamComplete(pair, result);
}
catch (ExecutionException | InterruptedException | TimeoutException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
throw Throwables.unchecked(cause);
}
}
private void markStreamFailure(Pair<InetAddressAndPort, InetAddressAndPort> pair, Throwable cause)
{
TimeUUID planId;
if (cause instanceof StreamException)
planId = ((StreamException) cause).finalState.planId;
else
planId = null;
streamResults.computeIfPresent(pair, (p, result) -> result.streamFailed(planId));
}
/**
* This shouldn't throw an exception, even if we fail to notify peers of the streaming failure.
*/
private void maybeCleanupFailedStreams(Throwable cause)
{
try
{
boolean purgeable = TransferTrackingService.instance().purger.test(this);
if (!purgeable)
return;
notifyFailure();
TransferTrackingService.instance().scheduleCleanup();
}
catch (Throwable t)
{
if (cause != null)
t.addSuppressed(cause);
logger.error("{} Failed to notify peers of stream failure", logPrefix(), t);
}
}
private void streamComplete(Pair<InetAddressAndPort, InetAddressAndPort> pair, SingleTransferResult result) throws ExecutionException, InterruptedException, TimeoutException
{
streamResults.put(pair, result);
logger.info("{} Completed streaming for pair {}, {}", logPrefix(), pair, this);
maybeActivate();
}
synchronized void maybeActivate()
{
// If any activations have already been sent out, send new activations to any received plans that have not yet
// been activated
boolean anyActivated = false;
Set<Pair<InetAddressAndPort, InetAddressAndPort>> awaitingActivation = new HashSet<>();
for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> entry : streamResults.entrySet())
{
Pair<InetAddressAndPort, InetAddressAndPort> peer = entry.getKey();
SingleTransferResult result = entry.getValue();
if (result.state == COMMITTING || result.state == COMMITTED)
{
anyActivated = true;
}
else if (result.state == STREAM_COMPLETE)
awaitingActivation.add(peer);
}
if (anyActivated && !awaitingActivation.isEmpty())
{
logger.debug("{} Transfer already activated on some peers, sending activations to remaining: {}", logPrefix(), awaitingActivation);
activate(awaitingActivation);
return;
}
// If no activations have been sent out, check whether we have enough planIds back to meet the required CL
else if (sufficient())
{
Set<Pair<InetAddressAndPort, InetAddressAndPort>> pairs = new HashSet<>();
for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, SingleTransferResult> entry : streamResults.entrySet())
{
Pair<InetAddressAndPort, InetAddressAndPort> pair = entry.getKey();
SingleTransferResult result = entry.getValue();
if (result.state == STREAM_COMPLETE)
pairs.add(pair);
}
logger.debug("{} Transfer meets consistency level {}, sending activations to {}", logPrefix(), cl, pairs);
activate(pairs);
return;
}
logger.debug("{} Nothing to activate", logPrefix());
}
protected void prepare(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets)
{
// First phase ensures data is present on disk, then second phase does the actual import. This ensures that if
// something goes wrong (like a topology change during import), we don't have divergence.
class Prepare extends AsyncFuture<Void> implements RequestCallbackWithFailure<ActivationResponse>
{
final AtomicInteger responses = new AtomicInteger(0);
public Prepare()
{
responses.addAndGet(targets.size());
}
@Override
public void onResponse(Message<ActivationResponse> msg)
{
logger.debug("{} Got prepare response from: {}", logPrefix(), msg.from());
if (responses.decrementAndGet() == 0)
trySuccess(null);
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailure failure)
{
logger.debug("{} Got prepare failure {} from {}", logPrefix(), failure, from);
// Any failure fails the whole transfer here, so marking all results for streams to this remote as
// failed is harmless and allows the purging logic to clean up doomed transfer artifacts.
streamResults.forEach((pair, result) -> {
if (pair.right.equals(from) && result.canTransition(PREPARE_FAILED))
streamResults.put(pair, result.prepareFailed());
});
tryFailure(new RuntimeException("Tracked transfer failed during PREPARE on " + from + " due to " + failure.reason));
}
}
Prepare prepare = new Prepare();
for (Pair<InetAddressAndPort, InetAddressAndPort> target : targets)
{
ActivationRequest activation = createActivation(target, Phase.PREPARE);
Message<ActivationRequest> msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation);
logger.debug("{} Sending prepare {} to peer {}", logPrefix(), activation, target.right);
MessagingService.instance().sendWithCallback(msg, target.right, prepare);
streamResults.computeIfPresent(target, (peer0, result) -> result.preparing());
}
try
{
prepare.get();
}
catch (InterruptedException | ExecutionException e)
{
Throwable cause = e instanceof ExecutionException ? e.getCause() : e;
throw Throwables.unchecked(cause);
}
}
@Override
protected ActivationRequest createActivation(Pair<InetAddressAndPort, InetAddressAndPort> pair, Phase phase)
{
return new ActivationRequest(StreamOperation.IMPORT, pair, phase, id(), ClusterMetadata.current().myNodeId(), range, keyspace, streamResults.get(pair).planId());
}
private SingleTransferResult streamTask(InetAddressAndPort to) throws StreamException, ExecutionException, InterruptedException, TimeoutException
{
StreamPlan plan = new StreamPlan(StreamOperation.IMPORT);
// No need to flush, only using non-live SSTables already on disk
plan.flushBeforeTransfer(false);
for (SSTableReader sstable : sstables)
{
List<Range<Token>> ranges = Collections.singletonList(range);
List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(ranges);
long estimatedKeys = sstable.estimatedKeysForRanges(ranges);
OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.IMPORT, sstable.ref(), positions, ranges, estimatedKeys);
plan.transferStreams(to, Collections.singleton(stream));
}
long timeout = DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds();
logger.info("{} Starting streaming transfer {} to peer {}", logPrefix(), this, to);
StreamResultFuture execute = plan.execute();
StreamState state;
try
{
state = execute.get(timeout, TimeUnit.MILLISECONDS);
logger.debug("{} Completed streaming transfer {} to peer {}", logPrefix(), this, to);
}
catch (InterruptedException | ExecutionException | TimeoutException e)
{
logger.error("Stream session failed with error", e);
throw e;
}
if (state.hasFailedSession() || state.hasAbortedSession())
throw new StreamException(state, "Stream failed due to failed or aborted sessions");
// If the SSTable doesn't contain any rows in the provided range, no streams delivered, nothing to activate
if (state.sessions().isEmpty())
return SingleTransferResult.Noop();
return SingleTransferResult.StreamComplete(plan.planId());
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
TrackedImportTransfer that = (TrackedImportTransfer) o;
return Objects.equals(keyspace, that.keyspace) && Objects.equals(range, that.range) && cl == that.cl && Objects.equals(streamResults, that.streamResults);
}
@Override
public int hashCode()
{
return Objects.hash(keyspace, range, cl, streamResults);
}
@Override
public String toString()
{
return "TrackedImportTransfer{" +
"keyspace='" + keyspace + '\'' +
", range=" + range +
", cl=" + cl +
", streamResults=" + streamResults +
", sstables=" + sstables +
", streamResults=" + streamResults +
'}';
}
}

18
src/java/org/apache/cassandra/replication/CoordinatedTransfers.java → src/java/org/apache/cassandra/replication/TrackedImportTransfers.java
View File

@ -33,20 +33,20 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Interval;
/**
* Factory and container for creating multiple {@link CoordinatedTransfer} instances from a collection
* Factory and container for creating multiple {@link TrackedImportTransfer} instances from a collection
* of SSTables, partitioned by {@link MutationTrackingService.KeyspaceShards}, which are aligned to replica ownership
* ranges. Each shard receives its own CoordinatedTransfer instance, which can be executed independently.
*/
class CoordinatedTransfers implements Iterable<CoordinatedTransfer>
public class TrackedImportTransfers implements Iterable<TrackedImportTransfer>
{
private final Collection<CoordinatedTransfer> transfers;
private final Collection<TrackedImportTransfer> transfers;
private CoordinatedTransfers(Collection<CoordinatedTransfer> transfers)
private TrackedImportTransfers(Collection<TrackedImportTransfer> transfers)
{
this.transfers = transfers;
}
static CoordinatedTransfers create(String keyspace, MutationTrackingService.KeyspaceShards shards, Collection<SSTableReader> sstables, ConsistencyLevel cl)
static TrackedImportTransfers create(String keyspace, MutationTrackingService.KeyspaceShards shards, Collection<SSTableReader> sstables, ConsistencyLevel cl)
{
// Clean up incoming SSTables to remove any existing untrusted CoordinatorLogOffsets
for (SSTableReader sstable : sstables)
@ -62,7 +62,7 @@ class CoordinatedTransfers implements Iterable<CoordinatedTransfer>
}
SSTableIntervalTree intervals = SSTableIntervalTree.buildSSTableIntervalTree(sstables);
List<CoordinatedTransfer> transfers = new ArrayList<>();
List<TrackedImportTransfer> transfers = new ArrayList<>();
shards.forEachShard(shard -> {
Range<Token> range = shard.tokenRange();
@ -70,14 +70,14 @@ class CoordinatedTransfers implements Iterable<CoordinatedTransfer>
if (sstablesForRange.isEmpty())
return;
CoordinatedTransfer transfer = new CoordinatedTransfer(keyspace, range, shard.participants, sstablesForRange, cl, shard::nextId);
TrackedImportTransfer transfer = new TrackedImportTransfer(keyspace, range, shard.participants, sstablesForRange, cl, shard::nextId);
transfers.add(transfer);
});
return new CoordinatedTransfers(transfers);
return new TrackedImportTransfers(transfers);
}
@Override
public Iterator<CoordinatedTransfer> iterator()
public Iterator<TrackedImportTransfer> iterator()
{
return transfers.iterator();
}

127
src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java Normal file
View File

@ -0,0 +1,127 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.SyncStat;
import org.apache.cassandra.repair.SyncTask;
import org.apache.cassandra.repair.SyncTasks;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.Pair;
/**
* Repair sync tasks (that stream SSTable contents) must be integrated with Mutation Tracking's bulk transfer handling
* because any data that is not present on all instances must be expressed as unreconciled in the log, since read
* reconciliations depend on the log state to guarantee monotonicity of subsequent reads. Streaming sessions for full
* repair can complete on some instances before others, so we need to represent those completed sessions as unreconciled
* in the log.
* <p>
* Tracked repairs mostly follow the default coordination process. They are notable in two ways. The first is that
* {@link #prepare} is essentially a no-op. Sync completion assumes that streamed SSTables have arrived at replicas.
* The second is that activation is based on the {@link #bounds} implied by the {@link SyncTask} owned by the transfer.
*/
public class TrackedRepairTransfer extends CoordinatedTransfer
{
private static final Logger logger = LoggerFactory.getLogger(TrackedRepairTransfer.class);
public TrackedRepairTransfer(SyncTasks.ShardedSyncTask shardedTask)
{
this(shardedTask.task.getTransferId(), shardedTask.participants, shardedTask.task, shardedTask.keyspace, shardedTask.range);
}
private TrackedRepairTransfer(ShortMutationId id, Participants participants, SyncTask task, String keyspace, Range<Token> range)
{
super(id, keyspace, range);
Set<Integer> replicaNodeIds = participants.asSet();
Set<InetAddressAndPort> participating = new HashSet<>();
participating.add(task.nodePair().coordinator);
participating.add(task.nodePair().peer);
streamResults.put(Pair.create(task.nodePair().coordinator, task.nodePair().peer), SingleTransferResult.Init());
ClusterMetadata cm = ClusterMetadata.current();
for (Integer replicaNodeId : replicaNodeIds)
{
InetAddressAndPort addr = cm.directory.endpoint(new NodeId(replicaNodeId));
// Need to activate on all replicas, not just ones with SyncTasks. For replicas that don't receive any data
// as part of a repair, they still need to activate the transfer ID as a no-op, to allow read
// reconciliations to complete.
if (!participating.contains(addr))
streamResults.put(Pair.create(addr, addr), SingleTransferResult.EmptySync());
}
}
/**
* When every {@link SyncTask} for a repair has completed, follow the bulk transfer activation path to safely make
* the new data live and tracked in the log. This needs to include all replicas of {@link #id()}, even those that
* did not receive anything as part of the repair. Otherwise, any read reconciliation will fail to complete.
*/
public void activate(SyncStat stat)
{
Preconditions.checkNotNull(stat.planId, "A complete sync should have a plan ID");
logger.debug("{} Activating {}", logPrefix(), this);
Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(stat.nodes.coordinator, stat.nodes.peer);
// Account for syncs where data streams both ways. (i.e. We need to activate both receivers.)
streamResults.put(pair, SingleTransferResult.StreamComplete(stat.planId));
streamResults.put(pair.reverse(), SingleTransferResult.StreamComplete(stat.planId));
activate(streamResults.keySet());
}
@Override
protected void prepare(Collection<Pair<InetAddressAndPort, InetAddressAndPort>> targets)
{
// There should be no need to prepare on stream completion for repair, as not all activations will involve a
// local pending transfer to begin with.
for (Pair<InetAddressAndPort, InetAddressAndPort> target : targets)
streamResults.computeIfPresent(target, (pair, result) -> result.preparing());
}
@Override
protected ActivationRequest createActivation(Pair<InetAddressAndPort, InetAddressAndPort> pair, ActivationRequest.Phase phase)
{
return new ActivationRequest(StreamOperation.REPAIR, pair, phase, id(), ClusterMetadata.current().myNodeId(), range, keyspace, streamResults.get(pair).planId());
}
@Override
public String toString()
{
return "TrackedRepairTransfer{" +
"keyspace='" + keyspace + '\'' +
", range=" + range +
", streamResults=" + streamResults +
'}';
}
}

197
src/java/org/apache/cassandra/replication/TransferActivation.java
View File

@ -1,197 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.io.IOException;
import java.util.Objects;
import com.google.common.base.Preconditions;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.NoPayload;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.TimeUUID;
/**
* Activation transitions a {@link PendingLocalTransfer} from being pending (durable on disk but not yet visible to
* reads) to live (visible to reads and compactions), by associating the streaming plan ID with a mutation ID, referred
* to as the transfer ID.
* <p>
* See {@link CoordinatedTransfer} for the lifecycle of a transfer and when a {@link TransferActivation} is sent.
*/
public class TransferActivation
{
public final TimeUUID planId;
public final ShortMutationId transferId;
public final NodeId coordinatorId;
public final Phase phase;
public enum Phase
{
PREPARE(0),
COMMIT(1);
private final int id;
private static final Phase[] ids;
static
{
ids = new Phase[values().length];
for (Phase phase : values())
ids[phase.id] = phase;
}
Phase(int id)
{
this.id = id;
}
static Phase from(int id)
{
Phase phase = ids[id];
Preconditions.checkState(phase.id == id);
return phase;
}
}
public TransferActivation(CoordinatedTransfer transfer, InetAddressAndPort peer, Phase phase)
{
this(transfer.streamResults.get(peer).planId(), transfer.id(), ClusterMetadata.current().myNodeId(), phase);
}
TransferActivation(TimeUUID planId, ShortMutationId transferId, NodeId coordinatorId, Phase phase)
{
Preconditions.checkArgument(!transferId.isNone());
Preconditions.checkNotNull(planId);
Preconditions.checkNotNull(coordinatorId);
this.planId = planId;
this.transferId = transferId;
this.coordinatorId = coordinatorId;
this.phase = phase;
}
ShortMutationId id()
{
return transferId;
}
public boolean isPrepare()
{
return phase == Phase.PREPARE;
}
public boolean isCommit()
{
return phase == Phase.COMMIT;
}
public void apply()
{
MutationTrackingService.instance.activateLocal(this);
}
public static final Serializer serializer = new Serializer();
public static class Serializer implements IVersionedSerializer<TransferActivation>
{
@Override
public void serialize(TransferActivation activate, DataOutputPlus out, int version) throws IOException
{
TimeUUID.Serializer.instance.serialize(activate.planId, out, version);
ShortMutationId.serializer.serialize(activate.id(), out, version);
NodeId.messagingSerializer.serialize(activate.coordinatorId, out, version);
out.writeByte(activate.phase.id);
}
@Override
public TransferActivation deserialize(DataInputPlus in, int version) throws IOException
{
TimeUUID planId = TimeUUID.Serializer.instance.deserialize(in, version);
ShortMutationId id = ShortMutationId.serializer.deserialize(in, version);
NodeId coordinatorId = NodeId.messagingSerializer.deserialize(in, version);
Phase phase = Phase.from(in.readByte());
return new TransferActivation(planId, id, coordinatorId, phase);
}
@Override
public long serializedSize(TransferActivation activate, int version)
{
long size = 0;
size += TimeUUID.Serializer.instance.serializedSize(activate.planId, version);
size += ShortMutationId.serializer.serializedSize(activate.id(), version);
size += NodeId.messagingSerializer.serializedSize(activate.coordinatorId, version);
size += TypeSizes.BYTE_SIZE; // Enum ordinal
return size;
}
}
public static class VerbHandler implements IVerbHandler<TransferActivation>
{
@Override
public void doVerb(Message<TransferActivation> msg) throws IOException
{
LocalTransfers.instance().executor.submit(() -> {
try
{
msg.payload.apply();
MessagingService.instance().respond(NoPayload.noPayload, msg);
}
catch (Throwable t)
{
MessagingService.instance().respondWithFailure(RequestFailureReason.forException(t), msg);
}
});
}
}
public static final VerbHandler verbHandler = new VerbHandler();
@Override
public String toString()
{
return "TransferActivation{" +
", planId=" + planId +
", transferId=" + transferId +
", coordinatorId=" + coordinatorId +
", phase=" + phase +
'}';
}
@Override
public boolean equals(Object o)
{
if (o == null || getClass() != o.getClass()) return false;
TransferActivation that = (TransferActivation) o;
return Objects.equals(planId, that.planId) && Objects.equals(transferId, that.transferId) && Objects.equals(coordinatorId, that.coordinatorId) && phase == that.phase;
}
@Override
public int hashCode()
{
return Objects.hash(planId, transferId, coordinatorId, phase);
}
}

2
src/java/org/apache/cassandra/replication/TransferFailed.java
View File

@ -28,7 +28,7 @@ import org.apache.cassandra.utils.TimeUUID;
/**
* Notification from coordinator to replicas when a bulk data transfer fails, triggering cleanup of the pending
* transfer state.
* @see CoordinatedTransfer
* @see TrackedImportTransfer
* @see PendingLocalTransfer
*/
public class TransferFailed

194
src/java/org/apache/cassandra/replication/LocalTransfers.java → src/java/org/apache/cassandra/replication/TransferTrackingService.java
View File

@ -18,25 +18,39 @@
package org.apache.cassandra.replication;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.FutureCallback;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.concurrent.ExecutorPlus;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.NoPayload;
import org.apache.cassandra.repair.RepairJob;
import org.apache.cassandra.repair.SyncStat;
import org.apache.cassandra.repair.SyncTask;
import org.apache.cassandra.repair.SyncTasks;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.AsyncPromise;
import org.apache.cassandra.utils.concurrent.Future;
import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory;
@ -50,9 +64,9 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFac
* TODO: Make changes to pending set durable with SystemKeyspace.savePendingLocalTransfer(transfer)?
* TODO: Add vtable for visibility into local and coordinated transfers
*/
public class LocalTransfers
public class TransferTrackingService
{
private static final Logger logger = LoggerFactory.getLogger(LocalTransfers.class);
private static final Logger logger = LoggerFactory.getLogger(TransferTrackingService.class);
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final Map<ShortMutationId, CoordinatedTransfer> coordinating = new ConcurrentHashMap<>();
@ -60,19 +74,18 @@ public class LocalTransfers
final ExecutorPlus executor = executorFactory().pooled("LocalTrackedTransfers", Integer.MAX_VALUE);
private static final LocalTransfers instance = new LocalTransfers();
static LocalTransfers instance()
private static final TransferTrackingService instance = new TransferTrackingService();
public static TransferTrackingService instance()
{
return instance;
}
void save(CoordinatedTransfer transfer)
void save(TrackedImportTransfer transfer)
{
lock.writeLock().lock();
try
{
CoordinatedTransfer existing = coordinating.put(transfer.id(), transfer);
Preconditions.checkState(existing == null);
saveInternal(transfer);
}
finally
{
@ -80,18 +93,12 @@ public class LocalTransfers
}
}
void activating(CoordinatedTransfer transfer)
private void saveInternal(CoordinatedTransfer transfer)
{
Preconditions.checkNotNull(transfer.id());
lock.writeLock().lock();
try
{
coordinating.put(transfer.id(), transfer);
}
finally
{
lock.writeLock().unlock();
}
Preconditions.checkNotNull(transfer.id(), "Cannot coordinate a transfar with no ID");
logger.debug("{} Saving {}", transfer.logPrefix(), transfer);
CoordinatedTransfer existing = coordinating.put(transfer.id(), transfer);
Preconditions.checkState(existing == null, "Attempted to save transfer multiple times");
}
void received(PendingLocalTransfer transfer)
@ -111,6 +118,132 @@ public class LocalTransfers
}
}
/**
* Track a repair as a set of {@link TrackedRepairTransfer} instances corresponding to sync tasks prior to task
* execution so when the syncs are done, we can activate them via {@link ActivationRequest} or fail by
* sending {@link TransferFailed} to all replicas. In other words, one {@link RepairJob} will have as many
* transfers as sync tasks.
*/
public void onRepairSyncExecution(SyncTasks tasks)
{
lock.writeLock().lock();
try
{
tasks.apply((SyncTasks.ShardedSyncTask shardedTask) -> {
TrackedRepairTransfer transfer = new TrackedRepairTransfer(shardedTask);
saveInternal(transfer);
});
}
finally
{
lock.writeLock().unlock();
}
}
/**
* Begin activation for transfers whose repair sync tasks have completed.
*/
public Future<List<SyncStat>> onRepairSyncCompletion(RepairJob job, Future<List<SyncStat>> syncCompletion, Executor executor)
{
AsyncPromise<List<SyncStat>> activationFuture = new AsyncPromise<>();
syncCompletion.addCallback(new FutureCallback<>()
{
@Override
public void onSuccess(List<SyncStat> syncs)
{
logger.info("Completed syncs {}. Coordinating: {} Local: {}", syncs, coordinating, local);
List<Pair<SyncStat, TrackedRepairTransfer>> transfersToActivate = new ArrayList<>();
lock.writeLock().lock();
try
{
// Look up transfers while holding the lock
for (SyncStat sync : syncs)
{
TrackedRepairTransfer transfer = (TrackedRepairTransfer) coordinating.get(sync.transferId);
transfersToActivate.add(Pair.create(sync, transfer));
}
}
finally
{
lock.writeLock().unlock();
}
// Activate transfers WITHOUT holding the lock (activate() acquires its own locks and can block)
for (Pair<SyncStat, TrackedRepairTransfer> pair : transfersToActivate)
{
TrackedRepairTransfer transfer = pair.right;
try
{
logger.info("{} Activating transfer...", transfer.logPrefix());
transfer.activate(pair.left);
}
catch (Throwable t)
{
// Note: cleanup will be triggered automatically when the async COMMIT responses complete
logger.error("{} Failed to activate transfer", transfer.logPrefix(), t);
activationFuture.tryFailure(t);
}
}
// Activation succeeded, complete the future with the sync stats
activationFuture.trySuccess(syncs);
}
@Override
public void onFailure(Throwable t)
{
logger.error("Failed to complete sync tasks. Cleaning up pending transfers... ", t);
lock.writeLock().lock();
try
{
Set<ShortMutationId> transferIds = new HashSet<>();
for (SyncTask task : job.getSyncTasks())
{
ShortMutationId transferId = task.getTransferId();
Preconditions.checkNotNull(transferId);
transferIds.add(transferId);
TimeUUID planId = task.getPlanId();
if (planId == null)
continue;
CoordinatedTransfer transfer = coordinating.get(transferId);
Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(task.nodePair().coordinator, task.nodePair().peer);
transfer.streamResults.put(pair, CoordinatedTransfer.SingleTransferResult.Init().streamFailed(planId));
transfer.streamResults.put(pair.reverse(), CoordinatedTransfer.SingleTransferResult.Init().streamFailed(planId));
}
for (ShortMutationId transferId : transferIds)
{
CoordinatedTransfer transfer = coordinating.get(transferId);
try
{
transfer.notifyFailure();
}
catch (Throwable t0)
{
logger.error("{} Failed to notify peers of repair failure", transfer.logPrefix(), t0);
}
}
scheduleCleanup();
activationFuture.tryFailure(t);
}
finally
{
lock.writeLock().unlock();
}
}
}, executor);
return activationFuture;
}
Purger purger = new Purger();
static class Purger
@ -126,7 +259,7 @@ public class LocalTransfers
boolean failedBeforeActivation = false;
boolean noneActivated = true;
boolean allComplete = true;
for (CoordinatedTransfer.SingleTransferResult result : transfer.streamResults.values())
for (TrackedImportTransfer.SingleTransferResult result : transfer.streamResults.values())
{
switch (result.state)
{
@ -230,12 +363,18 @@ public class LocalTransfers
if (transfer.id() != null)
coordinating.remove(transfer.id());
CoordinatedTransfer.SingleTransferResult localPending = transfer.streamResults.get(FBUtilities.getBroadcastAddressAndPort());
PendingLocalTransfer localTransfer;
TimeUUID planId;
if (localPending != null && (planId = localPending.planId()) != null && (localTransfer = local.get(planId)) != null)
purge(localTransfer);
for (Map.Entry<Pair<InetAddressAndPort, InetAddressAndPort>, CoordinatedTransfer.SingleTransferResult> result : transfer.streamResults.entrySet())
{
if (result.getKey().right.equals(FBUtilities.getBroadcastAddressAndPort()))
{
CoordinatedTransfer.SingleTransferResult localPending = transfer.streamResults.get(result.getKey());
PendingLocalTransfer localTransfer;
TimeUUID planId;
if (localPending != null && (planId = localPending.planId()) != null && (localTransfer = local.get(planId)) != null)
purge(localTransfer);
}
}
}
finally
{
@ -270,7 +409,8 @@ public class LocalTransfers
}
}
@Nullable CoordinatedTransfer getActivatedTransfer(ShortMutationId transferId)
@Nullable
CoordinatedTransfer getActivatedTransfer(ShortMutationId transferId)
{
lock.readLock().lock();
try
@ -284,7 +424,7 @@ public class LocalTransfers
}
public static IVerbHandler<TransferFailed> verbHandler = message -> {
LocalTransfers.instance().purge(message.payload);
TransferTrackingService.instance().purge(message.payload);
MessagingService.instance().respond(NoPayload.noPayload, message);
};
}

10
src/java/org/apache/cassandra/replication/UnreconciledMutations.java
View File

@ -34,8 +34,8 @@ import org.agrona.collections.Int2ObjectHashMap;
import org.apache.cassandra.db.Mutation;
import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.dht.AbstractBounds;
import org.apache.cassandra.dht.Bounds;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.TableId;
/**
@ -151,9 +151,9 @@ public class UnreconciledMutations
statesSet.remove(state);
}
public void activatedTransfer(ShortMutationId id, Collection<SSTableReader> sstables)
public void activatedTransfer(ShortMutationId id, Bounds<Token> bounds)
{
transfers.add(id, sstables);
transfers.add(id, bounds);
}
public UnreconciledMutations copy()
@ -274,10 +274,10 @@ public class UnreconciledMutations
result.addDirectly(mutation);
continue;
}
CoordinatedTransfer transfer = LocalTransfers.instance().getActivatedTransfer(id);
CoordinatedTransfer transfer = TransferTrackingService.instance().getActivatedTransfer(id);
if (transfer != null)
{
result.transfers.add(transfer.id(), transfer.sstables);
result.transfers.add(transfer.id(), transfer.bounds());
continue;
}

13
src/java/org/apache/cassandra/streaming/SessionSummary.java
View File

@ -57,6 +57,7 @@ public class SessionSummary
this.sendingSummaries = sendingSummaries;
}
@Override
public boolean equals(Object o)
{
if (this == o) return true;
@ -70,6 +71,7 @@ public class SessionSummary
return sendingSummaries.equals(summary.sendingSummaries);
}
@Override
public int hashCode()
{
int result = coordinator.hashCode();
@ -79,6 +81,17 @@ public class SessionSummary
return result;
}
@Override
public String toString()
{
return "SessionSummary{" +
"coordinator=" + coordinator +
", peer=" + peer +
", receivingSummaries=" + receivingSummaries +
", sendingSummaries=" + sendingSummaries +
'}';
}
public static IPartitionerDependentSerializer<SessionSummary> serializer = new IPartitionerDependentSerializer<SessionSummary>()
{
public void serialize(SessionSummary summary, DataOutputPlus out, int version) throws IOException

13
src/java/org/apache/cassandra/streaming/StreamOperation.java
View File

@ -27,7 +27,7 @@ public enum StreamOperation
REBUILD("Rebuild", false, true, false),
BULK_LOAD("Bulk Load", true, false, false),
REPAIR("Repair", true, false, true),
TRACKED_TRANSFER("Tracked Transfer", false, false, false);
IMPORT("Import", false, false, false);
private final String description;
private final boolean requiresViewBuild;
@ -47,6 +47,17 @@ public enum StreamOperation
this.requiresBarrierTransaction = requiresBarrierTransaction;
}
/**
* For tracked keyspaces, streams that are received while serving reads must be tracked in the log, to ensure that
* read reconciliation can represent the new data.
*
* @see org.apache.cassandra.replication.CoordinatedTransfer
*/
public boolean isTrackable()
{
return this == REPAIR || this == IMPORT;
}
public static StreamOperation fromString(String text)
{
for (StreamOperation b : StreamOperation.values())

13
src/java/org/apache/cassandra/streaming/StreamPlan.java
View File

@ -116,10 +116,8 @@ public class StreamPlan
// TODO: add flag for fully reconciled data only if this is for a tracked keyspace
session.addStreamRequest(keyspace, fullRanges, transientRanges, Arrays.asList(columnFamilies));
// Automatically include mutation logs for tracked keyspaces
if (isTrackedReplicationEnabled(keyspace)) {
if (includeMutationLogs(keyspace, session))
session.addMutationLogRequest(keyspace, fullRanges, transientRanges);
}
return this;
}
@ -136,9 +134,7 @@ public class StreamPlan
public StreamPlan transferRanges(InetAddressAndPort to, String keyspace, RangesAtEndpoint replicas, String... columnFamilies)
{
StreamSession session = coordinator.getOrCreateOutboundSession(to);
// Automatically include mutation logs for tracked keyspaces
ReconciledKeyspaceOffsets reconciledKeyspaceOffsets = isTrackedReplicationEnabled(keyspace)
ReconciledKeyspaceOffsets reconciledKeyspaceOffsets = includeMutationLogs(keyspace, session)
? session.addMutationLogTransfer(keyspace, replicas)
: null;
@ -147,6 +143,11 @@ public class StreamPlan
return this;
}
private boolean includeMutationLogs(String keyspace, StreamSession session)
{
return isTrackedReplicationEnabled(keyspace) && session.getStreamOperation() != StreamOperation.REPAIR;
}
/**
* Add transfer task to send given streams
*

1
src/java/org/apache/cassandra/streaming/StreamSession.java
View File

@ -1020,6 +1020,7 @@ public class StreamSession
manifest.keyspaceRanges.forEach((keyspace, ranges) ->
logTransfer.addKeyspaceRanges(keyspace, ranges));
}
/**
* In the case where we have an error checking disk space we allow the Operation to continue.
* In the case where we do _not_ have available space, this method raises a RTE.

9
src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java
View File

@ -130,11 +130,11 @@ public class ReplicaGroups
}
/**
* This method is intended to be used on read/write path, not forRange.
* This method is intended to be used on read/write path, not forRange. Returns a {@link VersionedEndpoints.ForRange}
* with the full ownership range, which necessarily contains the input range but may be wider.
*/
public VersionedEndpoints.ForRange matchRange(Range<Token> range)
{
EndpointsForRange.Builder builder = new EndpointsForRange.Builder(range);
Epoch lastModified = Epoch.EMPTY;
// find a range containing the *right* token for the given range - Range is start exclusive so if we looked for the
// left one we could get the wrong range
@ -142,10 +142,9 @@ public class ReplicaGroups
if (pos >= 0 && pos < ranges.size() && ranges.get(pos).contains(range))
{
VersionedEndpoints.ForRange eps = endpoints.get(pos);
lastModified = eps.lastModified();
builder.addAll(eps.get(), ReplicaCollection.Builder.Conflict.ALL);
return VersionedEndpoints.forRange(eps.lastModified(), eps.get());
}
return VersionedEndpoints.forRange(lastModified, builder.build());
return VersionedEndpoints.forRange(lastModified, EndpointsForRange.empty(range));
}
public VersionedEndpoints.ForRange forRange(Token token)

5
src/java/org/apache/cassandra/utils/Pair.java
View File

@ -69,4 +69,9 @@ public class Pair<T1, T2>
{
return new Pair<X, Y>(x, y);
}
public Pair<T2, T1> reverse()
{
return new Pair<>(right, left);
}
}

BIN
test/data/serialization/5.1/service.SyncComplete.bin
View File

Binary file not shown.

6
test/distributed/org/apache/cassandra/distributed/impl/Instance.java
View File

@ -1036,11 +1036,13 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance
AccordService.instance().shutdownAndWait(1l, MINUTES);
});
// The periodic tasks in MTS might hit the CommitLog, so make sure those shut down first...
error = parallelRun(error, executor, MutationTrackingService.instance::shutdownBlocking);
error = parallelRun(error, executor, MutationJournal.instance::shutdownBlocking);
// CommitLog must shut down after Stage, or threads from the latter may attempt to use the former.
// (ex. A Mutation stage thread may attempt to add a mutation to the CommitLog.)
error = parallelRun(error, executor, CommitLog.instance::shutdownBlocking);
error = parallelRun(error, executor, MutationJournal.instance::shutdownBlocking);
error = parallelRun(error, executor, MutationTrackingService.instance::shutdownBlocking);
error = parallelRun(error, executor,
() -> shutdownAndWait(Collections.singletonList(JMXBroadcastExecutor.executor))
);

949
test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java
View File

@ -1,949 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.time.Duration;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import net.bytebuddy.implementation.bind.annotation.This;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.streaming.CassandraStreamReceiver;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.ICoordinator;
import org.apache.cassandra.distributed.api.IInstanceInitializer;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.shared.Uninterruptibles;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.distributed.test.sai.SAIUtil;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.CQLSSTableWriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.replication.ActivatedTransfers;
import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
import org.apache.cassandra.replication.MutationSummary;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.replication.TransferActivation;
import org.apache.cassandra.replication.UnknownShardException;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.assertj.core.api.Assertions;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
import static org.apache.cassandra.replication.TransferActivation.Phase.COMMIT;
import static org.apache.cassandra.replication.TransferActivation.Phase.PREPARE;
/**
* For now, tracked import with a replica down is not supported. The intention is to support this scenario by allowing
* users to provide a {@link ConsistencyLevel} for tracked import operations, where the import will complete if
* sufficient replicas acknowledge the transfer and activate it.
*/
public class BulkTransfersTest extends TestBaseImpl
{
private static final Logger logger = LoggerFactory.getLogger(BulkTransfersTest.class);
private static final String TABLE = "tbl";
private static final String KEYSPACE_TABLE = String.format("%s.%s", KEYSPACE, TABLE);
private static final String TABLE_SCHEMA_CQL = String.format(withKeyspace("CREATE TABLE %s." + TABLE + " (k int primary key, v int);"));
private static final int IMPORT_PK = 1;
private static final Token IMPORT_TOKEN = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(IMPORT_PK));
private static final int NODES = 3;
private static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EXIST = sstable -> {
Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers())
.isNotEmpty();
Assertions.assertThat(sstable.isRepaired()).isFalse();
};
private static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EMPTY = sstable -> {
Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers())
.isEmpty();
Assertions.assertThat(sstable.isRepaired()).isTrue();
};
private static final IIsolatedExecutor.SerializableConsumer<SSTableReader> NOOP = sstable -> {};
@Test
public void importHappyPath() throws Throwable
{
try (Cluster cluster = cluster())
{
createSchema(cluster);
doImport(cluster);
// All pending/ dirs should be empty, should have no SSTables left if all the transfers completed
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
// Verify transfer IDs exist before compaction, then compact, then verify they're removed
assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
// Run after compaction, to enforce offset persistence + broadcast
assertSummary(cluster, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 1;
assert s.unreconciledIds() == 0;
});
});
assertLocalSelect(cluster, rows -> assertRows(rows, row(1, 1)));
}
}
@Test
public void importIndexAlreadyPresent() throws Throwable
{
try (Cluster cluster = cluster())
{
createSchema(cluster);
String indexName = "v_idx";
String indexCql = withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'");
cluster.schemaChange(indexCql);
// This will add an SSTable that already has an SAI index, that needs to be distributed alongside the SSTable on transfer
IInvokableInstance importer = cluster.get(1);
long mark = importer.logs().mark();
doImport(cluster, importer, indexCql);
List<String> logs = importer.logs().grep(mark, "Submitting incremental index build of " + indexName).getResult();
Assertions.assertThat(logs).isEmpty();
// Index should exist and be queryable on all replicas after import
SAIUtil.assertIndexQueryable(cluster, KEYSPACE, indexName);
// Validate queries using the index
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1"));
assertRows(rows, row(1, 1));
});
}
}
@Test
public void importBuildsIndex() throws Throwable
{
try (Cluster cluster = cluster())
{
createSchema(cluster);
String indexName = "v_idx";
cluster.schemaChange(withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'"));
// This will add an SSTable that's missing an SAI index, and the index will be built during the import on
// the coordinator
IInvokableInstance importer = cluster.get(1);
long mark = importer.logs().mark();
doImport(cluster, importer);
List<String> logs = importer.logs().watchFor(mark, Duration.ofMinutes(1), "Submitting incremental index build of " + indexName).getResult();
Assertions.assertThat(logs).isNotEmpty();
// Index should exist and be queryable on all replicas after import
SAIUtil.assertIndexQueryable(cluster, KEYSPACE, indexName);
// Validate queries using the index
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1"));
assertRows(rows, row(1, 1));
});
}
}
@Test
@Ignore
public void importReplicaDown() throws Throwable
{
try (Cluster cluster = cluster())
{
createSchema(cluster);
Iterable<IInvokableInstance> down = Collections.singleton(cluster.get(3));
Iterable<IInvokableInstance> up = cluster.stream().filter(instance -> instance != down).collect(Collectors.toList());
for (IInvokableInstance instance : down)
instance.shutdown().get();
doImport(cluster);
cluster.get(3).startup();
// Transfers did not complete, files should still exist on up replicas
assertPendingDirs(up, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
});
assertPendingDirs(down, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
// Transfers did not complete, transfer IDs should not be removed
assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EXIST);
assertLocalSelect(up, rows -> assertRows(rows, row(1, 1)));
}
}
@Test
public void importMissedActivationPrepare() throws Throwable
{
importMissedActivation(PREPARE);
}
@Test
public void importMissedActivationCommit() throws Throwable
{
importMissedActivation(COMMIT);
}
public void importMissedActivation(TransferActivation.Phase phase) throws Throwable
{
int MISSED_ACTIVATION = 2;
try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
{
ByteBuddyInjections.SkipActivation.setup(cluster, phase);
createSchema(cluster);
Set<IInvokableInstance> missed = Collections.singleton(cluster.get(MISSED_ACTIVATION));
Iterable<IInvokableInstance> received = cluster.stream().filter(instance -> !missed.contains(instance)).collect(Collectors.toList());
Assertions.assertThatThrownBy(() -> doImport(cluster))
.hasMessageContaining("Failed adding SSTables")
.cause()
.hasMessageContaining("Failed streaming on 1 instance(s):")
.cause()
.hasMessageMatching("Tracked import failed during " + phase + " on " + cluster.get(MISSED_ACTIVATION).broadcastAddress() + " due to TIMEOUT")
.hasNoCause();
assertSummary(received, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == (phase == COMMIT ? 1 : 0);
});
});
assertSummary(missed, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == 0;
});
});
switch (phase)
{
case PREPARE:
// Activation did not start, files should be cleaned up
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
break;
case COMMIT:
// Activation did not complete, files should still exist on all replicas
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
});
break;
}
// If the activation is not everywhere, it shouldn't be purged on compaction
assertCompaction(cluster, received, TRANSFERS_EXIST, TRANSFERS_EXIST);
if (phase == PREPARE)
return;
// Permit activation of missed commits during read reconciliation
ByteBuddyInjections.SkipActivation.setup(cluster, null);
// Use coordinated query rather to confirm read reconciliation triggers activation
IInvokableInstance coordinator = cluster.get(3); // not initial transfer coordinator, but received activation
assertCoordinatedRead(coordinator, rows -> {
assertRows(rows, row(1, 1));
});
// Confirm others receive activation
assertLocalSelect(missed, rows -> {
assertRows(rows, row(1, 1));
});
assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
// Activation completed, files should be removed
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
}
}
/**
* If a replica is missing an activation, executes a data read, then discovers the missing transfer during read
* reconciliation, it needs to augment the client response with the new transfer.
*/
@Test
public void importMissingOnDataReplicaDuringAugment() throws Throwable
{
TransferActivation.Phase phase = COMMIT;
int MISSED_ACTIVATION = 2;
try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
{
ByteBuddyInjections.SkipActivation.setup(cluster, phase);
createSchema(cluster);
IInvokableInstance missed = cluster.get(MISSED_ACTIVATION);
Assertions.assertThatThrownBy(() -> doImport(cluster))
.hasMessageContaining("Failed adding SSTables")
.cause()
.hasMessageContaining("Failed streaming on 1 instance(s):")
.cause()
.hasMessageMatching("Tracked import failed during " + phase + " on " + missed.broadcastAddress() + " due to TIMEOUT")
.hasNoCause();
assertSummary(Collections.singleton(missed), summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == 0;
});
});
// Permit activation of missed commits during read reconciliation
ByteBuddyInjections.SkipActivation.setup(cluster, null);
// First read will fail due to failure to augment with transfer
long mark = missed.logs().mark();
Assertions.assertThatThrownBy(() -> {
assertCoordinatedRead(missed, rows -> {
assertRows(rows, row(1, 1));
});
}).isInstanceOf(missed.callOnInstance(() -> ReadTimeoutException.class)); // use instance classloader
List<String> logs = missed.logs().grep(mark, "Missing mutation ShortMutationId").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Retry succeeds
assertCoordinatedRead(missed, rows -> {
assertRows(rows, row(1, 1));
});
}
}
/*
* When an import fails, bounce must not move the pending SSTables into the live set.
*/
@Test
public void importBounceAfterPending() throws Throwable
{
IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3);
try (Cluster cluster = cluster(initializer))
{
ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT);
createSchema(cluster);
Assertions.assertThatThrownBy(() -> doImport(cluster))
.hasMessageContaining("Failed adding SSTables")
.cause()
.hasMessageContaining("Tracked import failed during COMMIT");
Runnable assertEmpty = () -> {
// Activation did not complete, files should still exist on all replicas
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
});
// No one has activated, so should not be present in any summary
assertSummary(cluster, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == 0;
});
});
assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS));
};
assertEmpty.run();
bounce(cluster);
assertEmpty.run();
}
}
@Test
public void importOutOfRange() throws Throwable
{
try (Cluster cluster = cluster())
{
createSchema(cluster, 1);
Set<IInvokableInstance> inRange = new HashSet<>();
Set<IInvokableInstance> outOfRange = new HashSet<>();
cluster.forEach(instance -> {
boolean importReplica = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
});
(importReplica ? inRange : outOfRange).add(instance);
});
logger.info("inRange: {}, outOfRange: {}", inRange, outOfRange);
Assertions.assertThat(inRange).hasSize(1);
IInvokableInstance onlyInRange = inRange.iterator().next();
// Reject import out of range
for (IInvokableInstance instance : outOfRange)
{
long mark = instance.logs().mark();
Consumer<List<String>> onResult = failedDirs -> Assertions.assertThat(failedDirs).hasSize(1);
doImport(cluster, instance, onResult, null);
instance.logs().grep(mark, "java.lang.RuntimeException: Key DecoratedKey(-4069959284402364209, 00000001) is not contained in the given ranges");
}
doImport(cluster, onlyInRange);
assertSummary(Collections.singleton(onlyInRange), summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 1;
assert s.unreconciledIds() == 0;
});
});
for (IInvokableInstance instance : outOfRange)
{
// Out of range shouldn't have any transfers
assertCompaction(cluster, Collections.singleton(instance), TRANSFERS_EMPTY, TRANSFERS_EMPTY);
// Run after compaction, to enforce offset persistence + broadcast
assertSummary(Collections.singleton(instance), summary -> {
Assertions.assertThat(summary).isNull();
});
}
}
}
/*
* Ensure that activation IDs attached to SSTables aren't spread across Token boundaries by compaction.
*
* For example:
* IMPORT_TOKEN is owned by replicas (A, B)
* OUTSIDE_IMPORT_TOKEN is owned by replicas (B, C)
* Execute import so (A, B) have IMPORT_TOKEN
* Execute plain write so (B, C) have OUTSIDE_IMPORT_TOKEN
* Do a major compaction on B so IMPORT_TOKEN and OUTSIDE_IMPORT_TOKEN are compacted together into the same SSTable
* Execute a data read for OUTSIDE_IMPORT_TOKEN against B, ensure it doesn't contain any activation IDs
*/
@Test
public void importActivationMergedByCompaction() throws Throwable
{
IInstanceInitializer initializer = (cl, tg, instance, gen) -> {
ByteBuddyInjections.SkipPurgeTransfers.install().initialise(cl, tg, instance, gen);
};
try (Cluster cluster = cluster(initializer))
{
createSchema(cluster, 2);
Set<IInvokableInstance> inImportRange = new HashSet<>();
cluster.forEach(instance -> {
logger.debug("Instance {} ring is {}", ClusterUtils.instanceId(instance), ClusterUtils.ring(instance));
boolean isInRange = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
});
if (isInRange)
inImportRange.add(instance);
});
Assertions.assertThat(inImportRange).hasSize(2);
// Find a partition key that's not owned by the same replicas as the import
Murmur3Partitioner.LongToken NON_IMPORT_TOKEN = new Murmur3Partitioner.LongToken(IMPORT_TOKEN.getLongValue() * 3);
int NON_IMPORT_PK = Int32Type.instance.compose(Murmur3Partitioner.LongToken.keyForToken(NON_IMPORT_TOKEN));
Set<IInvokableInstance> inNonImportRange = new HashSet<>();
cluster.forEach(instance -> {
boolean isInRange = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
return placement.writes.forToken(NON_IMPORT_TOKEN).get().containsSelf();
});
if (isInRange)
inNonImportRange.add(instance);
});
Assertions.assertThat(inNonImportRange).hasSize(2);
Assertions.assertThat(inNonImportRange).isNotEqualTo(inImportRange);
// Import: (A, B)
// Plain: (B, C)
IInvokableInstance A = null;
IInvokableInstance B = null;
IInvokableInstance C = null;
for (IInvokableInstance instance : cluster)
{
boolean isImport = inImportRange.contains(instance);
boolean isNonImport = inNonImportRange.contains(instance);
if (isImport && isNonImport)
B = instance;
else if (isImport)
A = instance;
else if (isNonImport)
C = instance;
}
Assertions.assertThat(A).isNotNull();
Assertions.assertThat(B).isNotNull();
Assertions.assertThat(C).isNotNull();
doImport(cluster, A);
assertLocalSelect(List.of(A, B), (IIsolatedExecutor.SerializableConsumer<Object[][]>) rows -> {
assertRows(rows, row(IMPORT_PK, IMPORT_PK));
});
ShortMutationId importTransferId = callSerialized(A, () -> ShortMutationId.serializer, () -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
ActivatedTransfers transfers = sstable.getCoordinatorLogOffsets().transfers();
if (!transfers.isEmpty())
return transfers.iterator().next();
}
return null;
});
Assertions.assertThat(importTransferId).isNotNull();
C.coordinator().execute(withKeyspace("INSERT INTO %s." + TABLE + "(k, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NON_IMPORT_PK, NON_IMPORT_PK);
assertCompaction(cluster, Collections.singleton(B), NOOP, NOOP);
// Reading from B for a range that doesn't include the import shouldn't include any transfer IDs, even though they've been compacted together
long mark = B.logs().mark();
Object[][] rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, NON_IMPORT_PK);
assertRows(rows, row(NON_IMPORT_PK, NON_IMPORT_PK));
Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isEmpty();
// But if the read range does include a transfer ID, it should have been added
mark = B.logs().mark();
rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, IMPORT_PK);
assertRows(rows, row(IMPORT_PK, IMPORT_PK));
Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isNotEmpty();
}
}
@Test
public void importFailedStreamCleanup() throws Throwable
{
int FAILED_STREAM = 3;
try (Cluster cluster = cluster(ByteBuddyInjections.FailIncomingStream.install(FAILED_STREAM)))
{
createSchema(cluster);
IInvokableInstance importer = cluster.get(1);
IInvokableInstance missed = cluster.get(FAILED_STREAM);
long mark = importer.logs().mark();
Assertions.assertThatThrownBy(() -> doImport(cluster, importer))
.isInstanceOf(RuntimeException.class)
.cause()
.isInstanceOf(RuntimeException.class)
.cause()
.hasMessageContaining("Remote peer " + missed.broadcastAddress() + " failed stream session");
List<String> logs = importer.logs().watchFor(mark, "Remote peer " + missed.broadcastAddress().toString() + " failed stream session").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Await cleanup of failed stream
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
assertLocalSelect(cluster, rows -> {
assertRows(rows); // empty
});
}
}
public static class ByteBuddyInjections
{
// Only skips direct transfer activation, not activation as part of read reconciliation
public static class SkipActivation
{
// null to not skip
public static volatile TransferActivation.Phase phase;
public static IInstanceInitializer install(int...nodes)
{
return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
for (int node : nodes)
if (node == num)
new ByteBuddy().rebase(TransferActivation.VerbHandler.class)
.method(named("doVerb"))
.intercept(MethodDelegation.to(ByteBuddyInjections.SkipActivation.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
};
}
// Need to set phase in each instance's classloader, otherwise assignment won't be visible to injected method body
public static void setup(Cluster cluster, TransferActivation.Phase phase)
{
logger.debug("Setting up phase {}", phase);
cluster.forEach(instance -> instance.runOnInstance(() -> ByteBuddyInjections.SkipActivation.phase = phase));
}
@SuppressWarnings("unused")
public static void doVerb(Message<TransferActivation> msg, @SuperCall Callable<?> zuper)
{
if (phase != null && msg.payload.phase == SkipActivation.phase)
{
logger.info("Skipping activation for test: {}", msg.payload);
return;
}
logger.info("Test running activation as usual: {}", msg.payload);
try
{
zuper.call();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
}
// ImmutableCoordinatorLogOffsets.Builder.purgeTransfers(Predicate)
public static class SkipPurgeTransfers
{
public static IInstanceInitializer install()
{
return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
new ByteBuddy().rebase(ImmutableCoordinatorLogOffsets.Builder.class)
.method(named("purgeTransfers").and(takesArguments(Predicate.class)))
.intercept(MethodDelegation.to(SkipPurgeTransfers.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
};
}
@SuppressWarnings("unused")
public static void purgeTransfers()
{
logger.debug("Skipping purgeTransfers for test");
}
}
// CassandraStreamReceiver.finished
public static class FailIncomingStream
{
@SuppressWarnings("unused")
private static volatile boolean enabled = true;
public static IInstanceInitializer install(int... nodes)
{
return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
for (int node : nodes)
if (node == num)
new ByteBuddy().rebase(CassandraStreamReceiver.class)
.method(named("finished").and(takesNoArguments()))
.intercept(MethodDelegation.to(FailIncomingStream.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
};
}
@SuppressWarnings("unused")
public static void finished(@This CassandraStreamReceiver self)
{
throw new RuntimeException("Failing incoming stream for test");
}
public static void toggle(Cluster cluster, boolean enable)
{
enabled = enable;
cluster.forEach(instance -> instance.runOnInstance(() -> FailIncomingStream.enabled = enable));
}
}
}
private static Cluster cluster() throws IOException
{
return cluster((cl, tg, instance, generation) -> {});
}
private static Cluster cluster(IInstanceInitializer initializer) throws IOException
{
return Cluster.build(NODES)
.withConfig(cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", "true")
.set("write_request_timeout", "1000ms")
.set("autocompaction_on_startup_enabled", false)
.set("repair_request_timeout", "2s")
.set("stream_transfer_task_timeout", "10s"))
.withInstanceInitializer(initializer)
.start();
}
private static void createSchema(Cluster cluster)
{
createSchema(cluster, NODES);
}
private static void createSchema(Cluster cluster, int rf)
{
cluster.schemaChange(String.format(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': " + rf + "} " +
"AND replication_type='tracked';")));
cluster.schemaChange(TABLE_SCHEMA_CQL);
}
private static void doImport(Cluster cluster) throws IOException
{
doImport(cluster, cluster.get(1));
}
private static void doImport(Cluster cluster, IInvokableInstance target) throws IOException
{
doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), null);
}
private static void doImport(Cluster cluster, IInvokableInstance target, @Nullable String createIndexCql) throws IOException
{
doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), createIndexCql);
}
private static void doImport(Cluster cluster, IInvokableInstance target, Consumer<List<String>> onFailedDirs, @Nullable String createIndexCql) throws IOException
{
String file = Files.createTempDirectory(MutationTrackingTest.class.getSimpleName()).toString();
// Needs to run outside of instance executor because creates schema
CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder()
.forTable(TABLE_SCHEMA_CQL)
.inDirectory(file)
.using("INSERT INTO " + KEYSPACE_TABLE + " (k, v) " + "VALUES (?, ?)");
if (createIndexCql != null)
{
builder.withIndexes(createIndexCql).withBuildIndexes(true);
}
try (CQLSSTableWriter writer = builder.build())
{
writer.addRow(IMPORT_PK, 1);
}
assertLocalSelect(cluster, rows -> {
assertRows(rows); // empty
});
List<String> failed = target.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
Set<String> paths = Set.of(file);
logger.info("Importing SSTables {}", paths);
return cfs.importNewSSTables(paths, true, true, true, true, true, true, true);
});
// Sleep for a while to make sure import completes
Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS);
onFailedDirs.accept(failed);
}
private static void assertCoordinatedRead(IInvokableInstance instance, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
{
ICoordinator coordinator = instance.coordinator();
String cql = "SELECT * FROM %s." + TABLE + " WHERE k = 1";
Object[][] rows = coordinator.execute(withKeyspace(cql), ConsistencyLevel.ALL);
onRows.accept(rows);
}
private static void assertPendingDirs(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<File> forPendingUuidDir)
{
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
Set<File> allPendingDirs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).getDirectories().getPendingLocations();
for (File pendingDir : allPendingDirs)
{
File[] pendingUuidDirs = pendingDir.listUnchecked(File::isDirectory);
for (File pendingUuidDir : pendingUuidDirs)
{
forPendingUuidDir.accept(pendingUuidDir);
}
}
});
}
}
private static void assertSummary(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<MutationSummary> onSummary)
{
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1));
TableId tableId = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).metadata().id;
MutationSummary summary;
try
{
summary = MutationTrackingService.instance.createSummaryForKey(key, tableId, false);
}
catch (UnknownShardException e)
{
summary = null;
}
logger.debug("Validating summary {}", summary);
onSummary.accept(summary);
});
}
}
private static void assertCompaction(Cluster cluster, Iterable<IInvokableInstance> validate,
IIsolatedExecutor.SerializableConsumer<SSTableReader> before,
IIsolatedExecutor.SerializableConsumer<SSTableReader> after)
{
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
logger.info("SSTable {} before compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
before.accept(sstable);
}
});
}
// Activation ID must be persisted and broadcast across all peers in the cluster for any to mark as persisted + reconciled
cluster.forEach(i -> {
i.runOnInstance(() -> {
MutationTrackingService.instance.persistLogStateForTesting();
MutationTrackingService.instance.broadcastOffsetsForTesting();
});
});
// Broadcast is async, wait until completion
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
logger.info("Triggering compaction on instance {}", cfs.metadata.keyspace);
CompactionManager.instance.performMaximal(cfs);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
logger.info("SSTable {} after compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
after.accept(sstable);
}
});
}
}
private static void assertLocalSelect(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
{
for (IInvokableInstance instance : validate)
{
{
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = 1"));
onRows.accept(rows);
}
{
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE));
onRows.accept(rows);
}
}
}
private static void bounce(Cluster cluster)
{
cluster.forEach(instance -> {
try
{
instance.shutdown().get();
}
catch (InterruptedException | ExecutionException e)
{
throw new RuntimeException(e);
}
instance.startup();
});
}
private static <T> T callSerialized(IInvokableInstance instance, IIsolatedExecutor.SerializableSupplier<IVersionedSerializer<T>> serializer, IIsolatedExecutor.SerializableCallable<T> callable)
{
ByteBuffer serialized = instance.callOnInstance(() -> {
T deserialized = callable.call();
IVersionedSerializer<T> serialize = serializer.get();
try
{
return serialize.serialize(deserialized, MessagingService.current_version);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
});
try
{
return serializer.get().deserialize(serialized, MessagingService.current_version);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}

409
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java Normal file
View File

@ -0,0 +1,409 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.assertj.core.api.Assertions;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICoordinator;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.distributed.shared.AssertUtils;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.distributed.shared.Uninterruptibles;
import org.apache.cassandra.exceptions.ReadTimeoutException;
import org.apache.cassandra.io.IVersionedSerializer;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.replication.ActivatedTransfers;
import org.apache.cassandra.replication.ActivationRequest;
import org.apache.cassandra.replication.ShortMutationId;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
import static org.apache.cassandra.replication.ActivationRequest.Phase.COMMIT;
import static org.apache.cassandra.replication.ActivationRequest.Phase.PREPARE;
import static org.assertj.core.api.Assertions.assertThat;
public class TrackedImportFailureTest extends TrackedTransferTestBase
{
private static final Logger logger = LoggerFactory.getLogger(TrackedImportFailureTest.class);
@Test
public void importMissedActivationPrepare() throws Throwable
{
importMissedActivation(PREPARE);
}
@Test
public void importMissedActivationCommit() throws Throwable
{
importMissedActivation(COMMIT);
}
public void importMissedActivation(ActivationRequest.Phase phase) throws Throwable
{
int MISSED_ACTIVATION = 2;
try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
{
TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase);
createSchema(cluster);
Set<IInvokableInstance> missed = Collections.singleton(cluster.get(MISSED_ACTIVATION));
Iterable<IInvokableInstance> received = cluster.stream().filter(instance -> !missed.contains(instance)).collect(Collectors.toList());
Assertions.assertThatThrownBy(() -> doImport(cluster))
.hasMessageContaining("Failed adding SSTables")
.cause()
.hasMessageContaining("Failed streaming on 1 instance(s):")
.cause()
.hasMessageMatching("Tracked transfer failed during " + phase + " on " + cluster.get(MISSED_ACTIVATION).broadcastAddress() + " due to TIMEOUT")
.hasNoCause();
assertSummary(received, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == (phase == COMMIT ? 1 : 0);
});
});
assertSummary(missed, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == 0;
});
});
switch (phase)
{
case PREPARE:
// Activation did not start, files should be cleaned up
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
break;
case COMMIT:
// Activation did not complete, files should still exist on all replicas
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
});
break;
}
// If the activation is not everywhere, it shouldn't be purged on compaction
assertCompaction(cluster, received, TRANSFERS_EXIST, TRANSFERS_EXIST);
if (phase == PREPARE)
return;
// Permit activation of missed commits during read reconciliation
TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, null);
// Use coordinated query rather to confirm read reconciliation triggers activation
IInvokableInstance coordinator = cluster.get(3); // not initial transfer coordinator, but received activation
assertCoordinatedRead(coordinator, rows -> {
assertRows(rows, row(1, 1));
});
// Confirm others receive activation
assertLocalSelect(missed, rows -> {
assertRows(rows, row(1, 1));
});
assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
// Activation completed, files should be removed
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
}
}
/**
* If a replica is missing an activation, executes a data read, then discovers the missing transfer during read
* reconciliation, it needs to augment the client response with the new transfer.
*/
@Test
public void importMissingOnDataReplicaDuringAugment() throws Throwable
{
ActivationRequest.Phase phase = COMMIT;
int MISSED_ACTIVATION = 2;
try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION)))
{
TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase);
createSchema(cluster);
IInvokableInstance missed = cluster.get(MISSED_ACTIVATION);
Assertions.assertThatThrownBy(() -> doImport(cluster))
.hasMessageContaining("Failed adding SSTables")
.cause()
.hasMessageContaining("Failed streaming on 1 instance(s):")
.cause()
.hasMessageMatching("Tracked transfer failed during " + phase + " on " + missed.broadcastAddress() + " due to TIMEOUT")
.hasNoCause();
assertSummary(Collections.singleton(missed), summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 0;
assert s.unreconciledIds() == 0;
});
});
// Permit activation of missed commits during read reconciliation
TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, null);
// First read will fail due to failure to augment with transfer
long mark = missed.logs().mark();
Assertions.assertThatThrownBy(() -> {
assertCoordinatedRead(missed, rows -> {
assertRows(rows, row(1, 1));
});
}).isInstanceOf(missed.callOnInstance(() -> ReadTimeoutException.class)); // use instance classloader
List<String> logs = missed.logs().grep(mark, "Missing mutation ShortMutationId").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Retry succeeds
assertCoordinatedRead(missed, rows -> {
assertRows(rows, row(1, 1));
});
}
}
/*
* Ensure that activation IDs attached to SSTables aren't spread across Token boundaries by compaction.
*
* For example:
* IMPORT_TOKEN is owned by replicas (A, B)
* OUTSIDE_IMPORT_TOKEN is owned by replicas (B, C)
* Execute import so (A, B) have IMPORT_TOKEN
* Execute plain write so (B, C) have OUTSIDE_IMPORT_TOKEN
* Do a major compaction on B so IMPORT_TOKEN and OUTSIDE_IMPORT_TOKEN are compacted together into the same SSTable
* Execute a data read for OUTSIDE_IMPORT_TOKEN against B, ensure it doesn't contain any activation IDs
*/
@Test
public void importActivationMergedByCompaction() throws Throwable
{
try (Cluster cluster = cluster((cl, tg, instance, gen) -> ByteBuddyInjections.SkipPurgeTransfers.install().initialise(cl, tg, instance, gen)))
{
createSchema(cluster, 2);
Set<IInvokableInstance> inImportRange = new HashSet<>();
cluster.forEach(instance -> {
logger.debug("Instance {} ring is {}", ClusterUtils.instanceId(instance), ClusterUtils.ring(instance));
boolean isInRange = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
});
if (isInRange)
inImportRange.add(instance);
});
Assertions.assertThat(inImportRange).hasSize(2);
// Find a partition key that's not owned by the same replicas as the import
Murmur3Partitioner.LongToken NON_IMPORT_TOKEN = new Murmur3Partitioner.LongToken(IMPORT_TOKEN.getLongValue() * 3);
int NON_IMPORT_PK = Int32Type.instance.compose(Murmur3Partitioner.LongToken.keyForToken(NON_IMPORT_TOKEN));
Set<IInvokableInstance> inNonImportRange = new HashSet<>();
cluster.forEach(instance -> {
boolean isInRange = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
return placement.writes.forToken(NON_IMPORT_TOKEN).get().containsSelf();
});
if (isInRange)
inNonImportRange.add(instance);
});
Assertions.assertThat(inNonImportRange).hasSize(2);
Assertions.assertThat(inNonImportRange).isNotEqualTo(inImportRange);
// Import: (A, B)
// Plain: (B, C)
IInvokableInstance A = null;
IInvokableInstance B = null;
IInvokableInstance C = null;
for (IInvokableInstance instance : cluster)
{
boolean isImport = inImportRange.contains(instance);
boolean isNonImport = inNonImportRange.contains(instance);
if (isImport && isNonImport)
B = instance;
else if (isImport)
A = instance;
else if (isNonImport)
C = instance;
}
Assertions.assertThat(A).isNotNull();
Assertions.assertThat(B).isNotNull();
Assertions.assertThat(C).isNotNull();
doImport(cluster, A);
assertLocalSelect(List.of(A, B), (IIsolatedExecutor.SerializableConsumer<Object[][]>) rows -> {
assertRows(rows, row(IMPORT_PK, IMPORT_PK));
});
ShortMutationId importTransferId = callSerialized(A, () -> ShortMutationId.serializer, () -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
ActivatedTransfers transfers = sstable.getCoordinatorLogOffsets().transfers();
if (!transfers.isEmpty())
return transfers.iterator().next();
}
return null;
});
Assertions.assertThat(importTransferId).isNotNull();
C.coordinator().execute(withKeyspace("INSERT INTO %s." + TABLE + "(k, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NON_IMPORT_PK, NON_IMPORT_PK);
assertCompaction(cluster, Collections.singleton(B), NOOP, NOOP);
// Reading from B for a range that doesn't include the import shouldn't include any transfer IDs, even though they've been compacted together
long mark = B.logs().mark();
Object[][] rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, NON_IMPORT_PK);
assertRows(rows, row(NON_IMPORT_PK, NON_IMPORT_PK));
Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isEmpty();
// But if the read range does include a transfer ID, it should have been added
mark = B.logs().mark();
rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, IMPORT_PK);
assertRows(rows, row(IMPORT_PK, IMPORT_PK));
Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isNotEmpty();
}
}
@Test
public void importFailedStreamCleanup() throws Throwable
{
int FAILED_STREAM = 3;
try (Cluster cluster = cluster(ByteBuddyInjections.FailIncomingStream.install(FAILED_STREAM)))
{
createSchema(cluster);
IInvokableInstance importer = cluster.get(1);
IInvokableInstance missed = cluster.get(FAILED_STREAM);
long mark = importer.logs().mark();
Assertions.assertThatThrownBy(() -> doImport(cluster, importer))
.isInstanceOf(RuntimeException.class)
.cause()
.isInstanceOf(RuntimeException.class)
.cause()
.hasMessageContaining("Remote peer " + missed.broadcastAddress() + " failed stream session");
List<String> logs = importer.logs().watchFor(mark, "Remote peer " + missed.broadcastAddress().toString() + " failed stream session").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Await cleanup of failed stream
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
assertPendingDirs(cluster, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
// empty
assertLocalSelect(cluster, AssertUtils::assertRows);
}
}
@Test
@Ignore("Reactivate this when we support CL < ALL for tracked imports")
public void importReplicaDown() throws Throwable
{
try (Cluster cluster = cluster())
{
String keyspace = "replica_down";
createSchema(cluster, keyspace);
Iterable<IInvokableInstance> down = Collections.singleton(cluster.get(3));
Iterable<IInvokableInstance> up = cluster.stream().filter(instance -> instance != down).collect(Collectors.toList());
for (IInvokableInstance instance : down)
instance.shutdown().get();
doImport(cluster, keyspace);
cluster.get(3).startup();
// Transfers did not complete, files should still exist on up replicas
assertPendingDirs(up, keyspace, (File pendingUuidDir) -> {
assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
});
assertPendingDirs(down, keyspace, (File pendingUuidDir) -> {
assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
// Transfers did not complete, transfer IDs should not be removed
assertCompaction(cluster, keyspace, cluster, TRANSFERS_EXIST, TRANSFERS_EXIST);
assertLocalSelect(up, keyspace, rows -> assertRows(rows, row(1, 1)));
}
}
private static void assertCoordinatedRead(IInvokableInstance instance, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
{
ICoordinator coordinator = instance.coordinator();
String cql = "SELECT * FROM %s." + TABLE + " WHERE k = 1";
Object[][] rows = coordinator.execute(withKeyspace(cql), ConsistencyLevel.ALL);
onRows.accept(rows);
}
private static <T> T callSerialized(IInvokableInstance instance, IIsolatedExecutor.SerializableSupplier<IVersionedSerializer<T>> serializer, IIsolatedExecutor.SerializableCallable<T> callable)
{
ByteBuffer serialized = instance.callOnInstance(() -> {
T deserialized = callable.call();
IVersionedSerializer<T> serialize = serializer.get();
try
{
return serialize.serialize(deserialized, MessagingService.current_version);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
});
try
{
return serializer.get().deserialize(serialized, MessagingService.current_version);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
}

203
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java Normal file
View File

@ -0,0 +1,203 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.test.sai.SAIUtil;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.tcm.ClusterMetadata;
import org.apache.cassandra.tcm.ownership.DataPlacement;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
import static org.assertj.core.api.Assertions.assertThat;
/**
* For now, tracked import with a replica down is not supported. The intention is to support this scenario by allowing
* users to provide a {@link ConsistencyLevel} for tracked import operations, where the import will complete if
* sufficient replicas acknowledge the transfer and activate it.
*/
public class TrackedImportTransferTest extends TrackedTransferTestBase
{
private static final Logger logger = LoggerFactory.getLogger(TrackedImportTransferTest.class);
private static Cluster cluster;
@BeforeClass
public static void setup() throws IOException
{
cluster = cluster();
}
@AfterClass
public static void teardown()
{
if (cluster != null)
cluster.close();
}
@Test
public void importHappyPath() throws Throwable
{
String keyspace = "happy_path";
createSchema(cluster, keyspace);
doImport(cluster, keyspace);
// All pending/ dirs should be empty, should have no SSTables left if all the transfers completed
assertPendingDirs(cluster, keyspace, (File pendingUuidDir) -> {
assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty();
});
// Verify transfer IDs exist before compaction, then compact, then verify they're removed
assertCompaction(cluster, keyspace, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY);
// Run after compaction, to enforce offset persistence + broadcast
assertSummary(cluster, keyspace, summary -> {
assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 1;
assert s.unreconciledIds() == 0;
});
});
assertLocalSelect(cluster, keyspace, rows -> assertRows(rows, row(1, 1)));
}
@Test
public void importIndexAlreadyPresent() throws Throwable
{
String keyspace = "index_present";
createSchema(cluster, keyspace);
String indexName = "v_idx";
String indexCql = withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'", keyspace);
cluster.schemaChange(indexCql);
// This will add an SSTable that already has an SAI index, that needs to be distributed alongside the SSTable on transfer
IInvokableInstance importer = cluster.get(1);
long mark = importer.logs().mark();
doImport(cluster, importer, keyspace, indexCql);
List<String> logs = importer.logs().grep(mark, "Submitting incremental index build of " + indexName).getResult();
assertThat(logs).isEmpty();
// Index should exist and be queryable on all replicas after import
SAIUtil.assertIndexQueryable(cluster, keyspace, indexName);
// Validate queries using the index
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1", keyspace));
assertRows(rows, row(1, 1));
});
}
@Test
public void importBuildsIndex() throws Throwable
{
String keyspace = "import_builds";
createSchema(cluster, keyspace);
String indexName = "v_idx";
cluster.schemaChange(withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'", keyspace));
// This will add an SSTable that's missing an SAI index, and the index will be built during the import on
// the coordinator
IInvokableInstance importer = cluster.get(1);
long mark = importer.logs().mark();
doImport(cluster, importer, keyspace);
List<String> logs = importer.logs().watchFor(mark, Duration.ofMinutes(1), "Submitting incremental index build of " + indexName).getResult();
assertThat(logs).isNotEmpty();
// Index should exist and be queryable on all replicas after import
SAIUtil.assertIndexQueryable(cluster, keyspace, indexName);
// Validate queries using the index
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1", keyspace));
assertRows(rows, row(1, 1));
});
}
@Test
public void importOutOfRange() throws Throwable
{
String keyspace = "out_of_range";
createSchema(cluster, keyspace, 1);
Set<IInvokableInstance> inRange = new HashSet<>();
Set<IInvokableInstance> outOfRange = new HashSet<>();
cluster.forEach(instance -> {
boolean importReplica = instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication);
return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf();
});
(importReplica ? inRange : outOfRange).add(instance);
});
logger.info("inRange: {}, outOfRange: {}", inRange, outOfRange);
assertThat(inRange).hasSize(1);
IInvokableInstance onlyInRange = inRange.iterator().next();
// Reject import out of range
for (IInvokableInstance instance : outOfRange)
{
long mark = instance.logs().mark();
Consumer<List<String>> onResult = failedDirs -> assertThat(failedDirs).hasSize(1);
doImport(cluster, instance, onResult, keyspace, null);
instance.logs().grep(mark, "java.lang.RuntimeException: Key DecoratedKey(-4069959284402364209, 00000001) is not contained in the given ranges");
}
doImport(cluster, onlyInRange, keyspace);
assertSummary(Collections.singleton(onlyInRange), keyspace, summary -> {
assertThat(summary).satisfies(s -> {
assert s.reconciledIds() == 1;
assert s.unreconciledIds() == 0;
});
});
for (IInvokableInstance instance : outOfRange)
{
// Out of range shouldn't have any transfers
assertCompaction(cluster, keyspace, Collections.singleton(instance), TRANSFERS_EMPTY, TRANSFERS_EMPTY);
// Run after compaction, to enforce offset persistence + broadcast
assertSummary(Collections.singleton(instance), keyspace, summary -> {
assertThat(summary).isNull();
});
}
}
}

32
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import org.junit.BeforeClass;
public class TrackedNonZeroCopyRepairTransferTest extends TrackedRepairTransferSuccessTestBase
{
@BeforeClass
public static void setup() throws IOException
{
cluster = cluster(NON_ZCS_CONFIG);
}
}

390
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java Normal file
View File

@ -0,0 +1,390 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import org.awaitility.Awaitility;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.streaming.CassandraStreamReceiver;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.replication.ActivationRequest;
import static net.bytebuddy.implementation.MethodDelegation.to;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
import static org.assertj.core.api.Assertions.assertThat;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
public class TrackedRepairFailureTest extends TrackedRepairTransferTestBase
{
@Test
public void testFullRepairPartiallyCompleteAnomaly() throws IOException, ExecutionException, InterruptedException, TimeoutException
{
try (Cluster cluster = cluster(StreamReceiverFailureHelper::install))
{
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
String TABLE_SCHEMA_CQL = "CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (k INT PRIMARY KEY, v INT)";
cluster.schemaChange(TABLE_SCHEMA_CQL);
IInvokableInstance COORDINATING = cluster.get(1);
IInvokableInstance RECEIVING = cluster.get(2);
IInvokableInstance MISSING = cluster.get(3);
/*
If we were to start this process with a normal write, that write would be added to the log. When repair
validation runs and finds the mismatching range, it streams the log. MISSING then receives the log and
applies the new mutation to the memtable, where it's visible to reads.
We want to emulate a situation where one node has a mutation that's not present in the log, hence the
roundabout write path. Once the mutation has completed and is repaired on the coordinator, it's been
(durably) reconciled on all replicas. Then, drop this SSTable on the other peers and we should have a full
repair digest mismatch.
The idea here is to emulate logical data corruption, where SSTables mismatch but the logs are in agreement.
*/
COORDINATING.coordinator().execute("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) " + "VALUES (?, ?)", ALL, 1, 1);
COORDINATING.flush(KEYSPACE);
Awaitility.waitAtMost(1, TimeUnit.MINUTES).pollDelay(1, TimeUnit.SECONDS)
.until(() -> {
boolean isRepaired = COORDINATING.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
Set<SSTableReader> sstables = cfs.getLiveSSTables();
if (sstables.size() != 1)
return false;
SSTableReader sstable = sstables.iterator().next();
return sstable.isRepaired();
});
if (!isRepaired)
COORDINATING.forceCompact(KEYSPACE, TABLE);
return isRepaired;
});
List.of(RECEIVING, MISSING).forEach(instance -> instance.runOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
cfs.truncateBlockingWithoutSnapshot();
}));
cluster.forEach(instance -> {
// Before repair, peers should have no data
Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1");
if (instance == COORDINATING)
assertRows(rows, row(1, 1));
else
assertRows(rows);
});
// Prevent repair stream from completing
MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(true));
// Run full repair from COORDINATING
{
ExecutorService repairExecutor = Executors.newSingleThreadExecutor();
Future<NodeToolResult> repair = repairExecutor.submit(() -> COORDINATING.nodetoolResult("repair", "--full", KEYSPACE));
Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS)
.until(() -> {
int finished = StreamReceiverFailureHelper.getFinishedRepairs(RECEIVING);
return finished > 0;
});
MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(true));
repair.get(10, TimeUnit.SECONDS).asserts().failure();
repairExecutor.shutdown();
}
// Even after partial repair, RECEIVED should not move its SSTable to the live set, since the repair failed
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1");
if (instance == COORDINATING)
assertRows(rows, row(1, 1));
else
assertRows(rows);
});
/*
At this point, the repair is complete and partially applied. RECEIVED has an SSTable it received from
repair, and MISSING has no SSTables. If we were to do a tracked data read against RECEIVED, we'd have an
emptpy summary but rows, and if we were to execute the same read against MISSING we'd have an entirely
empty response. This would break monotonicity if a client executes a QUORUM read against RECEIVED then
against missing, because the empty summaries lead to no reconciliation happening.
To provide monotonicity in this scenario, we integrate the full repair with bulk transfer machinery and
tag the SSTables with transfer IDs that can be included in summaries and reconciled. Then, the initial data
read against RECEIVED includes transfer IDs that are reconciled. Reconciliation detects that RECEIVED has an
SSTable that isn't present on MISSING and streams them, so the subsequent read against MISSING is up to
date.
*/
{
MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(false));
MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(false));
// Don't let coordinating act as a replica for the read
cluster.filters().inbound().to(ClusterUtils.instanceId(COORDINATING)).drop();
MutationTrackingReadReconciliationTest.awaitNodeDead(RECEIVING, COORDINATING);
}
// Repair did not succeed sync, so it did not proceed to activation, so it's not visible on RECEIVING.
{
Object[][] rows = RECEIVING.coordinator().execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1", QUORUM);
assertRows(rows); // empty
}
cluster.filters().reset();
MutationTrackingReadReconciliationTest.awaitNodeAlive(RECEIVING, COORDINATING);
// Another repair succeeds, all peers should now agree on the local data
long mark = COORDINATING.logs().mark();
COORDINATING.nodetoolResult("repair", "--full", KEYSPACE).asserts().success();
List<String> logs = COORDINATING.logs().grep(mark, "Activating .* for ").getResult();
assertThat(logs).isNotEmpty();
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1");
assertRows(rows, row(1, 1));
});
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1", ALL);
assertRows(rows, row(1, 1));
});
}
}
@Test
public void testFullRepairCleanupOnFailure() throws IOException, ExecutionException, InterruptedException, TimeoutException
{
try (Cluster cluster = cluster(StreamReceiverFailureHelper::install))
{
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
String TABLE_SCHEMA_CQL = "CREATE TABLE " + KEYSPACE + '.' + TABLE + " (k INT PRIMARY KEY, v INT)";
cluster.schemaChange(TABLE_SCHEMA_CQL);
IInvokableInstance COORDINATING = cluster.get(1);
IInvokableInstance RECEIVING = cluster.get(2);
IInvokableInstance MISSING = cluster.get(3);
// Write a single row to COORDINATING node only
COORDINATING.executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) VALUES (?, ?)", 1, 100);
// Before repair, only COORDINATING has data
for (IInvokableInstance instance : cluster)
{
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
if (instance == COORDINATING)
assertRows(rows, row(1, 100));
else
assertRows(rows); // empty
}
// Prevent repair stream from completing
MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(true));
// Run full repair from COORDINATING
{
ExecutorService repairExecutor = Executors.newSingleThreadExecutor();
Future<NodeToolResult> repair = repairExecutor.submit(() -> COORDINATING.nodetoolResult("repair", "--full", KEYSPACE));
Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS)
.until(() -> {
int finished = StreamReceiverFailureHelper.getFinishedRepairs(RECEIVING);
return finished > 0;
});
// Repair completed against RECEIVING, so it should have pending SSTables
{
List<String> pending = getPendingSSTablePaths(RECEIVING);
assertThat(pending).isNotEmpty();
}
MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(true));
repair.get(10, TimeUnit.SECONDS).asserts().failure();
repairExecutor.shutdown();
}
// No pending SSTables on COORDINATING because they're streamed from the live set
// No pending SSTables on MISSING because of stream failure injection
// No pending SSTables on RECEIVING because they were cleaned up when the repair failed
for (IInvokableInstance instance : cluster)
{
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
if (instance == COORDINATING)
assertRows(rows, row(1, 100));
else
assertRows(rows); // empty
List<String> pending = getPendingSSTablePaths(instance);
assertThat(pending).isEmpty();
}
}
}
@Test
public void testRepairFailsOnMissedActivation() throws IOException
{
try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(2, 3)))
{
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (k INT PRIMARY KEY, v INT)");
IInvokableInstance COORDINATING = cluster.get(1);
COORDINATING.executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) VALUES (?, ?)", 1, 100);
// Before repair, only instance 1 has data
for (IInvokableInstance instance : cluster)
{
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
if (instance == COORDINATING)
assertRows(rows, row(1, 100));
else
assertRows(rows); // empty
}
// Repair fails because activation is blocked on nodes 2 and 3
ByteBuddyInjections.SkipActivation.setup(cluster, ActivationRequest.Phase.COMMIT, true);
{
NodeToolResult repair = COORDINATING.nodetoolResult("repair", "--full", KEYSPACE);
repair.asserts().failure();
}
for (IInvokableInstance instance : cluster)
{
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
List<String> pending = getPendingSSTablePaths(instance);
if (instance == COORDINATING)
{
assertRows(rows, row(1, 100));
assertThat(pending).isEmpty();
}
else
{
// Nodes 2 and 3 should only have pending SSTables
assertRows(rows);
assertThat(pending).isNotEmpty();
}
}
// Re-enable activation; read reconciliation at ALL should activate the pending SSTables on nodes 2 and 3
ByteBuddyInjections.SkipActivation.setup(cluster, null);
COORDINATING.coordinator().execute("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1", ALL);
for (IInvokableInstance instance : cluster)
{
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1");
assertRows(rows, row(1, 100));
}
// Pending directory cleanup is scheduled, not synchronous with read reconciliation
for (IInvokableInstance instance : cluster)
{
Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS)
.until(() -> {
List<String> pending = getPendingSSTablePaths(instance);
return pending.isEmpty();
});
}
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1", ALL);
assertRows(rows, row(1, 100));
});
}
}
private static List<String> getPendingSSTablePaths(IInvokableInstance instance)
{
return instance.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE);
Set<File> pendingLocations = cfs.getDirectories().getPendingLocations();
List<String> pendingUuidDirs = new ArrayList<>();
for (File pendingDir : pendingLocations)
{
File[] uuidDirs = pendingDir.listUnchecked(File::isDirectory);
for (File dir : uuidDirs)
pendingUuidDirs.add(dir.absolutePath());
}
return pendingUuidDirs;
});
}
public static class StreamReceiverFailureHelper
{
private static final Logger logger = LoggerFactory.getLogger(StreamReceiverFailureHelper.class);
static AtomicBoolean shouldThrow = new AtomicBoolean(false);
static AtomicBoolean shouldWait = new AtomicBoolean(false);
static AtomicInteger count = new AtomicInteger(0);
/**
* {@link CassandraStreamReceiver#finished}
*/
@SuppressWarnings("resource")
public static void install(ClassLoader classLoader, Integer instanceNum)
{
new ByteBuddy().rebase(CassandraStreamReceiver.class)
.method(named("finished").and(takesNoArguments()))
.intercept(to(StreamReceiverFailureHelper.class))
.make()
.load(classLoader, ClassLoadingStrategy.Default.INJECTION);
}
public static void finished(@SuperCall Callable<Void> zuper) throws Exception
{
while (shouldWait.get())
{
if (shouldThrow.get())
throw new RuntimeException("Test: failing stream session");
logger.info("Test: blocking finish of stream session");
Thread.sleep(1_000); // TODO: Look for a way to do this without sleeping
}
zuper.call();
logger.info("Test: finished stream session");
count.incrementAndGet();
}
protected static int getFinishedRepairs(IInvokableInstance instance)
{
return instance.callOnInstance(() -> StreamReceiverFailureHelper.count.get());
}
}
}

178
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java Normal file
View File

@ -0,0 +1,178 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import org.junit.AfterClass;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
public abstract class TrackedRepairTransferSuccessTestBase extends TrackedRepairTransferTestBase
{
protected static Cluster cluster;
@AfterClass
public static void teardown()
{
if (cluster != null)
cluster.close();
}
@Test
public void testFullRepairShardAlignedSinglePlan() throws IOException
{
String keyspace = "full_repair_shard_aligned_single_plan";
testFullRepairSinglePlan(keyspace, cluster, KEY_200, 2, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsSinglePlan() throws IOException
{
String keyspace = "full_repair_single_plan";
testFullRepairSinglePlan(keyspace, cluster, KEY_200, 2, false, "repair", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedSinglePlanOptimized() throws IOException
{
String keyspace = "full_repair_shard_aligned_single_plan_optimized";
testFullRepairSinglePlan(keyspace, cluster, KEY_200, 3, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsSinglePlanOptimized() throws IOException
{
String keyspace = "full_repair_single_plan_optimized";
testFullRepairSinglePlan(keyspace, cluster, KEY_200, 3, true, "repair", "--optimise-streams", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedRemoteSender() throws IOException
{
String keyspace = "full_repair_shard_aligned_remote_sender";
testFullRepairRemoteSender(keyspace, cluster, 2, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsRemoteSender() throws IOException
{
String keyspace = "full_repair_remote_sender";
testFullRepairRemoteSender(keyspace, cluster, 2, false, "repair", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedRemoteSenderOptimized() throws IOException
{
String keyspace = "full_repair_aligned_optimized_remote_sender";
testFullRepairRemoteSender(keyspace, cluster, 3, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsRemoteSenderOptimized() throws IOException
{
String keyspace = "full_repair_optimized_remote_sender";
testFullRepairRemoteSender(keyspace, cluster, 3, true, "repair", "--optimise-streams", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedDuplicateSender() throws IOException
{
String keyspace = "full_repair_shard_aligned_duplicate_sender";
testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsDuplicateSender() throws IOException
{
String keyspace = "full_repair_duplicate_sender";
testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 3, false, "repair", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedDuplicateSenderOptimized() throws IOException
{
String keyspace = "full_repair_optimized_aligned_duplicate_sender";
testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsDuplicateSenderOptimized() throws IOException
{
String keyspace = "full_repair_optimized_duplicate_sender";
testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 6, true, "repair", "--optimise-streams", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedMultiSender() throws IOException
{
String keyspace = "full_repair_shard_aligned_multi_sender";
testFullRepairMultiSender(keyspace, cluster, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsMultiSender() throws IOException
{
String keyspace = "full_repair_multi_sender";
testFullRepairMultiSender(keyspace, cluster, 3, false, "repair", "--full", keyspace);
}
@Test
public void testFullRepairShardAlignedMultiSenderOptimized() throws IOException
{
String keyspace = "full_repair_aligned_optimized_multi_sender";
testFullRepairMultiSender(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairAcrossShardsMultiSenderOptimized() throws IOException
{
String keyspace = "full_repair_optimized_multi_sender";
testFullRepairMultiSender(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--full", keyspace);
}
@Test
public void testFullRepairMultiSenderSameTokenShardAligned() throws IOException
{
String keyspace = "full_repair_multi_sender_same_token_aligned";
testFullRepairMultiSenderSameToken(keyspace, cluster, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairMultiSenderSameTokenAcrossShards() throws IOException
{
String keyspace = "full_repair_multi_sender_same_token";
testFullRepairMultiSenderSameToken(keyspace, cluster, 3, false, "repair", "--full", keyspace);
}
@Test
public void testFullRepairMultiSenderSameTokenShardAlignedOptimized() throws IOException
{
String keyspace = "full_multi_sender_same_token_aligned_optimized";
testFullRepairMultiSenderSameToken(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace);
}
@Test
public void testFullRepairMultiSenderSameTokenAcrossShardsOptimized() throws IOException
{
String keyspace = "full_multi_sender_same_token_optimized";
testFullRepairMultiSenderSameToken(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--full", keyspace);
}
}

234
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java Normal file
View File

@ -0,0 +1,234 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import org.assertj.core.api.Assertions;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.NodeToolResult;
import org.apache.cassandra.distributed.shared.ClusterUtils;
import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
public abstract class TrackedRepairTransferTestBase extends TrackedTransferTestBase
{
public void testFullRepairSinglePlan(String keyspace, Cluster cluster, ByteBuffer key, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
{
cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
IInvokableInstance coordinator = cluster.get(1);
coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key);
// Write should only be present on instance 1
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
if (ClusterUtils.instanceId(instance) == 1)
assertRows(rows, row(key, 1));
else
assertRows(rows); // empty
});
long mark = coordinator.logs().mark();
NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
result.asserts().success();
List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Write visible on all instances after repair
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
assertRows(rows, row(key, 1));
});
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, key);
assertRows(rows, row(key, 1));
});
}
public void testFullRepairRemoteSender(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
{
cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
IInvokableInstance coordinator = cluster.get(1);
cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2)", KEY_201);
// Second key should only be present on instance 2
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
if (ClusterUtils.instanceId(instance) == 2)
assertRows(rows, row(KEY_201, 2));
else
assertRows(rows); // empty
});
long mark = coordinator.logs().mark();
NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
result.asserts().success();
List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Write visible on all instances after repair
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
assertRows(rows, row(KEY_201, 2));
});
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_201);
assertRows(rows, row(KEY_201, 2));
});
}
public void testFullRepairDuplicateSender(String keyspace, Cluster cluster, ByteBuffer key, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
{
cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
IInvokableInstance coordinator = cluster.get(1);
coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key);
cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key);
// Write should be missing on node 3
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
if (ClusterUtils.instanceId(instance) != 3)
assertRows(rows, row(key, 1));
else
assertRows(rows); // empty
});
long mark = coordinator.logs().mark();
coordinator.nodetoolResult(repairCommandAndArgs).asserts().success();
List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Write visible on all instances after repair
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key);
assertRows(rows, row(key, 1));
});
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, key);
assertRows(rows, row(key, 1));
});
}
public void testFullRepairMultiSender(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
{
cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
IInvokableInstance coordinator = cluster.get(1);
coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", KEY_200);
cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2)", KEY_201);
// First key should only be present on instance 1
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
if (ClusterUtils.instanceId(instance) == 1)
assertRows(rows, row(KEY_200, 1));
else
assertRows(rows); // empty
});
// Second key should only be present on instance 2
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
if (ClusterUtils.instanceId(instance) == 2)
assertRows(rows, row(KEY_201, 2));
else
assertRows(rows); // empty
});
long mark = coordinator.logs().mark();
NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
result.asserts().success();
List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Writes visible on all instances after repair
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
assertRows(rows, row(KEY_200, 1));
rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201);
assertRows(rows, row(KEY_201, 2));
});
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_200);
assertRows(rows, row(KEY_200, 1));
rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_201);
assertRows(rows, row(KEY_201, 2));
});
}
public void testFullRepairMultiSenderSameToken(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException
{
cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)");
IInvokableInstance coordinator = cluster.get(1);
coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1) USING TIMESTAMP 1", KEY_200);
cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2) USING TIMESTAMP 2", KEY_200);
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
if (ClusterUtils.instanceId(instance) == 1)
assertRows(rows, row(KEY_200, 1));
else if (ClusterUtils.instanceId(instance) == 2)
assertRows(rows, row(KEY_200, 2));
else
assertRows(rows); // empty
});
long mark = coordinator.logs().mark();
NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs);
result.asserts().success();
List<String> logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult();
Assertions.assertThat(logs).isNotEmpty();
// Writes visible on all instances after repair
cluster.forEach(instance -> {
Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200);
assertRows(rows, row(KEY_200, 2));
});
// Make sure all instances can successfully coordinate
cluster.forEach(instance -> {
Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_200);
assertRows(rows, row(KEY_200, 2));
});
}
}

124
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java Normal file
View File

@ -0,0 +1,124 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.util.function.Consumer;
import org.assertj.core.api.Assertions;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInstanceInitializer;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
import static org.apache.cassandra.replication.ActivationRequest.Phase.COMMIT;
public class TrackedTransferBounceTest extends TrackedTransferTestBase
{
/*
* When an import fails, bounce must not move the pending SSTables into the live set.
*/
@Test
public void testBounceAfterPendingImport() throws Throwable
{
IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3);
try (Cluster cluster = cluster(initializer))
{
ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT);
createSchema(cluster);
Assertions.assertThatThrownBy(() -> doImport(cluster))
.hasMessageContaining("Failed adding SSTables")
.cause()
.hasMessageContaining("Tracked transfer failed during COMMIT");
assertPendingActivation(cluster);
assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS));
bounce(cluster);
assertPendingActivation(cluster);
assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS));
}
}
@Test
public void testBounceAfterPendingShardAlignedZeroCopy() throws IOException
{
testBounceAfterPendingRepair(ZCS_CONFIG, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", KEYSPACE);
}
@Test
public void testBounceAfterPendingAcrossShardsZeroCopy() throws IOException
{
testBounceAfterPendingRepair(ZCS_CONFIG, "repair", "--full", KEYSPACE);
}
@Test
public void testBounceAfterPendingShardAlignedNonZeroCopy() throws IOException
{
testBounceAfterPendingRepair(NON_ZCS_CONFIG, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", KEYSPACE);
}
@Test
public void testBounceAfterPendingAcrossShardsNonZeroCopy() throws IOException
{
testBounceAfterPendingRepair(NON_ZCS_CONFIG, "repair", "--full", KEYSPACE);
}
/*
* When a repair fails, bounce must not move the pending SSTables into the live set.
*/
private static void testBounceAfterPendingRepair(Consumer<IInstanceConfig> config, String... repairCommandAndArgs) throws IOException
{
IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3);
try (Cluster cluster = cluster(config, initializer))
{
// Make sure we fail on commit...
ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT);
cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';");
cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (pk BLOB PRIMARY KEY, v INT)");
cluster.get(1).executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (pk, v) VALUES (?, 1)", KEY_100);
assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1));
assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
cluster.get(1).nodetoolResult(repairCommandAndArgs).asserts().failure().errorContains("Tracked transfer failed during COMMIT");
assertPendingActivation(cluster);
assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1));
assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
bounce(cluster);
assertPendingActivation(cluster);
assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1));
assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100));
}
}
}

546
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java Normal file
View File

@ -0,0 +1,546 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Predicate;
import javax.annotation.Nullable;
import org.assertj.core.api.Assertions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.bytebuddy.ByteBuddy;
import net.bytebuddy.dynamic.loading.ClassLoadingStrategy;
import net.bytebuddy.implementation.MethodDelegation;
import net.bytebuddy.implementation.bind.annotation.SuperCall;
import net.bytebuddy.implementation.bind.annotation.This;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.compaction.CompactionManager;
import org.apache.cassandra.db.streaming.CassandraStreamReceiver;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.Feature;
import org.apache.cassandra.distributed.api.IInstanceConfig;
import org.apache.cassandra.distributed.api.IInstanceInitializer;
import org.apache.cassandra.distributed.api.IInvokableInstance;
import org.apache.cassandra.distributed.api.IIsolatedExecutor;
import org.apache.cassandra.distributed.shared.AssertUtils;
import org.apache.cassandra.distributed.shared.Uninterruptibles;
import org.apache.cassandra.distributed.test.TestBaseImpl;
import org.apache.cassandra.io.sstable.CQLSSTableWriter;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.replication.ActivationRequest;
import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets;
import org.apache.cassandra.replication.MutationSummary;
import org.apache.cassandra.replication.MutationTrackingService;
import org.apache.cassandra.replication.UnknownShardException;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.ByteBufferUtil;
import static net.bytebuddy.matcher.ElementMatchers.named;
import static net.bytebuddy.matcher.ElementMatchers.takesArguments;
import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments;
import static org.junit.Assert.assertEquals;
public abstract class TrackedTransferTestBase extends TestBaseImpl
{
private static final Logger logger = LoggerFactory.getLogger(TrackedTransferTestBase.class);
protected static final Consumer<IInstanceConfig> CONFIG = cfg -> cfg.with(Feature.NETWORK)
.with(Feature.GOSSIP)
.set("mutation_tracking_enabled", true)
.set("write_request_timeout", "1000ms")
.set("autocompaction_on_startup_enabled", false)
.set("repair_request_timeout", "2s")
.set("stream_transfer_task_timeout", "10s");
protected static final Consumer<IInstanceConfig> ZCS_CONFIG = CONFIG.andThen(cfg -> cfg.set("stream_entire_sstables", true));
protected static final Consumer<IInstanceConfig> NON_ZCS_CONFIG = CONFIG.andThen(cfg -> cfg.set("stream_entire_sstables", false));
protected static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EXIST = sstable -> {
Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()).isNotEmpty();
Assertions.assertThat(sstable.isRepaired()).isFalse();
};
protected static final IIsolatedExecutor.SerializableConsumer<SSTableReader> TRANSFERS_EMPTY = sstable -> {
Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()).isEmpty();
Assertions.assertThat(sstable.isRepaired()).isTrue();
};
protected static final IIsolatedExecutor.SerializableConsumer<SSTableReader> NOOP = sstable -> {};
protected static final int NODES = 3;
protected static final String TABLE = "tbl";
protected static final int IMPORT_PK = 1;
protected static final Token IMPORT_TOKEN = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(IMPORT_PK));
// This should be aligned to a single shard: (min, -3074457345618258603]
protected final static long TOKEN_VALUE_100 = -4074457345618258601L;
protected final static Token TOKEN_100 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_100);
protected final static ByteBuffer KEY_100 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_100.getLongValue());
protected final static Range<Token> SHARD_ALIGNED_RANGE_1 = new Range<>(new Murmur3Partitioner.LongToken(TOKEN_VALUE_100 - 10), new Murmur3Partitioner.LongToken(TOKEN_VALUE_100 + 10));
// This should be aligned to a single shard: (-3074457345618258603,3074457345618258601]
protected final static long TOKEN_VALUE_200 = 1;
protected final static Token TOKEN_200 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_200);
protected final static ByteBuffer KEY_200 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_200.getLongValue());
protected final static long TOKEN_VALUE_201 = 2;
protected final static Token TOKEN_201 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_201);
protected final static ByteBuffer KEY_201 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_201.getLongValue());
protected final static Range<Token> SHARD_ALIGNED_RANGE_2 = new Range<>(new Murmur3Partitioner.LongToken(TOKEN_VALUE_200 - 10), new Murmur3Partitioner.LongToken(TOKEN_VALUE_200 + 10));
static
{
DecoratedKey reversed = Murmur3Partitioner.instance.decorateKey(KEY_100);
Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_100);
reversed = Murmur3Partitioner.instance.decorateKey(KEY_200);
Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_200);
reversed = Murmur3Partitioner.instance.decorateKey(KEY_201);
Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_201);
}
protected static Cluster cluster() throws IOException
{
return cluster((cl, tg, instance, generation) -> {});
}
protected static Cluster cluster(Consumer<IInstanceConfig> config) throws IOException
{
return Cluster.build(NODES).withConfig(config).start();
}
protected static Cluster cluster(Consumer<IInstanceConfig> config, IInstanceInitializer initializer) throws IOException
{
return Cluster.build(NODES).withConfig(config).withInstanceInitializer(initializer).start();
}
protected static Cluster cluster(IInstanceInitializer initializer) throws IOException
{
return Cluster.build(NODES).withConfig(CONFIG).withInstanceInitializer(initializer).start();
}
protected static Cluster cluster(BiConsumer<ClassLoader, Integer> initializer) throws IOException
{
return Cluster.build(NODES).withConfig(CONFIG).withInstanceInitializer(initializer).start();
}
protected static void assertPendingActivation(Cluster cluster)
{
assertPendingActivation(cluster, KEYSPACE);
}
protected static void assertPendingActivation(Cluster cluster, String keyspace)
{
// Activation did not complete, files should still exist on all replicas
assertPendingDirs(cluster, keyspace, (File pendingUuidDir) -> {
Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty();
});
// No one has activated, so should not be present in any summary
assertSummary(cluster, keyspace, summary -> {
Assertions.assertThat(summary).satisfies(s -> {
assertEquals(0, s.reconciledIds());
assertEquals(0, s.unreconciledIds());
});
});
}
protected static void assertPendingDirs(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<File> forPendingUuidDir)
{
assertPendingDirs(validate, KEYSPACE, forPendingUuidDir);
}
protected static void assertPendingDirs(Iterable<IInvokableInstance> validate, String keysapce, IIsolatedExecutor.SerializableConsumer<File> forPendingUuidDir)
{
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
Set<File> allPendingDirs = ColumnFamilyStore.getIfExists(keysapce, TABLE).getDirectories().getPendingLocations();
for (File pendingDir : allPendingDirs)
{
File[] pendingUuidDirs = pendingDir.listUnchecked(File::isDirectory);
for (File pendingUuidDir : pendingUuidDirs)
{
forPendingUuidDir.accept(pendingUuidDir);
}
}
});
}
}
protected static void assertSummary(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<MutationSummary> onSummary)
{
assertSummary(validate, KEYSPACE, onSummary);
}
protected static void assertSummary(Iterable<IInvokableInstance> validate, String keyspace, IIsolatedExecutor.SerializableConsumer<MutationSummary> onSummary)
{
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1));
TableId tableId = ColumnFamilyStore.getIfExists(keyspace, TABLE).metadata().id;
MutationSummary summary;
try
{
summary = MutationTrackingService.instance.createSummaryForKey(key, tableId, false);
}
catch (UnknownShardException e)
{
summary = null;
}
onSummary.accept(summary);
});
}
}
protected static void bounce(Cluster cluster)
{
cluster.forEach(instance -> {
try
{
instance.shutdown().get();
}
catch (InterruptedException | ExecutionException e)
{
throw new RuntimeException(e);
}
instance.startup();
});
}
protected static String withKeyspace(String replaceIn, String keyspace)
{
return String.format(replaceIn, keyspace);
}
protected static String tableSchema(String keyspace)
{
return String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v int)", keyspace, TABLE);
}
protected static String tableWithKeyspace(String keyspace)
{
return String.format("%s.%s", keyspace, TABLE);
}
protected static void createSchema(Cluster cluster)
{
createSchema(cluster, KEYSPACE, NODES);
}
protected static void createSchema(Cluster cluster, String keyspace)
{
createSchema(cluster, keyspace, NODES);
}
protected static void createSchema(Cluster cluster, int rf)
{
createSchema(cluster, KEYSPACE, rf);
}
protected static void createSchema(Cluster cluster, String keyspace, int rf)
{
cluster.schemaChange(String.format(withKeyspace("CREATE KEYSPACE %s WITH replication = " +
"{'class': 'SimpleStrategy', 'replication_factor': " + rf + "} " +
"AND replication_type='tracked'", keyspace)));
cluster.schemaChange(tableSchema(keyspace));
}
protected static void doImport(Cluster cluster) throws IOException
{
doImport(cluster, KEYSPACE);
}
protected static void doImport(Cluster cluster, String keyspace) throws IOException
{
doImport(cluster, cluster.get(1), keyspace);
}
protected static void doImport(Cluster cluster, IInvokableInstance target) throws IOException
{
doImport(cluster, target, KEYSPACE);
}
protected static void doImport(Cluster cluster, IInvokableInstance target, String keyspace) throws IOException
{
doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), keyspace, null);
}
protected static void doImport(Cluster cluster, IInvokableInstance target, String keyspace, @Nullable String createIndexCql) throws IOException
{
doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), keyspace, createIndexCql);
}
protected static void doImport(Cluster cluster, IInvokableInstance target, Consumer<List<String>> onFailedDirs, String keyspace, @Nullable String createIndexCql) throws IOException
{
String file = Files.createTempDirectory(MutationTrackingTest.class.getSimpleName()).toString();
// Needs to run outside of instance executor because creates schema
CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder()
.forTable(tableSchema(keyspace))
.inDirectory(file)
.using("INSERT INTO " + tableWithKeyspace(keyspace) + " (k, v) " + "VALUES (?, ?)");
if (createIndexCql != null)
{
builder.withIndexes(createIndexCql).withBuildIndexes(true);
}
try (CQLSSTableWriter writer = builder.build())
{
writer.addRow(IMPORT_PK, 1);
}
// empty
assertLocalSelect(cluster, keyspace, AssertUtils::assertRows);
List<String> failed = target.callOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
Set<String> paths = Set.of(file);
logger.info("Importing SSTables {}", paths);
return cfs.importNewSSTables(paths, true, true, true, true, true, true, true);
});
// Sleep for a while to make sure import completes
Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS);
onFailedDirs.accept(failed);
}
protected static void assertLocalSelect(Iterable<IInvokableInstance> validate, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
{
assertLocalSelect(validate, KEYSPACE, onRows);
}
protected static void assertLocalSelect(Iterable<IInvokableInstance> validate, String keyspace, IIsolatedExecutor.SerializableConsumer<Object[][]> onRows)
{
for (IInvokableInstance instance : validate)
{
{
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = 1", keyspace));
onRows.accept(rows);
}
{
Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE, keyspace));
onRows.accept(rows);
}
}
}
protected static void assertCompaction(Cluster cluster,
Iterable<IInvokableInstance> validate,
IIsolatedExecutor.SerializableConsumer<SSTableReader> before,
IIsolatedExecutor.SerializableConsumer<SSTableReader> after)
{
assertCompaction(cluster, KEYSPACE, validate, before, after);
}
protected static void assertCompaction(Cluster cluster,
String keyspace,
Iterable<IInvokableInstance> validate,
IIsolatedExecutor.SerializableConsumer<SSTableReader> before,
IIsolatedExecutor.SerializableConsumer<SSTableReader> after)
{
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
logger.info("SSTable {} before compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
before.accept(sstable);
}
});
}
// Activation ID must be persisted and broadcast across all peers in the cluster for any to mark as persisted + reconciled
cluster.forEach(i -> {
i.runOnInstance(() -> {
MutationTrackingService.instance.persistLogStateForTesting();
MutationTrackingService.instance.broadcastOffsetsForTesting();
});
});
// Broadcast is async, wait until completion
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
for (IInvokableInstance instance : validate)
{
instance.runOnInstance(() -> {
ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE);
logger.info("Triggering compaction on instance {}", cfs.metadata.keyspace);
CompactionManager.instance.performMaximal(cfs);
for (SSTableReader sstable : cfs.getLiveSSTables())
{
logger.info("SSTable {} after compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets());
after.accept(sstable);
}
});
}
}
public static class ByteBuddyInjections
{
// Only skips direct transfer activation, not activation as part of read reconciliation
public static class SkipActivation
{
// null to not skip
public static volatile ActivationRequest.Phase phase;
public static volatile boolean throwOnActivation = false;
@SuppressWarnings("resource")
public static IInstanceInitializer install(int...nodes)
{
return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
for (int node : nodes)
if (node == num)
new ByteBuddy().rebase(ActivationRequest.VerbHandler.class)
.method(named("doVerb"))
.intercept(MethodDelegation.to(ByteBuddyInjections.SkipActivation.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
};
}
// Need to set phase in each instance's classloader, otherwise assignment won't be visible to injected method body
public static void setup(Cluster cluster, ActivationRequest.Phase phase)
{
setup(cluster, phase, false);
}
public static void setup(Cluster cluster, ActivationRequest.Phase phase, boolean throwOnActivation)
{
logger.debug("Setting up phase {}, throwOnActivation {}", phase, throwOnActivation);
cluster.forEach(instance -> instance.runOnInstance(() -> {
SkipActivation.phase = phase;
SkipActivation.throwOnActivation = throwOnActivation;
}));
}
@SuppressWarnings("unused")
public static void doVerb(Message<ActivationRequest> msg, @SuperCall Callable<?> zuper)
{
if (phase != null && msg.payload.phase == SkipActivation.phase)
{
if (throwOnActivation)
{
// Avoid spamming logs with retries
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
throw new RuntimeException("Throwing on activation for test");
}
else
{
logger.info("Skipping activation for test: {}", msg.payload);
return;
}
}
logger.info("Test running activation as usual: {}", msg.payload);
try
{
zuper.call();
}
catch (Exception e)
{
throw new RuntimeException(e);
}
}
}
// ImmutableCoordinatorLogOffsets.Builder.purgeTransfers(Predicate)
public static class SkipPurgeTransfers
{
@SuppressWarnings("resource")
public static IInstanceInitializer install()
{
return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
new ByteBuddy().rebase(ImmutableCoordinatorLogOffsets.Builder.class)
.method(named("purgeTransfers").and(takesArguments(Predicate.class)))
.intercept(MethodDelegation.to(SkipPurgeTransfers.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
};
}
@SuppressWarnings("unused")
public static void purgeTransfers()
{
logger.debug("Skipping purgeTransfers for test");
}
}
// CassandraStreamReceiver.finished
public static class FailIncomingStream
{
@SuppressWarnings("unused")
private static volatile boolean enabled = true;
@SuppressWarnings("resource")
public static IInstanceInitializer install(int... nodes)
{
return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> {
for (int node : nodes)
if (node == num)
new ByteBuddy().rebase(CassandraStreamReceiver.class)
.method(named("finished").and(takesNoArguments()))
.intercept(MethodDelegation.to(FailIncomingStream.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
};
}
@SuppressWarnings("unused")
public static void finished(@This CassandraStreamReceiver self)
{
throw new RuntimeException("Failing incoming stream for test");
}
public static void toggle(Cluster cluster, boolean enable)
{
enabled = enable;
cluster.forEach(instance -> instance.runOnInstance(() -> FailIncomingStream.enabled = enable));
}
}
}
}

32
test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test.tracking;
import java.io.IOException;
import org.junit.BeforeClass;
public class TrackedZeroCopyRepairTransferTest extends TrackedRepairTransferSuccessTestBase
{
@BeforeClass
public static void setup() throws IOException
{
cluster = cluster(ZCS_CONFIG);
}
}

4
test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java
View File

@ -133,7 +133,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest
TreeResponse r1 = new TreeResponse(local, tree1);
TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2);
LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
NO_PENDING_REPAIR, true, true, PreviewKind.NONE, null);
NettyStreamingConnectionFactory.MAX_CONNECT_ATTEMPTS = 1;
try
{
@ -159,7 +159,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest
TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner()));
LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees),
NO_PENDING_REPAIR, true, true, PreviewKind.NONE);
NO_PENDING_REPAIR, true, true, PreviewKind.NONE, null);
StreamPlan plan = task.createStreamPlan();
assertEquals(NO_PENDING_REPAIR, plan.getPendingRepair());

4
test/unit/org/apache/cassandra/repair/RepairJobTest.java
View File

@ -284,7 +284,7 @@ public class RepairJobTest
// block syncComplete execution until test has verified session still retains the trees
CompletableFuture<?> future = new CompletableFuture<>();
session.registerSyncCompleteCallback(future::get);
ListenableFuture<List<SyncStat>> syncResults = job.executeTasks(syncTasks);
ListenableFuture<List<SyncStat>> syncResults = job.executeTasks(SyncTasks.untracked(syncTasks));
// Immediately following execution the internal execution queue should still retain the trees
long sizeDuringExecution = ObjectSizes.measureDeep(session);
@ -903,7 +903,7 @@ public class RepairJobTest
break;
case SYNC_REQ:
SyncRequest syncRequest = (SyncRequest) message.payload;
session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList())).from(to).build());
session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList(), null, null)).from(to).build());
break;
default:
break;

9
test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java
View File

@ -68,8 +68,8 @@ public class StreamingRepairTaskTest extends AbstractRepairTest
ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID);
RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges());
SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false);
StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false);
SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false, null);
StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, null), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false, null);
StreamPlan plan = task.createStreamPlan(request.dst);
Assert.assertFalse(plan.getFlushBeforeTransfer());
@ -81,8 +81,9 @@ public class StreamingRepairTaskTest extends AbstractRepairTest
TimeUUID sessionID = registerSession(cfs, false, true);
ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID);
RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges());
SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false);
StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false);
SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false, null);
StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, null), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false, null);
StreamPlan plan = task.createStreamPlan(request.dst);
Assert.assertTrue(plan.getFlushBeforeTransfer());

2
test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java
View File

@ -41,7 +41,7 @@ public class SymmetricRemoteSyncTaskTest extends AbstractRepairTest
{
public InstrumentedSymmetricRemoteSyncTask(InetAddressAndPort e1, InetAddressAndPort e2)
{
super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE);
super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE, null);
}
RepairMessage sentMessage = null;

5
test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java
View File

@ -169,7 +169,8 @@ public class RepairMessageSerializationsTest extends CassandraTestBase
InetAddressAndPort src = InetAddressAndPort.getByName("127.0.0.2");
InetAddressAndPort dst = InetAddressAndPort.getByName("127.0.0.3");
SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE, false);
// TODO: Do we want to test with a transfer ID?
SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE, false, null);
serializeRoundTrip(msg, SyncRequest.serializer);
}
@ -183,7 +184,7 @@ public class RepairMessageSerializationsTest extends CassandraTestBase
Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 5, 100)),
Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 500, 10))
));
SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries);
SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries, null, null);
serializeRoundTrip(msg, SyncResponse.serializer);
}

33
test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java → test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java
View File

@ -22,16 +22,24 @@ import java.io.IOException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.CassandraTestBase;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper;
import org.apache.cassandra.io.Serializers;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.streaming.StreamOperation;
import org.apache.cassandra.tcm.membership.NodeId;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
public class TransferActivationSerializationTest
public class ActivationRequestSerializationTest extends CassandraTestBase
{
private static final int VERSION = MessagingService.current_version;
@ -39,37 +47,44 @@ public class TransferActivationSerializationTest
public static void setUpClass()
{
DatabaseDescriptor.daemonInitialization();
ClusterMetadataTestHelper.setInstanceForTest();
}
@Test
public void testRoundtripPreparePhase() throws IOException
{
Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost());
TimeUUID planId = nextTimeUUID();
MutationId transferId = new MutationId(1L, 100, 200);
ShortMutationId transferId = new ShortMutationId(1L, 100);
NodeId coordinatorId = new NodeId(1);
TransferActivation.Phase phase = TransferActivation.Phase.PREPARE;
ActivationRequest.Phase phase = ActivationRequest.Phase.PREPARE;
String keyspace = "test_ks";
Range<Token> range = new Range<>(new ByteOrderedPartitioner.BytesToken("key1".getBytes()), new ByteOrderedPartitioner.BytesToken("key100".getBytes()));
TransferActivation activation = new TransferActivation(planId, transferId, coordinatorId, phase);
ActivationRequest activation = new ActivationRequest(StreamOperation.IMPORT, pair, phase, transferId, coordinatorId, range, keyspace, planId);
try (DataOutputBuffer output = new DataOutputBuffer())
{
Serializers.testSerde(output, TransferActivation.serializer, activation, VERSION);
Serializers.testSerde(output, ActivationRequest.serializer, activation, VERSION);
}
}
@Test
public void testRoundtripCommitPhase() throws IOException
{
Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost());
TimeUUID planId = nextTimeUUID();
MutationId transferId = new MutationId(2L, 300, 400);
ShortMutationId transferId = new ShortMutationId(2L, 300);
NodeId coordinatorId = new NodeId(2);
TransferActivation.Phase phase = TransferActivation.Phase.COMMIT;
ActivationRequest.Phase phase = ActivationRequest.Phase.COMMIT;
String keyspace = "test_ks";
Range<Token> range = new Range<>(new ByteOrderedPartitioner.BytesToken("key1".getBytes()), new ByteOrderedPartitioner.BytesToken("key100".getBytes()));
TransferActivation activation = new TransferActivation(planId, transferId, coordinatorId, phase);
ActivationRequest activation = new ActivationRequest(StreamOperation.IMPORT, pair, phase, transferId, coordinatorId, range, keyspace, planId);
try (DataOutputBuffer output = new DataOutputBuffer())
{
Serializers.testSerde(output, TransferActivation.serializer, activation, VERSION);
Serializers.testSerde(output, ActivationRequest.serializer, activation, VERSION);
}
}
}

84
test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java Normal file
View File

@ -0,0 +1,84 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.io.IOException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.Serializers;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.utils.Pair;
import static org.assertj.core.api.Assertions.assertThat;
public class ActivationResponseSerializationTest
{
private static final int VERSION = MessagingService.current_version;
@BeforeClass
public static void setUpClass()
{
DatabaseDescriptor.daemonInitialization();
}
@Test
public void testRoundtripSuccess() throws IOException
{
Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost());
ActivationResponse response = new ActivationResponse(pair);
try (DataOutputBuffer output = new DataOutputBuffer())
{
Serializers.testSerde(output, ActivationResponse.serializer, response, VERSION);
}
}
@Test
public void testRoundtripDifferentAddresses() throws IOException
{
InetAddressAndPort coordinator = InetAddressAndPort.getByName("127.0.0.1");
InetAddressAndPort peer = InetAddressAndPort.getByName("127.0.0.2");
Pair<InetAddressAndPort, InetAddressAndPort> pair = Pair.create(coordinator, peer);
ActivationResponse response = new ActivationResponse(pair);
try (DataOutputBuffer output = new DataOutputBuffer())
{
Serializers.testSerde(output, ActivationResponse.serializer, response, VERSION);
}
}
@Test
public void testEqualsAndHashCode() throws Exception
{
InetAddressAndPort addr1 = InetAddressAndPort.getByName("127.0.0.1");
InetAddressAndPort addr2 = InetAddressAndPort.getByName("127.0.0.2");
Pair<InetAddressAndPort, InetAddressAndPort> pair1 = Pair.create(addr1, addr2);
Pair<InetAddressAndPort, InetAddressAndPort> pair2 = Pair.create(addr1, addr2);
ActivationResponse resp1 = new ActivationResponse(pair1);
ActivationResponse resp2 = new ActivationResponse(pair2);
assertThat(resp1).isEqualTo(resp2);
assertThat(resp1.hashCode()).isEqualTo(resp2.hashCode());
}
}

344
test/unit/org/apache/cassandra/replication/LocalTransfersTest.java
View File

@ -1,344 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.util.Collection;
import java.util.Collections;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.utils.TimeUUID;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
public class LocalTransfersTest
{
private LocalTransfers localTransfers;
private ShortMutationId transferId;
private TimeUUID planId;
@BeforeClass
public static void setUpClass()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
}
@Before
public void setUp() throws Exception
{
localTransfers = new LocalTransfers();
transferId = new ShortMutationId(1, 100);
planId = nextTimeUUID();
}
private CoordinatedTransfer coordinatedTransfer(ShortMutationId transferId)
{
return coordinatedTransfer(transferId, new Range<>(tk(0), tk(1000)));
}
private CoordinatedTransfer coordinatedTransfer(ShortMutationId transferId, Range<Token> range)
{
MutationId mutationId = transferId != null
? new MutationId(transferId.logId(), transferId.offset(), (int) System.currentTimeMillis())
: null;
return new CoordinatedTransfer(range, mutationId);
}
private PendingLocalTransfer pendingTransfer(TimeUUID planId)
{
SSTableReader mockSSTable = mock(SSTableReader.class);
Collection<SSTableReader> sstables = Collections.singletonList(mockSSTable);
return new PendingLocalTransfer(planId, sstables);
}
private static Token tk(long token)
{
return new Murmur3Partitioner.LongToken(token);
}
@Test
public void testSaveCoordinatedTransfer()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
localTransfers.save(transfer);
localTransfers.activating(transfer);
CoordinatedTransfer loaded = localTransfers.getActivatedTransfer(transferId);
Assertions.assertThat(loaded).isEqualTo(transfer);
assertThatThrownBy(() -> localTransfers.save(transfer))
.isInstanceOf(IllegalStateException.class);
}
@Test
public void testActivatingTransfer()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
localTransfers.save(transfer);
localTransfers.activating(transfer);
CoordinatedTransfer retrieved = localTransfers.getActivatedTransfer(transferId);
assertThat(retrieved).isEqualTo(transfer);
}
@Test
public void testReceivedTransfer()
{
TimeUUID planId = nextTimeUUID();
PendingLocalTransfer transfer = pendingTransfer(planId);
localTransfers.received(transfer);
PendingLocalTransfer retrieved = localTransfers.getPendingTransfer(planId);
assertThat(retrieved).isEqualTo(transfer);
}
@Test
public void testReceivedEmptyTransferThrows()
{
assertThatThrownBy(() -> new PendingLocalTransfer(planId, Collections.emptyList()))
.isInstanceOf(IllegalStateException.class);
}
@Test
public void testGetPendingTransferNotFound()
{
PendingLocalTransfer retrieved = localTransfers.getPendingTransfer(planId);
assertThat(retrieved).isNull();
}
@Test
public void testGetActivatedTransferNotFound()
{
CoordinatedTransfer retrieved = localTransfers.getActivatedTransfer(transferId);
assertThat(retrieved).isNull();
}
@Test
public void testPurgingTransferNotStarted()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// All streams in INIT state - should NOT be purgeable (stream hasn't started yet)
CoordinatedTransfer.SingleTransferResult result = CoordinatedTransfer.SingleTransferResult.Init();
transfer.streamResults.put(mock(InetAddressAndPort.class), result);
Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferAllStreamsComplete()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// All streams in STREAM_COMPLETE state - should NOT be purgeable (no failures)
CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferPrepareFailed()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(PREPARE_FAILED, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferAllActivationCommitted()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// All streams in ACTIVATE_COMMITTED state - should be purgeable (allComplete = true)
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedCommittedAndNoop()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// Mix of ACTIVATE_COMMITTED and STREAM_NOOP - should be purgeable (allComplete = true)
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_NOOP, null);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferActivationPartialCommitted()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// One stream in ACTIVATE_PREPARING - should NOT be purgeable
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTING, planId);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferAllStreamsFailed()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// All streams in STREAM_FAILED state - should be purgeable (noneActivated = true)
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_FAILED, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_FAILED, planId);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedInitAndFailed()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// Mix of INIT and STREAM_FAILED - should be purgeable (has failure, none activated)
CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.Init();
CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedCompleteAndFailed()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// Mix of STREAM_COMPLETE and STREAM_FAILED - should be purgeable (has failure, none activated)
CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedStreamingCompleteAndPreparing()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// Mix of STREAM_COMPLETE and ACTIVATE_PREPARING - should NOT be purgeable
// (noneActivated = false because of ACTIVATE_PREPARING, allComplete = false)
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferMixedCommittingCommitted()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTING, planId);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferWithNullTransferId()
{
CoordinatedTransfer transfer = coordinatedTransfer(null);
// All streams complete but transferId is null - should NOT be purgeable
CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, null);
CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, null);
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
// allComplete = true, but transferId is null, so should not purge
Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferNoopOnly()
{
CoordinatedTransfer transfer = coordinatedTransfer(transferId);
// All streams in STREAM_NOOP - should be purgeable (both noneActivated and allComplete are true)
CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.Noop();
CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Noop();
transfer.streamResults.put(mock(InetAddressAndPort.class), result1);
transfer.streamResults.put(mock(InetAddressAndPort.class), result2);
Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue();
}
}

265
test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java Normal file
View File

@ -0,0 +1,265 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.repair.RepairJobDesc;
import org.apache.cassandra.repair.SharedContext;
import org.apache.cassandra.repair.SyncTask;
import org.apache.cassandra.repair.SyncTasks;
import org.apache.cassandra.repair.SymmetricRemoteSyncTask;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.ReplicationType;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.TimeUUID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class MutationTrackingServiceTest
{
private static final String TEST_KEYSPACE = "test_ks";
private static final String TEST_TABLE = "test_table";
private static final InetAddressAndPort LOCAL = InetAddressAndPort.getByNameUnchecked("127.0.0.1");
private static final InetAddressAndPort REMOTE = InetAddressAndPort.getByNameUnchecked("127.0.0.2");
@BeforeClass
public static void setup() throws IOException
{
DatabaseDescriptor.daemonInitialization();
SchemaLoader.prepareServer();
SchemaLoader.createKeyspace(TEST_KEYSPACE, KeyspaceParams.simple(1, ReplicationType.tracked), SchemaLoader.standardCFMD(TEST_KEYSPACE, TEST_TABLE));
}
@Test
public void testAlignToShardBoundariesSingleTaskWithinSingleShard()
{
MutationTrackingService service = MutationTrackingService.TestAccess.create();
// Create a single shard covering a-z
Set<Range<Token>> shardRanges = new HashSet<>();
shardRanges.add(range("a", "z"));
MutationTrackingService.KeyspaceShards shards =
MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
// Input task completely within the shard
List<SyncTask> inputTasks = Collections.singletonList(createSyncTask(range("d", "m")));
Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
// Should have one shard entry
AtomicInteger entries = new AtomicInteger(0);
result.apply((shardedTask) -> entries.incrementAndGet());
assertEquals(1, entries.get());
// Get all tasks from the result
List<SyncTask> allTasks = new ArrayList<>();
result.apply((shardedTask) -> allTasks.add(shardedTask.task));
assertEquals(1, allTasks.size());
SyncTask resultTask = allTasks.get(0);
assertEquals(1, resultTask.rangesToSync.size());
assertTrue(resultTask.rangesToSync.contains(range("d", "m")));
// Should have a transfer ID assigned
assertNotNull(resultTask.getTransferId());
}
@Test
public void testAlignToShardBoundariesSingleTaskSpanningMultipleShards()
{
MutationTrackingService service = MutationTrackingService.TestAccess.create();
// Create two shards
Set<Range<Token>> shardRanges = new HashSet<>();
shardRanges.add(range("a", "m"));
shardRanges.add(range("m", "z"));
MutationTrackingService.KeyspaceShards shards =
MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
// Input task spans both shards
List<SyncTask> inputTasks = Collections.singletonList(createSyncTask(range("d", "s")));
Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
// Should be split into two shard entries
AtomicInteger entries = new AtomicInteger(0);
result.apply((shardedTask) -> entries.incrementAndGet());
assertEquals(2, entries.get());
// Collect all ranges from all tasks
Set<Range<Token>> allRanges = new HashSet<>();
result.apply((shardedTask) -> allRanges.addAll(shardedTask.task.rangesToSync));
// Should contain the two split pieces
assertEquals(2, allRanges.size());
assertTrue(allRanges.contains(range("d", "m")));
assertTrue(allRanges.contains(range("m", "s")));
}
@Test
public void testAlignToShardBoundariesMultipleTasksAcrossMultipleShards()
{
MutationTrackingService service = MutationTrackingService.TestAccess.create();
// Create three shards
Set<Range<Token>> shardRanges = new HashSet<>();
shardRanges.add(range("a", "h"));
shardRanges.add(range("h", "p"));
shardRanges.add(range("p", "z"));
MutationTrackingService.KeyspaceShards shards =
MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
// Multiple tasks, some spanning shards
List<SyncTask> inputTasks = Arrays.asList(
createSyncTask(range("b", "e")), // Within shard 1
createSyncTask(range("f", "j")), // Spans shard 1 and 2
createSyncTask(range("q", "s")) // Within shard 3
);
Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
// Should have 4 entries (one per sync task)
AtomicInteger entries = new AtomicInteger(0);
result.apply((shardedTask) -> entries.incrementAndGet());
assertEquals(4, entries.get());
// Collect all ranges from all tasks
Set<Range<Token>> allRanges = new HashSet<>();
result.apply((shardedTask) -> allRanges.addAll(shardedTask.task.rangesToSync));
// Should have split ranges: b-e, f-h (shard 1), h-j (shard 2), q-s (shard 3)
assertTrue(allRanges.contains(range("b", "e")));
assertTrue(allRanges.contains(range("f", "h")));
assertTrue(allRanges.contains(range("h", "j")));
assertTrue(allRanges.contains(range("q", "s")));
}
@Test
public void testAlignToShardBoundariesTaskWithMultipleRangesSpanningShards()
{
MutationTrackingService service = MutationTrackingService.TestAccess.create();
// Create two shards
Set<Range<Token>> shardRanges = new HashSet<>();
shardRanges.add(range("a", "m"));
shardRanges.add(range("m", "z"));
MutationTrackingService.KeyspaceShards shards =
MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
// Single task with multiple ranges spanning both shards
List<Range<Token>> ranges = Arrays.asList(
range("b", "e"), // Shard 1
range("f", "p") // Spans both shards
);
SyncTask task = createSyncTask(ranges, LOCAL, REMOTE);
List<SyncTask> inputTasks = Collections.singletonList(task);
Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
// Should be split into two shard entries
AtomicInteger entries = new AtomicInteger(0);
result.forEach((shardedTask) -> entries.incrementAndGet());
assertEquals(2, entries.get());
// Collect all ranges
Set<Range<Token>> allRanges = new HashSet<>();
result.forEach((shardedTask) -> allRanges.addAll(shardedTask.rangesToSync));
// Should have: b-e, f-m (shard 1), m-p (shard 2)
assertTrue(allRanges.contains(range("b", "e")));
assertTrue(allRanges.contains(range("f", "m")));
assertTrue(allRanges.contains(range("m", "p")));
}
@Test
public void testAlignToShardBoundariesPreservesTaskType()
{
MutationTrackingService service = MutationTrackingService.TestAccess.create();
// Create two shards
Set<Range<Token>> shardRanges = new HashSet<>();
shardRanges.add(range("a", "m"));
shardRanges.add(range("m", "z"));
MutationTrackingService.KeyspaceShards shards =
MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges);
MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards);
// Task spanning both shards
List<SyncTask> inputTasks = Collections.singletonList(createSyncTask(range("d", "s")));
Keyspace keyspace = Keyspace.open(TEST_KEYSPACE);
SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks);
// All resulting tasks should be the same type as the input
result.apply((shardedTask) -> assertTrue("Task should be SymmetricRemoteSyncTask", shardedTask.task instanceof SymmetricRemoteSyncTask));
}
private static Token tk(String key)
{
return new ByteOrderedPartitioner.BytesToken(ByteBufferUtil.bytes(key));
}
private static Range<Token> range(String left, String right)
{
return new Range<>(tk(left), tk(right));
}
private static SyncTask createSyncTask(Range<Token> range)
{
return createSyncTask(Collections.singletonList(range), LOCAL, REMOTE);
}
private static SyncTask createSyncTask(List<Range<Token>> ranges, InetAddressAndPort local, InetAddressAndPort remote)
{
SharedContext ctx = SharedContext.Global.instance;
TimeUUID sessionId = TimeUUID.Generator.nextTimeUUID();
RepairJobDesc desc = new RepairJobDesc(sessionId, TimeUUID.Generator.nextTimeUUID(),TEST_KEYSPACE, TEST_TABLE, ranges);
return new SymmetricRemoteSyncTask(ctx, desc, local, remote, ranges, PreviewKind.NONE, null);
}
}

342
test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java Normal file
View File

@ -0,0 +1,342 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.replication;
import java.util.Collection;
import java.util.Collections;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.utils.Pair;
import org.apache.cassandra.utils.TimeUUID;
import org.assertj.core.api.Assertions;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED;
import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP;
import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
public class TransferTrackingServiceTest
{
private TransferTrackingService transferTrackingService;
private ShortMutationId transferId;
private TimeUUID planId;
@BeforeClass
public static void setUpClass()
{
DatabaseDescriptor.daemonInitialization();
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
}
@Before
public void setUp() throws Exception
{
transferTrackingService = new TransferTrackingService();
transferId = new ShortMutationId(1, 100);
planId = nextTimeUUID();
}
private TrackedImportTransfer coordinatedTransfer(ShortMutationId transferId)
{
return coordinatedTransfer(transferId, new Range<>(tk(0), tk(1000)));
}
private TrackedImportTransfer coordinatedTransfer(ShortMutationId transferId, Range<Token> range)
{
MutationId mutationId = transferId != null
? new MutationId(transferId.logId(), transferId.offset(), (int) System.currentTimeMillis())
: null;
return new TrackedImportTransfer(range, mutationId);
}
private PendingLocalTransfer pendingTransfer(TimeUUID planId)
{
SSTableReader mockSSTable = mock(SSTableReader.class);
Collection<SSTableReader> sstables = Collections.singletonList(mockSSTable);
return new PendingLocalTransfer(planId, sstables);
}
private static Token tk(long token)
{
return new Murmur3Partitioner.LongToken(token);
}
@Test
public void testSaveCoordinatedTransfer()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
transferTrackingService.save(transfer);
CoordinatedTransfer loaded = transferTrackingService.getActivatedTransfer(transferId);
Assertions.assertThat(loaded).isEqualTo(transfer);
assertThatThrownBy(() -> transferTrackingService.save(transfer))
.isInstanceOf(IllegalStateException.class);
}
@Test
public void testActivatingTransfer()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
transferTrackingService.save(transfer);
CoordinatedTransfer retrieved = transferTrackingService.getActivatedTransfer(transferId);
assertThat(retrieved).isEqualTo(transfer);
}
@Test
public void testReceivedTransfer()
{
TimeUUID planId = nextTimeUUID();
PendingLocalTransfer transfer = pendingTransfer(planId);
transferTrackingService.received(transfer);
PendingLocalTransfer retrieved = transferTrackingService.getPendingTransfer(planId);
assertThat(retrieved).isEqualTo(transfer);
}
@Test
public void testReceivedEmptyTransferThrows()
{
assertThatThrownBy(() -> new PendingLocalTransfer(planId, Collections.emptyList()))
.isInstanceOf(IllegalStateException.class);
}
@Test
public void testGetPendingTransferNotFound()
{
PendingLocalTransfer retrieved = transferTrackingService.getPendingTransfer(planId);
assertThat(retrieved).isNull();
}
@Test
public void testGetActivatedTransferNotFound()
{
CoordinatedTransfer retrieved = transferTrackingService.getActivatedTransfer(transferId);
assertThat(retrieved).isNull();
}
@Test
public void testPurgingTransferNotStarted()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// All streams in INIT state - should NOT be purgeable (stream hasn't started yet)
TrackedImportTransfer.SingleTransferResult result = TrackedImportTransfer.SingleTransferResult.Init();
transfer.streamResults.put(mock(Pair.class), result);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferAllStreamsComplete()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// All streams in STREAM_COMPLETE state - should NOT be purgeable (no failures)
TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferPrepareFailed()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(PREPARE_FAILED, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferAllActivationCommitted()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// All streams in ACTIVATE_COMMITTED state - should be purgeable (allComplete = true)
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedCommittedAndNoop()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// Mix of ACTIVATE_COMMITTED and STREAM_NOOP - should be purgeable (allComplete = true)
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_NOOP, null);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferActivationPartialCommitted()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// One stream in ACTIVATE_PREPARING - should NOT be purgeable
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTING, planId);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferAllStreamsFailed()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// All streams in STREAM_FAILED state - should be purgeable (noneActivated = true)
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_FAILED, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_FAILED, planId);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedInitAndFailed()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// Mix of INIT and STREAM_FAILED - should be purgeable (has failure, none activated)
TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.Init();
TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedCompleteAndFailed()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// Mix of STREAM_COMPLETE and STREAM_FAILED - should be purgeable (has failure, none activated)
TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID());
TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID());
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
@Test
public void testPurgingTransferMixedStreamingCompleteAndPreparing()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// Mix of STREAM_COMPLETE and ACTIVATE_PREPARING - should NOT be purgeable
// (noneActivated = false because of ACTIVATE_PREPARING, allComplete = false)
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferMixedCommittingCommitted()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTING, planId);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferWithNullTransferId()
{
TrackedImportTransfer transfer = coordinatedTransfer(null);
// All streams complete but transferId is null - should NOT be purgeable
TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, null);
TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, null);
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
// allComplete = true, but transferId is null, so should not purge
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse();
}
@Test
public void testPurgingTransferNoopOnly()
{
TrackedImportTransfer transfer = coordinatedTransfer(transferId);
// All streams in STREAM_NOOP - should be purgeable (both noneActivated and allComplete are true)
TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.Noop();
TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Noop();
transfer.streamResults.put(mock(Pair.class), result1);
transfer.streamResults.put(mock(Pair.class), result2);
Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue();
}
}

41
test/unit/org/apache/cassandra/schema/MockSchema.java
View File

@ -222,15 +222,7 @@ public class MockSchema
first.retainable().getKey().slice(),
last.retainable().getKey().slice());
StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS);
try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE))
{
new StatsComponent(metadataComponents).save(descriptor);
out.flush();
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
writeStatsMetadata(descriptor, metadataComponents);
BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components)
.setTableMetadataRef(cfs.metadata)
.setDataFile(fileHandle.sharedCopy())
@ -253,7 +245,7 @@ public class MockSchema
}
else if (BtiFormat.is(format))
{
Set<Component> components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC);
Set<Component> components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC, Components.STATS);
for (Component component : components)
{
File file = descriptor.fileFor(component);
@ -268,9 +260,17 @@ public class MockSchema
collector.update(DeletionTime.build(timestamp, minLocalDeletionTime));
BufferDecoratedKey first = readerBounds(firstToken);
BufferDecoratedKey last = readerBounds(lastToken);
StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level)
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, ImmutableCoordinatorLogOffsets.NONE, header, first.retainable().getKey(), last.retainable().getKey())
.get(MetadataType.STATS);
Map<MetadataType, MetadataComponent> metadataComponents = collector.sstableLevel(level)
.finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(),
0.01f,
UNREPAIRED_SSTABLE,
null,
ImmutableCoordinatorLogOffsets.NONE,
header,
first.retainable().getKey(),
last.retainable().getKey());
StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS);
writeStatsMetadata(descriptor, metadataComponents);
BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components)
.setTableMetadataRef(cfs.metadata)
.setDataFile(fileHandle.sharedCopy())
@ -278,7 +278,7 @@ public class MockSchema
.setRowIndexFile(fileHandle.sharedCopy())
.setFilter(FilterFactory.AlwaysPresent)
.setMaxDataAge(1L)
.setStatsMetadata(metadata)
.setStatsMetadata(statsMetadata)
.setOpenReason(SSTableReader.OpenReason.NORMAL)
.setSerializationHeader(header)
.setFirst(readerBounds(firstToken))
@ -295,6 +295,19 @@ public class MockSchema
}
}
private static void writeStatsMetadata(Descriptor descriptor, Map<MetadataType, MetadataComponent> metadataComponents)
{
try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE))
{
new StatsComponent(metadataComponents).save(descriptor);
out.flush();
}
catch (IOException ioe)
{
throw new RuntimeException(ioe);
}
}
private static void maybeSetDataLength(Descriptor descriptor, long size)
{
if (size > 0)

7
test/unit/org/apache/cassandra/service/SerializationsTest.java
View File

@ -192,7 +192,8 @@ public class SerializationsTest extends AbstractSerializationsTester
InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT);
InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT);
SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE, false);
// TODO: Do we want to test with a transfer ID?
SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE, false, null);
testRepairMessageWrite("service.SyncRequest.bin", SyncRequest.serializer, message);
}
@ -228,9 +229,9 @@ public class SerializationsTest extends AbstractSerializationsTester
Lists.newArrayList(new StreamSummary(TABLE_ID, emptyList(), 5, 100)),
Lists.newArrayList(new StreamSummary(TABLE_ID, emptyList(), 500, 10))
));
SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries);
SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries, RANDOM_UUID, null);
// sync fail
SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList());
SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList(), null, null);
testRepairMessageWrite("service.SyncComplete.bin", SyncResponse.serializer, success, fail);
}