From d4826900b96c8975e69adc78f71ffbbc938e4a8e Mon Sep 17 00:00:00 2001 From: Abe Ratnofsky Date: Fri, 12 Dec 2025 12:32:44 -0500 Subject: [PATCH] CEP-45: Full Repair Support for Tracked Keyspaces Tracked keyspaces cannot accept new data without first registering it in the log. Any unreconciled data that isn't present in the log will break read monotonicity, since mutation tracking uses a single data read and can only read reconcile mutation IDs that are present in the log. Full repair sync tasks also deliver data to replicas, and require integration with the log just like imports do. The general design of this integration is to give repair SyncTasks the same two-phase commit as import transfers, where we stream SSTables to a pending directory, then once sufficient streams complete successfully, we "activate" those streams and move them out of the pending directory and into the live set. patch by Caleb Rackliffe; reviewed by Abe Ratnofsky for CASSANDRA-21066 Co-authored-by: Caleb Rackliffe Co-authored-by: Abe Ratnofsky --- .../cassandra/db/ColumnFamilyStore.java | 1 + .../apache/cassandra/db/SSTableImporter.java | 13 +- .../CassandraEntireSSTableStreamReader.java | 2 +- .../db/streaming/CassandraStreamManager.java | 3 +- .../db/streaming/CassandraStreamReader.java | 4 +- .../db/streaming/CassandraStreamReceiver.java | 8 +- .../io/sstable/format/SSTableWriter.java | 16 +- src/java/org/apache/cassandra/net/Verb.java | 13 +- .../repair/AsymmetricRemoteSyncTask.java | 39 +- .../repair/CompletableRemoteSyncTask.java | 6 +- .../cassandra/repair/LocalSyncTask.java | 31 +- .../apache/cassandra/repair/RepairJob.java | 45 +- .../repair/RepairMessageVerbHandler.java | 5 +- .../cassandra/repair/RepairSession.java | 10 +- .../cassandra/repair/StreamingRepairTask.java | 23 +- .../repair/SymmetricRemoteSyncTask.java | 33 +- .../org/apache/cassandra/repair/SyncStat.java | 37 +- .../org/apache/cassandra/repair/SyncTask.java | 73 +- .../apache/cassandra/repair/SyncTasks.java | 106 ++ .../repair/messages/SyncRequest.java | 39 +- .../repair/messages/SyncResponse.java | 54 +- .../cassandra/repair/state/SyncState.java | 17 +- .../replication/ActivatedTransfers.java | 10 +- .../replication/ActivationRequest.java | 260 +++++ .../replication/ActivationResponse.java | 94 ++ .../replication/ActiveLogReconciler.java | 7 +- .../replication/CoordinatedTransfer.java | 566 +++-------- .../cassandra/replication/CoordinatorLog.java | 59 +- .../ImmutableCoordinatorLogOffsets.java | 14 - .../replication/MutationTrackingService.java | 198 +++- .../replication/PendingLocalTransfer.java | 26 +- .../apache/cassandra/replication/Shard.java | 22 +- .../replication/TrackedImportTransfer.java | 421 ++++++++ ...sfers.java => TrackedImportTransfers.java} | 18 +- .../replication/TrackedRepairTransfer.java | 127 +++ .../replication/TransferActivation.java | 197 ---- .../cassandra/replication/TransferFailed.java | 2 +- ...fers.java => TransferTrackingService.java} | 194 +++- .../replication/UnreconciledMutations.java | 10 +- .../cassandra/streaming/SessionSummary.java | 13 + .../cassandra/streaming/StreamOperation.java | 13 +- .../cassandra/streaming/StreamPlan.java | 13 +- .../cassandra/streaming/StreamSession.java | 1 + .../tcm/ownership/ReplicaGroups.java | 9 +- src/java/org/apache/cassandra/utils/Pair.java | 5 + .../5.1/service.SyncComplete.bin | Bin 258 -> 258 bytes .../cassandra/distributed/impl/Instance.java | 6 +- .../test/tracking/BulkTransfersTest.java | 949 ------------------ .../tracking/TrackedImportFailureTest.java | 409 ++++++++ .../tracking/TrackedImportTransferTest.java | 203 ++++ .../TrackedNonZeroCopyRepairTransferTest.java | 32 + .../tracking/TrackedRepairFailureTest.java | 390 +++++++ .../TrackedRepairTransferSuccessTestBase.java | 178 ++++ .../TrackedRepairTransferTestBase.java | 234 +++++ .../tracking/TrackedTransferBounceTest.java | 124 +++ .../tracking/TrackedTransferTestBase.java | 546 ++++++++++ .../TrackedZeroCopyRepairTransferTest.java | 32 + .../cassandra/repair/LocalSyncTaskTest.java | 4 +- .../cassandra/repair/RepairJobTest.java | 4 +- .../repair/StreamingRepairTaskTest.java | 9 +- .../repair/SymmetricRemoteSyncTaskTest.java | 2 +- .../RepairMessageSerializationsTest.java | 5 +- ...> ActivationRequestSerializationTest.java} | 33 +- .../ActivationResponseSerializationTest.java | 84 ++ .../replication/LocalTransfersTest.java | 344 ------- .../MutationTrackingServiceTest.java | 265 +++++ .../TransferTrackingServiceTest.java | 342 +++++++ .../apache/cassandra/schema/MockSchema.java | 41 +- .../cassandra/service/SerializationsTest.java | 7 +- 69 files changed, 4912 insertions(+), 2188 deletions(-) create mode 100644 src/java/org/apache/cassandra/repair/SyncTasks.java create mode 100644 src/java/org/apache/cassandra/replication/ActivationRequest.java create mode 100644 src/java/org/apache/cassandra/replication/ActivationResponse.java create mode 100644 src/java/org/apache/cassandra/replication/TrackedImportTransfer.java rename src/java/org/apache/cassandra/replication/{CoordinatedTransfers.java => TrackedImportTransfers.java} (76%) create mode 100644 src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java delete mode 100644 src/java/org/apache/cassandra/replication/TransferActivation.java rename src/java/org/apache/cassandra/replication/{LocalTransfers.java => TransferTrackingService.java} (52%) delete mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java rename test/unit/org/apache/cassandra/replication/{TransferActivationSerializationTest.java => ActivationRequestSerializationTest.java} (50%) create mode 100644 test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java delete mode 100644 test/unit/org/apache/cassandra/replication/LocalTransfersTest.java create mode 100644 test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java create mode 100644 test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index f4631e689a..f289e881b3 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1690,6 +1690,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner * * param @ filename - filename just flushed to disk */ + @VisibleForTesting public void addSSTable(SSTableReader sstable) { assert sstable.getColumnFamilyName().equals(name); diff --git a/src/java/org/apache/cassandra/db/SSTableImporter.java b/src/java/org/apache/cassandra/db/SSTableImporter.java index 1725dcbe15..1bdb168620 100644 --- a/src/java/org/apache/cassandra/db/SSTableImporter.java +++ b/src/java/org/apache/cassandra/db/SSTableImporter.java @@ -235,7 +235,7 @@ public class SSTableImporter cfs.indexManager.buildSSTableAttachedIndexesBlocking(newSSTables); if (isTracked) - TrackedBulkTransfer.execute(cfs.keyspace.getName(), newSSTables); + MutationTrackingService.instance.executeTransfers(cfs.keyspace.getName(), newSSTables, ConsistencyLevel.ALL); else cfs.getTracker().addSSTables(newSSTables); @@ -253,17 +253,6 @@ public class SSTableImporter return failedDirectories; } - /** - * TODO: Support user-defined consistency level for import, for import with replicas down - */ - private static class TrackedBulkTransfer - { - private static void execute(String keyspace, Set sstables) - { - MutationTrackingService.instance.executeTransfers(keyspace, sstables, ConsistencyLevel.ALL); - } - } - /** * Check the state of this node and throws an {@link InterruptedException} if it is currently draining * diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java index 535c5ff7de..0defc3e822 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraEntireSSTableStreamReader.java @@ -160,7 +160,7 @@ public class CassandraEntireSSTableStreamReader implements IStreamReader private File getDataDir(ColumnFamilyStore cfs, long totalSize) throws IOException { - boolean isTracked = cfs.metadata().replicationType().isTracked(); + boolean isTracked = cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable(); Directories.DataDirectory localDir = cfs.getDirectories().getWriteableLocation(totalSize); if (localDir == null) diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java index 2fc802b036..5aece1dc7e 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamManager.java @@ -130,9 +130,10 @@ public class CassandraStreamManager implements TableStreamManager Set sstables = Sets.newHashSet(); SSTableIntervalTree intervalTree = buildSSTableIntervalTree(ImmutableList.copyOf(view.select(SSTableSet.CANONICAL))); Predicate predicate; + // reconciledKeyspaceOffsets are only included when mutation logs are streamed, since we include logs + // for all unreconciled mutations, and SSTables for all reconciled mutations if (reconciledKeyspaceOffsets != null) { - // TODO: relax these restrictions as repair support is add Preconditions.checkArgument(previewKind == PreviewKind.NONE); Preconditions.checkArgument(pendingRepair == ActiveRepairService.NO_PENDING_REPAIR); predicate = getSSTablePredicateForKeyspaceRanges(reconciledKeyspaceOffsets); diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index 579fc9b29b..cbfc3bfe7c 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -62,7 +62,6 @@ import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.ProgressInfo; -import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException; import org.apache.cassandra.streaming.StreamReceiver; import org.apache.cassandra.streaming.StreamSession; @@ -191,9 +190,8 @@ public class CassandraStreamReader implements IStreamReader ILifecycleTransaction txn = createTxn(); RangeAwareSSTableWriter writer; - if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER) + if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable()) { - Preconditions.checkState(cfs.metadata().replicationType().isTracked()); writer = new RangeAwarePendingSSTableWriter(cfs, estimatedKeys, repairedAt, pendingRepair, coordinatorLogOffsets, format, sstableLevel, totalSize, txn, getHeader(cfs.metadata()), session.planId()); } else diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 89f869ee06..62f53d7363 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -54,7 +54,6 @@ import org.apache.cassandra.service.accord.TimeOnlyRequestBookkeeping.LatencyReq import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.replication.PendingLocalTransfer; import org.apache.cassandra.streaming.IncomingStream; -import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.streaming.StreamReceiver; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.tcm.ClusterMetadata; @@ -135,9 +134,8 @@ public class CassandraStreamReceiver implements StreamReceiver sstables.addAll(finished); receivedEntireSSTable = file.isEntireSSTable(); - if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER) + if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable()) { - Preconditions.checkState(cfs.metadata().replicationType().isTracked()); PendingLocalTransfer transfer = new PendingLocalTransfer(cfs.metadata().id, session.planId(), sstables); MutationTrackingService.instance.received(transfer); } @@ -266,8 +264,8 @@ public class CassandraStreamReceiver implements StreamReceiver // add sstables (this will build non-SSTable-attached secondary indexes too, see CASSANDRA-10130) logger.debug("[Stream #{}] Received {} sstables from {} ({})", session.planId(), readers.size(), session.peer, readers); - // Don't mark as live until activated by the stream coordinator - if (session.streamOperation() == StreamOperation.TRACKED_TRANSFER) + // SSTables involved in a coordinated transfer become live when the transfer is activated + if (cfs.metadata().replicationType().isTracked() && session.streamOperation().isTrackable()) return; cfs.addSSTables(readers); diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index 2506d0a9a6..979cacc8af 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -35,15 +35,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets; -import org.apache.cassandra.replication.MutationTrackingService; -import org.apache.cassandra.service.ActiveRepairService; -import org.apache.cassandra.utils.Clock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.compaction.OperationType; import org.apache.cassandra.db.lifecycle.ILifecycleTransaction; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.AbstractBounds; @@ -62,6 +59,10 @@ import org.apache.cassandra.io.sstable.metadata.MetadataComponent; import org.apache.cassandra.io.sstable.metadata.MetadataType; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; import org.apache.cassandra.io.util.MmappedRegionsCache; +import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.utils.Clock; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.Transactional; @@ -336,9 +337,10 @@ public abstract class SSTableWriter extends SSTable implements Transactional protected Map finalizeMetadata() { - // Migration from incremental repair to mutation tracking will be supported, but support for mixing - // incremental repair and mutation tracking is not planned - if (metadata().replicationType().isTracked() && repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE) + // Reconciliation should not occur before activation for coordinated transfer streams for tracked keyspaces. + boolean reconcile = txn.opType() != OperationType.STREAM; + + if (metadata().replicationType().isTracked() && repairedAt == ActiveRepairService.UNREPAIRED_SSTABLE && reconcile) { Preconditions.checkState(Objects.equals(pendingRepair, ActiveRepairService.NO_PENDING_REPAIR)); if (MutationTrackingService.instance.isDurablyReconciled(coordinatorLogOffsets)) diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index 6719e71e27..a30a0361c9 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -74,12 +74,13 @@ import org.apache.cassandra.repair.messages.SyncRequest; import org.apache.cassandra.repair.messages.SyncResponse; import org.apache.cassandra.repair.messages.ValidationRequest; import org.apache.cassandra.repair.messages.ValidationResponse; +import org.apache.cassandra.replication.ActivationResponse; import org.apache.cassandra.replication.BroadcastLogOffsets; import org.apache.cassandra.replication.ForwardedWrite; -import org.apache.cassandra.replication.LocalTransfers; +import org.apache.cassandra.replication.TransferTrackingService; import org.apache.cassandra.replication.PullMutationsRequest; import org.apache.cassandra.replication.PushMutationRequest; -import org.apache.cassandra.replication.TransferActivation; +import org.apache.cassandra.replication.ActivationRequest; import org.apache.cassandra.replication.TransferFailed; import org.apache.cassandra.schema.SchemaMutationsSerializer; import org.apache.cassandra.schema.SchemaPullVerbHandler; @@ -339,10 +340,10 @@ public enum Verb TRACKED_SUMMARY_RSP (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler ), TRACKED_SUMMARY_REQ (911, P3, readTimeout, READ, () -> TrackedRead.SummaryRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_SUMMARY_RSP ), - TRACKED_TRANSFER_ACTIVATE_RSP (912, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER), - TRACKED_TRANSFER_ACTIVATE_REQ (913, P1, repairTimeout, ANTI_ENTROPY, () -> TransferActivation.serializer, () -> TransferActivation.verbHandler, TRACKED_TRANSFER_ACTIVATE_RSP), - TRACKED_TRANSFER_FAILED_RSP (914, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER), - TRACKED_TRANSFER_FAILED_REQ (915, P1, repairTimeout, ANTI_ENTROPY, () -> TransferFailed.serializer, () -> LocalTransfers.verbHandler, TRACKED_TRANSFER_FAILED_RSP), + TRACKED_TRANSFER_ACTIVATE_RSP (912, P1, repairTimeout, REQUEST_RESPONSE, () -> ActivationResponse.serializer, RESPONSE_HANDLER), + TRACKED_TRANSFER_ACTIVATE_REQ (913, P1, repairTimeout, ANTI_ENTROPY, () -> ActivationRequest.serializer, () -> ActivationRequest.verbHandler, TRACKED_TRANSFER_ACTIVATE_RSP), + TRACKED_TRANSFER_FAILED_RSP (914, P1, repairTimeout, REQUEST_RESPONSE, () -> NoPayload.serializer, RESPONSE_HANDLER), + TRACKED_TRANSFER_FAILED_REQ (915, P1, repairTimeout, ANTI_ENTROPY, () -> TransferFailed.serializer, () -> TransferTrackingService.verbHandler, TRACKED_TRANSFER_FAILED_RSP), // accord ACCORD_SIMPLE_RSP (119, P2, writeTimeout, IMMEDIATE, () -> accordEmbedded(EnumSerializer.simpleReply), AccordService::responseHandlerOrNoop ), diff --git a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java index 475c389678..fa42242387 100644 --- a/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/AsymmetricRemoteSyncTask.java @@ -18,16 +18,21 @@ package org.apache.cassandra.repair; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import com.google.common.base.Preconditions; + import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RepairException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.tracing.Tracing; /** @@ -38,29 +43,46 @@ import org.apache.cassandra.tracing.Tracing; */ public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRemoteSyncTask { - public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List> differences, PreviewKind previewKind) + public AsymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort to, InetAddressAndPort from, List> differences, PreviewKind previewKind, ShortMutationId transferId) { - super(ctx, desc, to, from, differences, previewKind); + super(ctx, desc, to, from, differences, previewKind, transferId); + } + + @Override + public SyncTask withRanges(Collection> newRanges) + { + List> rangeList = newRanges instanceof List ? (List>) newRanges : new ArrayList<>(newRanges); + return new AsymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, previewKind, transferId); + } + + @Override + public SyncTask withTransferId(ShortMutationId transferId) + { + Preconditions.checkState(this.transferId == null); + return new AsymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, transferId); } public void startSync() { InetAddressAndPort local = ctx.broadcastAddressAndPort(); - SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true); + SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, true, transferId); String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst); Tracing.traceRepair(message); sendRequest(request, request.src); } - public void syncComplete(boolean success, List summaries) + public void syncComplete(SyncResponse response) { - if (success) + if (response.success) { - trySuccess(stat.withSummaries(summaries)); + trySuccess(stat.withSummaries(response.summaries, response.planId, response.transferId)); } else { - tryFailure(RepairException.warn(desc, previewKind, String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer))); + String message = transferId == null + ? String.format("Sync failed between %s and %s", nodePair.coordinator, nodePair.peer) + : String.format("Sync failed between %s and %s for transfer %s", nodePair.coordinator, nodePair.peer, transferId); + tryFailure(RepairException.warn(desc, previewKind, message)); } } @@ -70,6 +92,7 @@ public class AsymmetricRemoteSyncTask extends SyncTask implements CompletableRem return "AsymmetricRemoteSyncTask{" + "rangesToSync=" + rangesToSync + ", nodePair=" + nodePair + + ", transfer ID=" + transferId + '}'; } } diff --git a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java index c4fe6c8b20..66791315c2 100644 --- a/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/CompletableRemoteSyncTask.java @@ -18,11 +18,9 @@ package org.apache.cassandra.repair; -import java.util.List; - -import org.apache.cassandra.streaming.SessionSummary; +import org.apache.cassandra.repair.messages.SyncResponse; public interface CompletableRemoteSyncTask { - void syncComplete(boolean success, List summaries); + void syncComplete(SyncResponse response); } diff --git a/src/java/org/apache/cassandra/repair/LocalSyncTask.java b/src/java/org/apache/cassandra/repair/LocalSyncTask.java index 379ba4b2a1..bd15e86baa 100644 --- a/src/java/org/apache/cassandra/repair/LocalSyncTask.java +++ b/src/java/org/apache/cassandra/repair/LocalSyncTask.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.repair; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -30,6 +32,7 @@ import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.ProgressInfo; import org.apache.cassandra.streaming.StreamEvent; @@ -66,9 +69,9 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote, List> diff, TimeUUID pendingRepair, - boolean requestRanges, boolean transferRanges, PreviewKind previewKind) + boolean requestRanges, boolean transferRanges, PreviewKind previewKind, ShortMutationId transferId) { - super(ctx, desc, local, remote, diff, previewKind); + super(ctx, desc, local, remote, diff, previewKind, transferId); Preconditions.checkArgument(requestRanges || transferRanges, "Nothing to do in a sync job"); Preconditions.checkArgument(local.equals(ctx.broadcastAddressAndPort())); @@ -77,6 +80,27 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler this.transferRanges = transferRanges; } + public LocalSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort local, InetAddressAndPort remote, + List> diff, TimeUUID pendingRepair, + boolean requestRanges, boolean transferRanges, PreviewKind previewKind) + { + this(ctx, desc, local, remote, diff, pendingRepair, requestRanges, transferRanges, previewKind, null); + } + + @Override + public SyncTask withRanges(Collection> newRanges) + { + List> rangeList = newRanges instanceof List ? (List>) newRanges : new ArrayList<>(newRanges); + return new LocalSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, pendingRepair, requestRanges, transferRanges, previewKind, transferId); + } + + @Override + public SyncTask withTransferId(ShortMutationId transferId) + { + Preconditions.checkState(this.transferId == null); + return new LocalSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, pendingRepair, requestRanges, transferRanges, previewKind, transferId); + } + @VisibleForTesting StreamPlan createStreamPlan() { @@ -167,7 +191,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler status, desc.sessionId, nodePair.coordinator, nodePair.peer, desc.columnFamily); logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message); Tracing.traceRepair(message); - trySuccess(result.hasAbortedSession() ? stat : stat.withSummaries(result.createSummaries())); + trySuccess(result.hasAbortedSession() ? stat : stat.withSummaries(result.createSummaries(), result.planId, transferId)); finished(); } } @@ -190,6 +214,7 @@ public class LocalSyncTask extends SyncTask implements StreamEventHandler ", transferRanges=" + transferRanges + ", rangesToSync=" + rangesToSync + ", nodePair=" + nodePair + + ", transfer ID=" + transferId + '}'; } diff --git a/src/java/org/apache/cassandra/repair/RepairJob.java b/src/java/org/apache/cassandra/repair/RepairJob.java index 28640a60a4..0a0411a982 100644 --- a/src/java/org/apache/cassandra/repair/RepairJob.java +++ b/src/java/org/apache/cassandra/repair/RepairJob.java @@ -50,6 +50,7 @@ import org.apache.cassandra.repair.asymmetric.HostDifferences; import org.apache.cassandra.repair.asymmetric.PreferedNodeFilter; import org.apache.cassandra.repair.asymmetric.ReduceHelper; import org.apache.cassandra.repair.state.JobState; +import org.apache.cassandra.replication.TransferTrackingService; import org.apache.cassandra.schema.SystemDistributedKeyspace; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.accord.IAccordService; @@ -120,7 +121,6 @@ public class RepairJob extends AsyncFuture implements Runnable if ((!session.repairData && !session.repairPaxos) && !metadata.requiresAccordSupport()) throw new IllegalArgumentException(String.format("Cannot run accord only repair on %s.%s, which isn't configured for accord operations", cfs.keyspace.getName(), cfs.name)); - } public long getNowInSeconds() @@ -136,6 +136,11 @@ public class RepairJob extends AsyncFuture implements Runnable } } + public Collection getSyncTasks() + { + return syncTasks; + } + @Override public void run() { @@ -254,6 +259,11 @@ public class RepairJob extends AsyncFuture implements Runnable // that there are in memory at once. When all validations complete, submit sync tasks out of the scheduler. syncResults = session.validationScheduler.schedule(() -> createSyncTasks(accordRepair, allSnapshotTasks, allEndpoints), taskExecutor) .flatMap(this::executeTasks, taskExecutor); + + // For tracked keyspaces, we need to ensure sync'd data is present in the log + boolean isTracked = cfs.metadata().replicationType().isTracked(); + if (isTracked) + syncResults = TransferTrackingService.instance().onRepairSyncCompletion(this, syncResults, taskExecutor); } else { @@ -306,7 +316,7 @@ public class RepairJob extends AsyncFuture implements Runnable }, taskExecutor); } - private Future> createSyncTasks(Future accordRepair, Future allSnapshotTasks, List allEndpoints) + private Future createSyncTasks(Future accordRepair, Future allSnapshotTasks, List allEndpoints) { Future> treeResponses; if (allSnapshotTasks != null) @@ -330,9 +340,17 @@ public class RepairJob extends AsyncFuture implements Runnable return a; }); - return treeResponses.map(session.optimiseStreams && !session.pullRepair - ? this::createOptimisedSyncingSyncTasks - : this::createStandardSyncTasks, taskExecutor); + return treeResponses.map(trees -> { + List syncTasks; + if (session.optimiseStreams && !session.pullRepair) + syncTasks = createOptimisedSyncingSyncTasks(trees); + else + syncTasks = createStandardSyncTasks(trees); + + return ks.getMetadata().params.replicationType.isTracked() + ? SyncTasks.tracked(ks, syncTasks) + : SyncTasks.untracked(syncTasks); + }, taskExecutor); } public synchronized void abort(@Nullable Throwable reason) @@ -413,18 +431,18 @@ public class RepairJob extends AsyncFuture implements Runnable continue; task = new LocalSyncTask(ctx, desc, self.endpoint, remote.endpoint, differences, isIncremental ? desc.parentSessionId : null, - requestRanges, transferRanges, previewKind); + requestRanges, transferRanges, previewKind, null); } else if (isTransient.test(r1.endpoint) || isTransient.test(r2.endpoint)) { // Stream only from transient replica TreeResponse streamFrom = isTransient.test(r1.endpoint) ? r1 : r2; TreeResponse streamTo = isTransient.test(r1.endpoint) ? r2 : r1; - task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind); + task = new AsymmetricRemoteSyncTask(ctx, desc, streamTo.endpoint, streamFrom.endpoint, differences, previewKind, null); } else { - task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind); + task = new SymmetricRemoteSyncTask(ctx, desc, r1.endpoint, r2.endpoint, differences, previewKind, null); } syncTasks.add(task); } @@ -437,7 +455,7 @@ public class RepairJob extends AsyncFuture implements Runnable } @VisibleForTesting - Future> executeTasks(List tasks) + Future> executeTasks(SyncTasks tasks) { try { @@ -447,10 +465,13 @@ public class RepairJob extends AsyncFuture implements Runnable if (!tasks.isEmpty()) state.phase.streamSubmitted(); + if (cfs.metadata().replicationType().isTracked()) + TransferTrackingService.instance().onRepairSyncExecution(tasks); + for (SyncTask task : tasks) { if (!task.isLocal()) - session.trackSyncCompletion(Pair.create(desc, task.nodePair()), (CompletableRemoteSyncTask) task); + session.trackSyncCompletion(new SyncTask.SyncTaskId(desc, task.nodePair(), task.transferId), (CompletableRemoteSyncTask) task); taskExecutor.execute(task); } @@ -535,11 +556,11 @@ public class RepairJob extends AsyncFuture implements Runnable if (address.equals(local)) { task = new LocalSyncTask(ctx, desc, address, fetchFrom, toFetch, isIncremental ? desc.parentSessionId : null, - true, false, previewKind); + true, false, previewKind, null); } else { - task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind); + task = new AsymmetricRemoteSyncTask(ctx, desc, address, fetchFrom, toFetch, previewKind, null); } syncTasks.add(task); diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index 200c246fb3..a0ed59d979 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -311,7 +311,7 @@ public class RepairMessageVerbHandler implements IVerbHandler logErrorAndSendFailureResponse("Unknown repair " + desc.parentSessionId, message); return; } - SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst); + SyncState state = new SyncState(ctx.clock(), desc, request.initiator, request.src, request.dst, request.transferId); if (!register(message, participate, state, participate::register, participate::sync)) @@ -324,7 +324,8 @@ public class RepairMessageVerbHandler implements IVerbHandler request.ranges, isIncremental(desc.parentSessionId) ? desc.parentSessionId : null, request.previewKind, - request.asymmetric); + request.asymmetric, + request.transferId); task.run(); sendAck(message); } diff --git a/src/java/org/apache/cassandra/repair/RepairSession.java b/src/java/org/apache/cassandra/repair/RepairSession.java index 106950bb4a..8e52fbf6f3 100644 --- a/src/java/org/apache/cassandra/repair/RepairSession.java +++ b/src/java/org/apache/cassandra/repair/RepairSession.java @@ -134,7 +134,7 @@ public class RepairSession extends AsyncFuture implements I // Each validation task waits response from replica in validating ConcurrentMap (keyed by CF name and endpoint address) private final ConcurrentMap, ValidationTask> validating = new ConcurrentHashMap<>(); // Remote syncing jobs wait response in syncingTasks map - private final ConcurrentMap, CompletableRemoteSyncTask> syncingTasks = new ConcurrentHashMap<>(); + private final ConcurrentMap syncingTasks = new ConcurrentHashMap<>(); // Tasks(snapshot, validate request, differencing, ...) are run on taskExecutor public final SafeExecutor taskExecutor; @@ -224,7 +224,7 @@ public class RepairSession extends AsyncFuture implements I validating.put(key, task); } - public synchronized void trackSyncCompletion(Pair key, CompletableRemoteSyncTask task) + public synchronized void trackSyncCompletion(SyncTask.SyncTaskId key, CompletableRemoteSyncTask task) { if (terminated) return; @@ -269,7 +269,7 @@ public class RepairSession extends AsyncFuture implements I public void syncComplete(RepairJobDesc desc, Message message) { SyncNodePair nodes = message.payload.nodes; - CompletableRemoteSyncTask task = syncingTasks.remove(Pair.create(desc, nodes)); + CompletableRemoteSyncTask task = syncingTasks.remove(new SyncTask.SyncTaskId(desc, nodes, message.payload.transferId)); // replies without a callback get dropped, so if in mixed mode this should be ignored ctx.messaging().send(message.emptyResponse(), message.from()); if (task == null) @@ -277,11 +277,11 @@ public class RepairSession extends AsyncFuture implements I if (logger.isDebugEnabled()) logger.debug("{} Repair completed between {} and {} on {}", previewKind.logPrefix(getId()), nodes.coordinator, nodes.peer, desc.columnFamily); - task.syncComplete(message.payload.success, message.payload.summaries); + task.syncComplete(message.payload); } @VisibleForTesting - Map, CompletableRemoteSyncTask> getSyncingTasks() + Map getSyncingTasks() { return Collections.unmodifiableMap(syncingTasks); } diff --git a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java index 0f84d66893..cb67c2ebfc 100644 --- a/src/java/org/apache/cassandra/repair/StreamingRepairTask.java +++ b/src/java/org/apache/cassandra/repair/StreamingRepairTask.java @@ -20,6 +20,8 @@ package org.apache.cassandra.repair; import java.util.Collections; import java.util.Collection; +import javax.annotation.Nullable; + import com.google.common.annotations.VisibleForTesting; import org.apache.cassandra.locator.RangesAtEndpoint; @@ -32,6 +34,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SyncResponse; import org.apache.cassandra.repair.state.SyncState; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.streaming.StreamEvent; import org.apache.cassandra.streaming.StreamEventHandler; @@ -65,7 +68,20 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler private final TimeUUID pendingRepair; private final PreviewKind previewKind; - public StreamingRepairTask(SharedContext ctx, SyncState state, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, Collection> ranges, TimeUUID pendingRepair, PreviewKind previewKind, boolean asymmetric) + @Nullable + private final ShortMutationId transferId; + + public StreamingRepairTask(SharedContext ctx, + SyncState state, + RepairJobDesc desc, + InetAddressAndPort initiator, + InetAddressAndPort src, + InetAddressAndPort dst, + Collection> ranges, + TimeUUID pendingRepair, + PreviewKind previewKind, + boolean asymmetric, + ShortMutationId transferId) { this.ctx = ctx; this.state = state; @@ -77,6 +93,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler this.asymmetric = asymmetric; this.pendingRepair = pendingRepair; this.previewKind = previewKind; + this.transferId = transferId; } public void run() @@ -120,7 +137,7 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler { logger.info("[repair #{}] streaming task succeed, returning response to {}", desc.sessionId, initiator); this.state.phase.success(); - RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries()), SYNC_RSP, initiator); + RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, true, state.createSummaries(), state.planId, transferId), SYNC_RSP, initiator); } /** @@ -130,6 +147,6 @@ public class StreamingRepairTask implements Runnable, StreamEventHandler public void onFailure(Throwable t) { this.state.phase.fail(t); - RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList()), SYNC_RSP, initiator); + RepairMessage.sendMessageWithRetries(ctx, new SyncResponse(desc, src, dst, false, Collections.emptyList(), null, transferId), SYNC_RSP, initiator); } } diff --git a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java index 95b8959407..ed954bd627 100644 --- a/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java +++ b/src/java/org/apache/cassandra/repair/SymmetricRemoteSyncTask.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.repair; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import com.google.common.base.Preconditions; @@ -28,8 +30,9 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RepairException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.messages.SyncResponse; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.PreviewKind; -import org.apache.cassandra.streaming.SessionSummary; import org.apache.cassandra.tracing.Tracing; /** @@ -42,16 +45,30 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo { private static final Logger logger = LoggerFactory.getLogger(SymmetricRemoteSyncTask.class); - public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List> differences, PreviewKind previewKind) + public SymmetricRemoteSyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort r1, InetAddressAndPort r2, List> differences, PreviewKind previewKind, ShortMutationId transferId) { - super(ctx, desc, r1, r2, differences, previewKind); + super(ctx, desc, r1, r2, differences, previewKind, transferId); + } + + @Override + public SyncTask withRanges(Collection> newRanges) + { + List> rangeList = newRanges instanceof List ? (List>) newRanges : new ArrayList<>(newRanges); + return new SymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangeList, previewKind, transferId); + } + + @Override + public SyncTask withTransferId(ShortMutationId transferId) + { + Preconditions.checkState(this.transferId == null); + return new SymmetricRemoteSyncTask(ctx, desc, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, transferId); } @Override protected void startSync() { InetAddressAndPort local = ctx.broadcastAddressAndPort(); - SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false); + SyncRequest request = new SyncRequest(desc, local, nodePair.coordinator, nodePair.peer, rangesToSync, previewKind, false, transferId); Preconditions.checkArgument(nodePair.coordinator.equals(request.src)); String message = String.format("Forwarding streaming repair of %d ranges to %s (to be streamed with %s)", request.ranges.size(), request.src, request.dst); logger.info("{} {}", previewKind.logPrefix(desc.sessionId), message); @@ -59,11 +76,12 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo sendRequest(request, request.src); } - public void syncComplete(boolean success, List summaries) + @Override + public void syncComplete(SyncResponse response) { - if (success) + if (response.success) { - trySuccess(stat.withSummaries(summaries)); + trySuccess(stat.withSummaries(response.summaries, response.planId, response.transferId)); } else { @@ -78,6 +96,7 @@ public class SymmetricRemoteSyncTask extends SyncTask implements CompletableRemo return "SymmetricRemoteSyncTask{" + "rangesToSync=" + rangesToSync + ", nodePair=" + nodePair + + ", transfer ID=" + transferId + '}'; } } diff --git a/src/java/org/apache/cassandra/repair/SyncStat.java b/src/java/org/apache/cassandra/repair/SyncStat.java index 2241915ab4..b1231a8569 100644 --- a/src/java/org/apache/cassandra/repair/SyncStat.java +++ b/src/java/org/apache/cassandra/repair/SyncStat.java @@ -20,9 +20,13 @@ package org.apache.cassandra.repair; import java.util.Collection; import java.util.List; +import javax.annotation.Nullable; + import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.SessionSummary; +import org.apache.cassandra.utils.TimeUUID; /** * Statistics about synchronizing two replica @@ -32,21 +36,48 @@ public class SyncStat public final SyncNodePair nodes; public final Collection> differences; public final List summaries; + public final TimeUUID planId; + + @Nullable + public final ShortMutationId transferId; public SyncStat(SyncNodePair nodes, Collection> differences) { this(nodes, differences, null); } - public SyncStat(SyncNodePair nodes, Collection> differences, List summaries) + private SyncStat(SyncNodePair nodes, Collection> differences, List summaries) { this.nodes = nodes; this.summaries = summaries; this.differences = differences; + this.planId = null; + this.transferId = null; } - public SyncStat withSummaries(List summaries) + private SyncStat(SyncNodePair nodes, Collection> differences, List summaries, TimeUUID planId, ShortMutationId transferId) { - return new SyncStat(nodes, differences, summaries); + this.nodes = nodes; + this.summaries = summaries; + this.differences = differences; + this.planId = planId; + this.transferId = transferId; + } + + public SyncStat withSummaries(List summaries, TimeUUID planId, ShortMutationId transferId) + { + return new SyncStat(nodes, differences, summaries, planId, transferId); + } + + @Override + public String toString() + { + return "SyncStat{" + + "nodes=" + nodes + + ", differences=" + differences + + ", summaries=" + summaries + + ", planId=" + planId + + ", transfer ID=" + transferId + + '}'; } } diff --git a/src/java/org/apache/cassandra/repair/SyncTask.java b/src/java/org/apache/cassandra/repair/SyncTask.java index a3b1a57493..9f986bc26b 100644 --- a/src/java/org/apache/cassandra/repair/SyncTask.java +++ b/src/java/org/apache/cassandra/repair/SyncTask.java @@ -18,13 +18,16 @@ package org.apache.cassandra.repair; +import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import org.apache.cassandra.utils.concurrent.AsyncFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +37,11 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.PreviewKind; import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.AsyncFuture; import static org.apache.cassandra.net.Verb.SYNC_REQ; import static org.apache.cassandra.repair.messages.RepairMessage.notDone; @@ -50,11 +56,13 @@ public abstract class SyncTask extends AsyncFuture implements Runnable public final List> rangesToSync; protected final PreviewKind previewKind; protected final SyncNodePair nodePair; + protected final ShortMutationId transferId; + protected volatile TimeUUID planId; protected volatile long startTime = Long.MIN_VALUE; protected final SyncStat stat; - protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List> rangesToSync, PreviewKind previewKind) + protected SyncTask(SharedContext ctx, RepairJobDesc desc, InetAddressAndPort primaryEndpoint, InetAddressAndPort peer, List> rangesToSync, PreviewKind previewKind, ShortMutationId transferId) { Preconditions.checkArgument(!peer.equals(primaryEndpoint), "Sending and receiving node are the same: %s", peer); this.ctx = ctx; @@ -62,16 +70,47 @@ public abstract class SyncTask extends AsyncFuture implements Runnable this.rangesToSync = rangesToSync; this.nodePair = new SyncNodePair(primaryEndpoint, peer); this.previewKind = previewKind; + this.transferId = transferId; this.stat = new SyncStat(nodePair, rangesToSync); + + addCallback((syncStat, failure) -> { + if (syncStat != null && syncStat.planId != null) + this.planId = syncStat.planId; + else if (failure instanceof org.apache.cassandra.streaming.StreamException) + this.planId = ((org.apache.cassandra.streaming.StreamException) failure).finalState.planId; + }); } protected abstract void startSync(); + /** + * Creates a new SyncTask with the same parameters but different ranges. + * Used for splitting sync tasks on shard boundaries. + */ + public abstract SyncTask withRanges(Collection> newRanges); + + public abstract SyncTask withTransferId(ShortMutationId transferId); + public SyncNodePair nodePair() { return nodePair; } + public ShortMutationId getTransferId() + { + return transferId; + } + + /** + * Returns the planId associated with this sync task's streaming operation. + * The planId is captured when the task completes (successfully or with a StreamException). + * @return the planId if streaming has completed, null otherwise + */ + public TimeUUID getPlanId() + { + return planId; + } + /** * Compares trees, and triggers repairs for any ranges that mismatch. */ @@ -118,4 +157,34 @@ public abstract class SyncTask extends AsyncFuture implements Runnable to, this::tryFailure); } + + public static class SyncTaskId + { + RepairJobDesc desc; + SyncNodePair pair; + + @Nullable + ShortMutationId transferId; + + public SyncTaskId(RepairJobDesc desc, SyncNodePair pair, ShortMutationId transferId) + { + this.desc = desc; + this.pair = pair; + this.transferId = transferId; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + SyncTaskId that = (SyncTaskId) o; + return Objects.equals(desc, that.desc) && Objects.equals(pair, that.pair) && Objects.equals(transferId, that.transferId); + } + + @Override + public int hashCode() + { + return Objects.hash(desc, pair, transferId); + } + } } diff --git a/src/java/org/apache/cassandra/repair/SyncTasks.java b/src/java/org/apache/cassandra/repair/SyncTasks.java new file mode 100644 index 0000000000..d5d60a81d1 --- /dev/null +++ b/src/java/org/apache/cassandra/repair/SyncTasks.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.AbstractCollection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.replication.Participants; +import org.apache.cassandra.replication.Shard; + +public class SyncTasks extends AbstractCollection +{ + private final List shardedTasks = new ArrayList<>(); + + public static class ShardedSyncTask + { + public final String keyspace; + public final Participants participants; + public final SyncTask task; + public final Range range; + + private ShardedSyncTask(String keyspace, Participants participants, SyncTask task, Range range) + { + this.keyspace = keyspace; + this.participants = participants; + this.task = task; + this.range = range; + } + } + + static SyncTasks untracked(Collection tasks) + { + SyncTasks syncTasks = new SyncTasks(); + tasks.forEach(t -> syncTasks.shardedTasks.add(new ShardedSyncTask(null, null, t, null))); + return syncTasks; + } + + /** + * Mutation Tracking manages tracking metadata within shards that are each responsible for a piece of the owned + * token space. Executing a full repair across an entire node's ownership will span multiple shards, so repair sync + * tasks need to be split to each align within a single tracking shard. + */ + static SyncTasks tracked(Keyspace keyspace, List tasks) + { + return MutationTrackingService.instance.alignToShardBoundaries(keyspace, tasks); + } + + public void add(Shard shard, SyncTask task) + { + // Narrow the ultimate scope of activation to the ranges in the sync tasks rather than the entire shard. + Set> ranges = new HashSet<>(task.rangesToSync); + Range span = span(ranges); + shardedTasks.add(new ShardedSyncTask(shard.keyspace, shard.participants, task, span)); + } + + public static Range span(Set> ranges) + { + List> normalized = Range.normalize(ranges); + Token min = normalized.get(0).left; + Token max = normalized.get(normalized.size() - 1).right; + return new Range<>(min, max); + } + + public void apply(Consumer consumer) + { + shardedTasks.forEach(consumer); + } + + @Override + public Iterator iterator() + { + return shardedTasks.stream().map(shardedSyncTask -> shardedSyncTask.task).iterator(); + } + + @Override + public int size() + { + return shardedTasks.size(); + } +} diff --git a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java index 137147d898..b9b09ee48b 100644 --- a/src/java/org/apache/cassandra/repair/messages/SyncRequest.java +++ b/src/java/org/apache/cassandra/repair/messages/SyncRequest.java @@ -23,6 +23,8 @@ import java.util.Collection; import java.util.List; import java.util.Objects; +import javax.annotation.Nullable; + import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.IPartitioner; @@ -32,7 +34,9 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.PreviewKind; import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; @@ -51,6 +55,9 @@ public class SyncRequest extends RepairMessage public final Collection> ranges; public final PreviewKind previewKind; public final boolean asymmetric; + + @Nullable + public final ShortMutationId transferId; public SyncRequest(RepairJobDesc desc, InetAddressAndPort initiator, @@ -58,7 +65,8 @@ public class SyncRequest extends RepairMessage InetAddressAndPort dst, Collection> ranges, PreviewKind previewKind, - boolean asymmetric) + boolean asymmetric, + ShortMutationId transferId) { super(desc); this.initiator = initiator; @@ -67,6 +75,7 @@ public class SyncRequest extends RepairMessage this.ranges = ranges; this.previewKind = previewKind; this.asymmetric = asymmetric; + this.transferId = transferId; } @Override @@ -81,13 +90,14 @@ public class SyncRequest extends RepairMessage dst.equals(req.dst) && ranges.equals(req.ranges) && previewKind == req.previewKind && - asymmetric == req.asymmetric; + asymmetric == req.asymmetric && + transferId == req.transferId; } @Override public int hashCode() { - return Objects.hash(desc, initiator, src, dst, ranges, previewKind); + return Objects.hash(desc, initiator, src, dst, ranges, previewKind, transferId); } public static final IVersionedSerializer serializer = new IVersionedSerializer() @@ -103,6 +113,13 @@ public class SyncRequest extends RepairMessage AbstractBounds.tokenSerializer.serialize(range, out, version); out.writeInt(message.previewKind.getSerializationVal()); out.writeBoolean(message.asymmetric); + + if (version >= MessagingService.Version.VERSION_52.value) + { + out.writeBoolean(message.transferId != null); + if (message.transferId != null) + ShortMutationId.serializer.serialize(message.transferId, out, version); + } } public SyncRequest deserialize(DataInputPlus in, int version) throws IOException @@ -118,7 +135,12 @@ public class SyncRequest extends RepairMessage ranges.add((Range) AbstractBounds.tokenSerializer.deserialize(in, partitioner, version)); PreviewKind previewKind = PreviewKind.deserialize(in.readInt()); boolean asymmetric = in.readBoolean(); - return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric); + + ShortMutationId transferId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean() + ? ShortMutationId.serializer.deserialize(in, version) + : null; + + return new SyncRequest(desc, initiator, src, dst, ranges, previewKind, asymmetric, transferId); } public long serializedSize(SyncRequest message, int version) @@ -132,6 +154,14 @@ public class SyncRequest extends RepairMessage size += AbstractBounds.tokenSerializer.serializedSize(range, version); size += TypeSizes.sizeof(message.previewKind.getSerializationVal()); size += TypeSizes.sizeof(message.asymmetric); + + if (version >= MessagingService.Version.VERSION_52.value) + { + size += TypeSizes.sizeof(false); + if (message.transferId != null) + size += ShortMutationId.serializer.serializedSize(message.transferId, version); + } + return size; } }; @@ -146,6 +176,7 @@ public class SyncRequest extends RepairMessage ", ranges=" + ranges + ", previewKind=" + previewKind + ", asymmetric=" + asymmetric + + ", transfer ID=" + transferId + "} " + super.toString(); } } diff --git a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java index e7b5446bad..338a6e02e6 100644 --- a/src/java/org/apache/cassandra/repair/messages/SyncResponse.java +++ b/src/java/org/apache/cassandra/repair/messages/SyncResponse.java @@ -22,15 +22,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import javax.annotation.Nullable; + import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.IPartitionerDependentSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.repair.RepairJobDesc; import org.apache.cassandra.repair.SyncNodePair; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.streaming.SessionSummary; +import org.apache.cassandra.utils.TimeUUID; /** * @@ -45,20 +50,29 @@ public class SyncResponse extends RepairMessage public final List summaries; - public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List summaries) + @Nullable + public final TimeUUID planId; + @Nullable + public final ShortMutationId transferId; + + public SyncResponse(RepairJobDesc desc, SyncNodePair nodes, boolean success, List summaries, TimeUUID planId, ShortMutationId transferId) { super(desc); this.nodes = nodes; this.success = success; this.summaries = summaries; + this.planId = planId; + this.transferId = transferId; } - public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List summaries) + public SyncResponse(RepairJobDesc desc, InetAddressAndPort endpoint1, InetAddressAndPort endpoint2, boolean success, List summaries, TimeUUID planId, ShortMutationId transferId) { super(desc); this.summaries = summaries; this.nodes = new SyncNodePair(endpoint1, endpoint2); this.success = success; + this.planId = planId; + this.transferId = transferId; } @Override @@ -70,13 +84,15 @@ public class SyncResponse extends RepairMessage return desc.equals(other.desc) && success == other.success && nodes.equals(other.nodes) && - summaries.equals(other.summaries); + summaries.equals(other.summaries) && + Objects.equals(planId, other.planId) && + Objects.equals(transferId, other.transferId); } @Override public int hashCode() { - return Objects.hash(desc, success, nodes, summaries); + return Objects.hash(desc, success, nodes, summaries, planId, transferId); } public static final IPartitionerDependentSerializer serializer = new IPartitionerDependentSerializer() @@ -92,6 +108,17 @@ public class SyncResponse extends RepairMessage { SessionSummary.serializer.serialize(summary, out, version); } + + if (version >= MessagingService.Version.VERSION_52.value) + { + out.writeBoolean(message.planId != null); + if (message.planId != null) + TimeUUID.Serializer.instance.serialize(message.planId, out); + + out.writeBoolean(message.transferId != null); + if (message.transferId != null) + ShortMutationId.serializer.serialize(message.transferId, out, version); + } } @Override @@ -108,7 +135,13 @@ public class SyncResponse extends RepairMessage summaries.add(SessionSummary.serializer.deserialize(in, partitioner, version)); } - return new SyncResponse(desc, nodes, success, summaries); + TimeUUID planId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean() + ? TimeUUID.Serializer.instance.deserialize(in) : null; + + ShortMutationId transferId = version >= MessagingService.Version.VERSION_52.value && in.readBoolean() + ? ShortMutationId.serializer.deserialize(in, version) : null; + + return new SyncResponse(desc, nodes, success, summaries, planId, transferId); } public long serializedSize(SyncResponse message, int version) @@ -123,6 +156,17 @@ public class SyncResponse extends RepairMessage size += SessionSummary.serializer.serializedSize(summary, version); } + if (version >= MessagingService.Version.VERSION_52.value) + { + size += TypeSizes.sizeof(false); + if (message.planId != null) + size += TimeUUID.Serializer.instance.serializedSize(message.planId); + + size += TypeSizes.sizeof(false); + if (message.transferId != null) + size += ShortMutationId.serializer.serializedSize(message.transferId, version); + } + return size; } }; diff --git a/src/java/org/apache/cassandra/repair/state/SyncState.java b/src/java/org/apache/cassandra/repair/state/SyncState.java index 9c88aa1877..d2d49d73bb 100644 --- a/src/java/org/apache/cassandra/repair/state/SyncState.java +++ b/src/java/org/apache/cassandra/repair/state/SyncState.java @@ -20,8 +20,11 @@ package org.apache.cassandra.repair.state; import java.util.Objects; +import javax.annotation.Nullable; + import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.replication.ShortMutationId; import org.apache.cassandra.utils.Clock; public class SyncState extends AbstractState @@ -31,9 +34,9 @@ public class SyncState extends AbstractState public final Phase phase = new Phase(); - public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst) + public SyncState(Clock clock, RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, ShortMutationId transferId) { - super(clock, new Id(desc, initiator, src, dst), State.class); + super(clock, new Id(desc, initiator, src, dst, transferId), State.class); } public final class Phase extends BaseSkipPhase @@ -58,13 +61,17 @@ public class SyncState extends AbstractState { public final RepairJobDesc desc; public final InetAddressAndPort initiator, src, dst; + + @Nullable + public final ShortMutationId transferId; - public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst) + public Id(RepairJobDesc desc, InetAddressAndPort initiator, InetAddressAndPort src, InetAddressAndPort dst, ShortMutationId transferId) { this.desc = desc; this.initiator = initiator; this.src = src; this.dst = dst; + this.transferId = transferId; } @Override @@ -73,13 +80,13 @@ public class SyncState extends AbstractState if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Id id = (Id) o; - return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst); + return desc.equals(id.desc) && initiator.equals(id.initiator) && src.equals(id.src) && dst.equals(id.dst) && Objects.equals(transferId, id.transferId); } @Override public int hashCode() { - return Objects.hash(desc, initiator, src, dst); + return Objects.hash(desc, initiator, src, dst, transferId); } } } diff --git a/src/java/org/apache/cassandra/replication/ActivatedTransfers.java b/src/java/org/apache/cassandra/replication/ActivatedTransfers.java index bcee80ecba..76e7a40303 100644 --- a/src/java/org/apache/cassandra/replication/ActivatedTransfers.java +++ b/src/java/org/apache/cassandra/replication/ActivatedTransfers.java @@ -87,11 +87,6 @@ public class ActivatedTransfers implements Iterable this.bounds = bounds; } - private ActivatedTransfer(ShortMutationId id, Collection sstables) - { - this(id, covering(sstables)); - } - public static final IVersionedSerializer serializer = new IVersionedSerializer<>() { @Override @@ -151,7 +146,6 @@ public class ActivatedTransfers implements Iterable transfers.removeIf(transfer -> transfer.id.offset() == offset); } - @VisibleForTesting public void add(ShortMutationId transferId, Bounds bounds) { transfers.add(new ActivatedTransfer(transferId, bounds)); @@ -159,7 +153,7 @@ public class ActivatedTransfers implements Iterable public void add(ShortMutationId transferId, Collection sstables) { - transfers.add(new ActivatedTransfer(transferId, sstables)); + transfers.add(new ActivatedTransfer(transferId, covering(sstables))); } public void addAll(ActivatedTransfers other) @@ -192,7 +186,7 @@ public class ActivatedTransfers implements Iterable return transfers.isEmpty(); } - private static Bounds covering(Collection sstables) + public static Bounds covering(Collection sstables) { Preconditions.checkArgument(!sstables.isEmpty()); Iterator iter = sstables.iterator(); diff --git a/src/java/org/apache/cassandra/replication/ActivationRequest.java b/src/java/org/apache/cassandra/replication/ActivationRequest.java new file mode 100644 index 0000000000..1b9f260739 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/ActivationRequest.java @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.Objects; + +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.TimeUUID; + +/** + * Activation transitions a {@link PendingLocalTransfer} from being pending (durable on disk but not yet visible to + * reads) to live (visible to reads and compactions), by associating the streaming plan ID with a mutation ID, referred + * to as the transfer ID. + *

+ * For tracked repairs, activation may occur without a plan ID and therefore without a {@link PendingLocalTransfer}. In + * this case, activation skips this step (as the data is already present) and simply updates log offsets. + *

+ * See {@link CoordinatedTransfer} for the lifecycle of a transfer and when a {@link ActivationRequest} is sent. + */ +public class ActivationRequest +{ + public final StreamOperation operation; + public final Pair pair; + public final Phase phase; + public final ShortMutationId transferId; + public final NodeId coordinatorId; + public final String keyspace; + public final Range range; + + @Nullable + public final TimeUUID planId; + + public enum Phase + { + PREPARE(0), + COMMIT(1); + + private final int id; + private static final Phase[] ids; + static + { + ids = new Phase[values().length]; + for (Phase phase : values()) + ids[phase.id] = phase; + } + + Phase(int id) + { + this.id = id; + } + + static Phase from(int id) + { + Phase phase = ids[id]; + Preconditions.checkState(phase.id == id); + return phase; + } + } + + public ActivationRequest(StreamOperation operation, + Pair pair, + Phase phase, + ShortMutationId transferId, + NodeId coordinatorId, + Range range, + String keyspace, + TimeUUID planId) + { + Preconditions.checkArgument(operation.isTrackable(), "Operation " + operation.getDescription() + " is not trackable"); + Preconditions.checkNotNull(pair, "Activations require an address pair"); + Preconditions.checkArgument(!transferId.isNone(), "Activations require a transfer ID"); + Preconditions.checkNotNull(coordinatorId, "Activations require a coordinator node ID"); + + this.operation = operation; + this.pair = pair; + this.transferId = transferId; + this.coordinatorId = coordinatorId; + this.phase = phase; + this.keyspace = keyspace; + this.range = range; + this.planId = planId; + } + + ShortMutationId id() + { + return transferId; + } + + public boolean isPrepare() + { + return phase == Phase.PREPARE; + } + + public boolean isCommit() + { + return phase == Phase.COMMIT; + } + + public void apply() + { + MutationTrackingService.instance.activateLocal(this); + } + + public static final Serializer serializer = new Serializer(); + + public static class Serializer implements IVersionedSerializer + { + @Override + public void serialize(ActivationRequest request, DataOutputPlus out, int version) throws IOException + { + out.writeUTF(request.operation.getDescription()); + + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(request.pair.left, out, version); + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(request.pair.right, out, version); + + ShortMutationId.serializer.serialize(request.id(), out, version); + NodeId.messagingSerializer.serialize(request.coordinatorId, out, version); + out.writeByte(request.phase.id); + out.writeUTF(request.keyspace); + Range.serializer.serialize(request.range, out, null); + + out.writeBoolean(request.planId != null); + if (request.planId != null) + TimeUUID.Serializer.instance.serialize(request.planId, out, version); + } + + @Override + public ActivationRequest deserialize(DataInputPlus in, int version) throws IOException + { + StreamOperation operation = StreamOperation.fromString(in.readUTF()); + + InetAddressAndPort sender = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version); + InetAddressAndPort receiver = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version); + + ShortMutationId id = ShortMutationId.serializer.deserialize(in, version); + NodeId coordinatorId = NodeId.messagingSerializer.deserialize(in, version); + Phase phase = Phase.from(in.readByte()); + String keyspace = in.readUTF(); + Range range = Range.serializer.deserialize(in, null); + + TimeUUID planId = in.readBoolean() ? TimeUUID.Serializer.instance.deserialize(in, version) : null; + + return new ActivationRequest(operation, Pair.create(sender, receiver), phase, id, coordinatorId, range, keyspace, planId); + } + + @Override + public long serializedSize(ActivationRequest request, int version) + { + long size = 0; + + size += TypeSizes.sizeof(request.operation.getDescription()); + + size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(request.pair.left, version); + size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(request.pair.right, version); + + size += ShortMutationId.serializer.serializedSize(request.id(), version); + size += NodeId.messagingSerializer.serializedSize(request.coordinatorId, version); + size += TypeSizes.BYTE_SIZE; // Enum ordinal + + size += TypeSizes.sizeof(request.keyspace); + size += Range.serializer.serializedSize(request.range, null); + + size += TypeSizes.BOOL_SIZE; + if (request.planId != null) + size += TimeUUID.Serializer.instance.serializedSize(request.planId, version); + + return size; + } + } + + public static class VerbHandler implements IVerbHandler + { + private static final Logger logger = LoggerFactory.getLogger(VerbHandler.class); + + @Override + public void doVerb(Message msg) throws IOException + { + TransferTrackingService.instance().executor.submit(() -> { + try + { + msg.payload.apply(); + MessagingService.instance().respond(new ActivationResponse(msg.payload.pair), msg); + } + catch (Throwable t) + { + logger.error("Local activation of {} failed due to error", msg.payload, t); + MessagingService.instance().respondWithFailure(RequestFailureReason.forException(t), msg); + } + }); + } + } + + public static final VerbHandler verbHandler = new VerbHandler(); + + @Override + public String toString() + { + return "ActivationRequest{" + + "operation=" + operation + + ", pair=" + pair + + ", phase=" + phase + + ", transferId=" + transferId + + ", coordinatorId=" + coordinatorId + + ", keyspace='" + keyspace + '\'' + + ", range=" + range + + ", planId=" + planId + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + ActivationRequest that = (ActivationRequest) o; + return operation == that.operation && Objects.equals(pair, that.pair) && phase == that.phase && Objects.equals(transferId, that.transferId) && Objects.equals(coordinatorId, that.coordinatorId) && Objects.equals(keyspace, that.keyspace) && Objects.equals(range, that.range) && Objects.equals(planId, that.planId); + } + + @Override + public int hashCode() + { + return Objects.hash(operation, pair, phase, transferId, coordinatorId, keyspace, range, planId); + } +} diff --git a/src/java/org/apache/cassandra/replication/ActivationResponse.java b/src/java/org/apache/cassandra/replication/ActivationResponse.java new file mode 100644 index 0000000000..b7c6584f28 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/ActivationResponse.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.Objects; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.utils.Pair; + +public class ActivationResponse +{ + public final Pair syncPair; + + public ActivationResponse(Pair syncPair) + { + Preconditions.checkNotNull(syncPair, "Activations require a sync node address pair"); + + this.syncPair = syncPair; + } + + public static final Serializer serializer = new Serializer(); + + @Override + public String toString() + { + return "ActivationResponse{" + + "syncPair=" + syncPair + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + ActivationResponse that = (ActivationResponse) o; + return Objects.equals(syncPair, that.syncPair); + } + + @Override + public int hashCode() + { + return Objects.hash(syncPair); + } + + public static class Serializer implements IVersionedSerializer + { + @Override + public void serialize(ActivationResponse activate, DataOutputPlus out, int version) throws IOException + { + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(activate.syncPair.left, out, version); + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(activate.syncPair.right, out, version); + } + + @SuppressWarnings("SuspiciousNameCombination") + @Override + public ActivationResponse deserialize(DataInputPlus in, int version) throws IOException + { + InetAddressAndPort left = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version); + InetAddressAndPort right = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version); + return new ActivationResponse(Pair.create(left, right)); + } + + @Override + public long serializedSize(ActivationResponse activate, int version) + { + long size = 0; + size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(activate.syncPair.left, version); + size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(activate.syncPair.right, version); + return size; + } + } +} diff --git a/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java index e7d5504e90..ccc597d546 100644 --- a/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java +++ b/src/java/org/apache/cassandra/replication/ActiveLogReconciler.java @@ -17,7 +17,6 @@ */ package org.apache.cassandra.replication; -import java.util.Collections; import java.io.IOException; import java.util.concurrent.TimeUnit; @@ -132,7 +131,7 @@ public final class ActiveLogReconciler implements Shutdownable { private static Task from(ShortMutationId id, InetAddressAndPort toHost) { - CoordinatedTransfer transfer = LocalTransfers.instance().getActivatedTransfer(id); + CoordinatedTransfer transfer = TransferTrackingService.instance().getActivatedTransfer(id); if (transfer != null) return new TransferTask(transfer, toHost); else @@ -246,10 +245,10 @@ public final class ActiveLogReconciler implements Shutdownable void send() { logger.debug("Sending activation to {}", toHost); - LocalTransfers.instance().executor.submit(() -> { + TransferTrackingService.instance().executor.submit(() -> { try { - transfer.activateOn(Collections.singleton(toHost)); + transfer.activate(toHost); onResponse(null); } catch (Throwable t) diff --git a/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java b/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java index d82b27c9b2..65456705dd 100644 --- a/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java +++ b/src/java/org/apache/cassandra/replication/CoordinatedTransfer.java @@ -20,66 +20,40 @@ package org.apache.cassandra.replication; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.EnumSet; -import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Supplier; - +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import javax.annotation.CheckReturnValue; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; -import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.streaming.CassandraOutgoingFile; +import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RequestFailure; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.NoPayload; import org.apache.cassandra.net.RequestCallbackWithFailure; import org.apache.cassandra.net.Verb; -import org.apache.cassandra.streaming.OutgoingStream; -import org.apache.cassandra.streaming.StreamException; -import org.apache.cassandra.streaming.StreamOperation; -import org.apache.cassandra.streaming.StreamPlan; -import org.apache.cassandra.streaming.StreamResultFuture; -import org.apache.cassandra.streaming.StreamState; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.Throwables; import org.apache.cassandra.utils.TimeUUID; import org.apache.cassandra.utils.concurrent.AsyncFuture; -import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.INIT; import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP; -import static org.apache.cassandra.replication.TransferActivation.Phase; +import static org.apache.cassandra.replication.ActivationRequest.Phase; /** * Orchestrates the lifecycle of a tracked bulk data transfer for a single replica set, where the current instance is @@ -96,8 +70,8 @@ import static org.apache.cassandra.replication.TransferActivation.Phase; * *

  • * Activation {@link Phase#PREPARE} - * The coordinator sends PREPARE messages to verify replicas have the data persisted on disk and are ready for - * activation. + * The coordinator optinally sends PREPARE messages to verify replicas have the data persisted on disk and are + * ready for activation. Implementations may customize this via {@link #prepare} *
    • *
  • * Activation {@link Phase#COMMIT} @@ -107,57 +81,41 @@ import static org.apache.cassandra.replication.TransferActivation.Phase; * reconciliation processes (read reconciliation and background reconciliation). *
    • * - * - * For simplicity, the coordinator streams to itself rather than using direct file copy. This ensures we can use the - * same lifecycle management for crash-safety and atomic add. - *

    + *

    * If a tracked data read is executed on a replica that's missing an activation, the read reconciliation process will * apply the missing activation during reconciliation and a subsequent read will succeed. To minimize the gap between * activations across replicas, avoid expensive operations like file copies or index builds during - * {@link TransferActivation#apply()}. + * {@link ActivationRequest#apply()}. */ -public class CoordinatedTransfer +public abstract class CoordinatedTransfer { private static final Logger logger = LoggerFactory.getLogger(CoordinatedTransfer.class); + protected final String keyspace; + protected final Range range; + String logPrefix() { - return String.format("[CoordinatedTransfer #%s]", id); + return String.format("[%s #%s]", getClass().getSimpleName(), id); } private final ShortMutationId id; - private final String keyspace; - private final Range range; - private final ConsistencyLevel cl; - final Collection sstables; - final ConcurrentMap streamResults; + final ConcurrentMap, SingleTransferResult> streamResults; - @VisibleForTesting - CoordinatedTransfer(Range range, MutationId id) + public CoordinatedTransfer(ShortMutationId id, String keyspace, Range range) { - this.keyspace = null; - this.range = range; - this.sstables = Collections.emptyList(); - this.cl = null; this.id = id; this.streamResults = new ConcurrentHashMap<>(); - } - - CoordinatedTransfer(String keyspace, Range range, Participants participants, Collection sstables, ConsistencyLevel cl, Supplier nextId) - { this.keyspace = keyspace; this.range = range; - this.sstables = sstables; - this.cl = cl; - this.id = nextId.get(); + } - ClusterMetadata cm = ClusterMetadata.current(); + public CoordinatedTransfer(ShortMutationId id, Participants participants, String keyspace, Range range) + { + this.id = id; this.streamResults = new ConcurrentHashMap<>(participants.size()); - for (int i = 0; i < participants.size(); i++) - { - InetAddressAndPort addr = cm.directory.getNodeAddresses(new NodeId(participants.get(i))).broadcastAddress; - this.streamResults.put(addr, SingleTransferResult.Init()); - } + this.keyspace = keyspace; + this.range = range; } ShortMutationId id() @@ -165,298 +123,76 @@ public class CoordinatedTransfer return id; } - void execute() + Bounds bounds() { - logger.debug("{} Executing tracked bulk transfer {}", logPrefix(), this); - LocalTransfers.instance().save(this); - stream(); + return new Bounds<>(range.left.nextValidToken(), range.right); } - private void stream() + public boolean isCommitted() { - // TODO: Don't stream multiple copies over the WAN, send one copy and indicate forwarding - List> streaming = new ArrayList<>(streamResults.size()); - for (InetAddressAndPort to : streamResults.keySet()) + for (SingleTransferResult result : streamResults.values()) { - Future stream = LocalTransfers.instance().executor.submit(() -> { - stream(to); - return null; - }); - streaming.add(stream); + if (result.state != SingleTransferResult.State.COMMITTED) + return false; } - - // Wait for all streams to complete, so we can clean up after failures. If we exit at the first failure, a - // future stream can complete. - LinkedList failures = null; - for (Future stream : streaming) - { - try - { - stream.get(); - } - catch (InterruptedException | ExecutionException e) - { - if (failures == null) - failures = new LinkedList<>(); - failures.add(e); - logger.error("{} Failed transfer due to", logPrefix(), e); - } - } - - if (failures != null && !failures.isEmpty()) - { - Throwable failure = failures.element(); - Throwable cause = failure instanceof ExecutionException ? failure.getCause() : failure; - maybeCleanupFailedStreams(cause); - - String msg = String.format("Failed streaming on %s instance(s): %s", failures.size(), failures); - throw new RuntimeException(msg, Throwables.unchecked(cause)); - } - - logger.info("{} All streaming completed successfully", logPrefix()); + return true; } - private boolean sufficient() + protected abstract ActivationRequest createActivation(Pair pair, ActivationRequest.Phase phase); + + final void activate(InetAddressAndPort peer) { - AbstractReplicationStrategy ars = Keyspace.open(keyspace).getReplicationStrategy(); - int blockFor = cl.blockFor(ars); - int responses = 0; - for (Map.Entry entry : streamResults.entrySet()) - { - if (entry.getValue().state == STREAM_COMPLETE) - responses++; - } - return responses >= blockFor; + activate(streamResults.keySet().stream().filter(pair -> pair.right.equals(peer)).collect(Collectors.toList())); } - void stream(InetAddressAndPort to) + final void activate(Collection> pairs) { - SingleTransferResult result; - try + // There's no reason to try to re-activate already COMMITTED peers... + List> uncommittedPairs = new ArrayList<>(pairs.size()); + + for (Pair pair : pairs) + if (streamResults.get(pair).state != COMMITTED) + uncommittedPairs.add(pair); + + if (uncommittedPairs.isEmpty()) { - result = streamTask(to); - } - catch (StreamException | ExecutionException | InterruptedException | TimeoutException e) - { - Throwable cause = e instanceof ExecutionException ? e.getCause() : e; - markStreamFailure(to, cause); - throw Throwables.unchecked(cause); - } - - try - { - streamComplete(to, result); - } - catch (ExecutionException | InterruptedException | TimeoutException e) - { - Throwable cause = e instanceof ExecutionException ? e.getCause() : e; - throw Throwables.unchecked(cause); - } - } - - private void notifyFailure() throws ExecutionException, InterruptedException - { - class NotifyFailure extends AsyncFuture implements RequestCallbackWithFailure - { - final Set responses = ConcurrentHashMap.newKeySet(streamResults.size()); - - @Override - public void onResponse(Message msg) - { - responses.remove(msg.from()); - if (responses.isEmpty()) - trySuccess(null); - } - - @Override - public void onFailure(InetAddressAndPort from, RequestFailure failure) - { - tryFailure(failure.failure); - } - } - - NotifyFailure notifyFailure = new NotifyFailure(); - for (Map.Entry entry : streamResults.entrySet()) - { - InetAddressAndPort to = entry.getKey(); - // Coordinator cleans up CoordinatedTransfer and PendingLocalTransfer separately, does not need to notify - if (FBUtilities.getBroadcastAddressAndPort().equals(to)) - continue; - - SingleTransferResult result = entry.getValue(); - if (result.planId == null) - { - logger.warn("{} Skipping notification of transfer failure to {} due to unknown planId", logPrefix(), to); - continue; - } - - logger.debug("{}, Notifying {} of transfer failure for plan {}", logPrefix(), to, result.planId); - notifyFailure.responses.add(to); - Message msg = Message.out(Verb.TRACKED_TRANSFER_FAILED_REQ, new TransferFailed(result.planId)); - MessagingService.instance().sendWithCallback(msg, to, notifyFailure); - } - notifyFailure.get(); - } - - private void markStreamFailure(InetAddressAndPort to, Throwable cause) - { - TimeUUID planId; - if (cause instanceof StreamException) - planId = ((StreamException) cause).finalState.planId; - else - planId = null; - streamResults.computeIfPresent(to, (peer, result) -> result.streamFailed(planId)); - } - - /** - * This shouldn't throw an exception, even if we fail to notify peers of the streaming failure. - */ - private void maybeCleanupFailedStreams(Throwable cause) - { - try - { - boolean purgeable = LocalTransfers.instance().purger.test(this); - if (!purgeable) - return; - - notifyFailure(); - LocalTransfers.instance().scheduleCleanup(); - } - catch (Throwable t) - { - if (cause != null) - t.addSuppressed(cause); - logger.error("{} Failed to notify peers of stream failure", logPrefix(), t); - } - } - - private void streamComplete(InetAddressAndPort to, SingleTransferResult result) throws ExecutionException, InterruptedException, TimeoutException - { - streamResults.put(to, result); - logger.info("{} Completed streaming to {}, {}", logPrefix(), to, this); - maybeActivate(); - } - - synchronized void maybeActivate() - { - // If any activations have already been sent out, send new activations to any received plans that have not yet - // been activated - boolean anyActivated = false; - Set awaitingActivation = new HashSet<>(); - for (Map.Entry entry : streamResults.entrySet()) - { - InetAddressAndPort peer = entry.getKey(); - SingleTransferResult result = entry.getValue(); - if (result.state == COMMITTING || result.state == COMMITTED) - { - anyActivated = true; - } - else if (result.state == STREAM_COMPLETE) - awaitingActivation.add(peer); - } - if (anyActivated && !awaitingActivation.isEmpty()) - { - logger.debug("{} Transfer already activated on some peers, sending activations to remaining: {}", logPrefix(), awaitingActivation); - activateOn(awaitingActivation); - return; - } - // If no activations have been sent out, check whether we have enough planIds back to meet the required CL - else if (sufficient()) - { - Set peers = new HashSet<>(); - for (Map.Entry entry : streamResults.entrySet()) - { - InetAddressAndPort peer = entry.getKey(); - SingleTransferResult result = entry.getValue(); - if (result.state == STREAM_COMPLETE) - peers.add(peer); - } - logger.debug("{} Transfer meets consistency level {}, sending activations to {}", logPrefix(), cl, peers); - activateOn(peers); + logAlreadyCommitted(pairs); return; } - logger.debug("{} Nothing to activate", logPrefix()); + activateInternal(uncommittedPairs); } - void activateOn(Collection peers) + private synchronized void activateInternal(Collection> targets) { - Preconditions.checkState(!peers.isEmpty()); - logger.debug("{} Activating {} on {}", logPrefix(), this, peers); - LocalTransfers.instance().activating(this); + logger.debug("{} Activating {} for {}", logPrefix(), this, targets); - // First phase ensures data is present on disk, then second phase does the actual import. This ensures that if - // something goes wrong (like a topology change during import), we don't have divergence. - class Prepare extends AsyncFuture implements RequestCallbackWithFailure - { - final Set responses = ConcurrentHashMap.newKeySet(); - - public Prepare() - { - responses.addAll(peers); - } - - @Override - public void onResponse(Message msg) - { - logger.debug("{} Got response from: {}", logPrefix(), msg.from()); - responses.remove(msg.from()); - if (responses.isEmpty()) - trySuccess(null); - } - - @Override - public void onFailure(InetAddressAndPort from, RequestFailure failure) - { - logger.debug("{} Got failure {} from {}", logPrefix(), failure, from); - CoordinatedTransfer.this.streamResults.computeIfPresent(from, (peer, result) -> result.prepareFailed()); - tryFailure(new RuntimeException("Tracked import failed during PREPARE on " + from + " due to " + failure.reason)); - } - } - - Prepare prepare = new Prepare(); - for (InetAddressAndPort peer : peers) - { - TransferActivation activation = new TransferActivation(this, peer, Phase.PREPARE); - Message msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation); - logger.debug("{} Sending {} to peer {}", logPrefix(), activation, peer); - MessagingService.instance().sendWithCallback(msg, peer, prepare); - CoordinatedTransfer.this.streamResults.computeIfPresent(peer, (peer0, result) -> result.preparing()); - } - try - { - prepare.get(); - } - catch (InterruptedException | ExecutionException e) - { - Throwable cause = e instanceof ExecutionException ? e.getCause() : e; - throw Throwables.unchecked(cause); - } - logger.debug("{} Activation prepare complete for {}", logPrefix(), peers); + prepare(targets); + logger.debug("{} Activation prepare complete for {}", logPrefix(), targets); // Acknowledgement of activation is equivalent to a remote write acknowledgement. The imported SSTables // are now part of the live set, visible to reads. - class Commit extends AsyncFuture implements RequestCallbackWithFailure + class Commit extends AsyncFuture implements RequestCallbackWithFailure { - final Set responses = ConcurrentHashMap.newKeySet(); + final AtomicInteger responses = new AtomicInteger(0); - private Commit(Collection peers) + private Commit(Collection> pairs) { - responses.addAll(peers); + responses.addAndGet(pairs.size()); } @Override - public void onResponse(Message msg) + public void onResponse(Message msg) { logger.debug("{} Activation successfully applied on {}", logPrefix(), msg.from()); - CoordinatedTransfer.this.streamResults.computeIfPresent(msg.from(), (peer, result) -> result.committed()); + streamResults.computeIfPresent(msg.payload.syncPair, (peer, result) -> result.committed()); MutationTrackingService.instance.receivedActivationResponse(CoordinatedTransfer.this, msg.from()); - responses.remove(msg.from()); - if (responses.isEmpty()) + + if (responses.decrementAndGet() == 0) { // All activations complete, schedule cleanup to purge pending SSTables - LocalTransfers.instance().scheduleCleanup(); + TransferTrackingService.instance().scheduleCleanup(); trySuccess(null); } } @@ -464,22 +200,21 @@ public class CoordinatedTransfer @Override public void onFailure(InetAddressAndPort from, RequestFailure failure) { - logger.error("{} Failed activation on {} due to {}", logPrefix(), from, failure); - MutationTrackingService.instance.retryFailedTransfer(CoordinatedTransfer.this, from, failure.failure); + logger.error("{} Failed activation commit on {} due to {}", logPrefix(), from, failure); // TODO(expected): should only fail if we don't meet requested CL - tryFailure(new RuntimeException("Tracked import failed during COMMIT on " + from + " due to " + failure.reason)); + tryFailure(new RuntimeException("Tracked transfer failed during COMMIT on " + from + " due to " + failure.reason)); } } - Commit commit = new Commit(peers); - for (InetAddressAndPort peer : peers) + Commit commit = new Commit(targets); + for (Pair target : targets) { - TransferActivation activation = new TransferActivation(this, peer, Phase.COMMIT); - Message msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation); + ActivationRequest activation = createActivation(target, Phase.COMMIT); + Message msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation); - logger.debug("{} Sending {} to peer {}", logPrefix(), activation, peer); - MessagingService.instance().sendWithCallback(msg, peer, commit); - CoordinatedTransfer.this.streamResults.computeIfPresent(peer, (peer0, result) -> result.committing()); + logger.debug("{} Sending commit {} to peer {}", logPrefix(), activation, target.right); + MessagingService.instance().sendWithCallback(msg, target.right, commit); + streamResults.computeIfPresent(target, (peer0, result) -> result.committing()); } try @@ -491,17 +226,70 @@ public class CoordinatedTransfer Throwable cause = e instanceof ExecutionException ? e.getCause() : e; throw Throwables.unchecked(cause); } - logger.debug("{} Activation commit complete for {}", logPrefix(), peers); + logger.debug("{} Activation commit complete for {}", logPrefix(), targets); } - public boolean isCommitted() + protected abstract void prepare(Collection> targets); + + private void logAlreadyCommitted(Collection> pairs) { - for (SingleTransferResult result : streamResults.values()) + logger.debug("Transfer {} for {} is already committed. Skipping activation...", this, pairs); + } + + /** + * Notify all replicas that this transfer failed, triggering cleanup of pending SSTables. + * This is used by both {@link TrackedRepairTransfer} and {@link TrackedImportTransfer}. + */ + protected void notifyFailure() throws ExecutionException, InterruptedException + { + class NotifyFailure extends AsyncFuture implements RequestCallbackWithFailure { - if (result.state != COMMITTED) - return false; + // TODO: Does this actually work? What if there is a race between notification and callbacks where we decrement to zero before submitting a second request? + // It seems like we should add the pair up-front that actally have a plan ID...? + final AtomicInteger responses = new AtomicInteger(0); + + @Override + public void onResponse(Message msg) + { + if (responses.decrementAndGet() == 0) + trySuccess(null); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailure failure) + { + // Log but don't fail - best effort cleanup + logger.warn("{} Failed to notify {} of transfer failure: {}", logPrefix(), from, failure); + if (responses.decrementAndGet() == 0) + trySuccess(null); + } } - return true; + + NotifyFailure notifyFailure = new NotifyFailure(); + for (Map.Entry, SingleTransferResult> entry : streamResults.entrySet()) + { + InetAddressAndPort to = entry.getKey().right; + // Coordinator cleans up CoordinatedTransfer and PendingLocalTransfer separately, does not need to notify + if (FBUtilities.getBroadcastAddressAndPort().equals(to)) + continue; + + SingleTransferResult result = entry.getValue(); + if (result.planId() == null) + { + // No planId means streaming never completed, so there's nothing to clean up on the replica + logger.debug("{} Skipping notification of transfer failure to {} - no planId", logPrefix(), to); + continue; + } + + logger.debug("{} Notifying {} of transfer failure for plan {}", logPrefix(), to, result.planId()); + notifyFailure.responses.incrementAndGet(); + Message msg = Message.out(Verb.TRACKED_TRANSFER_FAILED_REQ, new TransferFailed(result.planId())); + MessagingService.instance().sendWithCallback(msg, to, notifyFailure); + } + + // Only wait if we actually sent notifications + if (notifyFailure.responses.get() > 0) + notifyFailure.get(); } /** @@ -511,6 +299,7 @@ public class CoordinatedTransfer *

  • {@link State#INIT}: Transfer created, not yet streaming.
    • *
  • {@link State#STREAM_COMPLETE}: Streaming successful, SSTables received on replica in pending directory.
    • *
  • {@link State#STREAM_NOOP}: No data streamed (e.g., SSTable contains no rows in target range).
    • + *
  • {@link State#EMPTY_SYNC}: No repair sync (e.g., No Merkle tree disagreement).
    • *
  • {@link State#STREAM_FAILED}: Streaming failed, may not have a streaming plan ID yet.
    • *
  • {@link State#PREPARING}: Preparing for activation (first phase).
    • *
  • {@link State#PREPARE_FAILED}: Prepare failed, aborting transfer.
    • @@ -523,8 +312,10 @@ public class CoordinatedTransfer * ┌────────────────┐ * ↓ │ * INIT ──┬──→ STREAM_COMPLETE ──→ PREPARING ──┬──→ COMMITTING ──→ COMMITTED - * │ │ - * ├──→ STREAM_NOOP └──→ PREPARE_FAILED + * │ ↑ │ + * ├──→ EMPTY_SYNC ─────────────┘ │ + * │ └──→ PREPARE_FAILED + * ├──→ STREAM_NOOP * │ * └──→ STREAM_FAILED * @@ -543,17 +334,19 @@ public class CoordinatedTransfer PREPARING, PREPARE_FAILED, COMMITTING, - COMMITTED; + COMMITTED, + EMPTY_SYNC; EnumSet transitionFrom; static { INIT.transitionFrom = EnumSet.noneOf(State.class); + EMPTY_SYNC.transitionFrom = EnumSet.noneOf(State.class); STREAM_NOOP.transitionFrom = EnumSet.of(INIT); STREAM_FAILED.transitionFrom = EnumSet.of(INIT); STREAM_COMPLETE.transitionFrom = EnumSet.of(INIT); - PREPARING.transitionFrom = EnumSet.of(STREAM_COMPLETE, COMMITTING); + PREPARING.transitionFrom = EnumSet.of(STREAM_COMPLETE, COMMITTING, EMPTY_SYNC); PREPARE_FAILED.transitionFrom = EnumSet.of(PREPARING); COMMITTING.transitionFrom = EnumSet.of(PREPARING); COMMITTED.transitionFrom = EnumSet.of(COMMITTING); @@ -570,14 +363,14 @@ public class CoordinatedTransfer this.planId = planId; } - private boolean canTransition(SingleTransferResult.State to) + boolean canTransition(State to) { return to.transitionFrom.contains(state); } public static SingleTransferResult Init() { - return new SingleTransferResult(INIT, null); + return new SingleTransferResult(State.INIT, null); } @VisibleForTesting @@ -589,7 +382,13 @@ public class CoordinatedTransfer @VisibleForTesting static SingleTransferResult Noop() { - return new SingleTransferResult(STREAM_NOOP, null); + return new SingleTransferResult(State.STREAM_NOOP, null); + } + + @VisibleForTesting + static SingleTransferResult EmptySync() + { + return new SingleTransferResult(State.EMPTY_SYNC, null); } @CheckReturnValue @@ -607,31 +406,31 @@ public class CoordinatedTransfer @CheckReturnValue public SingleTransferResult streamFailed(TimeUUID planId) { - return transition(STREAM_FAILED, planId); + return transition(State.STREAM_FAILED, planId); } @CheckReturnValue public SingleTransferResult preparing() { - return transition(PREPARING, this.planId); + return transition(State.PREPARING, this.planId); } @CheckReturnValue public SingleTransferResult prepareFailed() { - return transition(PREPARE_FAILED, this.planId); + return transition(State.PREPARE_FAILED, this.planId); } @CheckReturnValue public SingleTransferResult committing() { - return transition(COMMITTING, this.planId); + return transition(State.COMMITTING, this.planId); } @CheckReturnValue public SingleTransferResult committed() { - return transition(COMMITTED, this.planId); + return transition(State.COMMITTED, this.planId); } public TimeUUID planId() @@ -648,73 +447,4 @@ public class CoordinatedTransfer '}'; } } - - private SingleTransferResult streamTask(InetAddressAndPort to) throws StreamException, ExecutionException, InterruptedException, TimeoutException - { - StreamPlan plan = new StreamPlan(StreamOperation.TRACKED_TRANSFER); - - // No need to flush, only using non-live SSTables already on disk - plan.flushBeforeTransfer(false); - - for (SSTableReader sstable : sstables) - { - List> ranges = Collections.singletonList(range); - List positions = sstable.getPositionsForRanges(ranges); - long estimatedKeys = sstable.estimatedKeysForRanges(ranges); - OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.TRACKED_TRANSFER, sstable.ref(), positions, ranges, estimatedKeys); - plan.transferStreams(to, Collections.singleton(stream)); - } - - long timeout = DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds(); - - logger.info("{} Starting streaming transfer {} to peer {}", logPrefix(), this, to); - StreamResultFuture execute = plan.execute(); - StreamState state; - try - { - state = execute.get(timeout, TimeUnit.MILLISECONDS); - logger.debug("{} Completed streaming transfer {} to peer {}", logPrefix(), this, to); - } - catch (InterruptedException | ExecutionException | TimeoutException e) - { - logger.error("Stream session failed with error", e); - throw e; - } - - if (state.hasFailedSession() || state.hasAbortedSession()) - throw new StreamException(state, "Stream failed due to failed or aborted sessions"); - - // If the SSTable doesn't contain any rows in the provided range, no streams delivered, nothing to activate - if (state.sessions().isEmpty()) - return SingleTransferResult.Noop(); - - return SingleTransferResult.StreamComplete(plan.planId()); - } - - @Override - public boolean equals(Object o) - { - if (o == null || getClass() != o.getClass()) return false; - CoordinatedTransfer transfer = (CoordinatedTransfer) o; - return Objects.equals(keyspace, transfer.keyspace) && Objects.equals(range, transfer.range) && Objects.equals(streamResults, transfer.streamResults) && Objects.equals(sstables, transfer.sstables) && cl == transfer.cl && Objects.equals(id, transfer.id); - } - - @Override - public int hashCode() - { - return Objects.hash(keyspace, range, streamResults, sstables, cl, id); - } - - @Override - public String toString() - { - return "CoordinatedTransfer{" + - "id=" + id + - ", keyspace='" + keyspace + '\'' + - ", range=" + range + - ", cl=" + cl + - ", sstables=" + sstables + - ", streamResults=" + streamResults + - '}'; - } } diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLog.java b/src/java/org/apache/cassandra/replication/CoordinatorLog.java index 56180c2594..70479c12be 100644 --- a/src/java/org/apache/cassandra/replication/CoordinatorLog.java +++ b/src/java/org/apache/cassandra/replication/CoordinatorLog.java @@ -44,6 +44,7 @@ import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.metrics.MutationTrackingMetrics; @@ -329,12 +330,31 @@ public abstract class CoordinatorLog MutationTrackingMetrics.instance.writeTimeOffsetsDiscovered.inc(); unreconciledMutations.finishWriting(mutation); + maybeMoveOffset(offset); + } + finally + { + lock.writeLock().unlock(); + } + } - if (remoteReplicasWitnessed(offset)) - { - reconciledOffsets.add(offset); - unreconciledMutations.remove(offset); - } + /* + - On local replicas after they've completed activation (onHostId == me) + */ + void finishActivation(Bounds bounds, ActivationRequest activation) + { + logger.trace("witnessed local transfer {}", activation.id()); + + lock.writeLock().lock(); + try + { + int offset = activation.id().offset(); + // we've raced with another write, no need to do anything else + if (!witnessedOffsets.get(localNodeId).add(offset)) + return; + + unreconciledMutations.activatedTransfer(activation.id(), bounds); + maybeMoveOffset(offset); } finally { @@ -358,33 +378,12 @@ public abstract class CoordinatorLog return othersWitnessed(offset, localNodeId); } - /* - - On local replicas after they've completed activation (onHostId == me) - */ - void finishActivation(PendingLocalTransfer transfer, TransferActivation activation) + private void maybeMoveOffset(int offset) { - logger.trace("witnessed local transfer {}", activation.id()); - - lock.writeLock().lock(); - try + if (remoteReplicasWitnessed(offset)) { - int offset = activation.id().offset(); - // we've raced with another write, no need to do anything else - if (!witnessedOffsets.get(localNodeId).add(offset)) - return; - - // This is the only difference with finishWriting - can we consolidate these methods? - unreconciledMutations.activatedTransfer(activation.id(), transfer.sstables); - - if (remoteReplicasWitnessed(offset)) - { - reconciledOffsets.add(offset); - unreconciledMutations.remove(offset); - } - } - finally - { - lock.writeLock().unlock(); + reconciledOffsets.add(offset); + unreconciledMutations.remove(offset); } } diff --git a/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java b/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java index 61bc98dcb0..f20c4be45d 100644 --- a/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java +++ b/src/java/org/apache/cassandra/replication/ImmutableCoordinatorLogOffsets.java @@ -19,14 +19,12 @@ package org.apache.cassandra.replication; import java.io.IOException; -import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.function.Predicate; import javax.annotation.concurrent.NotThreadSafe; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterators; import org.slf4j.Logger; @@ -36,7 +34,6 @@ import org.agrona.collections.Long2ObjectHashMap; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.MessagingService; @@ -161,7 +158,6 @@ public class ImmutableCoordinatorLogOffsets implements CoordinatorLogOffsets bounds) { if (transferId.isNone()) @@ -172,16 +168,6 @@ public class ImmutableCoordinatorLogOffsets implements CoordinatorLogOffsets sstables) - { - if (transferId.isNone()) - return this; - if (transfers == null) - transfers = new ActivatedTransfers(); - transfers.add(transferId, sstables); - return this; - } - public Builder addTransfers(ActivatedTransfers other) { if (other.isEmpty()) diff --git a/src/java/org/apache/cassandra/replication/MutationTrackingService.java b/src/java/org/apache/cassandra/replication/MutationTrackingService.java index 18394bdf53..7dc59c7988 100644 --- a/src/java/org/apache/cassandra/replication/MutationTrackingService.java +++ b/src/java/org/apache/cassandra/replication/MutationTrackingService.java @@ -18,6 +18,7 @@ package org.apache.cassandra.replication; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -28,6 +29,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -46,28 +48,35 @@ import org.apache.cassandra.concurrent.ScheduledExecutorPlus; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Splitter; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.MutationTrackingMetrics; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.repair.SyncTask; +import org.apache.cassandra.repair.SyncTasks; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.reads.tracked.TrackedLocalReads; +import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.membership.NodeId; import org.apache.cassandra.tcm.ClusterMetadataService; import org.apache.cassandra.tcm.listeners.ChangeListener; @@ -234,6 +243,35 @@ public class MutationTrackingService } } + // Requires that ranges is aligned to a single shard + public MutationId nextMutationId(String keyspace, Collection> ranges) + { + shardLock.readLock().lock(); + try + { + KeyspaceShards shards = getOrCreateShards(keyspace); + Shard shard = null; + for (Range range : ranges) + { + Shard curShard = shards.lookUp(range); + if (curShard == null) + throw new UnknownShardException(range, shards.groups); + if (shard == null) + shard = curShard; + else if (shard != curShard) + throw new IllegalStateException(String.format("Cannot generate a mutation ID for ranges (%s) that span across more than one shard (%s, %s)", ranges, shard, curShard)); + } + Preconditions.checkNotNull(shard); + MutationId id = shard.nextId(); + logger.trace("Created new mutation id {}", id); + return id; + } + finally + { + shardLock.readLock().unlock(); + } + } + public void sentWriteRequest(Mutation mutation, IntHashSet toHostIds) { Preconditions.checkArgument(!mutation.id().isNone()); @@ -375,10 +413,10 @@ public class MutationTrackingService logger.info("Creating tracked bulk transfers for keyspace '{}' SSTables {}...", keyspace, sstables); KeyspaceShards shards = checkNotNull(keyspaceShards.get(keyspace)); - CoordinatedTransfers transfers = CoordinatedTransfers.create(keyspace, shards, sstables, cl); + TrackedImportTransfers transfers = TrackedImportTransfers.create(keyspace, shards, sstables, cl); logger.info("Split input SSTables into transfers {}", transfers); - for (CoordinatedTransfer transfer : transfers) + for (TrackedImportTransfer transfer : transfers) transfer.execute(); } finally @@ -390,23 +428,52 @@ public class MutationTrackingService public void received(PendingLocalTransfer transfer) { logger.debug("Received pending transfer for tracked table {}", transfer); - LocalTransfers.instance().received(transfer); + TransferTrackingService.instance().received(transfer); } - void activateLocal(TransferActivation activation) + void activateLocal(ActivationRequest request) { - PendingLocalTransfer pending = LocalTransfers.instance().getPendingTransfer(activation.planId); - if (pending == null) - throw new IllegalStateException(String.format("Cannot activate unknown local pending transfer %s", activation)); - pending.activate(activation); + boolean committed = false; + String keyspace = request.keyspace; + Bounds bounds; + PendingLocalTransfer pending = null; + + if (request.operation == StreamOperation.REPAIR) + { + bounds = new Bounds<>(request.range.left.nextValidToken(), request.range.right); + + // A sync task does not necessarily stream to both replicas, which means there may be a plan ID without a + // pending local transfer. In this case, we simply treat this as an already committed transfer and update + // the required offsets in the log (unless we're just preparing). + committed = request.isCommit(); + + // If we have no plan ID, it means this replica did not participate in a sync. + if (request.planId != null) + pending = TransferTrackingService.instance().getPendingTransfer(request.planId); + } + else if (request.operation == StreamOperation.IMPORT) + { + pending = TransferTrackingService.instance().getPendingTransfer(request.planId); + if (pending == null) + throw new IllegalStateException(String.format("Cannot activate unknown local pending transfer %s", request)); + + bounds = ActivatedTransfers.covering(pending.sstables); + } + else + { + throw new IllegalArgumentException("Cannot activate transfer for stream operation " + request.operation); + } + + if (pending != null) + committed = pending.activate(request, bounds); shardLock.readLock().lock(); try { - if (activation.isCommit()) + if (committed) { - keyspaceShards.get(pending.keyspace).lookUp(pending.range).finishActivation(pending, activation); - incomingMutations.invokeListeners(activation.transferId); + keyspaceShards.get(keyspace).lookUp(request.range).finishActivation(bounds, request); + incomingMutations.invokeListeners(request.transferId); } } finally @@ -780,6 +847,66 @@ public class MutationTrackingService forEachKeyspace(keyspace -> keyspace.collectDurablyReconciledOffsets(into)); } + public SyncTasks alignToShardBoundaries(Keyspace keyspace, List tasks) + { + Preconditions.checkArgument(keyspace.getMetadata().replicationStrategy.replicationType.isTracked(), "Keyspace " + keyspace.getName() + " is not tracked"); + + KeyspaceShards shards = keyspaceShards.get(keyspace.getName()); + Map> tasksByShard = new HashMap<>(); + + // Shard ranges do not wrap, so unwrap the task ranges before we start comparing them. + for (SyncTask task : unwrapped(tasks)) + { + Set intersectingShards = new HashSet<>(); + shards.forEachIntersectingShard(task.rangesToSync, intersectingShards::add); + for (Shard shard : intersectingShards) + { + // Ensure that we don't expand outside the ranges of the original sync tasks. + Set> intersectingSyncRanges = new HashSet<>(); + for (Range syncRange : task.rangesToSync) + intersectingSyncRanges.addAll(syncRange.intersectionWith(shard.range)); + + if (!intersectingSyncRanges.isEmpty()) + tasksByShard.computeIfAbsent(shard, key -> new ArrayList<>()).add(task.withRanges(intersectingSyncRanges)); + } + } + + SyncTasks into = new SyncTasks(); + + for (Map.Entry> entry : tasksByShard.entrySet()) + { + Shard shard = entry.getKey(); + Collection syncTasks = entry.getValue(); + + // Assign a new transfer ID to each sync task and add to the tasks container + for (SyncTask task : syncTasks) + into.add(shard, task.withTransferId(shard.nextId())); + } + + return into; + } + + private static List unwrapped(Collection tasks) + { + List unwrapped = new ArrayList<>(); + + for (SyncTask task : tasks) + { + List> unwrappedRanges = new ArrayList<>(); + for (Range range : task.rangesToSync) + { + if (range.isTrulyWrapAround()) + unwrappedRanges.addAll(range.unwrap()); + else + unwrappedRanges.add(range); + } + + unwrapped.add(task.withRanges(unwrappedRanges)); + } + + return unwrapped; + } + public static class KeyspaceShards { private enum UpdateDecision @@ -993,6 +1120,14 @@ public class MutationTrackingService }); } + private void forEachIntersectingShard(Collection> ranges, Consumer consumer) + { + shards.forEach((range0, shard) -> { + if (shard.range.intersects(ranges)) + consumer.accept(shard); + }); + } + void collectShardReconciledOffsetsToBuilder(ReconciledLogSnapshot.Builder builder) { ReconciledKeyspaceOffsets.Builder keyspaceBuilder = builder.getKeyspaceBuilder(keyspace); @@ -1098,8 +1233,9 @@ public class MutationTrackingService private static class ReplicatedOffsetsBroadcaster { // TODO (later): a more intelligent heuristic for scheduling broadcasts - private static final long TRANSIENT_BROADCAST_INTERVAL_MILLIS = 200; - private static final long DURABLE_BROADCAST_INTERVAL_MILLIS = 60_000; + // TODO: Revert before merge, just increased frequency for test + private static final long TRANSIENT_BROADCAST_INTERVAL_MILLIS = 1_000; + private static final long DURABLE_BROADCAST_INTERVAL_MILLIS = 1_000; private volatile boolean isPaused = false; @@ -1148,11 +1284,13 @@ public class MutationTrackingService private static class LogStatePersister implements Runnable { // TODO (expected): consider a different interval - private static final long PERSIST_INTERVAL_MINUTES = 1; + // TODO: Revert before merge, just increased frequency for test + // private static final long PERSIST_INTERVAL_MILLIS = 60_000; + private static final long PERSIST_INTERVAL_MILLIS = 1_000; void start() { - executor.scheduleWithFixedDelay(this, PERSIST_INTERVAL_MINUTES, PERSIST_INTERVAL_MINUTES, TimeUnit.MINUTES); + executor.scheduleWithFixedDelay(this, PERSIST_INTERVAL_MILLIS, PERSIST_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); } @Override @@ -1222,5 +1360,35 @@ public class MutationTrackingService { return service.keyspaceShards.get(keyspace); } + + /** + * Creates a test KeyspaceShards with the given shard ranges. + * The shards are created with minimal configuration suitable for testing. + */ + public static KeyspaceShards createTestKeyspaceShards(String keyspace, Set> shardRanges) + { + Map, Shard> shards = new HashMap<>(); + Map, VersionedEndpoints.ForRange> groups = new HashMap<>(); + + int localNodeId = 1; + AtomicInteger hostLogId = new AtomicInteger(0); + LongSupplier logId = () -> CoordinatorLogId.asLong(localNodeId, hostLogId.getAndIncrement()); + Participants participants = new Participants(List.of(localNodeId)); + for (Range range : shardRanges) + { + shards.put(range, new Shard(localNodeId, keyspace, range, participants, logId, (s, l) -> {})); + groups.put(range, VersionedEndpoints.forRange(Epoch.EMPTY, EndpointsForRange.empty(range))); + } + + return new KeyspaceShards(keyspace, shards, new ReplicaGroups(groups)); + } + + /** + * Sets the keyspace shards for testing purposes. + */ + public static void setKeyspaceShards(MutationTrackingService service, String keyspace, KeyspaceShards shards) + { + service.keyspaceShards.put(keyspace, shards); + } } } diff --git a/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java b/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java index a2c0d833a7..05ad017311 100644 --- a/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java +++ b/src/java/org/apache/cassandra/replication/PendingLocalTransfer.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -48,7 +49,7 @@ import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** * Represents a bulk data transfer received on a replica, from completion of streaming into the pending location, * through activation when it's made visible to reads. Pending transfers are identified by their streaming plan ID, - * and made live by {@link TransferActivation} which associates the streaming plan with a transfer ID that can be + * and made live by {@link ActivationRequest} which associates the streaming plan with a transfer ID that can be * represented in mutation summaries. */ public class PendingLocalTransfer @@ -133,16 +134,16 @@ public class PendingLocalTransfer * preserve is a transfer is only added to the live set iff the transfer ID is present in its mutation summaries. *

    * We don't validate checksums here, mostly because a transfer can be activated during a read, if one replica - * missed the TransferActivation. Transfers should not be pending for very long, and should be protected by + * missed the {@link ActivationRequest}. Transfers should not be pending for very long, and should be protected by * internode integrity checks provided by TLS. *

    * Synchronized to prevent a single activation from running multiple times if requested during read reconciliation * and in the background via {@link ActiveLogReconciler}. */ - public synchronized void activate(TransferActivation activation) + public synchronized boolean activate(ActivationRequest request, Bounds bounds) { if (activated) - return; + return false; Preconditions.checkState(isFullReplica()); @@ -152,21 +153,19 @@ public class PendingLocalTransfer Preconditions.checkNotNull(cfs); Preconditions.checkState(!sstables.isEmpty()); - if (activation.isPrepare()) + if (request.isPrepare()) { - logger.info("{} Not adding SSTables to live set for dryRun {}", logPrefix(), activation); - return; + logger.info("{} Not adding SSTables to live set for dryRun {}", logPrefix(), request); + return false; } // Modify SSTables metadata to durably set transfer ID before importing - ImmutableCoordinatorLogOffsets logOffsets = new ImmutableCoordinatorLogOffsets.Builder() - .addTransfer(activation.transferId, sstables) - .build(); + ImmutableCoordinatorLogOffsets logOffsets = + new ImmutableCoordinatorLogOffsets.Builder().addTransfer(request.transferId, bounds).build(); // Ensure no lingering mutation IDs, only activation IDs for (SSTableReader sstable : sstables) { - Preconditions.checkState(sstable.getCoordinatorLogOffsets().mutations().isEmpty()); try { sstable.mutateCoordinatorLogOffsetsAndReload(logOffsets); @@ -185,7 +184,7 @@ public class PendingLocalTransfer // Retain the original SSTables in pending/ dir on the coordinator, so future streams can get the originals, and // we don't need to isolate activated SSTables during compaction - boolean isCoordinator = activation.transferId.hostId == ClusterMetadata.current().myNodeId().id(); + boolean isCoordinator = request.transferId.hostId == ClusterMetadata.current().myNodeId().id(); logger.debug("{} {} pending SSTables for activation to {}", isCoordinator ? "Copying" : "Moving", logPrefix(), dst); dst.createFileIfNotExists(); @@ -210,7 +209,8 @@ public class PendingLocalTransfer long finishedActivation = currentTimeMillis(); logger.info("{} Finished activating transfer {} in {} ms", logPrefix(), this, finishedActivation - startedActivation); - LocalTransfers.instance().scheduleCleanup(); + TransferTrackingService.instance().scheduleCleanup(); + return true; } @Override diff --git a/src/java/org/apache/cassandra/replication/Shard.java b/src/java/org/apache/cassandra/replication/Shard.java index 5f7f1e7ee6..f282edf426 100644 --- a/src/java/org/apache/cassandra/replication/Shard.java +++ b/src/java/org/apache/cassandra/replication/Shard.java @@ -41,6 +41,7 @@ import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.InetAddressAndPort; @@ -58,9 +59,9 @@ public class Shard private static final Logger logger = LoggerFactory.getLogger(Shard.class); final int localNodeId; - final String keyspace; - final Range range; - final Participants participants; + public final String keyspace; + public final Range range; + public final Participants participants; private final LongSupplier logIdProvider; private final BiConsumer onNewLog; private final NonBlockingHashMapLong logs; @@ -180,9 +181,9 @@ public class Shard getOrCreate(mutationId).receivedWriteResponse(mutationId, fromHostId); } - void finishActivation(PendingLocalTransfer transfer, TransferActivation activation) + void finishActivation(Bounds bounds, ActivationRequest activation) { - getOrCreate(activation.transferId).finishActivation(transfer, activation); + getOrCreate(activation.transferId).finishActivation(bounds, activation); } void receivedActivationResponse(CoordinatedTransfer transfer, InetAddressAndPort onHost) @@ -406,6 +407,17 @@ public class Shard logs.values().forEach(log -> keyspaceBuilder.put(log.logId, log.collectReconciledOffsets(), range)); } + @Override + public String toString() + { + return "Shard{" + + "participants=" + participants + + ", range=" + range + + ", keyspace='" + keyspace + '\'' + + ", localNodeId=" + localNodeId + + '}'; + } + public DebugInfo getDebugInfo() { SortedMap logDebugState = new TreeMap<>(Comparator.comparing(CoordinatorLogId::asLong)); diff --git a/src/java/org/apache/cassandra/replication/TrackedImportTransfer.java b/src/java/org/apache/cassandra/replication/TrackedImportTransfer.java new file mode 100644 index 0000000000..b0fd787adb --- /dev/null +++ b/src/java/org/apache/cassandra/replication/TrackedImportTransfer.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.replication; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.streaming.CassandraOutgoingFile; +import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailure; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.RequestCallbackWithFailure; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.streaming.OutgoingStream; +import org.apache.cassandra.streaming.StreamException; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.streaming.StreamPlan; +import org.apache.cassandra.streaming.StreamResultFuture; +import org.apache.cassandra.streaming.StreamState; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.Throwables; +import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.AsyncFuture; +import org.apache.cassandra.utils.concurrent.Future; + +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE; +import static org.apache.cassandra.replication.ActivationRequest.Phase; + +/** + * Tracked imports mostly follow the default coordination process. They are notable in two ways. The first is that + * {@link #prepare} strictly verifies that streaming has completed on replicas. The second is that activation is based + * on the {@link #bounds} generated from {@link #sstables} rather than a sync range (as is the case with repair). + *

    + * For simplicity, the coordinator streams to itself rather than using direct file copy. This ensures we can use the + * same lifecycle management for crash-safety and atomic add. + */ +public class TrackedImportTransfer extends CoordinatedTransfer +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedImportTransfer.class); + + final Collection sstables; + private final ConsistencyLevel cl; + + @VisibleForTesting + TrackedImportTransfer(Range range, MutationId id) + { + super(id, null, range); + this.sstables = Collections.emptyList(); + this.cl = null; + } + + TrackedImportTransfer(String keyspace, Range range, Participants participants, Collection sstables, ConsistencyLevel cl, Supplier nextId) + { + super(nextId.get(), participants, keyspace, range); + this.sstables = sstables; + this.cl = cl; + + ClusterMetadata cm = ClusterMetadata.current(); + + for (int i = 0; i < participants.size(); i++) + { + InetAddressAndPort addr = cm.directory.getNodeAddresses(new NodeId(participants.get(i))).broadcastAddress; + this.streamResults.put(Pair.create(FBUtilities.getBroadcastAddressAndPort(), addr), SingleTransferResult.Init()); + } + } + + @Override + Bounds bounds() + { + return ActivatedTransfers.covering(sstables); + } + + void execute() + { + logger.debug("{} Executing tracked import transfer {}", logPrefix(), this); + TransferTrackingService.instance().save(this); + stream(); + } + + private void stream() + { + // TODO: Don't stream multiple copies over the WAN, send one copy and indicate forwarding + List> streaming = new ArrayList<>(streamResults.size()); + for (Pair pair : streamResults.keySet()) + { + Future stream = TransferTrackingService.instance().executor.submit(() -> { + stream(pair); + return null; + }); + streaming.add(stream); + } + + // Wait for all streams to complete, so we can clean up after failures. If we exit at the first failure, a + // future stream can complete. + LinkedList failures = null; + for (Future stream : streaming) + { + try + { + stream.get(); + } + catch (InterruptedException | ExecutionException e) + { + if (failures == null) + failures = new LinkedList<>(); + failures.add(e); + logger.error("{} Failed transfer due to", logPrefix(), e); + } + } + + if (failures != null && !failures.isEmpty()) + { + Throwable failure = failures.element(); + Throwable cause = failure instanceof ExecutionException ? failure.getCause() : failure; + maybeCleanupFailedStreams(cause); + + String msg = String.format("Failed streaming on %s instance(s): %s", failures.size(), failures); + throw new RuntimeException(msg, Throwables.unchecked(cause)); + } + + logger.info("{} All streaming completed successfully", logPrefix()); + } + + private boolean sufficient() + { + AbstractReplicationStrategy ars = Keyspace.open(keyspace).getReplicationStrategy(); + int blockFor = cl.blockFor(ars); + int responses = 0; + for (SingleTransferResult result : streamResults.values()) + { + if (result.state == STREAM_COMPLETE) + responses++; + } + return responses >= blockFor; + } + + void stream(Pair pair) + { + SingleTransferResult result; + try + { + result = streamTask(pair.right); + } + catch (StreamException | ExecutionException | InterruptedException | TimeoutException e) + { + Throwable cause = e instanceof ExecutionException ? e.getCause() : e; + markStreamFailure(pair, cause); + throw Throwables.unchecked(cause); + } + + try + { + streamComplete(pair, result); + } + catch (ExecutionException | InterruptedException | TimeoutException e) + { + Throwable cause = e instanceof ExecutionException ? e.getCause() : e; + throw Throwables.unchecked(cause); + } + } + + private void markStreamFailure(Pair pair, Throwable cause) + { + TimeUUID planId; + if (cause instanceof StreamException) + planId = ((StreamException) cause).finalState.planId; + else + planId = null; + streamResults.computeIfPresent(pair, (p, result) -> result.streamFailed(planId)); + } + + /** + * This shouldn't throw an exception, even if we fail to notify peers of the streaming failure. + */ + private void maybeCleanupFailedStreams(Throwable cause) + { + try + { + boolean purgeable = TransferTrackingService.instance().purger.test(this); + if (!purgeable) + return; + + notifyFailure(); + TransferTrackingService.instance().scheduleCleanup(); + } + catch (Throwable t) + { + if (cause != null) + t.addSuppressed(cause); + logger.error("{} Failed to notify peers of stream failure", logPrefix(), t); + } + } + + private void streamComplete(Pair pair, SingleTransferResult result) throws ExecutionException, InterruptedException, TimeoutException + { + streamResults.put(pair, result); + logger.info("{} Completed streaming for pair {}, {}", logPrefix(), pair, this); + maybeActivate(); + } + + synchronized void maybeActivate() + { + // If any activations have already been sent out, send new activations to any received plans that have not yet + // been activated + boolean anyActivated = false; + Set> awaitingActivation = new HashSet<>(); + for (Map.Entry, SingleTransferResult> entry : streamResults.entrySet()) + { + Pair peer = entry.getKey(); + SingleTransferResult result = entry.getValue(); + if (result.state == COMMITTING || result.state == COMMITTED) + { + anyActivated = true; + } + else if (result.state == STREAM_COMPLETE) + awaitingActivation.add(peer); + } + if (anyActivated && !awaitingActivation.isEmpty()) + { + logger.debug("{} Transfer already activated on some peers, sending activations to remaining: {}", logPrefix(), awaitingActivation); + activate(awaitingActivation); + return; + } + // If no activations have been sent out, check whether we have enough planIds back to meet the required CL + else if (sufficient()) + { + Set> pairs = new HashSet<>(); + for (Map.Entry, SingleTransferResult> entry : streamResults.entrySet()) + { + Pair pair = entry.getKey(); + SingleTransferResult result = entry.getValue(); + if (result.state == STREAM_COMPLETE) + pairs.add(pair); + } + logger.debug("{} Transfer meets consistency level {}, sending activations to {}", logPrefix(), cl, pairs); + activate(pairs); + return; + } + + logger.debug("{} Nothing to activate", logPrefix()); + } + + protected void prepare(Collection> targets) + { + // First phase ensures data is present on disk, then second phase does the actual import. This ensures that if + // something goes wrong (like a topology change during import), we don't have divergence. + class Prepare extends AsyncFuture implements RequestCallbackWithFailure + { + final AtomicInteger responses = new AtomicInteger(0); + + public Prepare() + { + responses.addAndGet(targets.size()); + } + + @Override + public void onResponse(Message msg) + { + logger.debug("{} Got prepare response from: {}", logPrefix(), msg.from()); + if (responses.decrementAndGet() == 0) + trySuccess(null); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailure failure) + { + logger.debug("{} Got prepare failure {} from {}", logPrefix(), failure, from); + + // Any failure fails the whole transfer here, so marking all results for streams to this remote as + // failed is harmless and allows the purging logic to clean up doomed transfer artifacts. + streamResults.forEach((pair, result) -> { + if (pair.right.equals(from) && result.canTransition(PREPARE_FAILED)) + streamResults.put(pair, result.prepareFailed()); + }); + + tryFailure(new RuntimeException("Tracked transfer failed during PREPARE on " + from + " due to " + failure.reason)); + } + } + + Prepare prepare = new Prepare(); + for (Pair target : targets) + { + ActivationRequest activation = createActivation(target, Phase.PREPARE); + Message msg = Message.out(Verb.TRACKED_TRANSFER_ACTIVATE_REQ, activation); + logger.debug("{} Sending prepare {} to peer {}", logPrefix(), activation, target.right); + MessagingService.instance().sendWithCallback(msg, target.right, prepare); + streamResults.computeIfPresent(target, (peer0, result) -> result.preparing()); + } + try + { + prepare.get(); + } + catch (InterruptedException | ExecutionException e) + { + Throwable cause = e instanceof ExecutionException ? e.getCause() : e; + throw Throwables.unchecked(cause); + } + } + + @Override + protected ActivationRequest createActivation(Pair pair, Phase phase) + { + return new ActivationRequest(StreamOperation.IMPORT, pair, phase, id(), ClusterMetadata.current().myNodeId(), range, keyspace, streamResults.get(pair).planId()); + } + + private SingleTransferResult streamTask(InetAddressAndPort to) throws StreamException, ExecutionException, InterruptedException, TimeoutException + { + StreamPlan plan = new StreamPlan(StreamOperation.IMPORT); + + // No need to flush, only using non-live SSTables already on disk + plan.flushBeforeTransfer(false); + + for (SSTableReader sstable : sstables) + { + List> ranges = Collections.singletonList(range); + List positions = sstable.getPositionsForRanges(ranges); + long estimatedKeys = sstable.estimatedKeysForRanges(ranges); + OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.IMPORT, sstable.ref(), positions, ranges, estimatedKeys); + plan.transferStreams(to, Collections.singleton(stream)); + } + + long timeout = DatabaseDescriptor.getStreamTransferTaskTimeout().toMilliseconds(); + + logger.info("{} Starting streaming transfer {} to peer {}", logPrefix(), this, to); + StreamResultFuture execute = plan.execute(); + StreamState state; + try + { + state = execute.get(timeout, TimeUnit.MILLISECONDS); + logger.debug("{} Completed streaming transfer {} to peer {}", logPrefix(), this, to); + } + catch (InterruptedException | ExecutionException | TimeoutException e) + { + logger.error("Stream session failed with error", e); + throw e; + } + + if (state.hasFailedSession() || state.hasAbortedSession()) + throw new StreamException(state, "Stream failed due to failed or aborted sessions"); + + // If the SSTable doesn't contain any rows in the provided range, no streams delivered, nothing to activate + if (state.sessions().isEmpty()) + return SingleTransferResult.Noop(); + + return SingleTransferResult.StreamComplete(plan.planId()); + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + TrackedImportTransfer that = (TrackedImportTransfer) o; + return Objects.equals(keyspace, that.keyspace) && Objects.equals(range, that.range) && cl == that.cl && Objects.equals(streamResults, that.streamResults); + } + + @Override + public int hashCode() + { + return Objects.hash(keyspace, range, cl, streamResults); + } + + @Override + public String toString() + { + return "TrackedImportTransfer{" + + "keyspace='" + keyspace + '\'' + + ", range=" + range + + ", cl=" + cl + + ", streamResults=" + streamResults + + ", sstables=" + sstables + + ", streamResults=" + streamResults + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/replication/CoordinatedTransfers.java b/src/java/org/apache/cassandra/replication/TrackedImportTransfers.java similarity index 76% rename from src/java/org/apache/cassandra/replication/CoordinatedTransfers.java rename to src/java/org/apache/cassandra/replication/TrackedImportTransfers.java index 2adb546fc5..250c240e0a 100644 --- a/src/java/org/apache/cassandra/replication/CoordinatedTransfers.java +++ b/src/java/org/apache/cassandra/replication/TrackedImportTransfers.java @@ -33,20 +33,20 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.utils.Interval; /** - * Factory and container for creating multiple {@link CoordinatedTransfer} instances from a collection + * Factory and container for creating multiple {@link TrackedImportTransfer} instances from a collection * of SSTables, partitioned by {@link MutationTrackingService.KeyspaceShards}, which are aligned to replica ownership * ranges. Each shard receives its own CoordinatedTransfer instance, which can be executed independently. */ -class CoordinatedTransfers implements Iterable +public class TrackedImportTransfers implements Iterable { - private final Collection transfers; + private final Collection transfers; - private CoordinatedTransfers(Collection transfers) + private TrackedImportTransfers(Collection transfers) { this.transfers = transfers; } - static CoordinatedTransfers create(String keyspace, MutationTrackingService.KeyspaceShards shards, Collection sstables, ConsistencyLevel cl) + static TrackedImportTransfers create(String keyspace, MutationTrackingService.KeyspaceShards shards, Collection sstables, ConsistencyLevel cl) { // Clean up incoming SSTables to remove any existing untrusted CoordinatorLogOffsets for (SSTableReader sstable : sstables) @@ -62,7 +62,7 @@ class CoordinatedTransfers implements Iterable } SSTableIntervalTree intervals = SSTableIntervalTree.buildSSTableIntervalTree(sstables); - List transfers = new ArrayList<>(); + List transfers = new ArrayList<>(); shards.forEachShard(shard -> { Range range = shard.tokenRange(); @@ -70,14 +70,14 @@ class CoordinatedTransfers implements Iterable if (sstablesForRange.isEmpty()) return; - CoordinatedTransfer transfer = new CoordinatedTransfer(keyspace, range, shard.participants, sstablesForRange, cl, shard::nextId); + TrackedImportTransfer transfer = new TrackedImportTransfer(keyspace, range, shard.participants, sstablesForRange, cl, shard::nextId); transfers.add(transfer); }); - return new CoordinatedTransfers(transfers); + return new TrackedImportTransfers(transfers); } @Override - public Iterator iterator() + public Iterator iterator() { return transfers.iterator(); } diff --git a/src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java b/src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java new file mode 100644 index 0000000000..c1e7479cfa --- /dev/null +++ b/src/java/org/apache/cassandra/replication/TrackedRepairTransfer.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.replication; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.SyncStat; +import org.apache.cassandra.repair.SyncTask; +import org.apache.cassandra.repair.SyncTasks; +import org.apache.cassandra.streaming.StreamOperation; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.utils.Pair; + +/** + * Repair sync tasks (that stream SSTable contents) must be integrated with Mutation Tracking's bulk transfer handling + * because any data that is not present on all instances must be expressed as unreconciled in the log, since read + * reconciliations depend on the log state to guarantee monotonicity of subsequent reads. Streaming sessions for full + * repair can complete on some instances before others, so we need to represent those completed sessions as unreconciled + * in the log. + *

    + * Tracked repairs mostly follow the default coordination process. They are notable in two ways. The first is that + * {@link #prepare} is essentially a no-op. Sync completion assumes that streamed SSTables have arrived at replicas. + * The second is that activation is based on the {@link #bounds} implied by the {@link SyncTask} owned by the transfer. + */ +public class TrackedRepairTransfer extends CoordinatedTransfer +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedRepairTransfer.class); + + public TrackedRepairTransfer(SyncTasks.ShardedSyncTask shardedTask) + { + this(shardedTask.task.getTransferId(), shardedTask.participants, shardedTask.task, shardedTask.keyspace, shardedTask.range); + } + + private TrackedRepairTransfer(ShortMutationId id, Participants participants, SyncTask task, String keyspace, Range range) + { + super(id, keyspace, range); + + Set replicaNodeIds = participants.asSet(); + Set participating = new HashSet<>(); + + participating.add(task.nodePair().coordinator); + participating.add(task.nodePair().peer); + streamResults.put(Pair.create(task.nodePair().coordinator, task.nodePair().peer), SingleTransferResult.Init()); + + ClusterMetadata cm = ClusterMetadata.current(); + for (Integer replicaNodeId : replicaNodeIds) + { + InetAddressAndPort addr = cm.directory.endpoint(new NodeId(replicaNodeId)); + + // Need to activate on all replicas, not just ones with SyncTasks. For replicas that don't receive any data + // as part of a repair, they still need to activate the transfer ID as a no-op, to allow read + // reconciliations to complete. + if (!participating.contains(addr)) + streamResults.put(Pair.create(addr, addr), SingleTransferResult.EmptySync()); + } + } + + /** + * When every {@link SyncTask} for a repair has completed, follow the bulk transfer activation path to safely make + * the new data live and tracked in the log. This needs to include all replicas of {@link #id()}, even those that + * did not receive anything as part of the repair. Otherwise, any read reconciliation will fail to complete. + */ + public void activate(SyncStat stat) + { + Preconditions.checkNotNull(stat.planId, "A complete sync should have a plan ID"); + + logger.debug("{} Activating {}", logPrefix(), this); + + Pair pair = Pair.create(stat.nodes.coordinator, stat.nodes.peer); + // Account for syncs where data streams both ways. (i.e. We need to activate both receivers.) + streamResults.put(pair, SingleTransferResult.StreamComplete(stat.planId)); + streamResults.put(pair.reverse(), SingleTransferResult.StreamComplete(stat.planId)); + + activate(streamResults.keySet()); + } + + @Override + protected void prepare(Collection> targets) + { + // There should be no need to prepare on stream completion for repair, as not all activations will involve a + // local pending transfer to begin with. + for (Pair target : targets) + streamResults.computeIfPresent(target, (pair, result) -> result.preparing()); + } + + @Override + protected ActivationRequest createActivation(Pair pair, ActivationRequest.Phase phase) + { + return new ActivationRequest(StreamOperation.REPAIR, pair, phase, id(), ClusterMetadata.current().myNodeId(), range, keyspace, streamResults.get(pair).planId()); + } + + @Override + public String toString() + { + return "TrackedRepairTransfer{" + + "keyspace='" + keyspace + '\'' + + ", range=" + range + + ", streamResults=" + streamResults + + '}'; + } +} diff --git a/src/java/org/apache/cassandra/replication/TransferActivation.java b/src/java/org/apache/cassandra/replication/TransferActivation.java deleted file mode 100644 index ba0c918286..0000000000 --- a/src/java/org/apache/cassandra/replication/TransferActivation.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.replication; - -import java.io.IOException; -import java.util.Objects; - -import com.google.common.base.Preconditions; - -import org.apache.cassandra.db.TypeSizes; -import org.apache.cassandra.exceptions.RequestFailureReason; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.util.DataInputPlus; -import org.apache.cassandra.io.util.DataOutputPlus; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.net.IVerbHandler; -import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.net.NoPayload; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.membership.NodeId; -import org.apache.cassandra.utils.TimeUUID; - -/** - * Activation transitions a {@link PendingLocalTransfer} from being pending (durable on disk but not yet visible to - * reads) to live (visible to reads and compactions), by associating the streaming plan ID with a mutation ID, referred - * to as the transfer ID. - *

    - * See {@link CoordinatedTransfer} for the lifecycle of a transfer and when a {@link TransferActivation} is sent. - */ -public class TransferActivation -{ - public final TimeUUID planId; - public final ShortMutationId transferId; - public final NodeId coordinatorId; - public final Phase phase; - - public enum Phase - { - PREPARE(0), - COMMIT(1); - - private final int id; - private static final Phase[] ids; - static - { - ids = new Phase[values().length]; - for (Phase phase : values()) - ids[phase.id] = phase; - } - - Phase(int id) - { - this.id = id; - } - - static Phase from(int id) - { - Phase phase = ids[id]; - Preconditions.checkState(phase.id == id); - return phase; - } - } - - public TransferActivation(CoordinatedTransfer transfer, InetAddressAndPort peer, Phase phase) - { - this(transfer.streamResults.get(peer).planId(), transfer.id(), ClusterMetadata.current().myNodeId(), phase); - } - - TransferActivation(TimeUUID planId, ShortMutationId transferId, NodeId coordinatorId, Phase phase) - { - Preconditions.checkArgument(!transferId.isNone()); - Preconditions.checkNotNull(planId); - Preconditions.checkNotNull(coordinatorId); - this.planId = planId; - this.transferId = transferId; - this.coordinatorId = coordinatorId; - this.phase = phase; - } - - ShortMutationId id() - { - return transferId; - } - - public boolean isPrepare() - { - return phase == Phase.PREPARE; - } - - public boolean isCommit() - { - return phase == Phase.COMMIT; - } - - public void apply() - { - MutationTrackingService.instance.activateLocal(this); - } - - public static final Serializer serializer = new Serializer(); - - public static class Serializer implements IVersionedSerializer - { - @Override - public void serialize(TransferActivation activate, DataOutputPlus out, int version) throws IOException - { - TimeUUID.Serializer.instance.serialize(activate.planId, out, version); - ShortMutationId.serializer.serialize(activate.id(), out, version); - NodeId.messagingSerializer.serialize(activate.coordinatorId, out, version); - out.writeByte(activate.phase.id); - } - - @Override - public TransferActivation deserialize(DataInputPlus in, int version) throws IOException - { - TimeUUID planId = TimeUUID.Serializer.instance.deserialize(in, version); - ShortMutationId id = ShortMutationId.serializer.deserialize(in, version); - NodeId coordinatorId = NodeId.messagingSerializer.deserialize(in, version); - Phase phase = Phase.from(in.readByte()); - return new TransferActivation(planId, id, coordinatorId, phase); - } - - @Override - public long serializedSize(TransferActivation activate, int version) - { - long size = 0; - size += TimeUUID.Serializer.instance.serializedSize(activate.planId, version); - size += ShortMutationId.serializer.serializedSize(activate.id(), version); - size += NodeId.messagingSerializer.serializedSize(activate.coordinatorId, version); - size += TypeSizes.BYTE_SIZE; // Enum ordinal - return size; - } - } - - public static class VerbHandler implements IVerbHandler - { - @Override - public void doVerb(Message msg) throws IOException - { - LocalTransfers.instance().executor.submit(() -> { - try - { - msg.payload.apply(); - MessagingService.instance().respond(NoPayload.noPayload, msg); - } - catch (Throwable t) - { - MessagingService.instance().respondWithFailure(RequestFailureReason.forException(t), msg); - } - }); - } - } - - public static final VerbHandler verbHandler = new VerbHandler(); - - @Override - public String toString() - { - return "TransferActivation{" + - ", planId=" + planId + - ", transferId=" + transferId + - ", coordinatorId=" + coordinatorId + - ", phase=" + phase + - '}'; - } - - @Override - public boolean equals(Object o) - { - if (o == null || getClass() != o.getClass()) return false; - TransferActivation that = (TransferActivation) o; - return Objects.equals(planId, that.planId) && Objects.equals(transferId, that.transferId) && Objects.equals(coordinatorId, that.coordinatorId) && phase == that.phase; - } - - @Override - public int hashCode() - { - return Objects.hash(planId, transferId, coordinatorId, phase); - } -} diff --git a/src/java/org/apache/cassandra/replication/TransferFailed.java b/src/java/org/apache/cassandra/replication/TransferFailed.java index 6a956df870..ef86ba8a77 100644 --- a/src/java/org/apache/cassandra/replication/TransferFailed.java +++ b/src/java/org/apache/cassandra/replication/TransferFailed.java @@ -28,7 +28,7 @@ import org.apache.cassandra.utils.TimeUUID; /** * Notification from coordinator to replicas when a bulk data transfer fails, triggering cleanup of the pending * transfer state. - * @see CoordinatedTransfer + * @see TrackedImportTransfer * @see PendingLocalTransfer */ public class TransferFailed diff --git a/src/java/org/apache/cassandra/replication/LocalTransfers.java b/src/java/org/apache/cassandra/replication/TransferTrackingService.java similarity index 52% rename from src/java/org/apache/cassandra/replication/LocalTransfers.java rename to src/java/org/apache/cassandra/replication/TransferTrackingService.java index 876bbeb47e..8c85cbbb3e 100644 --- a/src/java/org/apache/cassandra/replication/LocalTransfers.java +++ b/src/java/org/apache/cassandra/replication/TransferTrackingService.java @@ -18,25 +18,39 @@ package org.apache.cassandra.replication; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.annotation.Nullable; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.FutureCallback; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.concurrent.ExecutorPlus; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.NoPayload; +import org.apache.cassandra.repair.RepairJob; +import org.apache.cassandra.repair.SyncStat; +import org.apache.cassandra.repair.SyncTask; +import org.apache.cassandra.repair.SyncTasks; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.TimeUUID; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; @@ -50,9 +64,9 @@ import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFac * TODO: Make changes to pending set durable with SystemKeyspace.savePendingLocalTransfer(transfer)? * TODO: Add vtable for visibility into local and coordinated transfers */ -public class LocalTransfers +public class TransferTrackingService { - private static final Logger logger = LoggerFactory.getLogger(LocalTransfers.class); + private static final Logger logger = LoggerFactory.getLogger(TransferTrackingService.class); private final ReadWriteLock lock = new ReentrantReadWriteLock(); private final Map coordinating = new ConcurrentHashMap<>(); @@ -60,19 +74,18 @@ public class LocalTransfers final ExecutorPlus executor = executorFactory().pooled("LocalTrackedTransfers", Integer.MAX_VALUE); - private static final LocalTransfers instance = new LocalTransfers(); - static LocalTransfers instance() + private static final TransferTrackingService instance = new TransferTrackingService(); + public static TransferTrackingService instance() { return instance; } - void save(CoordinatedTransfer transfer) + void save(TrackedImportTransfer transfer) { lock.writeLock().lock(); try { - CoordinatedTransfer existing = coordinating.put(transfer.id(), transfer); - Preconditions.checkState(existing == null); + saveInternal(transfer); } finally { @@ -80,18 +93,12 @@ public class LocalTransfers } } - void activating(CoordinatedTransfer transfer) + private void saveInternal(CoordinatedTransfer transfer) { - Preconditions.checkNotNull(transfer.id()); - lock.writeLock().lock(); - try - { - coordinating.put(transfer.id(), transfer); - } - finally - { - lock.writeLock().unlock(); - } + Preconditions.checkNotNull(transfer.id(), "Cannot coordinate a transfar with no ID"); + logger.debug("{} Saving {}", transfer.logPrefix(), transfer); + CoordinatedTransfer existing = coordinating.put(transfer.id(), transfer); + Preconditions.checkState(existing == null, "Attempted to save transfer multiple times"); } void received(PendingLocalTransfer transfer) @@ -111,6 +118,132 @@ public class LocalTransfers } } + /** + * Track a repair as a set of {@link TrackedRepairTransfer} instances corresponding to sync tasks prior to task + * execution so when the syncs are done, we can activate them via {@link ActivationRequest} or fail by + * sending {@link TransferFailed} to all replicas. In other words, one {@link RepairJob} will have as many + * transfers as sync tasks. + */ + public void onRepairSyncExecution(SyncTasks tasks) + { + lock.writeLock().lock(); + try + { + tasks.apply((SyncTasks.ShardedSyncTask shardedTask) -> { + TrackedRepairTransfer transfer = new TrackedRepairTransfer(shardedTask); + saveInternal(transfer); + }); + } + finally + { + lock.writeLock().unlock(); + } + } + + /** + * Begin activation for transfers whose repair sync tasks have completed. + */ + public Future> onRepairSyncCompletion(RepairJob job, Future> syncCompletion, Executor executor) + { + AsyncPromise> activationFuture = new AsyncPromise<>(); + + syncCompletion.addCallback(new FutureCallback<>() + { + @Override + public void onSuccess(List syncs) + { + logger.info("Completed syncs {}. Coordinating: {} Local: {}", syncs, coordinating, local); + + List> transfersToActivate = new ArrayList<>(); + + lock.writeLock().lock(); + try + { + // Look up transfers while holding the lock + for (SyncStat sync : syncs) + { + TrackedRepairTransfer transfer = (TrackedRepairTransfer) coordinating.get(sync.transferId); + transfersToActivate.add(Pair.create(sync, transfer)); + } + } + finally + { + lock.writeLock().unlock(); + } + + // Activate transfers WITHOUT holding the lock (activate() acquires its own locks and can block) + for (Pair pair : transfersToActivate) + { + TrackedRepairTransfer transfer = pair.right; + + try + { + logger.info("{} Activating transfer...", transfer.logPrefix()); + transfer.activate(pair.left); + } + catch (Throwable t) + { + // Note: cleanup will be triggered automatically when the async COMMIT responses complete + logger.error("{} Failed to activate transfer", transfer.logPrefix(), t); + activationFuture.tryFailure(t); + } + } + + // Activation succeeded, complete the future with the sync stats + activationFuture.trySuccess(syncs); + } + + @Override + public void onFailure(Throwable t) + { + logger.error("Failed to complete sync tasks. Cleaning up pending transfers... ", t); + + lock.writeLock().lock(); + try + { + Set transferIds = new HashSet<>(); + for (SyncTask task : job.getSyncTasks()) + { + ShortMutationId transferId = task.getTransferId(); + Preconditions.checkNotNull(transferId); + transferIds.add(transferId); + + TimeUUID planId = task.getPlanId(); + if (planId == null) + continue; + + CoordinatedTransfer transfer = coordinating.get(transferId); + Pair pair = Pair.create(task.nodePair().coordinator, task.nodePair().peer); + transfer.streamResults.put(pair, CoordinatedTransfer.SingleTransferResult.Init().streamFailed(planId)); + transfer.streamResults.put(pair.reverse(), CoordinatedTransfer.SingleTransferResult.Init().streamFailed(planId)); + } + + for (ShortMutationId transferId : transferIds) + { + CoordinatedTransfer transfer = coordinating.get(transferId); + try + { + transfer.notifyFailure(); + } + catch (Throwable t0) + { + logger.error("{} Failed to notify peers of repair failure", transfer.logPrefix(), t0); + } + } + + scheduleCleanup(); + activationFuture.tryFailure(t); + } + finally + { + lock.writeLock().unlock(); + } + } + }, executor); + + return activationFuture; + } + Purger purger = new Purger(); static class Purger @@ -126,7 +259,7 @@ public class LocalTransfers boolean failedBeforeActivation = false; boolean noneActivated = true; boolean allComplete = true; - for (CoordinatedTransfer.SingleTransferResult result : transfer.streamResults.values()) + for (TrackedImportTransfer.SingleTransferResult result : transfer.streamResults.values()) { switch (result.state) { @@ -230,12 +363,18 @@ public class LocalTransfers if (transfer.id() != null) coordinating.remove(transfer.id()); - - CoordinatedTransfer.SingleTransferResult localPending = transfer.streamResults.get(FBUtilities.getBroadcastAddressAndPort()); - PendingLocalTransfer localTransfer; - TimeUUID planId; - if (localPending != null && (planId = localPending.planId()) != null && (localTransfer = local.get(planId)) != null) - purge(localTransfer); + + for (Map.Entry, CoordinatedTransfer.SingleTransferResult> result : transfer.streamResults.entrySet()) + { + if (result.getKey().right.equals(FBUtilities.getBroadcastAddressAndPort())) + { + CoordinatedTransfer.SingleTransferResult localPending = transfer.streamResults.get(result.getKey()); + PendingLocalTransfer localTransfer; + TimeUUID planId; + if (localPending != null && (planId = localPending.planId()) != null && (localTransfer = local.get(planId)) != null) + purge(localTransfer); + } + } } finally { @@ -270,7 +409,8 @@ public class LocalTransfers } } - @Nullable CoordinatedTransfer getActivatedTransfer(ShortMutationId transferId) + @Nullable + CoordinatedTransfer getActivatedTransfer(ShortMutationId transferId) { lock.readLock().lock(); try @@ -284,7 +424,7 @@ public class LocalTransfers } public static IVerbHandler verbHandler = message -> { - LocalTransfers.instance().purge(message.payload); + TransferTrackingService.instance().purge(message.payload); MessagingService.instance().respond(NoPayload.noPayload, message); }; } diff --git a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java index 22c1550230..61b5e1746a 100644 --- a/src/java/org/apache/cassandra/replication/UnreconciledMutations.java +++ b/src/java/org/apache/cassandra/replication/UnreconciledMutations.java @@ -34,8 +34,8 @@ import org.agrona.collections.Int2ObjectHashMap; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.TableId; /** @@ -151,9 +151,9 @@ public class UnreconciledMutations statesSet.remove(state); } - public void activatedTransfer(ShortMutationId id, Collection sstables) + public void activatedTransfer(ShortMutationId id, Bounds bounds) { - transfers.add(id, sstables); + transfers.add(id, bounds); } public UnreconciledMutations copy() @@ -274,10 +274,10 @@ public class UnreconciledMutations result.addDirectly(mutation); continue; } - CoordinatedTransfer transfer = LocalTransfers.instance().getActivatedTransfer(id); + CoordinatedTransfer transfer = TransferTrackingService.instance().getActivatedTransfer(id); if (transfer != null) { - result.transfers.add(transfer.id(), transfer.sstables); + result.transfers.add(transfer.id(), transfer.bounds()); continue; } diff --git a/src/java/org/apache/cassandra/streaming/SessionSummary.java b/src/java/org/apache/cassandra/streaming/SessionSummary.java index 8bb1a1eb81..6887947c3c 100644 --- a/src/java/org/apache/cassandra/streaming/SessionSummary.java +++ b/src/java/org/apache/cassandra/streaming/SessionSummary.java @@ -57,6 +57,7 @@ public class SessionSummary this.sendingSummaries = sendingSummaries; } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -70,6 +71,7 @@ public class SessionSummary return sendingSummaries.equals(summary.sendingSummaries); } + @Override public int hashCode() { int result = coordinator.hashCode(); @@ -79,6 +81,17 @@ public class SessionSummary return result; } + @Override + public String toString() + { + return "SessionSummary{" + + "coordinator=" + coordinator + + ", peer=" + peer + + ", receivingSummaries=" + receivingSummaries + + ", sendingSummaries=" + sendingSummaries + + '}'; + } + public static IPartitionerDependentSerializer serializer = new IPartitionerDependentSerializer() { public void serialize(SessionSummary summary, DataOutputPlus out, int version) throws IOException diff --git a/src/java/org/apache/cassandra/streaming/StreamOperation.java b/src/java/org/apache/cassandra/streaming/StreamOperation.java index 45dd7e9a2c..5e9f8316aa 100644 --- a/src/java/org/apache/cassandra/streaming/StreamOperation.java +++ b/src/java/org/apache/cassandra/streaming/StreamOperation.java @@ -27,7 +27,7 @@ public enum StreamOperation REBUILD("Rebuild", false, true, false), BULK_LOAD("Bulk Load", true, false, false), REPAIR("Repair", true, false, true), - TRACKED_TRANSFER("Tracked Transfer", false, false, false); + IMPORT("Import", false, false, false); private final String description; private final boolean requiresViewBuild; @@ -47,6 +47,17 @@ public enum StreamOperation this.requiresBarrierTransaction = requiresBarrierTransaction; } + /** + * For tracked keyspaces, streams that are received while serving reads must be tracked in the log, to ensure that + * read reconciliation can represent the new data. + * + * @see org.apache.cassandra.replication.CoordinatedTransfer + */ + public boolean isTrackable() + { + return this == REPAIR || this == IMPORT; + } + public static StreamOperation fromString(String text) { for (StreamOperation b : StreamOperation.values()) diff --git a/src/java/org/apache/cassandra/streaming/StreamPlan.java b/src/java/org/apache/cassandra/streaming/StreamPlan.java index 4d24992626..8fa5a9a231 100644 --- a/src/java/org/apache/cassandra/streaming/StreamPlan.java +++ b/src/java/org/apache/cassandra/streaming/StreamPlan.java @@ -116,10 +116,8 @@ public class StreamPlan // TODO: add flag for fully reconciled data only if this is for a tracked keyspace session.addStreamRequest(keyspace, fullRanges, transientRanges, Arrays.asList(columnFamilies)); - // Automatically include mutation logs for tracked keyspaces - if (isTrackedReplicationEnabled(keyspace)) { + if (includeMutationLogs(keyspace, session)) session.addMutationLogRequest(keyspace, fullRanges, transientRanges); - } return this; } @@ -136,9 +134,7 @@ public class StreamPlan public StreamPlan transferRanges(InetAddressAndPort to, String keyspace, RangesAtEndpoint replicas, String... columnFamilies) { StreamSession session = coordinator.getOrCreateOutboundSession(to); - - // Automatically include mutation logs for tracked keyspaces - ReconciledKeyspaceOffsets reconciledKeyspaceOffsets = isTrackedReplicationEnabled(keyspace) + ReconciledKeyspaceOffsets reconciledKeyspaceOffsets = includeMutationLogs(keyspace, session) ? session.addMutationLogTransfer(keyspace, replicas) : null; @@ -147,6 +143,11 @@ public class StreamPlan return this; } + private boolean includeMutationLogs(String keyspace, StreamSession session) + { + return isTrackedReplicationEnabled(keyspace) && session.getStreamOperation() != StreamOperation.REPAIR; + } + /** * Add transfer task to send given streams * diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index e93dec699f..4489f27bea 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -1020,6 +1020,7 @@ public class StreamSession manifest.keyspaceRanges.forEach((keyspace, ranges) -> logTransfer.addKeyspaceRanges(keyspace, ranges)); } + /** * In the case where we have an error checking disk space we allow the Operation to continue. * In the case where we do _not_ have available space, this method raises a RTE. diff --git a/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java b/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java index dc43ae2ee1..fa7f4a9bd8 100644 --- a/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java +++ b/src/java/org/apache/cassandra/tcm/ownership/ReplicaGroups.java @@ -130,11 +130,11 @@ public class ReplicaGroups } /** - * This method is intended to be used on read/write path, not forRange. + * This method is intended to be used on read/write path, not forRange. Returns a {@link VersionedEndpoints.ForRange} + * with the full ownership range, which necessarily contains the input range but may be wider. */ public VersionedEndpoints.ForRange matchRange(Range range) { - EndpointsForRange.Builder builder = new EndpointsForRange.Builder(range); Epoch lastModified = Epoch.EMPTY; // find a range containing the *right* token for the given range - Range is start exclusive so if we looked for the // left one we could get the wrong range @@ -142,10 +142,9 @@ public class ReplicaGroups if (pos >= 0 && pos < ranges.size() && ranges.get(pos).contains(range)) { VersionedEndpoints.ForRange eps = endpoints.get(pos); - lastModified = eps.lastModified(); - builder.addAll(eps.get(), ReplicaCollection.Builder.Conflict.ALL); + return VersionedEndpoints.forRange(eps.lastModified(), eps.get()); } - return VersionedEndpoints.forRange(lastModified, builder.build()); + return VersionedEndpoints.forRange(lastModified, EndpointsForRange.empty(range)); } public VersionedEndpoints.ForRange forRange(Token token) diff --git a/src/java/org/apache/cassandra/utils/Pair.java b/src/java/org/apache/cassandra/utils/Pair.java index cb0952947f..c08b1a8b3d 100644 --- a/src/java/org/apache/cassandra/utils/Pair.java +++ b/src/java/org/apache/cassandra/utils/Pair.java @@ -69,4 +69,9 @@ public class Pair { return new Pair(x, y); } + + public Pair reverse() + { + return new Pair<>(right, left); + } } diff --git a/test/data/serialization/5.1/service.SyncComplete.bin b/test/data/serialization/5.1/service.SyncComplete.bin index 5e27e345fdd589b2af7accc3b3b70c3b6b8c7b78..006f0a1cc711cc6c97778ff53929ad7f263b24c0 100644 GIT binary patch delta 59 vcmZo-YGRs@ZD=qv^1weU!4Dlw_azVWT-9S>U| shutdownAndWait(Collections.singletonList(JMXBroadcastExecutor.executor)) ); diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java deleted file mode 100644 index 2c85dd81b9..0000000000 --- a/test/distributed/org/apache/cassandra/distributed/test/tracking/BulkTransfersTest.java +++ /dev/null @@ -1,949 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.cassandra.distributed.test.tracking; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.time.Duration; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -import javax.annotation.Nullable; - -import org.junit.Ignore; -import org.junit.Test; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import net.bytebuddy.ByteBuddy; -import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; -import net.bytebuddy.implementation.MethodDelegation; -import net.bytebuddy.implementation.bind.annotation.SuperCall; -import net.bytebuddy.implementation.bind.annotation.This; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.DecoratedKey; -import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.marshal.Int32Type; -import org.apache.cassandra.db.streaming.CassandraStreamReceiver; -import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.distributed.Cluster; -import org.apache.cassandra.distributed.api.ConsistencyLevel; -import org.apache.cassandra.distributed.api.Feature; -import org.apache.cassandra.distributed.api.ICoordinator; -import org.apache.cassandra.distributed.api.IInstanceInitializer; -import org.apache.cassandra.distributed.api.IInvokableInstance; -import org.apache.cassandra.distributed.api.IIsolatedExecutor; -import org.apache.cassandra.distributed.shared.ClusterUtils; -import org.apache.cassandra.distributed.shared.Uninterruptibles; -import org.apache.cassandra.distributed.test.TestBaseImpl; -import org.apache.cassandra.distributed.test.sai.SAIUtil; -import org.apache.cassandra.exceptions.ReadTimeoutException; -import org.apache.cassandra.io.IVersionedSerializer; -import org.apache.cassandra.io.sstable.CQLSSTableWriter; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.File; -import org.apache.cassandra.net.Message; -import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.replication.ActivatedTransfers; -import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets; -import org.apache.cassandra.replication.MutationSummary; -import org.apache.cassandra.replication.MutationTrackingService; -import org.apache.cassandra.replication.ShortMutationId; -import org.apache.cassandra.replication.TransferActivation; -import org.apache.cassandra.replication.UnknownShardException; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ownership.DataPlacement; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.assertj.core.api.Assertions; - -import static net.bytebuddy.matcher.ElementMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; -import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments; -import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; -import static org.apache.cassandra.distributed.shared.AssertUtils.row; -import static org.apache.cassandra.replication.TransferActivation.Phase.COMMIT; -import static org.apache.cassandra.replication.TransferActivation.Phase.PREPARE; - -/** - * For now, tracked import with a replica down is not supported. The intention is to support this scenario by allowing - * users to provide a {@link ConsistencyLevel} for tracked import operations, where the import will complete if - * sufficient replicas acknowledge the transfer and activate it. - */ -public class BulkTransfersTest extends TestBaseImpl -{ - private static final Logger logger = LoggerFactory.getLogger(BulkTransfersTest.class); - - private static final String TABLE = "tbl"; - private static final String KEYSPACE_TABLE = String.format("%s.%s", KEYSPACE, TABLE); - private static final String TABLE_SCHEMA_CQL = String.format(withKeyspace("CREATE TABLE %s." + TABLE + " (k int primary key, v int);")); - - private static final int IMPORT_PK = 1; - private static final Token IMPORT_TOKEN = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(IMPORT_PK)); - private static final int NODES = 3; - - private static final IIsolatedExecutor.SerializableConsumer TRANSFERS_EXIST = sstable -> { - Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()) - .isNotEmpty(); - Assertions.assertThat(sstable.isRepaired()).isFalse(); - }; - private static final IIsolatedExecutor.SerializableConsumer TRANSFERS_EMPTY = sstable -> { - Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()) - .isEmpty(); - Assertions.assertThat(sstable.isRepaired()).isTrue(); - }; - private static final IIsolatedExecutor.SerializableConsumer NOOP = sstable -> {}; - - @Test - public void importHappyPath() throws Throwable - { - try (Cluster cluster = cluster()) - { - createSchema(cluster); - doImport(cluster); - - // All pending/ dirs should be empty, should have no SSTables left if all the transfers completed - assertPendingDirs(cluster, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); - }); - - // Verify transfer IDs exist before compaction, then compact, then verify they're removed - assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY); - - // Run after compaction, to enforce offset persistence + broadcast - assertSummary(cluster, summary -> { - Assertions.assertThat(summary).satisfies(s -> { - assert s.reconciledIds() == 1; - assert s.unreconciledIds() == 0; - }); - }); - - assertLocalSelect(cluster, rows -> assertRows(rows, row(1, 1))); - } - } - - @Test - public void importIndexAlreadyPresent() throws Throwable - { - try (Cluster cluster = cluster()) - { - createSchema(cluster); - - String indexName = "v_idx"; - String indexCql = withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'"); - cluster.schemaChange(indexCql); - - // This will add an SSTable that already has an SAI index, that needs to be distributed alongside the SSTable on transfer - IInvokableInstance importer = cluster.get(1); - long mark = importer.logs().mark(); - doImport(cluster, importer, indexCql); - List logs = importer.logs().grep(mark, "Submitting incremental index build of " + indexName).getResult(); - Assertions.assertThat(logs).isEmpty(); - - // Index should exist and be queryable on all replicas after import - SAIUtil.assertIndexQueryable(cluster, KEYSPACE, indexName); - - // Validate queries using the index - cluster.forEach(instance -> { - Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1")); - assertRows(rows, row(1, 1)); - }); - } - } - - @Test - public void importBuildsIndex() throws Throwable - { - try (Cluster cluster = cluster()) - { - createSchema(cluster); - - String indexName = "v_idx"; - cluster.schemaChange(withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'")); - - // This will add an SSTable that's missing an SAI index, and the index will be built during the import on - // the coordinator - IInvokableInstance importer = cluster.get(1); - long mark = importer.logs().mark(); - doImport(cluster, importer); - List logs = importer.logs().watchFor(mark, Duration.ofMinutes(1), "Submitting incremental index build of " + indexName).getResult(); - Assertions.assertThat(logs).isNotEmpty(); - - // Index should exist and be queryable on all replicas after import - SAIUtil.assertIndexQueryable(cluster, KEYSPACE, indexName); - - // Validate queries using the index - cluster.forEach(instance -> { - Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1")); - assertRows(rows, row(1, 1)); - }); - } - } - - @Test - @Ignore - public void importReplicaDown() throws Throwable - { - try (Cluster cluster = cluster()) - { - createSchema(cluster); - - Iterable down = Collections.singleton(cluster.get(3)); - Iterable up = cluster.stream().filter(instance -> instance != down).collect(Collectors.toList()); - for (IInvokableInstance instance : down) - instance.shutdown().get(); - - doImport(cluster); - - cluster.get(3).startup(); - - // Transfers did not complete, files should still exist on up replicas - assertPendingDirs(up, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty(); - }); - assertPendingDirs(down, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); - }); - - // Transfers did not complete, transfer IDs should not be removed - assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EXIST); - - assertLocalSelect(up, rows -> assertRows(rows, row(1, 1))); - } - } - - @Test - public void importMissedActivationPrepare() throws Throwable - { - importMissedActivation(PREPARE); - } - - @Test - public void importMissedActivationCommit() throws Throwable - { - importMissedActivation(COMMIT); - } - - public void importMissedActivation(TransferActivation.Phase phase) throws Throwable - { - int MISSED_ACTIVATION = 2; - try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION))) - { - ByteBuddyInjections.SkipActivation.setup(cluster, phase); - createSchema(cluster); - - Set missed = Collections.singleton(cluster.get(MISSED_ACTIVATION)); - Iterable received = cluster.stream().filter(instance -> !missed.contains(instance)).collect(Collectors.toList()); - - Assertions.assertThatThrownBy(() -> doImport(cluster)) - .hasMessageContaining("Failed adding SSTables") - .cause() - .hasMessageContaining("Failed streaming on 1 instance(s):") - .cause() - .hasMessageMatching("Tracked import failed during " + phase + " on " + cluster.get(MISSED_ACTIVATION).broadcastAddress() + " due to TIMEOUT") - .hasNoCause(); - - assertSummary(received, summary -> { - Assertions.assertThat(summary).satisfies(s -> { - assert s.reconciledIds() == 0; - assert s.unreconciledIds() == (phase == COMMIT ? 1 : 0); - }); - }); - assertSummary(missed, summary -> { - Assertions.assertThat(summary).satisfies(s -> { - assert s.reconciledIds() == 0; - assert s.unreconciledIds() == 0; - }); - }); - - switch (phase) - { - case PREPARE: - // Activation did not start, files should be cleaned up - Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); - assertPendingDirs(cluster, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); - }); - break; - case COMMIT: - // Activation did not complete, files should still exist on all replicas - assertPendingDirs(cluster, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty(); - }); - break; - } - - // If the activation is not everywhere, it shouldn't be purged on compaction - assertCompaction(cluster, received, TRANSFERS_EXIST, TRANSFERS_EXIST); - - if (phase == PREPARE) - return; - - // Permit activation of missed commits during read reconciliation - ByteBuddyInjections.SkipActivation.setup(cluster, null); - - // Use coordinated query rather to confirm read reconciliation triggers activation - IInvokableInstance coordinator = cluster.get(3); // not initial transfer coordinator, but received activation - assertCoordinatedRead(coordinator, rows -> { - assertRows(rows, row(1, 1)); - }); - - // Confirm others receive activation - assertLocalSelect(missed, rows -> { - assertRows(rows, row(1, 1)); - }); - - assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY); - - // Activation completed, files should be removed - assertPendingDirs(cluster, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); - }); - } - } - - /** - * If a replica is missing an activation, executes a data read, then discovers the missing transfer during read - * reconciliation, it needs to augment the client response with the new transfer. - */ - @Test - public void importMissingOnDataReplicaDuringAugment() throws Throwable - { - TransferActivation.Phase phase = COMMIT; - int MISSED_ACTIVATION = 2; - try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION))) - { - ByteBuddyInjections.SkipActivation.setup(cluster, phase); - createSchema(cluster); - - IInvokableInstance missed = cluster.get(MISSED_ACTIVATION); - - Assertions.assertThatThrownBy(() -> doImport(cluster)) - .hasMessageContaining("Failed adding SSTables") - .cause() - .hasMessageContaining("Failed streaming on 1 instance(s):") - .cause() - .hasMessageMatching("Tracked import failed during " + phase + " on " + missed.broadcastAddress() + " due to TIMEOUT") - .hasNoCause(); - - assertSummary(Collections.singleton(missed), summary -> { - Assertions.assertThat(summary).satisfies(s -> { - assert s.reconciledIds() == 0; - assert s.unreconciledIds() == 0; - }); - }); - - // Permit activation of missed commits during read reconciliation - ByteBuddyInjections.SkipActivation.setup(cluster, null); - - // First read will fail due to failure to augment with transfer - long mark = missed.logs().mark(); - Assertions.assertThatThrownBy(() -> { - assertCoordinatedRead(missed, rows -> { - assertRows(rows, row(1, 1)); - }); - }).isInstanceOf(missed.callOnInstance(() -> ReadTimeoutException.class)); // use instance classloader - - List logs = missed.logs().grep(mark, "Missing mutation ShortMutationId").getResult(); - Assertions.assertThat(logs).isNotEmpty(); - - // Retry succeeds - assertCoordinatedRead(missed, rows -> { - assertRows(rows, row(1, 1)); - }); - } - } - - /* - * When an import fails, bounce must not move the pending SSTables into the live set. - */ - @Test - public void importBounceAfterPending() throws Throwable - { - IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3); - try (Cluster cluster = cluster(initializer)) - { - ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT); - createSchema(cluster); - - Assertions.assertThatThrownBy(() -> doImport(cluster)) - .hasMessageContaining("Failed adding SSTables") - .cause() - .hasMessageContaining("Tracked import failed during COMMIT"); - - Runnable assertEmpty = () -> { - // Activation did not complete, files should still exist on all replicas - assertPendingDirs(cluster, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty(); - }); - - // No one has activated, so should not be present in any summary - assertSummary(cluster, summary -> { - Assertions.assertThat(summary).satisfies(s -> { - assert s.reconciledIds() == 0; - assert s.unreconciledIds() == 0; - }); - }); - - assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS)); - }; - - assertEmpty.run(); - - bounce(cluster); - - assertEmpty.run(); - } - } - - @Test - public void importOutOfRange() throws Throwable - { - try (Cluster cluster = cluster()) - { - createSchema(cluster, 1); - - Set inRange = new HashSet<>(); - Set outOfRange = new HashSet<>(); - cluster.forEach(instance -> { - boolean importReplica = instance.callOnInstance(() -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication); - return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf(); - }); - (importReplica ? inRange : outOfRange).add(instance); - }); - logger.info("inRange: {}, outOfRange: {}", inRange, outOfRange); - - Assertions.assertThat(inRange).hasSize(1); - IInvokableInstance onlyInRange = inRange.iterator().next(); - - // Reject import out of range - for (IInvokableInstance instance : outOfRange) - { - long mark = instance.logs().mark(); - Consumer> onResult = failedDirs -> Assertions.assertThat(failedDirs).hasSize(1); - doImport(cluster, instance, onResult, null); - instance.logs().grep(mark, "java.lang.RuntimeException: Key DecoratedKey(-4069959284402364209, 00000001) is not contained in the given ranges"); - } - - doImport(cluster, onlyInRange); - - assertSummary(Collections.singleton(onlyInRange), summary -> { - Assertions.assertThat(summary).satisfies(s -> { - assert s.reconciledIds() == 1; - assert s.unreconciledIds() == 0; - }); - }); - - for (IInvokableInstance instance : outOfRange) - { - // Out of range shouldn't have any transfers - assertCompaction(cluster, Collections.singleton(instance), TRANSFERS_EMPTY, TRANSFERS_EMPTY); - - // Run after compaction, to enforce offset persistence + broadcast - assertSummary(Collections.singleton(instance), summary -> { - Assertions.assertThat(summary).isNull(); - }); - } - } - } - - /* - * Ensure that activation IDs attached to SSTables aren't spread across Token boundaries by compaction. - * - * For example: - * IMPORT_TOKEN is owned by replicas (A, B) - * OUTSIDE_IMPORT_TOKEN is owned by replicas (B, C) - * Execute import so (A, B) have IMPORT_TOKEN - * Execute plain write so (B, C) have OUTSIDE_IMPORT_TOKEN - * Do a major compaction on B so IMPORT_TOKEN and OUTSIDE_IMPORT_TOKEN are compacted together into the same SSTable - * Execute a data read for OUTSIDE_IMPORT_TOKEN against B, ensure it doesn't contain any activation IDs - */ - @Test - public void importActivationMergedByCompaction() throws Throwable - { - IInstanceInitializer initializer = (cl, tg, instance, gen) -> { - ByteBuddyInjections.SkipPurgeTransfers.install().initialise(cl, tg, instance, gen); - }; - try (Cluster cluster = cluster(initializer)) - { - createSchema(cluster, 2); - - Set inImportRange = new HashSet<>(); - cluster.forEach(instance -> { - logger.debug("Instance {} ring is {}", ClusterUtils.instanceId(instance), ClusterUtils.ring(instance)); - boolean isInRange = instance.callOnInstance(() -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication); - return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf(); - }); - if (isInRange) - inImportRange.add(instance); - }); - Assertions.assertThat(inImportRange).hasSize(2); - - // Find a partition key that's not owned by the same replicas as the import - Murmur3Partitioner.LongToken NON_IMPORT_TOKEN = new Murmur3Partitioner.LongToken(IMPORT_TOKEN.getLongValue() * 3); - int NON_IMPORT_PK = Int32Type.instance.compose(Murmur3Partitioner.LongToken.keyForToken(NON_IMPORT_TOKEN)); - - Set inNonImportRange = new HashSet<>(); - cluster.forEach(instance -> { - boolean isInRange = instance.callOnInstance(() -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication); - return placement.writes.forToken(NON_IMPORT_TOKEN).get().containsSelf(); - }); - if (isInRange) - inNonImportRange.add(instance); - }); - Assertions.assertThat(inNonImportRange).hasSize(2); - Assertions.assertThat(inNonImportRange).isNotEqualTo(inImportRange); - - // Import: (A, B) - // Plain: (B, C) - IInvokableInstance A = null; - IInvokableInstance B = null; - IInvokableInstance C = null; - for (IInvokableInstance instance : cluster) - { - boolean isImport = inImportRange.contains(instance); - boolean isNonImport = inNonImportRange.contains(instance); - if (isImport && isNonImport) - B = instance; - else if (isImport) - A = instance; - else if (isNonImport) - C = instance; - } - Assertions.assertThat(A).isNotNull(); - Assertions.assertThat(B).isNotNull(); - Assertions.assertThat(C).isNotNull(); - - doImport(cluster, A); - assertLocalSelect(List.of(A, B), (IIsolatedExecutor.SerializableConsumer) rows -> { - assertRows(rows, row(IMPORT_PK, IMPORT_PK)); - }); - - ShortMutationId importTransferId = callSerialized(A, () -> ShortMutationId.serializer, () -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - for (SSTableReader sstable : cfs.getLiveSSTables()) - { - ActivatedTransfers transfers = sstable.getCoordinatorLogOffsets().transfers(); - if (!transfers.isEmpty()) - return transfers.iterator().next(); - } - return null; - }); - Assertions.assertThat(importTransferId).isNotNull(); - C.coordinator().execute(withKeyspace("INSERT INTO %s." + TABLE + "(k, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NON_IMPORT_PK, NON_IMPORT_PK); - assertCompaction(cluster, Collections.singleton(B), NOOP, NOOP); - - // Reading from B for a range that doesn't include the import shouldn't include any transfer IDs, even though they've been compacted together - long mark = B.logs().mark(); - Object[][] rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, NON_IMPORT_PK); - assertRows(rows, row(NON_IMPORT_PK, NON_IMPORT_PK)); - Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isEmpty(); - - // But if the read range does include a transfer ID, it should have been added - mark = B.logs().mark(); - rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, IMPORT_PK); - assertRows(rows, row(IMPORT_PK, IMPORT_PK)); - Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isNotEmpty(); - } - } - - @Test - public void importFailedStreamCleanup() throws Throwable - { - int FAILED_STREAM = 3; - try (Cluster cluster = cluster(ByteBuddyInjections.FailIncomingStream.install(FAILED_STREAM))) - { - createSchema(cluster); - - IInvokableInstance importer = cluster.get(1); - IInvokableInstance missed = cluster.get(FAILED_STREAM); - - long mark = importer.logs().mark(); - Assertions.assertThatThrownBy(() -> doImport(cluster, importer)) - .isInstanceOf(RuntimeException.class) - .cause() - .isInstanceOf(RuntimeException.class) - .cause() - .hasMessageContaining("Remote peer " + missed.broadcastAddress() + " failed stream session"); - List logs = importer.logs().watchFor(mark, "Remote peer " + missed.broadcastAddress().toString() + " failed stream session").getResult(); - Assertions.assertThat(logs).isNotEmpty(); - - // Await cleanup of failed stream - Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); - - assertPendingDirs(cluster, (File pendingUuidDir) -> { - Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); - }); - - assertLocalSelect(cluster, rows -> { - assertRows(rows); // empty - }); - } - } - - public static class ByteBuddyInjections - { - // Only skips direct transfer activation, not activation as part of read reconciliation - public static class SkipActivation - { - // null to not skip - public static volatile TransferActivation.Phase phase; - - public static IInstanceInitializer install(int...nodes) - { - return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> { - for (int node : nodes) - if (node == num) - new ByteBuddy().rebase(TransferActivation.VerbHandler.class) - .method(named("doVerb")) - .intercept(MethodDelegation.to(ByteBuddyInjections.SkipActivation.class)) - .make() - .load(cl, ClassLoadingStrategy.Default.INJECTION); - }; - } - - // Need to set phase in each instance's classloader, otherwise assignment won't be visible to injected method body - public static void setup(Cluster cluster, TransferActivation.Phase phase) - { - logger.debug("Setting up phase {}", phase); - cluster.forEach(instance -> instance.runOnInstance(() -> ByteBuddyInjections.SkipActivation.phase = phase)); - } - - @SuppressWarnings("unused") - public static void doVerb(Message msg, @SuperCall Callable zuper) - { - if (phase != null && msg.payload.phase == SkipActivation.phase) - { - logger.info("Skipping activation for test: {}", msg.payload); - return; - } - - logger.info("Test running activation as usual: {}", msg.payload); - - try - { - zuper.call(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - } - - // ImmutableCoordinatorLogOffsets.Builder.purgeTransfers(Predicate) - public static class SkipPurgeTransfers - { - public static IInstanceInitializer install() - { - return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> { - new ByteBuddy().rebase(ImmutableCoordinatorLogOffsets.Builder.class) - .method(named("purgeTransfers").and(takesArguments(Predicate.class))) - .intercept(MethodDelegation.to(SkipPurgeTransfers.class)) - .make() - .load(cl, ClassLoadingStrategy.Default.INJECTION); - }; - } - - @SuppressWarnings("unused") - public static void purgeTransfers() - { - logger.debug("Skipping purgeTransfers for test"); - } - } - - // CassandraStreamReceiver.finished - public static class FailIncomingStream - { - @SuppressWarnings("unused") - private static volatile boolean enabled = true; - - public static IInstanceInitializer install(int... nodes) - { - return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> { - for (int node : nodes) - if (node == num) - new ByteBuddy().rebase(CassandraStreamReceiver.class) - .method(named("finished").and(takesNoArguments())) - .intercept(MethodDelegation.to(FailIncomingStream.class)) - .make() - .load(cl, ClassLoadingStrategy.Default.INJECTION); - }; - } - - @SuppressWarnings("unused") - public static void finished(@This CassandraStreamReceiver self) - { - throw new RuntimeException("Failing incoming stream for test"); - } - - public static void toggle(Cluster cluster, boolean enable) - { - enabled = enable; - cluster.forEach(instance -> instance.runOnInstance(() -> FailIncomingStream.enabled = enable)); - } - } - } - - private static Cluster cluster() throws IOException - { - return cluster((cl, tg, instance, generation) -> {}); - } - - private static Cluster cluster(IInstanceInitializer initializer) throws IOException - { - return Cluster.build(NODES) - .withConfig(cfg -> cfg.with(Feature.NETWORK) - .with(Feature.GOSSIP) - .set("mutation_tracking_enabled", "true") - .set("write_request_timeout", "1000ms") - .set("autocompaction_on_startup_enabled", false) - .set("repair_request_timeout", "2s") - .set("stream_transfer_task_timeout", "10s")) - .withInstanceInitializer(initializer) - .start(); - } - - private static void createSchema(Cluster cluster) - { - createSchema(cluster, NODES); - } - - private static void createSchema(Cluster cluster, int rf) - { - cluster.schemaChange(String.format(withKeyspace("CREATE KEYSPACE %s WITH replication = " + - "{'class': 'SimpleStrategy', 'replication_factor': " + rf + "} " + - "AND replication_type='tracked';"))); - cluster.schemaChange(TABLE_SCHEMA_CQL); - } - - private static void doImport(Cluster cluster) throws IOException - { - doImport(cluster, cluster.get(1)); - } - - private static void doImport(Cluster cluster, IInvokableInstance target) throws IOException - { - doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), null); - } - - private static void doImport(Cluster cluster, IInvokableInstance target, @Nullable String createIndexCql) throws IOException - { - doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), createIndexCql); - } - - private static void doImport(Cluster cluster, IInvokableInstance target, Consumer> onFailedDirs, @Nullable String createIndexCql) throws IOException - { - String file = Files.createTempDirectory(MutationTrackingTest.class.getSimpleName()).toString(); - - // Needs to run outside of instance executor because creates schema - CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder() - .forTable(TABLE_SCHEMA_CQL) - .inDirectory(file) - .using("INSERT INTO " + KEYSPACE_TABLE + " (k, v) " + "VALUES (?, ?)"); - - if (createIndexCql != null) - { - builder.withIndexes(createIndexCql).withBuildIndexes(true); - } - - try (CQLSSTableWriter writer = builder.build()) - { - writer.addRow(IMPORT_PK, 1); - } - - assertLocalSelect(cluster, rows -> { - assertRows(rows); // empty - }); - - List failed = target.callOnInstance(() -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - Set paths = Set.of(file); - logger.info("Importing SSTables {}", paths); - return cfs.importNewSSTables(paths, true, true, true, true, true, true, true); - }); - - // Sleep for a while to make sure import completes - Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS); - onFailedDirs.accept(failed); - } - - private static void assertCoordinatedRead(IInvokableInstance instance, IIsolatedExecutor.SerializableConsumer onRows) - { - ICoordinator coordinator = instance.coordinator(); - String cql = "SELECT * FROM %s." + TABLE + " WHERE k = 1"; - Object[][] rows = coordinator.execute(withKeyspace(cql), ConsistencyLevel.ALL); - onRows.accept(rows); - } - - private static void assertPendingDirs(Iterable validate, IIsolatedExecutor.SerializableConsumer forPendingUuidDir) - { - for (IInvokableInstance instance : validate) - { - instance.runOnInstance(() -> { - Set allPendingDirs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).getDirectories().getPendingLocations(); - for (File pendingDir : allPendingDirs) - { - File[] pendingUuidDirs = pendingDir.listUnchecked(File::isDirectory); - for (File pendingUuidDir : pendingUuidDirs) - { - forPendingUuidDir.accept(pendingUuidDir); - } - } - }); - } - } - - private static void assertSummary(Iterable validate, IIsolatedExecutor.SerializableConsumer onSummary) - { - for (IInvokableInstance instance : validate) - { - instance.runOnInstance(() -> { - DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1)); - TableId tableId = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE).metadata().id; - MutationSummary summary; - try - { - summary = MutationTrackingService.instance.createSummaryForKey(key, tableId, false); - } - catch (UnknownShardException e) - { - summary = null; - } - logger.debug("Validating summary {}", summary); - onSummary.accept(summary); - }); - } - } - - private static void assertCompaction(Cluster cluster, Iterable validate, - IIsolatedExecutor.SerializableConsumer before, - IIsolatedExecutor.SerializableConsumer after) - { - for (IInvokableInstance instance : validate) - { - instance.runOnInstance(() -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - for (SSTableReader sstable : cfs.getLiveSSTables()) - { - logger.info("SSTable {} before compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets()); - before.accept(sstable); - } - }); - } - - // Activation ID must be persisted and broadcast across all peers in the cluster for any to mark as persisted + reconciled - cluster.forEach(i -> { - i.runOnInstance(() -> { - MutationTrackingService.instance.persistLogStateForTesting(); - MutationTrackingService.instance.broadcastOffsetsForTesting(); - }); - }); - - // Broadcast is async, wait until completion - Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); - - for (IInvokableInstance instance : validate) - { - instance.runOnInstance(() -> { - ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); - logger.info("Triggering compaction on instance {}", cfs.metadata.keyspace); - CompactionManager.instance.performMaximal(cfs); - - for (SSTableReader sstable : cfs.getLiveSSTables()) - { - logger.info("SSTable {} after compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets()); - after.accept(sstable); - } - }); - } - } - - private static void assertLocalSelect(Iterable validate, IIsolatedExecutor.SerializableConsumer onRows) - { - for (IInvokableInstance instance : validate) - { - { - Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = 1")); - onRows.accept(rows); - } - { - Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE)); - onRows.accept(rows); - } - } - } - - private static void bounce(Cluster cluster) - { - cluster.forEach(instance -> { - try - { - instance.shutdown().get(); - } - catch (InterruptedException | ExecutionException e) - { - throw new RuntimeException(e); - } - instance.startup(); - }); - } - - private static T callSerialized(IInvokableInstance instance, IIsolatedExecutor.SerializableSupplier> serializer, IIsolatedExecutor.SerializableCallable callable) - { - ByteBuffer serialized = instance.callOnInstance(() -> { - T deserialized = callable.call(); - IVersionedSerializer serialize = serializer.get(); - try - { - return serialize.serialize(deserialized, MessagingService.current_version); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - }); - try - { - return serializer.get().deserialize(serialized, MessagingService.current_version); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - } -} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java new file mode 100644 index 0000000000..b6e6ce290f --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportFailureTest.java @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.assertj.core.api.Assertions; + +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.ClusterUtils; +import org.apache.cassandra.distributed.shared.Uninterruptibles; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.ActivatedTransfers; +import org.apache.cassandra.replication.ActivationRequest; +import org.apache.cassandra.replication.ShortMutationId; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ownership.DataPlacement; + +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; +import static org.apache.cassandra.replication.ActivationRequest.Phase.COMMIT; +import static org.apache.cassandra.replication.ActivationRequest.Phase.PREPARE; +import static org.assertj.core.api.Assertions.assertThat; + +public class TrackedImportFailureTest extends TrackedTransferTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedImportFailureTest.class); + + @Test + public void importMissedActivationPrepare() throws Throwable + { + importMissedActivation(PREPARE); + } + + @Test + public void importMissedActivationCommit() throws Throwable + { + importMissedActivation(COMMIT); + } + + public void importMissedActivation(ActivationRequest.Phase phase) throws Throwable + { + int MISSED_ACTIVATION = 2; + try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION))) + { + TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase); + createSchema(cluster); + + Set missed = Collections.singleton(cluster.get(MISSED_ACTIVATION)); + Iterable received = cluster.stream().filter(instance -> !missed.contains(instance)).collect(Collectors.toList()); + + Assertions.assertThatThrownBy(() -> doImport(cluster)) + .hasMessageContaining("Failed adding SSTables") + .cause() + .hasMessageContaining("Failed streaming on 1 instance(s):") + .cause() + .hasMessageMatching("Tracked transfer failed during " + phase + " on " + cluster.get(MISSED_ACTIVATION).broadcastAddress() + " due to TIMEOUT") + .hasNoCause(); + + assertSummary(received, summary -> { + Assertions.assertThat(summary).satisfies(s -> { + assert s.reconciledIds() == 0; + assert s.unreconciledIds() == (phase == COMMIT ? 1 : 0); + }); + }); + assertSummary(missed, summary -> { + Assertions.assertThat(summary).satisfies(s -> { + assert s.reconciledIds() == 0; + assert s.unreconciledIds() == 0; + }); + }); + + switch (phase) + { + case PREPARE: + // Activation did not start, files should be cleaned up + Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); + assertPendingDirs(cluster, (File pendingUuidDir) -> { + Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); + }); + break; + case COMMIT: + // Activation did not complete, files should still exist on all replicas + assertPendingDirs(cluster, (File pendingUuidDir) -> { + Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty(); + }); + break; + } + + // If the activation is not everywhere, it shouldn't be purged on compaction + assertCompaction(cluster, received, TRANSFERS_EXIST, TRANSFERS_EXIST); + + if (phase == PREPARE) + return; + + // Permit activation of missed commits during read reconciliation + TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, null); + + // Use coordinated query rather to confirm read reconciliation triggers activation + IInvokableInstance coordinator = cluster.get(3); // not initial transfer coordinator, but received activation + assertCoordinatedRead(coordinator, rows -> { + assertRows(rows, row(1, 1)); + }); + + // Confirm others receive activation + assertLocalSelect(missed, rows -> { + assertRows(rows, row(1, 1)); + }); + + assertCompaction(cluster, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY); + + // Activation completed, files should be removed + assertPendingDirs(cluster, (File pendingUuidDir) -> { + Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); + }); + } + } + + /** + * If a replica is missing an activation, executes a data read, then discovers the missing transfer during read + * reconciliation, it needs to augment the client response with the new transfer. + */ + @Test + public void importMissingOnDataReplicaDuringAugment() throws Throwable + { + ActivationRequest.Phase phase = COMMIT; + int MISSED_ACTIVATION = 2; + try (Cluster cluster = cluster(TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.install(MISSED_ACTIVATION))) + { + TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, phase); + createSchema(cluster); + + IInvokableInstance missed = cluster.get(MISSED_ACTIVATION); + + Assertions.assertThatThrownBy(() -> doImport(cluster)) + .hasMessageContaining("Failed adding SSTables") + .cause() + .hasMessageContaining("Failed streaming on 1 instance(s):") + .cause() + .hasMessageMatching("Tracked transfer failed during " + phase + " on " + missed.broadcastAddress() + " due to TIMEOUT") + .hasNoCause(); + + assertSummary(Collections.singleton(missed), summary -> { + Assertions.assertThat(summary).satisfies(s -> { + assert s.reconciledIds() == 0; + assert s.unreconciledIds() == 0; + }); + }); + + // Permit activation of missed commits during read reconciliation + TrackedTransferTestBase.ByteBuddyInjections.SkipActivation.setup(cluster, null); + + // First read will fail due to failure to augment with transfer + long mark = missed.logs().mark(); + Assertions.assertThatThrownBy(() -> { + assertCoordinatedRead(missed, rows -> { + assertRows(rows, row(1, 1)); + }); + }).isInstanceOf(missed.callOnInstance(() -> ReadTimeoutException.class)); // use instance classloader + + List logs = missed.logs().grep(mark, "Missing mutation ShortMutationId").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Retry succeeds + assertCoordinatedRead(missed, rows -> { + assertRows(rows, row(1, 1)); + }); + } + } + + /* + * Ensure that activation IDs attached to SSTables aren't spread across Token boundaries by compaction. + * + * For example: + * IMPORT_TOKEN is owned by replicas (A, B) + * OUTSIDE_IMPORT_TOKEN is owned by replicas (B, C) + * Execute import so (A, B) have IMPORT_TOKEN + * Execute plain write so (B, C) have OUTSIDE_IMPORT_TOKEN + * Do a major compaction on B so IMPORT_TOKEN and OUTSIDE_IMPORT_TOKEN are compacted together into the same SSTable + * Execute a data read for OUTSIDE_IMPORT_TOKEN against B, ensure it doesn't contain any activation IDs + */ + @Test + public void importActivationMergedByCompaction() throws Throwable + { + try (Cluster cluster = cluster((cl, tg, instance, gen) -> ByteBuddyInjections.SkipPurgeTransfers.install().initialise(cl, tg, instance, gen))) + { + createSchema(cluster, 2); + + Set inImportRange = new HashSet<>(); + cluster.forEach(instance -> { + logger.debug("Instance {} ring is {}", ClusterUtils.instanceId(instance), ClusterUtils.ring(instance)); + boolean isInRange = instance.callOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); + DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication); + return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf(); + }); + if (isInRange) + inImportRange.add(instance); + }); + Assertions.assertThat(inImportRange).hasSize(2); + + // Find a partition key that's not owned by the same replicas as the import + Murmur3Partitioner.LongToken NON_IMPORT_TOKEN = new Murmur3Partitioner.LongToken(IMPORT_TOKEN.getLongValue() * 3); + int NON_IMPORT_PK = Int32Type.instance.compose(Murmur3Partitioner.LongToken.keyForToken(NON_IMPORT_TOKEN)); + + Set inNonImportRange = new HashSet<>(); + cluster.forEach(instance -> { + boolean isInRange = instance.callOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); + DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication); + return placement.writes.forToken(NON_IMPORT_TOKEN).get().containsSelf(); + }); + if (isInRange) + inNonImportRange.add(instance); + }); + Assertions.assertThat(inNonImportRange).hasSize(2); + Assertions.assertThat(inNonImportRange).isNotEqualTo(inImportRange); + + // Import: (A, B) + // Plain: (B, C) + IInvokableInstance A = null; + IInvokableInstance B = null; + IInvokableInstance C = null; + for (IInvokableInstance instance : cluster) + { + boolean isImport = inImportRange.contains(instance); + boolean isNonImport = inNonImportRange.contains(instance); + if (isImport && isNonImport) + B = instance; + else if (isImport) + A = instance; + else if (isNonImport) + C = instance; + } + Assertions.assertThat(A).isNotNull(); + Assertions.assertThat(B).isNotNull(); + Assertions.assertThat(C).isNotNull(); + + doImport(cluster, A); + assertLocalSelect(List.of(A, B), (IIsolatedExecutor.SerializableConsumer) rows -> { + assertRows(rows, row(IMPORT_PK, IMPORT_PK)); + }); + + ShortMutationId importTransferId = callSerialized(A, () -> ShortMutationId.serializer, () -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); + for (SSTableReader sstable : cfs.getLiveSSTables()) + { + ActivatedTransfers transfers = sstable.getCoordinatorLogOffsets().transfers(); + if (!transfers.isEmpty()) + return transfers.iterator().next(); + } + return null; + }); + Assertions.assertThat(importTransferId).isNotNull(); + C.coordinator().execute(withKeyspace("INSERT INTO %s." + TABLE + "(k, v) VALUES (?, ?)"), ConsistencyLevel.ALL, NON_IMPORT_PK, NON_IMPORT_PK); + assertCompaction(cluster, Collections.singleton(B), NOOP, NOOP); + + // Reading from B for a range that doesn't include the import shouldn't include any transfer IDs, even though they've been compacted together + long mark = B.logs().mark(); + Object[][] rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, NON_IMPORT_PK); + assertRows(rows, row(NON_IMPORT_PK, NON_IMPORT_PK)); + Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isEmpty(); + + // But if the read range does include a transfer ID, it should have been added + mark = B.logs().mark(); + rows = B.coordinator().execute(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = ?"), ConsistencyLevel.ALL, IMPORT_PK); + assertRows(rows, row(IMPORT_PK, IMPORT_PK)); + Assertions.assertThat(B.logs().grep(mark, "Found overlapping activation ID ").getResult()).isNotEmpty(); + } + } + + @Test + public void importFailedStreamCleanup() throws Throwable + { + int FAILED_STREAM = 3; + try (Cluster cluster = cluster(ByteBuddyInjections.FailIncomingStream.install(FAILED_STREAM))) + { + createSchema(cluster); + + IInvokableInstance importer = cluster.get(1); + IInvokableInstance missed = cluster.get(FAILED_STREAM); + + long mark = importer.logs().mark(); + Assertions.assertThatThrownBy(() -> doImport(cluster, importer)) + .isInstanceOf(RuntimeException.class) + .cause() + .isInstanceOf(RuntimeException.class) + .cause() + .hasMessageContaining("Remote peer " + missed.broadcastAddress() + " failed stream session"); + List logs = importer.logs().watchFor(mark, "Remote peer " + missed.broadcastAddress().toString() + " failed stream session").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Await cleanup of failed stream + Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS); + + assertPendingDirs(cluster, (File pendingUuidDir) -> { + Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); + }); + + // empty + assertLocalSelect(cluster, AssertUtils::assertRows); + } + } + + @Test + @Ignore("Reactivate this when we support CL < ALL for tracked imports") + public void importReplicaDown() throws Throwable + { + try (Cluster cluster = cluster()) + { + String keyspace = "replica_down"; + createSchema(cluster, keyspace); + + Iterable down = Collections.singleton(cluster.get(3)); + Iterable up = cluster.stream().filter(instance -> instance != down).collect(Collectors.toList()); + for (IInvokableInstance instance : down) + instance.shutdown().get(); + + doImport(cluster, keyspace); + + cluster.get(3).startup(); + + // Transfers did not complete, files should still exist on up replicas + assertPendingDirs(up, keyspace, (File pendingUuidDir) -> { + assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty(); + }); + assertPendingDirs(down, keyspace, (File pendingUuidDir) -> { + assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); + }); + + // Transfers did not complete, transfer IDs should not be removed + assertCompaction(cluster, keyspace, cluster, TRANSFERS_EXIST, TRANSFERS_EXIST); + + assertLocalSelect(up, keyspace, rows -> assertRows(rows, row(1, 1))); + } + } + + private static void assertCoordinatedRead(IInvokableInstance instance, IIsolatedExecutor.SerializableConsumer onRows) + { + ICoordinator coordinator = instance.coordinator(); + String cql = "SELECT * FROM %s." + TABLE + " WHERE k = 1"; + Object[][] rows = coordinator.execute(withKeyspace(cql), ConsistencyLevel.ALL); + onRows.accept(rows); + } + + private static T callSerialized(IInvokableInstance instance, IIsolatedExecutor.SerializableSupplier> serializer, IIsolatedExecutor.SerializableCallable callable) + { + ByteBuffer serialized = instance.callOnInstance(() -> { + T deserialized = callable.call(); + IVersionedSerializer serialize = serializer.get(); + try + { + return serialize.serialize(deserialized, MessagingService.current_version); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + }); + try + { + return serializer.get().deserialize(serialized, MessagingService.current_version); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java new file mode 100644 index 0000000000..f311381ab1 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedImportTransferTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.test.sai.SAIUtil; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ownership.DataPlacement; + +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * For now, tracked import with a replica down is not supported. The intention is to support this scenario by allowing + * users to provide a {@link ConsistencyLevel} for tracked import operations, where the import will complete if + * sufficient replicas acknowledge the transfer and activate it. + */ +public class TrackedImportTransferTest extends TrackedTransferTestBase +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedImportTransferTest.class); + + private static Cluster cluster; + + @BeforeClass + public static void setup() throws IOException + { + cluster = cluster(); + } + + @AfterClass + public static void teardown() + { + if (cluster != null) + cluster.close(); + } + + @Test + public void importHappyPath() throws Throwable + { + String keyspace = "happy_path"; + createSchema(cluster, keyspace); + doImport(cluster, keyspace); + + // All pending/ dirs should be empty, should have no SSTables left if all the transfers completed + assertPendingDirs(cluster, keyspace, (File pendingUuidDir) -> { + assertThat(pendingUuidDir.listUnchecked(File::isFile)).isEmpty(); + }); + + // Verify transfer IDs exist before compaction, then compact, then verify they're removed + assertCompaction(cluster, keyspace, cluster, TRANSFERS_EXIST, TRANSFERS_EMPTY); + + // Run after compaction, to enforce offset persistence + broadcast + assertSummary(cluster, keyspace, summary -> { + assertThat(summary).satisfies(s -> { + assert s.reconciledIds() == 1; + assert s.unreconciledIds() == 0; + }); + }); + + assertLocalSelect(cluster, keyspace, rows -> assertRows(rows, row(1, 1))); + } + + @Test + public void importIndexAlreadyPresent() throws Throwable + { + String keyspace = "index_present"; + createSchema(cluster, keyspace); + + String indexName = "v_idx"; + String indexCql = withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'", keyspace); + cluster.schemaChange(indexCql); + + // This will add an SSTable that already has an SAI index, that needs to be distributed alongside the SSTable on transfer + IInvokableInstance importer = cluster.get(1); + long mark = importer.logs().mark(); + doImport(cluster, importer, keyspace, indexCql); + List logs = importer.logs().grep(mark, "Submitting incremental index build of " + indexName).getResult(); + assertThat(logs).isEmpty(); + + // Index should exist and be queryable on all replicas after import + SAIUtil.assertIndexQueryable(cluster, keyspace, indexName); + + // Validate queries using the index + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1", keyspace)); + assertRows(rows, row(1, 1)); + }); + } + + @Test + public void importBuildsIndex() throws Throwable + { + String keyspace = "import_builds"; + createSchema(cluster, keyspace); + + String indexName = "v_idx"; + cluster.schemaChange(withKeyspace("CREATE INDEX " + indexName + " ON %s." + TABLE + " (v) USING 'sai'", keyspace)); + + // This will add an SSTable that's missing an SAI index, and the index will be built during the import on + // the coordinator + IInvokableInstance importer = cluster.get(1); + long mark = importer.logs().mark(); + doImport(cluster, importer, keyspace); + List logs = importer.logs().watchFor(mark, Duration.ofMinutes(1), "Submitting incremental index build of " + indexName).getResult(); + assertThat(logs).isNotEmpty(); + + // Index should exist and be queryable on all replicas after import + SAIUtil.assertIndexQueryable(cluster, keyspace, indexName); + + // Validate queries using the index + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE v = 1", keyspace)); + assertRows(rows, row(1, 1)); + }); + } + + @Test + public void importOutOfRange() throws Throwable + { + String keyspace = "out_of_range"; + createSchema(cluster, keyspace, 1); + + Set inRange = new HashSet<>(); + Set outOfRange = new HashSet<>(); + cluster.forEach(instance -> { + boolean importReplica = instance.callOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE); + DataPlacement placement = ClusterMetadata.current().placements.get(cfs.keyspace.getMetadata().params.replication); + return placement.writes.forToken(IMPORT_TOKEN).get().containsSelf(); + }); + (importReplica ? inRange : outOfRange).add(instance); + }); + logger.info("inRange: {}, outOfRange: {}", inRange, outOfRange); + + assertThat(inRange).hasSize(1); + IInvokableInstance onlyInRange = inRange.iterator().next(); + + // Reject import out of range + for (IInvokableInstance instance : outOfRange) + { + long mark = instance.logs().mark(); + Consumer> onResult = failedDirs -> assertThat(failedDirs).hasSize(1); + doImport(cluster, instance, onResult, keyspace, null); + instance.logs().grep(mark, "java.lang.RuntimeException: Key DecoratedKey(-4069959284402364209, 00000001) is not contained in the given ranges"); + } + + doImport(cluster, onlyInRange, keyspace); + + assertSummary(Collections.singleton(onlyInRange), keyspace, summary -> { + assertThat(summary).satisfies(s -> { + assert s.reconciledIds() == 1; + assert s.unreconciledIds() == 0; + }); + }); + + for (IInvokableInstance instance : outOfRange) + { + // Out of range shouldn't have any transfers + assertCompaction(cluster, keyspace, Collections.singleton(instance), TRANSFERS_EMPTY, TRANSFERS_EMPTY); + + // Run after compaction, to enforce offset persistence + broadcast + assertSummary(Collections.singleton(instance), keyspace, summary -> { + assertThat(summary).isNull(); + }); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java new file mode 100644 index 0000000000..710e4e01f2 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedNonZeroCopyRepairTransferTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; + +import org.junit.BeforeClass; + +public class TrackedNonZeroCopyRepairTransferTest extends TrackedRepairTransferSuccessTestBase +{ + @BeforeClass + public static void setup() throws IOException + { + cluster = cluster(NON_ZCS_CONFIG); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java new file mode 100644 index 0000000000..9748968887 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairFailureTest.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import org.awaitility.Awaitility; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.streaming.CassandraStreamReceiver; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.distributed.shared.ClusterUtils; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.replication.ActivationRequest; + +import static net.bytebuddy.implementation.MethodDelegation.to; +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments; +import static org.assertj.core.api.Assertions.assertThat; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; + +public class TrackedRepairFailureTest extends TrackedRepairTransferTestBase +{ + @Test + public void testFullRepairPartiallyCompleteAnomaly() throws IOException, ExecutionException, InterruptedException, TimeoutException + { + try (Cluster cluster = cluster(StreamReceiverFailureHelper::install)) + { + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + String TABLE_SCHEMA_CQL = "CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (k INT PRIMARY KEY, v INT)"; + cluster.schemaChange(TABLE_SCHEMA_CQL); + + IInvokableInstance COORDINATING = cluster.get(1); + IInvokableInstance RECEIVING = cluster.get(2); + IInvokableInstance MISSING = cluster.get(3); + + /* + If we were to start this process with a normal write, that write would be added to the log. When repair + validation runs and finds the mismatching range, it streams the log. MISSING then receives the log and + applies the new mutation to the memtable, where it's visible to reads. + + We want to emulate a situation where one node has a mutation that's not present in the log, hence the + roundabout write path. Once the mutation has completed and is repaired on the coordinator, it's been + (durably) reconciled on all replicas. Then, drop this SSTable on the other peers and we should have a full + repair digest mismatch. + + The idea here is to emulate logical data corruption, where SSTables mismatch but the logs are in agreement. + */ + COORDINATING.coordinator().execute("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) " + "VALUES (?, ?)", ALL, 1, 1); + COORDINATING.flush(KEYSPACE); + Awaitility.waitAtMost(1, TimeUnit.MINUTES).pollDelay(1, TimeUnit.SECONDS) + .until(() -> { + boolean isRepaired = COORDINATING.callOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); + Set sstables = cfs.getLiveSSTables(); + if (sstables.size() != 1) + return false; + SSTableReader sstable = sstables.iterator().next(); + return sstable.isRepaired(); + }); + if (!isRepaired) + COORDINATING.forceCompact(KEYSPACE, TABLE); + return isRepaired; + }); + List.of(RECEIVING, MISSING).forEach(instance -> instance.runOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); + cfs.truncateBlockingWithoutSnapshot(); + })); + cluster.forEach(instance -> { + // Before repair, peers should have no data + Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1"); + if (instance == COORDINATING) + assertRows(rows, row(1, 1)); + else + assertRows(rows); + }); + + // Prevent repair stream from completing + MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(true)); + + // Run full repair from COORDINATING + { + ExecutorService repairExecutor = Executors.newSingleThreadExecutor(); + Future repair = repairExecutor.submit(() -> COORDINATING.nodetoolResult("repair", "--full", KEYSPACE)); + Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS) + .until(() -> { + int finished = StreamReceiverFailureHelper.getFinishedRepairs(RECEIVING); + return finished > 0; + }); + MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(true)); + repair.get(10, TimeUnit.SECONDS).asserts().failure(); + repairExecutor.shutdown(); + } + + // Even after partial repair, RECEIVED should not move its SSTable to the live set, since the repair failed + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1"); + if (instance == COORDINATING) + assertRows(rows, row(1, 1)); + else + assertRows(rows); + }); + + /* + At this point, the repair is complete and partially applied. RECEIVED has an SSTable it received from + repair, and MISSING has no SSTables. If we were to do a tracked data read against RECEIVED, we'd have an + emptpy summary but rows, and if we were to execute the same read against MISSING we'd have an entirely + empty response. This would break monotonicity if a client executes a QUORUM read against RECEIVED then + against missing, because the empty summaries lead to no reconciliation happening. + + To provide monotonicity in this scenario, we integrate the full repair with bulk transfer machinery and + tag the SSTables with transfer IDs that can be included in summaries and reconciled. Then, the initial data + read against RECEIVED includes transfer IDs that are reconciled. Reconciliation detects that RECEIVED has an + SSTable that isn't present on MISSING and streams them, so the subsequent read against MISSING is up to + date. + */ + + { + MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(false)); + MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(false)); + // Don't let coordinating act as a replica for the read + cluster.filters().inbound().to(ClusterUtils.instanceId(COORDINATING)).drop(); + MutationTrackingReadReconciliationTest.awaitNodeDead(RECEIVING, COORDINATING); + } + // Repair did not succeed sync, so it did not proceed to activation, so it's not visible on RECEIVING. + { + Object[][] rows = RECEIVING.coordinator().execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1", QUORUM); + assertRows(rows); // empty + } + cluster.filters().reset(); + MutationTrackingReadReconciliationTest.awaitNodeAlive(RECEIVING, COORDINATING); + + // Another repair succeeds, all peers should now agree on the local data + long mark = COORDINATING.logs().mark(); + COORDINATING.nodetoolResult("repair", "--full", KEYSPACE).asserts().success(); + List logs = COORDINATING.logs().grep(mark, "Activating .* for ").getResult(); + assertThat(logs).isNotEmpty(); + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1"); + assertRows(rows, row(1, 1)); + }); + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE k = 1", ALL); + assertRows(rows, row(1, 1)); + }); + } + } + + @Test + public void testFullRepairCleanupOnFailure() throws IOException, ExecutionException, InterruptedException, TimeoutException + { + try (Cluster cluster = cluster(StreamReceiverFailureHelper::install)) + { + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + String TABLE_SCHEMA_CQL = "CREATE TABLE " + KEYSPACE + '.' + TABLE + " (k INT PRIMARY KEY, v INT)"; + cluster.schemaChange(TABLE_SCHEMA_CQL); + + IInvokableInstance COORDINATING = cluster.get(1); + IInvokableInstance RECEIVING = cluster.get(2); + IInvokableInstance MISSING = cluster.get(3); + + // Write a single row to COORDINATING node only + COORDINATING.executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) VALUES (?, ?)", 1, 100); + + // Before repair, only COORDINATING has data + for (IInvokableInstance instance : cluster) + { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1"); + if (instance == COORDINATING) + assertRows(rows, row(1, 100)); + else + assertRows(rows); // empty + } + + // Prevent repair stream from completing + MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldWait.set(true)); + + // Run full repair from COORDINATING + { + ExecutorService repairExecutor = Executors.newSingleThreadExecutor(); + Future repair = repairExecutor.submit(() -> COORDINATING.nodetoolResult("repair", "--full", KEYSPACE)); + Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS) + .until(() -> { + int finished = StreamReceiverFailureHelper.getFinishedRepairs(RECEIVING); + return finished > 0; + }); + + // Repair completed against RECEIVING, so it should have pending SSTables + { + List pending = getPendingSSTablePaths(RECEIVING); + assertThat(pending).isNotEmpty(); + } + + MISSING.runOnInstance(() -> StreamReceiverFailureHelper.shouldThrow.set(true)); + repair.get(10, TimeUnit.SECONDS).asserts().failure(); + repairExecutor.shutdown(); + } + + // No pending SSTables on COORDINATING because they're streamed from the live set + // No pending SSTables on MISSING because of stream failure injection + // No pending SSTables on RECEIVING because they were cleaned up when the repair failed + for (IInvokableInstance instance : cluster) + { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1"); + if (instance == COORDINATING) + assertRows(rows, row(1, 100)); + else + assertRows(rows); // empty + + List pending = getPendingSSTablePaths(instance); + assertThat(pending).isEmpty(); + } + } + } + + @Test + public void testRepairFailsOnMissedActivation() throws IOException + { + try (Cluster cluster = cluster(ByteBuddyInjections.SkipActivation.install(2, 3))) + { + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (k INT PRIMARY KEY, v INT)"); + + IInvokableInstance COORDINATING = cluster.get(1); + + COORDINATING.executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (k, v) VALUES (?, ?)", 1, 100); + + // Before repair, only instance 1 has data + for (IInvokableInstance instance : cluster) + { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1"); + if (instance == COORDINATING) + assertRows(rows, row(1, 100)); + else + assertRows(rows); // empty + } + + // Repair fails because activation is blocked on nodes 2 and 3 + ByteBuddyInjections.SkipActivation.setup(cluster, ActivationRequest.Phase.COMMIT, true); + { + NodeToolResult repair = COORDINATING.nodetoolResult("repair", "--full", KEYSPACE); + repair.asserts().failure(); + } + for (IInvokableInstance instance : cluster) + { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1"); + List pending = getPendingSSTablePaths(instance); + if (instance == COORDINATING) + { + assertRows(rows, row(1, 100)); + assertThat(pending).isEmpty(); + } + else + { + // Nodes 2 and 3 should only have pending SSTables + assertRows(rows); + assertThat(pending).isNotEmpty(); + } + } + + // Re-enable activation; read reconciliation at ALL should activate the pending SSTables on nodes 2 and 3 + ByteBuddyInjections.SkipActivation.setup(cluster, null); + COORDINATING.coordinator().execute("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1", ALL); + + for (IInvokableInstance instance : cluster) + { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1"); + assertRows(rows, row(1, 100)); + } + + // Pending directory cleanup is scheduled, not synchronous with read reconciliation + for (IInvokableInstance instance : cluster) + { + Awaitility.waitAtMost(10, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS) + .until(() -> { + List pending = getPendingSSTablePaths(instance); + return pending.isEmpty(); + }); + } + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE k = 1", ALL); + assertRows(rows, row(1, 100)); + }); + } + } + + private static List getPendingSSTablePaths(IInvokableInstance instance) + { + return instance.callOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(KEYSPACE, TABLE); + Set pendingLocations = cfs.getDirectories().getPendingLocations(); + + List pendingUuidDirs = new ArrayList<>(); + for (File pendingDir : pendingLocations) + { + File[] uuidDirs = pendingDir.listUnchecked(File::isDirectory); + for (File dir : uuidDirs) + pendingUuidDirs.add(dir.absolutePath()); + } + return pendingUuidDirs; + }); + } + + public static class StreamReceiverFailureHelper + { + private static final Logger logger = LoggerFactory.getLogger(StreamReceiverFailureHelper.class); + + static AtomicBoolean shouldThrow = new AtomicBoolean(false); + static AtomicBoolean shouldWait = new AtomicBoolean(false); + static AtomicInteger count = new AtomicInteger(0); + + /** + * {@link CassandraStreamReceiver#finished} + */ + @SuppressWarnings("resource") + public static void install(ClassLoader classLoader, Integer instanceNum) + { + new ByteBuddy().rebase(CassandraStreamReceiver.class) + .method(named("finished").and(takesNoArguments())) + .intercept(to(StreamReceiverFailureHelper.class)) + .make() + .load(classLoader, ClassLoadingStrategy.Default.INJECTION); + } + + public static void finished(@SuperCall Callable zuper) throws Exception + { + while (shouldWait.get()) + { + if (shouldThrow.get()) + throw new RuntimeException("Test: failing stream session"); + + logger.info("Test: blocking finish of stream session"); + Thread.sleep(1_000); // TODO: Look for a way to do this without sleeping + } + zuper.call(); + logger.info("Test: finished stream session"); + count.incrementAndGet(); + } + + protected static int getFinishedRepairs(IInvokableInstance instance) + { + return instance.callOnInstance(() -> StreamReceiverFailureHelper.count.get()); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java new file mode 100644 index 0000000000..81413325ee --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferSuccessTestBase.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; + +import org.junit.AfterClass; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; + +public abstract class TrackedRepairTransferSuccessTestBase extends TrackedRepairTransferTestBase +{ + protected static Cluster cluster; + + @AfterClass + public static void teardown() + { + if (cluster != null) + cluster.close(); + } + + @Test + public void testFullRepairShardAlignedSinglePlan() throws IOException + { + String keyspace = "full_repair_shard_aligned_single_plan"; + testFullRepairSinglePlan(keyspace, cluster, KEY_200, 2, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsSinglePlan() throws IOException + { + String keyspace = "full_repair_single_plan"; + testFullRepairSinglePlan(keyspace, cluster, KEY_200, 2, false, "repair", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedSinglePlanOptimized() throws IOException + { + String keyspace = "full_repair_shard_aligned_single_plan_optimized"; + testFullRepairSinglePlan(keyspace, cluster, KEY_200, 3, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsSinglePlanOptimized() throws IOException + { + String keyspace = "full_repair_single_plan_optimized"; + testFullRepairSinglePlan(keyspace, cluster, KEY_200, 3, true, "repair", "--optimise-streams", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedRemoteSender() throws IOException + { + String keyspace = "full_repair_shard_aligned_remote_sender"; + testFullRepairRemoteSender(keyspace, cluster, 2, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsRemoteSender() throws IOException + { + String keyspace = "full_repair_remote_sender"; + testFullRepairRemoteSender(keyspace, cluster, 2, false, "repair", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedRemoteSenderOptimized() throws IOException + { + String keyspace = "full_repair_aligned_optimized_remote_sender"; + testFullRepairRemoteSender(keyspace, cluster, 3, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsRemoteSenderOptimized() throws IOException + { + String keyspace = "full_repair_optimized_remote_sender"; + testFullRepairRemoteSender(keyspace, cluster, 3, true, "repair", "--optimise-streams", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedDuplicateSender() throws IOException + { + String keyspace = "full_repair_shard_aligned_duplicate_sender"; + testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsDuplicateSender() throws IOException + { + String keyspace = "full_repair_duplicate_sender"; + testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 3, false, "repair", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedDuplicateSenderOptimized() throws IOException + { + String keyspace = "full_repair_optimized_aligned_duplicate_sender"; + testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsDuplicateSenderOptimized() throws IOException + { + String keyspace = "full_repair_optimized_duplicate_sender"; + testFullRepairDuplicateSender(keyspace, cluster, KEY_100, 6, true, "repair", "--optimise-streams", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedMultiSender() throws IOException + { + String keyspace = "full_repair_shard_aligned_multi_sender"; + testFullRepairMultiSender(keyspace, cluster, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsMultiSender() throws IOException + { + String keyspace = "full_repair_multi_sender"; + testFullRepairMultiSender(keyspace, cluster, 3, false, "repair", "--full", keyspace); + } + + @Test + public void testFullRepairShardAlignedMultiSenderOptimized() throws IOException + { + String keyspace = "full_repair_aligned_optimized_multi_sender"; + testFullRepairMultiSender(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairAcrossShardsMultiSenderOptimized() throws IOException + { + String keyspace = "full_repair_optimized_multi_sender"; + testFullRepairMultiSender(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--full", keyspace); + } + + @Test + public void testFullRepairMultiSenderSameTokenShardAligned() throws IOException + { + String keyspace = "full_repair_multi_sender_same_token_aligned"; + testFullRepairMultiSenderSameToken(keyspace, cluster, 3, false, "repair", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairMultiSenderSameTokenAcrossShards() throws IOException + { + String keyspace = "full_repair_multi_sender_same_token"; + testFullRepairMultiSenderSameToken(keyspace, cluster, 3, false, "repair", "--full", keyspace); + } + + @Test + public void testFullRepairMultiSenderSameTokenShardAlignedOptimized() throws IOException + { + String keyspace = "full_multi_sender_same_token_aligned_optimized"; + testFullRepairMultiSenderSameToken(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--start-token", SHARD_ALIGNED_RANGE_2.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_2.right.toString(), "--full", keyspace); + } + + @Test + public void testFullRepairMultiSenderSameTokenAcrossShardsOptimized() throws IOException + { + String keyspace = "full_multi_sender_same_token_optimized"; + testFullRepairMultiSenderSameToken(keyspace, cluster, 6, true, "repair", "--optimise-streams", "--full", keyspace); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java new file mode 100644 index 0000000000..759b57a1fb --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedRepairTransferTestBase.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import org.assertj.core.api.Assertions; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.NodeToolResult; +import org.apache.cassandra.distributed.shared.ClusterUtils; + +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; + +public abstract class TrackedRepairTransferTestBase extends TrackedTransferTestBase +{ + public void testFullRepairSinglePlan(String keyspace, Cluster cluster, ByteBuffer key, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException + { + cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)"); + + IInvokableInstance coordinator = cluster.get(1); + coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key); + + // Write should only be present on instance 1 + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key); + if (ClusterUtils.instanceId(instance) == 1) + assertRows(rows, row(key, 1)); + else + assertRows(rows); // empty + }); + + long mark = coordinator.logs().mark(); + NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs); + result.asserts().success(); + List logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Write visible on all instances after repair + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key); + assertRows(rows, row(key, 1)); + }); + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, key); + assertRows(rows, row(key, 1)); + }); + } + + public void testFullRepairRemoteSender(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException + { + cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)"); + + IInvokableInstance coordinator = cluster.get(1); + cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2)", KEY_201); + + // Second key should only be present on instance 2 + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201); + if (ClusterUtils.instanceId(instance) == 2) + assertRows(rows, row(KEY_201, 2)); + else + assertRows(rows); // empty + }); + + long mark = coordinator.logs().mark(); + NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs); + result.asserts().success(); + List logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Write visible on all instances after repair + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201); + assertRows(rows, row(KEY_201, 2)); + }); + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_201); + assertRows(rows, row(KEY_201, 2)); + }); + } + + public void testFullRepairDuplicateSender(String keyspace, Cluster cluster, ByteBuffer key, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException + { + cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)"); + + IInvokableInstance coordinator = cluster.get(1); + coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key); + cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", key); + + // Write should be missing on node 3 + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key); + if (ClusterUtils.instanceId(instance) != 3) + assertRows(rows, row(key, 1)); + else + assertRows(rows); // empty + }); + + long mark = coordinator.logs().mark(); + coordinator.nodetoolResult(repairCommandAndArgs).asserts().success(); + + List logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Write visible on all instances after repair + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", key); + assertRows(rows, row(key, 1)); + }); + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, key); + assertRows(rows, row(key, 1)); + }); + } + + public void testFullRepairMultiSender(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException + { + cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)"); + + IInvokableInstance coordinator = cluster.get(1); + coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1)", KEY_200); + cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2)", KEY_201); + + // First key should only be present on instance 1 + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200); + if (ClusterUtils.instanceId(instance) == 1) + assertRows(rows, row(KEY_200, 1)); + else + assertRows(rows); // empty + }); + + // Second key should only be present on instance 2 + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201); + if (ClusterUtils.instanceId(instance) == 2) + assertRows(rows, row(KEY_201, 2)); + else + assertRows(rows); // empty + }); + + long mark = coordinator.logs().mark(); + NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs); + result.asserts().success(); + List logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Writes visible on all instances after repair + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200); + assertRows(rows, row(KEY_200, 1)); + rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_201); + assertRows(rows, row(KEY_201, 2)); + }); + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_200); + assertRows(rows, row(KEY_200, 1)); + rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_201); + assertRows(rows, row(KEY_201, 2)); + }); + } + + public void testFullRepairMultiSenderSameToken(String keyspace, Cluster cluster, int syncs, boolean optimized, String... repairCommandAndArgs) throws IOException + { + cluster.schemaChange("CREATE KEYSPACE " + keyspace + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(keyspace) + " (pk BLOB PRIMARY KEY, v INT)"); + + IInvokableInstance coordinator = cluster.get(1); + coordinator.executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 1) USING TIMESTAMP 1", KEY_200); + cluster.get(2).executeInternal("INSERT INTO " + tableWithKeyspace(keyspace) + " (pk, v) VALUES (?, 2) USING TIMESTAMP 2", KEY_200); + + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200); + if (ClusterUtils.instanceId(instance) == 1) + assertRows(rows, row(KEY_200, 1)); + else if (ClusterUtils.instanceId(instance) == 2) + assertRows(rows, row(KEY_200, 2)); + else + assertRows(rows); // empty + }); + + long mark = coordinator.logs().mark(); + NodeToolResult result = coordinator.nodetoolResult(repairCommandAndArgs); + result.asserts().success(); + List logs = coordinator.logs().grep(mark, "Created " + syncs + (optimized ? " optimised" : "") + " sync tasks based on 3 merkle tree responses").getResult(); + Assertions.assertThat(logs).isNotEmpty(); + + // Writes visible on all instances after repair + cluster.forEach(instance -> { + Object[][] rows = instance.executeInternal("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", KEY_200); + assertRows(rows, row(KEY_200, 2)); + }); + + // Make sure all instances can successfully coordinate + cluster.forEach(instance -> { + Object[][] rows = instance.coordinator().execute("SELECT * FROM " + tableWithKeyspace(keyspace) + " WHERE pk = ?", ALL, KEY_200); + assertRows(rows, row(KEY_200, 2)); + }); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java new file mode 100644 index 0000000000..2da449eead --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferBounceTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.util.function.Consumer; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInstanceInitializer; + +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; +import static org.apache.cassandra.replication.ActivationRequest.Phase.COMMIT; + +public class TrackedTransferBounceTest extends TrackedTransferTestBase +{ + /* + * When an import fails, bounce must not move the pending SSTables into the live set. + */ + @Test + public void testBounceAfterPendingImport() throws Throwable + { + IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3); + try (Cluster cluster = cluster(initializer)) + { + ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT); + createSchema(cluster); + + Assertions.assertThatThrownBy(() -> doImport(cluster)) + .hasMessageContaining("Failed adding SSTables") + .cause() + .hasMessageContaining("Tracked transfer failed during COMMIT"); + + assertPendingActivation(cluster); + assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS)); + + bounce(cluster); + + assertPendingActivation(cluster); + assertLocalSelect(cluster, rows -> assertRows(rows, EMPTY_ROWS)); + } + } + + @Test + public void testBounceAfterPendingShardAlignedZeroCopy() throws IOException + { + testBounceAfterPendingRepair(ZCS_CONFIG, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", KEYSPACE); + } + + @Test + public void testBounceAfterPendingAcrossShardsZeroCopy() throws IOException + { + testBounceAfterPendingRepair(ZCS_CONFIG, "repair", "--full", KEYSPACE); + } + + @Test + public void testBounceAfterPendingShardAlignedNonZeroCopy() throws IOException + { + testBounceAfterPendingRepair(NON_ZCS_CONFIG, "repair", "--start-token", SHARD_ALIGNED_RANGE_1.left.toString(), "--end-token", SHARD_ALIGNED_RANGE_1.right.toString(), "--full", KEYSPACE); + } + + @Test + public void testBounceAfterPendingAcrossShardsNonZeroCopy() throws IOException + { + testBounceAfterPendingRepair(NON_ZCS_CONFIG, "repair", "--full", KEYSPACE); + } + + /* + * When a repair fails, bounce must not move the pending SSTables into the live set. + */ + private static void testBounceAfterPendingRepair(Consumer config, String... repairCommandAndArgs) throws IOException + { + IInstanceInitializer initializer = ByteBuddyInjections.SkipActivation.install(1, 2, 3); + + try (Cluster cluster = cluster(config, initializer)) + { + // Make sure we fail on commit... + ByteBuddyInjections.SkipActivation.setup(cluster, COMMIT); + + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3} AND replication_type='tracked';"); + cluster.schemaChange("CREATE TABLE " + tableWithKeyspace(KEYSPACE) + " (pk BLOB PRIMARY KEY, v INT)"); + + cluster.get(1).executeInternal("INSERT INTO " + tableWithKeyspace(KEYSPACE) + " (pk, v) VALUES (?, 1)", KEY_100); + + assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1)); + assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100)); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100)); + + cluster.get(1).nodetoolResult(repairCommandAndArgs).asserts().failure().errorContains("Tracked transfer failed during COMMIT"); + + assertPendingActivation(cluster); + assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1)); + assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100)); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100)); + + bounce(cluster); + + assertPendingActivation(cluster); + assertRows(cluster.get(1).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100), row(KEY_100, 1)); + assertRows(cluster.get(2).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100)); + assertRows(cluster.get(3).executeInternal("SELECT * FROM " + tableWithKeyspace(KEYSPACE) + " WHERE pk = ?", KEY_100)); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java new file mode 100644 index 0000000000..5b5a3b5a45 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedTransferTestBase.java @@ -0,0 +1,546 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import javax.annotation.Nullable; + +import org.assertj.core.api.Assertions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.dynamic.loading.ClassLoadingStrategy; +import net.bytebuddy.implementation.MethodDelegation; +import net.bytebuddy.implementation.bind.annotation.SuperCall; +import net.bytebuddy.implementation.bind.annotation.This; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.streaming.CassandraStreamReceiver; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInstanceInitializer; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.distributed.api.IIsolatedExecutor; +import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.distributed.shared.Uninterruptibles; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.io.sstable.CQLSSTableWriter; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.replication.ActivationRequest; +import org.apache.cassandra.replication.ImmutableCoordinatorLogOffsets; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.replication.UnknownShardException; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; +import static net.bytebuddy.matcher.ElementMatchers.takesNoArguments; +import static org.junit.Assert.assertEquals; + +public abstract class TrackedTransferTestBase extends TestBaseImpl +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedTransferTestBase.class); + + protected static final Consumer CONFIG = cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", true) + .set("write_request_timeout", "1000ms") + .set("autocompaction_on_startup_enabled", false) + .set("repair_request_timeout", "2s") + .set("stream_transfer_task_timeout", "10s"); + + protected static final Consumer ZCS_CONFIG = CONFIG.andThen(cfg -> cfg.set("stream_entire_sstables", true)); + protected static final Consumer NON_ZCS_CONFIG = CONFIG.andThen(cfg -> cfg.set("stream_entire_sstables", false)); + + protected static final IIsolatedExecutor.SerializableConsumer TRANSFERS_EXIST = sstable -> { + Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()).isNotEmpty(); + Assertions.assertThat(sstable.isRepaired()).isFalse(); + }; + protected static final IIsolatedExecutor.SerializableConsumer TRANSFERS_EMPTY = sstable -> { + Assertions.assertThat(sstable.getCoordinatorLogOffsets().transfers()).isEmpty(); + Assertions.assertThat(sstable.isRepaired()).isTrue(); + }; + protected static final IIsolatedExecutor.SerializableConsumer NOOP = sstable -> {}; + + protected static final int NODES = 3; + + protected static final String TABLE = "tbl"; + + protected static final int IMPORT_PK = 1; + protected static final Token IMPORT_TOKEN = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(IMPORT_PK)); + + // This should be aligned to a single shard: (min, -3074457345618258603] + protected final static long TOKEN_VALUE_100 = -4074457345618258601L; + protected final static Token TOKEN_100 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_100); + protected final static ByteBuffer KEY_100 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_100.getLongValue()); + + protected final static Range SHARD_ALIGNED_RANGE_1 = new Range<>(new Murmur3Partitioner.LongToken(TOKEN_VALUE_100 - 10), new Murmur3Partitioner.LongToken(TOKEN_VALUE_100 + 10)); + + // This should be aligned to a single shard: (-3074457345618258603,3074457345618258601] + protected final static long TOKEN_VALUE_200 = 1; + protected final static Token TOKEN_200 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_200); + protected final static ByteBuffer KEY_200 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_200.getLongValue()); + + protected final static long TOKEN_VALUE_201 = 2; + protected final static Token TOKEN_201 = new Murmur3Partitioner.LongToken(TOKEN_VALUE_201); + protected final static ByteBuffer KEY_201 = Murmur3Partitioner.LongToken.keyForToken(TOKEN_201.getLongValue()); + + protected final static Range SHARD_ALIGNED_RANGE_2 = new Range<>(new Murmur3Partitioner.LongToken(TOKEN_VALUE_200 - 10), new Murmur3Partitioner.LongToken(TOKEN_VALUE_200 + 10)); + + static + { + DecoratedKey reversed = Murmur3Partitioner.instance.decorateKey(KEY_100); + Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_100); + + reversed = Murmur3Partitioner.instance.decorateKey(KEY_200); + Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_200); + + reversed = Murmur3Partitioner.instance.decorateKey(KEY_201); + Assertions.assertThat(reversed.getToken()).isEqualTo(TOKEN_201); + } + + protected static Cluster cluster() throws IOException + { + return cluster((cl, tg, instance, generation) -> {}); + } + + protected static Cluster cluster(Consumer config) throws IOException + { + return Cluster.build(NODES).withConfig(config).start(); + } + + protected static Cluster cluster(Consumer config, IInstanceInitializer initializer) throws IOException + { + return Cluster.build(NODES).withConfig(config).withInstanceInitializer(initializer).start(); + } + + protected static Cluster cluster(IInstanceInitializer initializer) throws IOException + { + return Cluster.build(NODES).withConfig(CONFIG).withInstanceInitializer(initializer).start(); + } + + protected static Cluster cluster(BiConsumer initializer) throws IOException + { + return Cluster.build(NODES).withConfig(CONFIG).withInstanceInitializer(initializer).start(); + } + + protected static void assertPendingActivation(Cluster cluster) + { + assertPendingActivation(cluster, KEYSPACE); + } + + protected static void assertPendingActivation(Cluster cluster, String keyspace) + { + // Activation did not complete, files should still exist on all replicas + assertPendingDirs(cluster, keyspace, (File pendingUuidDir) -> { + Assertions.assertThat(pendingUuidDir.listUnchecked(File::isFile)).isNotEmpty(); + }); + + // No one has activated, so should not be present in any summary + assertSummary(cluster, keyspace, summary -> { + Assertions.assertThat(summary).satisfies(s -> { + assertEquals(0, s.reconciledIds()); + assertEquals(0, s.unreconciledIds()); + }); + }); + } + + protected static void assertPendingDirs(Iterable validate, IIsolatedExecutor.SerializableConsumer forPendingUuidDir) + { + assertPendingDirs(validate, KEYSPACE, forPendingUuidDir); + } + + protected static void assertPendingDirs(Iterable validate, String keysapce, IIsolatedExecutor.SerializableConsumer forPendingUuidDir) + { + for (IInvokableInstance instance : validate) + { + instance.runOnInstance(() -> { + Set allPendingDirs = ColumnFamilyStore.getIfExists(keysapce, TABLE).getDirectories().getPendingLocations(); + for (File pendingDir : allPendingDirs) + { + File[] pendingUuidDirs = pendingDir.listUnchecked(File::isDirectory); + for (File pendingUuidDir : pendingUuidDirs) + { + forPendingUuidDir.accept(pendingUuidDir); + } + } + }); + } + } + + protected static void assertSummary(Iterable validate, IIsolatedExecutor.SerializableConsumer onSummary) + { + assertSummary(validate, KEYSPACE, onSummary); + } + + protected static void assertSummary(Iterable validate, String keyspace, IIsolatedExecutor.SerializableConsumer onSummary) + { + for (IInvokableInstance instance : validate) + { + instance.runOnInstance(() -> { + DecoratedKey key = DatabaseDescriptor.getPartitioner().decorateKey(ByteBufferUtil.bytes(1)); + TableId tableId = ColumnFamilyStore.getIfExists(keyspace, TABLE).metadata().id; + MutationSummary summary; + try + { + summary = MutationTrackingService.instance.createSummaryForKey(key, tableId, false); + } + catch (UnknownShardException e) + { + summary = null; + } + onSummary.accept(summary); + }); + } + } + + protected static void bounce(Cluster cluster) + { + cluster.forEach(instance -> { + try + { + instance.shutdown().get(); + } + catch (InterruptedException | ExecutionException e) + { + throw new RuntimeException(e); + } + instance.startup(); + }); + } + + protected static String withKeyspace(String replaceIn, String keyspace) + { + return String.format(replaceIn, keyspace); + } + + protected static String tableSchema(String keyspace) + { + return String.format("CREATE TABLE %s.%s (k int PRIMARY KEY, v int)", keyspace, TABLE); + } + + protected static String tableWithKeyspace(String keyspace) + { + return String.format("%s.%s", keyspace, TABLE); + } + + protected static void createSchema(Cluster cluster) + { + createSchema(cluster, KEYSPACE, NODES); + } + + protected static void createSchema(Cluster cluster, String keyspace) + { + createSchema(cluster, keyspace, NODES); + } + + protected static void createSchema(Cluster cluster, int rf) + { + createSchema(cluster, KEYSPACE, rf); + } + + protected static void createSchema(Cluster cluster, String keyspace, int rf) + { + cluster.schemaChange(String.format(withKeyspace("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': " + rf + "} " + + "AND replication_type='tracked'", keyspace))); + cluster.schemaChange(tableSchema(keyspace)); + } + + protected static void doImport(Cluster cluster) throws IOException + { + doImport(cluster, KEYSPACE); + } + + protected static void doImport(Cluster cluster, String keyspace) throws IOException + { + doImport(cluster, cluster.get(1), keyspace); + } + + protected static void doImport(Cluster cluster, IInvokableInstance target) throws IOException + { + doImport(cluster, target, KEYSPACE); + } + + protected static void doImport(Cluster cluster, IInvokableInstance target, String keyspace) throws IOException + { + doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), keyspace, null); + } + + protected static void doImport(Cluster cluster, IInvokableInstance target, String keyspace, @Nullable String createIndexCql) throws IOException + { + doImport(cluster, target, failedDirs -> Assertions.assertThat(failedDirs).isEmpty(), keyspace, createIndexCql); + } + + protected static void doImport(Cluster cluster, IInvokableInstance target, Consumer> onFailedDirs, String keyspace, @Nullable String createIndexCql) throws IOException + { + String file = Files.createTempDirectory(MutationTrackingTest.class.getSimpleName()).toString(); + + // Needs to run outside of instance executor because creates schema + CQLSSTableWriter.Builder builder = CQLSSTableWriter.builder() + .forTable(tableSchema(keyspace)) + .inDirectory(file) + .using("INSERT INTO " + tableWithKeyspace(keyspace) + " (k, v) " + "VALUES (?, ?)"); + + if (createIndexCql != null) + { + builder.withIndexes(createIndexCql).withBuildIndexes(true); + } + + try (CQLSSTableWriter writer = builder.build()) + { + writer.addRow(IMPORT_PK, 1); + } + + // empty + assertLocalSelect(cluster, keyspace, AssertUtils::assertRows); + + List failed = target.callOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE); + Set paths = Set.of(file); + logger.info("Importing SSTables {}", paths); + return cfs.importNewSSTables(paths, true, true, true, true, true, true, true); + }); + + // Sleep for a while to make sure import completes + Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS); + onFailedDirs.accept(failed); + } + + protected static void assertLocalSelect(Iterable validate, IIsolatedExecutor.SerializableConsumer onRows) + { + assertLocalSelect(validate, KEYSPACE, onRows); + } + + protected static void assertLocalSelect(Iterable validate, String keyspace, IIsolatedExecutor.SerializableConsumer onRows) + { + for (IInvokableInstance instance : validate) + { + { + Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE + " WHERE k = 1", keyspace)); + onRows.accept(rows); + } + { + Object[][] rows = instance.executeInternal(withKeyspace("SELECT * FROM %s." + TABLE, keyspace)); + onRows.accept(rows); + } + } + } + + protected static void assertCompaction(Cluster cluster, + Iterable validate, + IIsolatedExecutor.SerializableConsumer before, + IIsolatedExecutor.SerializableConsumer after) + { + assertCompaction(cluster, KEYSPACE, validate, before, after); + } + protected static void assertCompaction(Cluster cluster, + String keyspace, + Iterable validate, + IIsolatedExecutor.SerializableConsumer before, + IIsolatedExecutor.SerializableConsumer after) + { + for (IInvokableInstance instance : validate) + { + instance.runOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE); + for (SSTableReader sstable : cfs.getLiveSSTables()) + { + logger.info("SSTable {} before compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets()); + before.accept(sstable); + } + }); + } + + // Activation ID must be persisted and broadcast across all peers in the cluster for any to mark as persisted + reconciled + cluster.forEach(i -> { + i.runOnInstance(() -> { + MutationTrackingService.instance.persistLogStateForTesting(); + MutationTrackingService.instance.broadcastOffsetsForTesting(); + }); + }); + + // Broadcast is async, wait until completion + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + + for (IInvokableInstance instance : validate) + { + instance.runOnInstance(() -> { + ColumnFamilyStore cfs = ColumnFamilyStore.getIfExists(keyspace, TABLE); + logger.info("Triggering compaction on instance {}", cfs.metadata.keyspace); + CompactionManager.instance.performMaximal(cfs); + + for (SSTableReader sstable : cfs.getLiveSSTables()) + { + logger.info("SSTable {} after compaction: {}", sstable.getFilename(), sstable.getCoordinatorLogOffsets()); + after.accept(sstable); + } + }); + } + } + + public static class ByteBuddyInjections + { + // Only skips direct transfer activation, not activation as part of read reconciliation + public static class SkipActivation + { + // null to not skip + public static volatile ActivationRequest.Phase phase; + + public static volatile boolean throwOnActivation = false; + + @SuppressWarnings("resource") + public static IInstanceInitializer install(int...nodes) + { + return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> { + for (int node : nodes) + if (node == num) + new ByteBuddy().rebase(ActivationRequest.VerbHandler.class) + .method(named("doVerb")) + .intercept(MethodDelegation.to(ByteBuddyInjections.SkipActivation.class)) + .make() + .load(cl, ClassLoadingStrategy.Default.INJECTION); + }; + } + + // Need to set phase in each instance's classloader, otherwise assignment won't be visible to injected method body + public static void setup(Cluster cluster, ActivationRequest.Phase phase) + { + setup(cluster, phase, false); + } + + public static void setup(Cluster cluster, ActivationRequest.Phase phase, boolean throwOnActivation) + { + logger.debug("Setting up phase {}, throwOnActivation {}", phase, throwOnActivation); + cluster.forEach(instance -> instance.runOnInstance(() -> { + SkipActivation.phase = phase; + SkipActivation.throwOnActivation = throwOnActivation; + })); + } + + @SuppressWarnings("unused") + public static void doVerb(Message msg, @SuperCall Callable zuper) + { + if (phase != null && msg.payload.phase == SkipActivation.phase) + { + if (throwOnActivation) + { + // Avoid spamming logs with retries + Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS); + throw new RuntimeException("Throwing on activation for test"); + } + else + { + logger.info("Skipping activation for test: {}", msg.payload); + return; + } + } + + logger.info("Test running activation as usual: {}", msg.payload); + + try + { + zuper.call(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + } + + // ImmutableCoordinatorLogOffsets.Builder.purgeTransfers(Predicate) + public static class SkipPurgeTransfers + { + @SuppressWarnings("resource") + public static IInstanceInitializer install() + { + return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> { + new ByteBuddy().rebase(ImmutableCoordinatorLogOffsets.Builder.class) + .method(named("purgeTransfers").and(takesArguments(Predicate.class))) + .intercept(MethodDelegation.to(SkipPurgeTransfers.class)) + .make() + .load(cl, ClassLoadingStrategy.Default.INJECTION); + }; + } + + @SuppressWarnings("unused") + public static void purgeTransfers() + { + logger.debug("Skipping purgeTransfers for test"); + } + } + + // CassandraStreamReceiver.finished + public static class FailIncomingStream + { + @SuppressWarnings("unused") + private static volatile boolean enabled = true; + + @SuppressWarnings("resource") + public static IInstanceInitializer install(int... nodes) + { + return (ClassLoader cl, ThreadGroup tg, int num, int generation) -> { + for (int node : nodes) + if (node == num) + new ByteBuddy().rebase(CassandraStreamReceiver.class) + .method(named("finished").and(takesNoArguments())) + .intercept(MethodDelegation.to(FailIncomingStream.class)) + .make() + .load(cl, ClassLoadingStrategy.Default.INJECTION); + }; + } + + @SuppressWarnings("unused") + public static void finished(@This CassandraStreamReceiver self) + { + throw new RuntimeException("Failing incoming stream for test"); + } + + public static void toggle(Cluster cluster, boolean enable) + { + enabled = enable; + cluster.forEach(instance -> instance.runOnInstance(() -> FailIncomingStream.enabled = enable)); + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java new file mode 100644 index 0000000000..75c95b4e3f --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/TrackedZeroCopyRepairTransferTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; + +import org.junit.BeforeClass; + +public class TrackedZeroCopyRepairTransferTest extends TrackedRepairTransferSuccessTestBase +{ + @BeforeClass + public static void setup() throws IOException + { + cluster = cluster(ZCS_CONFIG); + } +} diff --git a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java index c03fe2aac7..7594fe6b47 100644 --- a/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/LocalSyncTaskTest.java @@ -133,7 +133,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest TreeResponse r1 = new TreeResponse(local, tree1); TreeResponse r2 = new TreeResponse(InetAddressAndPort.getByName("127.0.0.2"), tree2); LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), - NO_PENDING_REPAIR, true, true, PreviewKind.NONE); + NO_PENDING_REPAIR, true, true, PreviewKind.NONE, null); NettyStreamingConnectionFactory.MAX_CONNECT_ATTEMPTS = 1; try { @@ -159,7 +159,7 @@ public class LocalSyncTaskTest extends AbstractRepairTest TreeResponse r2 = new TreeResponse(PARTICIPANT2, createInitialTree(desc, DatabaseDescriptor.getPartitioner())); LocalSyncTask task = new LocalSyncTask(SharedContext.Global.instance, desc, r1.endpoint, r2.endpoint, MerkleTrees.difference(r1.trees, r2.trees), - NO_PENDING_REPAIR, true, true, PreviewKind.NONE); + NO_PENDING_REPAIR, true, true, PreviewKind.NONE, null); StreamPlan plan = task.createStreamPlan(); assertEquals(NO_PENDING_REPAIR, plan.getPendingRepair()); diff --git a/test/unit/org/apache/cassandra/repair/RepairJobTest.java b/test/unit/org/apache/cassandra/repair/RepairJobTest.java index b01b26bd83..69610934cd 100644 --- a/test/unit/org/apache/cassandra/repair/RepairJobTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairJobTest.java @@ -284,7 +284,7 @@ public class RepairJobTest // block syncComplete execution until test has verified session still retains the trees CompletableFuture future = new CompletableFuture<>(); session.registerSyncCompleteCallback(future::get); - ListenableFuture> syncResults = job.executeTasks(syncTasks); + ListenableFuture> syncResults = job.executeTasks(SyncTasks.untracked(syncTasks)); // Immediately following execution the internal execution queue should still retain the trees long sizeDuringExecution = ObjectSizes.measureDeep(session); @@ -903,7 +903,7 @@ public class RepairJobTest break; case SYNC_REQ: SyncRequest syncRequest = (SyncRequest) message.payload; - session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList())).from(to).build()); + session.syncComplete(sessionJobDesc, Message.builder(Verb.SYNC_RSP, new SyncResponse(sessionJobDesc, new SyncNodePair(syncRequest.src, syncRequest.dst), true, Collections.emptyList(), null, null)).from(to).build()); break; default: break; diff --git a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java index 8aa39d06d9..7cbc26b803 100644 --- a/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/StreamingRepairTaskTest.java @@ -68,8 +68,8 @@ public class StreamingRepairTaskTest extends AbstractRepairTest ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges()); - SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false); - StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false); + SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false, null); + StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, null), desc, request.initiator, request.src, request.dst, request.ranges, desc.sessionId, PreviewKind.NONE, false, null); StreamPlan plan = task.createStreamPlan(request.dst); Assert.assertFalse(plan.getFlushBeforeTransfer()); @@ -81,8 +81,9 @@ public class StreamingRepairTaskTest extends AbstractRepairTest TimeUUID sessionID = registerSession(cfs, false, true); ActiveRepairService.ParentRepairSession prs = ActiveRepairService.instance().getParentRepairSession(sessionID); RepairJobDesc desc = new RepairJobDesc(sessionID, nextTimeUUID(), ks, tbl, prs.getRanges()); - SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false); - StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false); + + SyncRequest request = new SyncRequest(desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, prs.getRanges(), PreviewKind.NONE, false, null); + StreamingRepairTask task = new StreamingRepairTask(SharedContext.Global.instance, new SyncState(Clock.Global.clock(), desc, PARTICIPANT1, PARTICIPANT2, PARTICIPANT3, null), desc, request.initiator, request.src, request.dst, request.ranges, null, PreviewKind.NONE, false, null); StreamPlan plan = task.createStreamPlan(request.dst); Assert.assertTrue(plan.getFlushBeforeTransfer()); diff --git a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java index 37d4e09e78..ff840391c2 100644 --- a/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java +++ b/test/unit/org/apache/cassandra/repair/SymmetricRemoteSyncTaskTest.java @@ -41,7 +41,7 @@ public class SymmetricRemoteSyncTaskTest extends AbstractRepairTest { public InstrumentedSymmetricRemoteSyncTask(InetAddressAndPort e1, InetAddressAndPort e2) { - super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE); + super(SharedContext.Global.instance, DESC, e1, e2, RANGE_LIST, PreviewKind.NONE, null); } RepairMessage sentMessage = null; diff --git a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java index f15b2a6182..22f7cf8810 100644 --- a/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java +++ b/test/unit/org/apache/cassandra/repair/messages/RepairMessageSerializationsTest.java @@ -169,7 +169,8 @@ public class RepairMessageSerializationsTest extends CassandraTestBase InetAddressAndPort src = InetAddressAndPort.getByName("127.0.0.2"); InetAddressAndPort dst = InetAddressAndPort.getByName("127.0.0.3"); - SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE, false); + // TODO: Do we want to test with a transfer ID? + SyncRequest msg = new SyncRequest(buildRepairJobDesc(), initiator, src, dst, buildTokenRanges(), PreviewKind.NONE, false, null); serializeRoundTrip(msg, SyncRequest.serializer); } @@ -183,7 +184,7 @@ public class RepairMessageSerializationsTest extends CassandraTestBase Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 5, 100)), Lists.newArrayList(new StreamSummary(TableId.fromUUID(UUID.randomUUID()), emptyList(), 500, 10)) )); - SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries); + SyncResponse msg = new SyncResponse(buildRepairJobDesc(), new SyncNodePair(src, dst), true, summaries, null, null); serializeRoundTrip(msg, SyncResponse.serializer); } diff --git a/test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java b/test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java similarity index 50% rename from test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java rename to test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java index 57a51f87b8..424b57ddb9 100644 --- a/test/unit/org/apache/cassandra/replication/TransferActivationSerializationTest.java +++ b/test/unit/org/apache/cassandra/replication/ActivationRequestSerializationTest.java @@ -22,16 +22,24 @@ import java.io.IOException; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.cassandra.CassandraTestBase; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.io.Serializers; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.streaming.StreamOperation; import org.apache.cassandra.tcm.membership.NodeId; +import org.apache.cassandra.utils.Pair; import org.apache.cassandra.utils.TimeUUID; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; -public class TransferActivationSerializationTest +public class ActivationRequestSerializationTest extends CassandraTestBase { private static final int VERSION = MessagingService.current_version; @@ -39,37 +47,44 @@ public class TransferActivationSerializationTest public static void setUpClass() { DatabaseDescriptor.daemonInitialization(); + ClusterMetadataTestHelper.setInstanceForTest(); } @Test public void testRoundtripPreparePhase() throws IOException { + Pair pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost()); TimeUUID planId = nextTimeUUID(); - MutationId transferId = new MutationId(1L, 100, 200); + ShortMutationId transferId = new ShortMutationId(1L, 100); NodeId coordinatorId = new NodeId(1); - TransferActivation.Phase phase = TransferActivation.Phase.PREPARE; + ActivationRequest.Phase phase = ActivationRequest.Phase.PREPARE; + String keyspace = "test_ks"; + Range range = new Range<>(new ByteOrderedPartitioner.BytesToken("key1".getBytes()), new ByteOrderedPartitioner.BytesToken("key100".getBytes())); - TransferActivation activation = new TransferActivation(planId, transferId, coordinatorId, phase); + ActivationRequest activation = new ActivationRequest(StreamOperation.IMPORT, pair, phase, transferId, coordinatorId, range, keyspace, planId); try (DataOutputBuffer output = new DataOutputBuffer()) { - Serializers.testSerde(output, TransferActivation.serializer, activation, VERSION); + Serializers.testSerde(output, ActivationRequest.serializer, activation, VERSION); } } @Test public void testRoundtripCommitPhase() throws IOException { + Pair pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost()); TimeUUID planId = nextTimeUUID(); - MutationId transferId = new MutationId(2L, 300, 400); + ShortMutationId transferId = new ShortMutationId(2L, 300); NodeId coordinatorId = new NodeId(2); - TransferActivation.Phase phase = TransferActivation.Phase.COMMIT; + ActivationRequest.Phase phase = ActivationRequest.Phase.COMMIT; + String keyspace = "test_ks"; + Range range = new Range<>(new ByteOrderedPartitioner.BytesToken("key1".getBytes()), new ByteOrderedPartitioner.BytesToken("key100".getBytes())); - TransferActivation activation = new TransferActivation(planId, transferId, coordinatorId, phase); + ActivationRequest activation = new ActivationRequest(StreamOperation.IMPORT, pair, phase, transferId, coordinatorId, range, keyspace, planId); try (DataOutputBuffer output = new DataOutputBuffer()) { - Serializers.testSerde(output, TransferActivation.serializer, activation, VERSION); + Serializers.testSerde(output, ActivationRequest.serializer, activation, VERSION); } } } diff --git a/test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java b/test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java new file mode 100644 index 0000000000..14cc384e7f --- /dev/null +++ b/test/unit/org/apache/cassandra/replication/ActivationResponseSerializationTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.Serializers; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.Pair; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ActivationResponseSerializationTest +{ + private static final int VERSION = MessagingService.current_version; + + @BeforeClass + public static void setUpClass() + { + DatabaseDescriptor.daemonInitialization(); + } + + @Test + public void testRoundtripSuccess() throws IOException + { + Pair pair = Pair.create(InetAddressAndPort.getLocalHost(), InetAddressAndPort.getLocalHost()); + ActivationResponse response = new ActivationResponse(pair); + + try (DataOutputBuffer output = new DataOutputBuffer()) + { + Serializers.testSerde(output, ActivationResponse.serializer, response, VERSION); + } + } + + @Test + public void testRoundtripDifferentAddresses() throws IOException + { + InetAddressAndPort coordinator = InetAddressAndPort.getByName("127.0.0.1"); + InetAddressAndPort peer = InetAddressAndPort.getByName("127.0.0.2"); + Pair pair = Pair.create(coordinator, peer); + ActivationResponse response = new ActivationResponse(pair); + + try (DataOutputBuffer output = new DataOutputBuffer()) + { + Serializers.testSerde(output, ActivationResponse.serializer, response, VERSION); + } + } + + @Test + public void testEqualsAndHashCode() throws Exception + { + InetAddressAndPort addr1 = InetAddressAndPort.getByName("127.0.0.1"); + InetAddressAndPort addr2 = InetAddressAndPort.getByName("127.0.0.2"); + Pair pair1 = Pair.create(addr1, addr2); + Pair pair2 = Pair.create(addr1, addr2); + + ActivationResponse resp1 = new ActivationResponse(pair1); + ActivationResponse resp2 = new ActivationResponse(pair2); + + assertThat(resp1).isEqualTo(resp2); + assertThat(resp1.hashCode()).isEqualTo(resp2.hashCode()); + } +} diff --git a/test/unit/org/apache/cassandra/replication/LocalTransfersTest.java b/test/unit/org/apache/cassandra/replication/LocalTransfersTest.java deleted file mode 100644 index 465f280c49..0000000000 --- a/test/unit/org/apache/cassandra/replication/LocalTransfersTest.java +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.replication; - -import java.util.Collection; -import java.util.Collections; - -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.dht.Murmur3Partitioner; -import org.apache.cassandra.dht.Range; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.utils.TimeUUID; -import org.assertj.core.api.Assertions; - -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED; -import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP; -import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.Mockito.mock; - -public class LocalTransfersTest -{ - private LocalTransfers localTransfers; - private ShortMutationId transferId; - private TimeUUID planId; - - @BeforeClass - public static void setUpClass() - { - DatabaseDescriptor.daemonInitialization(); - DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); - } - - @Before - public void setUp() throws Exception - { - localTransfers = new LocalTransfers(); - transferId = new ShortMutationId(1, 100); - planId = nextTimeUUID(); - } - - private CoordinatedTransfer coordinatedTransfer(ShortMutationId transferId) - { - return coordinatedTransfer(transferId, new Range<>(tk(0), tk(1000))); - } - - private CoordinatedTransfer coordinatedTransfer(ShortMutationId transferId, Range range) - { - MutationId mutationId = transferId != null - ? new MutationId(transferId.logId(), transferId.offset(), (int) System.currentTimeMillis()) - : null; - return new CoordinatedTransfer(range, mutationId); - } - - private PendingLocalTransfer pendingTransfer(TimeUUID planId) - { - SSTableReader mockSSTable = mock(SSTableReader.class); - Collection sstables = Collections.singletonList(mockSSTable); - - return new PendingLocalTransfer(planId, sstables); - } - - private static Token tk(long token) - { - return new Murmur3Partitioner.LongToken(token); - } - - @Test - public void testSaveCoordinatedTransfer() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - localTransfers.save(transfer); - localTransfers.activating(transfer); - - CoordinatedTransfer loaded = localTransfers.getActivatedTransfer(transferId); - Assertions.assertThat(loaded).isEqualTo(transfer); - - assertThatThrownBy(() -> localTransfers.save(transfer)) - .isInstanceOf(IllegalStateException.class); - } - - @Test - public void testActivatingTransfer() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - localTransfers.save(transfer); - localTransfers.activating(transfer); - - CoordinatedTransfer retrieved = localTransfers.getActivatedTransfer(transferId); - assertThat(retrieved).isEqualTo(transfer); - } - - @Test - public void testReceivedTransfer() - { - TimeUUID planId = nextTimeUUID(); - PendingLocalTransfer transfer = pendingTransfer(planId); - localTransfers.received(transfer); - PendingLocalTransfer retrieved = localTransfers.getPendingTransfer(planId); - assertThat(retrieved).isEqualTo(transfer); - } - - @Test - public void testReceivedEmptyTransferThrows() - { - assertThatThrownBy(() -> new PendingLocalTransfer(planId, Collections.emptyList())) - .isInstanceOf(IllegalStateException.class); - } - - @Test - public void testGetPendingTransferNotFound() - { - PendingLocalTransfer retrieved = localTransfers.getPendingTransfer(planId); - assertThat(retrieved).isNull(); - } - - @Test - public void testGetActivatedTransferNotFound() - { - CoordinatedTransfer retrieved = localTransfers.getActivatedTransfer(transferId); - assertThat(retrieved).isNull(); - } - - @Test - public void testPurgingTransferNotStarted() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // All streams in INIT state - should NOT be purgeable (stream hasn't started yet) - CoordinatedTransfer.SingleTransferResult result = CoordinatedTransfer.SingleTransferResult.Init(); - transfer.streamResults.put(mock(InetAddressAndPort.class), result); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse(); - } - - @Test - public void testPurgingTransferAllStreamsComplete() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // All streams in STREAM_COMPLETE state - should NOT be purgeable (no failures) - CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID()); - CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID()); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse(); - } - - @Test - public void testPurgingTransferPrepareFailed() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(PREPARE_FAILED, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } - - @Test - public void testPurgingTransferAllActivationCommitted() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // All streams in ACTIVATE_COMMITTED state - should be purgeable (allComplete = true) - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } - - @Test - public void testPurgingTransferMixedCommittedAndNoop() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // Mix of ACTIVATE_COMMITTED and STREAM_NOOP - should be purgeable (allComplete = true) - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_NOOP, null); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } - - @Test - public void testPurgingTransferActivationPartialCommitted() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // One stream in ACTIVATE_PREPARING - should NOT be purgeable - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTING, planId); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse(); - } - - @Test - public void testPurgingTransferAllStreamsFailed() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // All streams in STREAM_FAILED state - should be purgeable (noneActivated = true) - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_FAILED, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_FAILED, planId); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } - - @Test - public void testPurgingTransferMixedInitAndFailed() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // Mix of INIT and STREAM_FAILED - should be purgeable (has failure, none activated) - CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.Init(); - CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID()); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } - - @Test - public void testPurgingTransferMixedCompleteAndFailed() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // Mix of STREAM_COMPLETE and STREAM_FAILED - should be purgeable (has failure, none activated) - CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.StreamComplete(nextTimeUUID()); - CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID()); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } - - @Test - public void testPurgingTransferMixedStreamingCompleteAndPreparing() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // Mix of STREAM_COMPLETE and ACTIVATE_PREPARING - should NOT be purgeable - // (noneActivated = false because of ACTIVATE_PREPARING, allComplete = false) - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(PREPARING, planId); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse(); - } - - @Test - public void testPurgingTransferMixedCommittingCommitted() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(COMMITTING, planId); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(COMMITTED, planId); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse(); - } - - @Test - public void testPurgingTransferWithNullTransferId() - { - CoordinatedTransfer transfer = coordinatedTransfer(null); - - // All streams complete but transferId is null - should NOT be purgeable - CoordinatedTransfer.SingleTransferResult result1 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, null); - CoordinatedTransfer.SingleTransferResult result2 = new CoordinatedTransfer.SingleTransferResult(STREAM_COMPLETE, null); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - // allComplete = true, but transferId is null, so should not purge - Assertions.assertThat(localTransfers.purger.test(transfer)).isFalse(); - } - - @Test - public void testPurgingTransferNoopOnly() - { - CoordinatedTransfer transfer = coordinatedTransfer(transferId); - - // All streams in STREAM_NOOP - should be purgeable (both noneActivated and allComplete are true) - CoordinatedTransfer.SingleTransferResult result1 = CoordinatedTransfer.SingleTransferResult.Noop(); - CoordinatedTransfer.SingleTransferResult result2 = CoordinatedTransfer.SingleTransferResult.Noop(); - - transfer.streamResults.put(mock(InetAddressAndPort.class), result1); - transfer.streamResults.put(mock(InetAddressAndPort.class), result2); - - Assertions.assertThat(localTransfers.purger.test(transfer)).isTrue(); - } -} diff --git a/test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java b/test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java new file mode 100644 index 0000000000..7d53a3f29b --- /dev/null +++ b/test/unit/org/apache/cassandra/replication/MutationTrackingServiceTest.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.repair.RepairJobDesc; +import org.apache.cassandra.repair.SharedContext; +import org.apache.cassandra.repair.SyncTask; +import org.apache.cassandra.repair.SyncTasks; +import org.apache.cassandra.repair.SymmetricRemoteSyncTask; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.TimeUUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class MutationTrackingServiceTest +{ + private static final String TEST_KEYSPACE = "test_ks"; + private static final String TEST_TABLE = "test_table"; + private static final InetAddressAndPort LOCAL = InetAddressAndPort.getByNameUnchecked("127.0.0.1"); + private static final InetAddressAndPort REMOTE = InetAddressAndPort.getByNameUnchecked("127.0.0.2"); + + @BeforeClass + public static void setup() throws IOException + { + DatabaseDescriptor.daemonInitialization(); + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(TEST_KEYSPACE, KeyspaceParams.simple(1, ReplicationType.tracked), SchemaLoader.standardCFMD(TEST_KEYSPACE, TEST_TABLE)); + } + + @Test + public void testAlignToShardBoundariesSingleTaskWithinSingleShard() + { + MutationTrackingService service = MutationTrackingService.TestAccess.create(); + + // Create a single shard covering a-z + Set> shardRanges = new HashSet<>(); + shardRanges.add(range("a", "z")); + MutationTrackingService.KeyspaceShards shards = + MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges); + MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards); + + // Input task completely within the shard + List inputTasks = Collections.singletonList(createSyncTask(range("d", "m"))); + + Keyspace keyspace = Keyspace.open(TEST_KEYSPACE); + SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks); + + // Should have one shard entry + AtomicInteger entries = new AtomicInteger(0); + result.apply((shardedTask) -> entries.incrementAndGet()); + assertEquals(1, entries.get()); + + // Get all tasks from the result + List allTasks = new ArrayList<>(); + result.apply((shardedTask) -> allTasks.add(shardedTask.task)); + + assertEquals(1, allTasks.size()); + + SyncTask resultTask = allTasks.get(0); + assertEquals(1, resultTask.rangesToSync.size()); + assertTrue(resultTask.rangesToSync.contains(range("d", "m"))); + + // Should have a transfer ID assigned + assertNotNull(resultTask.getTransferId()); + } + + @Test + public void testAlignToShardBoundariesSingleTaskSpanningMultipleShards() + { + MutationTrackingService service = MutationTrackingService.TestAccess.create(); + + // Create two shards + Set> shardRanges = new HashSet<>(); + shardRanges.add(range("a", "m")); + shardRanges.add(range("m", "z")); + MutationTrackingService.KeyspaceShards shards = + MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges); + MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards); + + // Input task spans both shards + List inputTasks = Collections.singletonList(createSyncTask(range("d", "s"))); + + Keyspace keyspace = Keyspace.open(TEST_KEYSPACE); + SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks); + + // Should be split into two shard entries + AtomicInteger entries = new AtomicInteger(0); + result.apply((shardedTask) -> entries.incrementAndGet()); + assertEquals(2, entries.get()); + + // Collect all ranges from all tasks + Set> allRanges = new HashSet<>(); + result.apply((shardedTask) -> allRanges.addAll(shardedTask.task.rangesToSync)); + + // Should contain the two split pieces + assertEquals(2, allRanges.size()); + assertTrue(allRanges.contains(range("d", "m"))); + assertTrue(allRanges.contains(range("m", "s"))); + } + + @Test + public void testAlignToShardBoundariesMultipleTasksAcrossMultipleShards() + { + MutationTrackingService service = MutationTrackingService.TestAccess.create(); + + // Create three shards + Set> shardRanges = new HashSet<>(); + shardRanges.add(range("a", "h")); + shardRanges.add(range("h", "p")); + shardRanges.add(range("p", "z")); + MutationTrackingService.KeyspaceShards shards = + MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges); + MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards); + + // Multiple tasks, some spanning shards + List inputTasks = Arrays.asList( + createSyncTask(range("b", "e")), // Within shard 1 + createSyncTask(range("f", "j")), // Spans shard 1 and 2 + createSyncTask(range("q", "s")) // Within shard 3 + ); + + Keyspace keyspace = Keyspace.open(TEST_KEYSPACE); + SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks); + + // Should have 4 entries (one per sync task) + AtomicInteger entries = new AtomicInteger(0); + result.apply((shardedTask) -> entries.incrementAndGet()); + assertEquals(4, entries.get()); + + // Collect all ranges from all tasks + Set> allRanges = new HashSet<>(); + result.apply((shardedTask) -> allRanges.addAll(shardedTask.task.rangesToSync)); + + // Should have split ranges: b-e, f-h (shard 1), h-j (shard 2), q-s (shard 3) + assertTrue(allRanges.contains(range("b", "e"))); + assertTrue(allRanges.contains(range("f", "h"))); + assertTrue(allRanges.contains(range("h", "j"))); + assertTrue(allRanges.contains(range("q", "s"))); + } + + @Test + public void testAlignToShardBoundariesTaskWithMultipleRangesSpanningShards() + { + MutationTrackingService service = MutationTrackingService.TestAccess.create(); + + // Create two shards + Set> shardRanges = new HashSet<>(); + shardRanges.add(range("a", "m")); + shardRanges.add(range("m", "z")); + MutationTrackingService.KeyspaceShards shards = + MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges); + MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards); + + // Single task with multiple ranges spanning both shards + List> ranges = Arrays.asList( + range("b", "e"), // Shard 1 + range("f", "p") // Spans both shards + ); + + SyncTask task = createSyncTask(ranges, LOCAL, REMOTE); + List inputTasks = Collections.singletonList(task); + + Keyspace keyspace = Keyspace.open(TEST_KEYSPACE); + SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks); + + // Should be split into two shard entries + AtomicInteger entries = new AtomicInteger(0); + result.forEach((shardedTask) -> entries.incrementAndGet()); + assertEquals(2, entries.get()); + + // Collect all ranges + Set> allRanges = new HashSet<>(); + result.forEach((shardedTask) -> allRanges.addAll(shardedTask.rangesToSync)); + + // Should have: b-e, f-m (shard 1), m-p (shard 2) + assertTrue(allRanges.contains(range("b", "e"))); + assertTrue(allRanges.contains(range("f", "m"))); + assertTrue(allRanges.contains(range("m", "p"))); + } + + @Test + public void testAlignToShardBoundariesPreservesTaskType() + { + MutationTrackingService service = MutationTrackingService.TestAccess.create(); + + // Create two shards + Set> shardRanges = new HashSet<>(); + shardRanges.add(range("a", "m")); + shardRanges.add(range("m", "z")); + MutationTrackingService.KeyspaceShards shards = + MutationTrackingService.TestAccess.createTestKeyspaceShards(TEST_KEYSPACE, shardRanges); + MutationTrackingService.TestAccess.setKeyspaceShards(service, TEST_KEYSPACE, shards); + + // Task spanning both shards + List inputTasks = Collections.singletonList(createSyncTask(range("d", "s"))); + + Keyspace keyspace = Keyspace.open(TEST_KEYSPACE); + SyncTasks result = service.alignToShardBoundaries(keyspace, inputTasks); + + // All resulting tasks should be the same type as the input + result.apply((shardedTask) -> assertTrue("Task should be SymmetricRemoteSyncTask", shardedTask.task instanceof SymmetricRemoteSyncTask)); + } + + private static Token tk(String key) + { + return new ByteOrderedPartitioner.BytesToken(ByteBufferUtil.bytes(key)); + } + + private static Range range(String left, String right) + { + return new Range<>(tk(left), tk(right)); + } + + private static SyncTask createSyncTask(Range range) + { + return createSyncTask(Collections.singletonList(range), LOCAL, REMOTE); + } + + private static SyncTask createSyncTask(List> ranges, InetAddressAndPort local, InetAddressAndPort remote) + { + SharedContext ctx = SharedContext.Global.instance; + TimeUUID sessionId = TimeUUID.Generator.nextTimeUUID(); + RepairJobDesc desc = new RepairJobDesc(sessionId, TimeUUID.Generator.nextTimeUUID(),TEST_KEYSPACE, TEST_TABLE, ranges); + return new SymmetricRemoteSyncTask(ctx, desc, local, remote, ranges, PreviewKind.NONE, null); + } +} diff --git a/test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java b/test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java new file mode 100644 index 0000000000..159dd3cffe --- /dev/null +++ b/test/unit/org/apache/cassandra/replication/TransferTrackingServiceTest.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.util.Collection; +import java.util.Collections; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.TimeUUID; +import org.assertj.core.api.Assertions; + +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTED; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.COMMITTING; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARE_FAILED; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.PREPARING; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_COMPLETE; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_FAILED; +import static org.apache.cassandra.replication.CoordinatedTransfer.SingleTransferResult.State.STREAM_NOOP; +import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +public class TransferTrackingServiceTest +{ + private TransferTrackingService transferTrackingService; + private ShortMutationId transferId; + private TimeUUID planId; + + @BeforeClass + public static void setUpClass() + { + DatabaseDescriptor.daemonInitialization(); + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + } + + @Before + public void setUp() throws Exception + { + transferTrackingService = new TransferTrackingService(); + transferId = new ShortMutationId(1, 100); + planId = nextTimeUUID(); + } + + private TrackedImportTransfer coordinatedTransfer(ShortMutationId transferId) + { + return coordinatedTransfer(transferId, new Range<>(tk(0), tk(1000))); + } + + private TrackedImportTransfer coordinatedTransfer(ShortMutationId transferId, Range range) + { + MutationId mutationId = transferId != null + ? new MutationId(transferId.logId(), transferId.offset(), (int) System.currentTimeMillis()) + : null; + return new TrackedImportTransfer(range, mutationId); + } + + private PendingLocalTransfer pendingTransfer(TimeUUID planId) + { + SSTableReader mockSSTable = mock(SSTableReader.class); + Collection sstables = Collections.singletonList(mockSSTable); + + return new PendingLocalTransfer(planId, sstables); + } + + private static Token tk(long token) + { + return new Murmur3Partitioner.LongToken(token); + } + + @Test + public void testSaveCoordinatedTransfer() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + transferTrackingService.save(transfer); + + CoordinatedTransfer loaded = transferTrackingService.getActivatedTransfer(transferId); + Assertions.assertThat(loaded).isEqualTo(transfer); + + assertThatThrownBy(() -> transferTrackingService.save(transfer)) + .isInstanceOf(IllegalStateException.class); + } + + @Test + public void testActivatingTransfer() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + transferTrackingService.save(transfer); + + CoordinatedTransfer retrieved = transferTrackingService.getActivatedTransfer(transferId); + assertThat(retrieved).isEqualTo(transfer); + } + + @Test + public void testReceivedTransfer() + { + TimeUUID planId = nextTimeUUID(); + PendingLocalTransfer transfer = pendingTransfer(planId); + transferTrackingService.received(transfer); + PendingLocalTransfer retrieved = transferTrackingService.getPendingTransfer(planId); + assertThat(retrieved).isEqualTo(transfer); + } + + @Test + public void testReceivedEmptyTransferThrows() + { + assertThatThrownBy(() -> new PendingLocalTransfer(planId, Collections.emptyList())) + .isInstanceOf(IllegalStateException.class); + } + + @Test + public void testGetPendingTransferNotFound() + { + PendingLocalTransfer retrieved = transferTrackingService.getPendingTransfer(planId); + assertThat(retrieved).isNull(); + } + + @Test + public void testGetActivatedTransferNotFound() + { + CoordinatedTransfer retrieved = transferTrackingService.getActivatedTransfer(transferId); + assertThat(retrieved).isNull(); + } + + @Test + public void testPurgingTransferNotStarted() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // All streams in INIT state - should NOT be purgeable (stream hasn't started yet) + TrackedImportTransfer.SingleTransferResult result = TrackedImportTransfer.SingleTransferResult.Init(); + transfer.streamResults.put(mock(Pair.class), result); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse(); + } + + @Test + public void testPurgingTransferAllStreamsComplete() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // All streams in STREAM_COMPLETE state - should NOT be purgeable (no failures) + TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID()); + TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID()); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse(); + } + + @Test + public void testPurgingTransferPrepareFailed() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(PREPARE_FAILED, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } + + @Test + public void testPurgingTransferAllActivationCommitted() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // All streams in ACTIVATE_COMMITTED state - should be purgeable (allComplete = true) + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } + + @Test + public void testPurgingTransferMixedCommittedAndNoop() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // Mix of ACTIVATE_COMMITTED and STREAM_NOOP - should be purgeable (allComplete = true) + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_NOOP, null); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } + + @Test + public void testPurgingTransferActivationPartialCommitted() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // One stream in ACTIVATE_PREPARING - should NOT be purgeable + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTING, planId); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse(); + } + + @Test + public void testPurgingTransferAllStreamsFailed() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // All streams in STREAM_FAILED state - should be purgeable (noneActivated = true) + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_FAILED, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_FAILED, planId); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } + + @Test + public void testPurgingTransferMixedInitAndFailed() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // Mix of INIT and STREAM_FAILED - should be purgeable (has failure, none activated) + TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.Init(); + TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID()); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } + + @Test + public void testPurgingTransferMixedCompleteAndFailed() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // Mix of STREAM_COMPLETE and STREAM_FAILED - should be purgeable (has failure, none activated) + TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.StreamComplete(nextTimeUUID()); + TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Init().streamFailed(nextTimeUUID()); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } + + @Test + public void testPurgingTransferMixedStreamingCompleteAndPreparing() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // Mix of STREAM_COMPLETE and ACTIVATE_PREPARING - should NOT be purgeable + // (noneActivated = false because of ACTIVATE_PREPARING, allComplete = false) + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(PREPARING, planId); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse(); + } + + @Test + public void testPurgingTransferMixedCommittingCommitted() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(COMMITTING, planId); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(COMMITTED, planId); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse(); + } + + @Test + public void testPurgingTransferWithNullTransferId() + { + TrackedImportTransfer transfer = coordinatedTransfer(null); + + // All streams complete but transferId is null - should NOT be purgeable + TrackedImportTransfer.SingleTransferResult result1 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, null); + TrackedImportTransfer.SingleTransferResult result2 = new TrackedImportTransfer.SingleTransferResult(STREAM_COMPLETE, null); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + // allComplete = true, but transferId is null, so should not purge + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isFalse(); + } + + @Test + public void testPurgingTransferNoopOnly() + { + TrackedImportTransfer transfer = coordinatedTransfer(transferId); + + // All streams in STREAM_NOOP - should be purgeable (both noneActivated and allComplete are true) + TrackedImportTransfer.SingleTransferResult result1 = TrackedImportTransfer.SingleTransferResult.Noop(); + TrackedImportTransfer.SingleTransferResult result2 = TrackedImportTransfer.SingleTransferResult.Noop(); + + transfer.streamResults.put(mock(Pair.class), result1); + transfer.streamResults.put(mock(Pair.class), result2); + + Assertions.assertThat(transferTrackingService.purger.test(transfer)).isTrue(); + } +} diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index b7720e051c..c64d55f46a 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -222,15 +222,7 @@ public class MockSchema first.retainable().getKey().slice(), last.retainable().getKey().slice()); StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS); - try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE)) - { - new StatsComponent(metadataComponents).save(descriptor); - out.flush(); - } - catch (IOException ioe) - { - throw new RuntimeException(ioe); - } + writeStatsMetadata(descriptor, metadataComponents); BigTableReader reader = new BigTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) .setDataFile(fileHandle.sharedCopy()) @@ -253,7 +245,7 @@ public class MockSchema } else if (BtiFormat.is(format)) { - Set components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC); + Set components = ImmutableSet.of(Components.DATA, BtiFormat.Components.PARTITION_INDEX, BtiFormat.Components.ROW_INDEX, Components.FILTER, Components.TOC, Components.STATS); for (Component component : components) { File file = descriptor.fileFor(component); @@ -268,9 +260,17 @@ public class MockSchema collector.update(DeletionTime.build(timestamp, minLocalDeletionTime)); BufferDecoratedKey first = readerBounds(firstToken); BufferDecoratedKey last = readerBounds(lastToken); - StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, ImmutableCoordinatorLogOffsets.NONE, header, first.retainable().getKey(), last.retainable().getKey()) - .get(MetadataType.STATS); + Map metadataComponents = collector.sstableLevel(level) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), + 0.01f, + UNREPAIRED_SSTABLE, + null, + ImmutableCoordinatorLogOffsets.NONE, + header, + first.retainable().getKey(), + last.retainable().getKey()); + StatsMetadata statsMetadata = (StatsMetadata) metadataComponents.get(MetadataType.STATS); + writeStatsMetadata(descriptor, metadataComponents); BtiTableReader reader = new BtiTableReader.Builder(descriptor).setComponents(components) .setTableMetadataRef(cfs.metadata) .setDataFile(fileHandle.sharedCopy()) @@ -278,7 +278,7 @@ public class MockSchema .setRowIndexFile(fileHandle.sharedCopy()) .setFilter(FilterFactory.AlwaysPresent) .setMaxDataAge(1L) - .setStatsMetadata(metadata) + .setStatsMetadata(statsMetadata) .setOpenReason(SSTableReader.OpenReason.NORMAL) .setSerializationHeader(header) .setFirst(readerBounds(firstToken)) @@ -295,6 +295,19 @@ public class MockSchema } } + private static void writeStatsMetadata(Descriptor descriptor, Map metadataComponents) + { + try (DataOutputStreamPlus out = descriptor.fileFor(Components.STATS).newOutputStream(File.WriteMode.OVERWRITE)) + { + new StatsComponent(metadataComponents).save(descriptor); + out.flush(); + } + catch (IOException ioe) + { + throw new RuntimeException(ioe); + } + } + private static void maybeSetDataLength(Descriptor descriptor, long size) { if (size > 0) diff --git a/test/unit/org/apache/cassandra/service/SerializationsTest.java b/test/unit/org/apache/cassandra/service/SerializationsTest.java index 00864f5a34..89de9c2fcd 100644 --- a/test/unit/org/apache/cassandra/service/SerializationsTest.java +++ b/test/unit/org/apache/cassandra/service/SerializationsTest.java @@ -192,7 +192,8 @@ public class SerializationsTest extends AbstractSerializationsTester InetAddressAndPort src = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.2", PORT); InetAddressAndPort dest = InetAddressAndPort.getByNameOverrideDefaults("127.0.0.3", PORT); - SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE, false); + // TODO: Do we want to test with a transfer ID? + SyncRequest message = new SyncRequest(DESC, local, src, dest, Collections.singleton(FULL_RANGE), PreviewKind.NONE, false, null); testRepairMessageWrite("service.SyncRequest.bin", SyncRequest.serializer, message); } @@ -228,9 +229,9 @@ public class SerializationsTest extends AbstractSerializationsTester Lists.newArrayList(new StreamSummary(TABLE_ID, emptyList(), 5, 100)), Lists.newArrayList(new StreamSummary(TABLE_ID, emptyList(), 500, 10)) )); - SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries); + SyncResponse success = new SyncResponse(DESC, src, dest, true, summaries, RANDOM_UUID, null); // sync fail - SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList()); + SyncResponse fail = new SyncResponse(DESC, src, dest, false, emptyList(), null, null); testRepairMessageWrite("service.SyncComplete.bin", SyncResponse.serializer, success, fail); }