diff --git a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java index 21e4292292..76bcd86e71 100644 --- a/src/java/org/apache/cassandra/batchlog/BatchlogManager.java +++ b/src/java/org/apache/cassandra/batchlog/BatchlogManager.java @@ -48,7 +48,6 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.cql3.UntypedResultSet.Row; import org.apache.cassandra.db.ColumnFamilyStore; -import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.SystemKeyspace; @@ -60,20 +59,17 @@ import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.RetryOnDifferentSystemException; import org.apache.cassandra.exceptions.WriteFailureException; import org.apache.cassandra.exceptions.WriteTimeoutException; -import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.hints.Hint; import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaLayout; -import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.Replicas; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.PreserveTimestamp; @@ -609,22 +605,21 @@ public class BatchlogManager implements BatchlogManagerMBean String ks = mutation.getKeyspaceName(); Token tk = mutation.key().getToken(); ClusterMetadata metadata = ClusterMetadata.current(); - KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaceMetadata(ks); + Keyspace keyspace = Keyspace.open(ks); // TODO: this logic could do with revisiting at some point, as it is unclear what its rationale is // we perform a local write, ignoring errors and inline in this thread (potentially slowing replay down) // effectively bumping CL for locally owned writes and also potentially stalling log replay if an error occurs - // once we decide how it should work, it can also probably be simplified, and avoid constructing a ReplicaPlan directly - ReplicaLayout.ForTokenWrite allReplias = ReplicaLayout.forTokenWriteLiveAndDown(metadata, keyspaceMetadata, tk); - ReplicaPlan.ForWrite replicaPlan = forReplayMutation(metadata, Keyspace.open(ks), tk); + ReplicaLayout.ForTokenWrite allReplicas = ReplicaLayout.forTokenWriteLiveAndDown(metadata, keyspace.getMetadata(), tk); + CoordinationPlan.ForWrite replayPlan = CoordinationPlan.forReplayMutation(metadata, keyspace, tk); - Replica selfReplica = allReplias.all().selfIfPresent(); + Replica selfReplica = allReplicas.all().selfIfPresent(); if (selfReplica != null) mutation.apply(); - for (Replica replica : allReplias.all()) + for (Replica replica : allReplicas.all()) { - if (replica == selfReplica || replicaPlan.liveAndDown().contains(replica)) + if (replica == selfReplica || replayPlan.replicas().liveAndDown().contains(replica)) continue; UUID hostId = metadata.directory.peerId(replica.endpoint()).toUUID(); @@ -635,26 +630,13 @@ public class BatchlogManager implements BatchlogManagerMBean } } - ReplayWriteResponseHandler handler = new ReplayWriteResponseHandler<>(replicaPlan, mutation, Dispatcher.RequestTime.forImmediateExecution()); + ReplayWriteResponseHandler handler = new ReplayWriteResponseHandler<>(replayPlan, mutation, Dispatcher.RequestTime.forImmediateExecution()); Message message = Message.outWithFlag(MUTATION_REQ, mutation, MessageFlag.CALL_BACK_ON_FAILURE); - for (Replica replica : replicaPlan.liveAndDown()) + for (Replica replica : replayPlan.replicas().liveAndDown()) MessagingService.instance().sendWriteWithCallback(message, replica, handler); return handler; } - public static ReplicaPlan.ForWrite forReplayMutation(ClusterMetadata metadata, Keyspace keyspace, Token token) - { - ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWriteLiveAndDown(metadata, keyspace.getMetadata(), token); - Replicas.temporaryAssertFull(liveAndDown.all()); // TODO in CASSANDRA-14549 - - Replica selfReplica = liveAndDown.all().selfIfPresent(); - ReplicaLayout.ForTokenWrite liveRemoteOnly = liveAndDown.filter(r -> FailureDetector.isReplicaAlive.test(r) && r != selfReplica); - - return new ReplicaPlan.ForWrite(keyspace, liveAndDown.replicationStrategy(), - ConsistencyLevel.ONE, liveRemoteOnly.pending(), liveRemoteOnly.all(), liveRemoteOnly.all(), liveRemoteOnly.all(), - (cm) -> forReplayMutation(cm, keyspace, token), - metadata.epoch); - } private static int gcgs(Collection mutations) { int gcgs = Integer.MAX_VALUE; @@ -672,16 +654,10 @@ public class BatchlogManager implements BatchlogManagerMBean private final Set undelivered = Collections.newSetFromMap(new ConcurrentHashMap<>()); // TODO: should we be hinting here, since presumably batch log will retry? Maintaining historical behaviour for the moment. - ReplayWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) + ReplayWriteResponseHandler(CoordinationPlan.ForWrite coordinationPlan, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) { - super(replicaPlan, null, WriteType.UNLOGGED_BATCH, hintOnFailure, requestTime); - Iterables.addAll(undelivered, replicaPlan.contacts().endpoints()); - } - - @Override - protected int blockFor() - { - return this.replicaPlan.contacts().size(); + super(coordinationPlan, null, WriteType.UNLOGGED_BATCH, hintOnFailure, requestTime); + Iterables.addAll(undelivered, replicaPlan().contacts().endpoints()); } @Override diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java index 818b79d237..ffa75d7208 100644 --- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java +++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java @@ -20,30 +20,48 @@ package org.apache.cassandra.locator; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; +import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; +import javax.annotation.Nullable; + +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.index.Index; import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; +import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.AbstractWriteResponseHandler; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.DatacenterSyncWriteResponseHandler; import org.apache.cassandra.service.DatacenterWriteResponseHandler; import org.apache.cassandra.service.WriteResponseHandler; +import org.apache.cassandra.service.paxos.Paxos; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tcm.compatibility.TokenRingUtils; @@ -52,6 +70,8 @@ import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; +import static org.apache.cassandra.locator.ReplicaLayout.forTokenWriteLiveAndDown; + /** * A abstract parent for all replication strategies. */ @@ -92,57 +112,49 @@ public abstract class AbstractReplicationStrategy public abstract DataPlacement calculateDataPlacement(Epoch epoch, List> ranges, ClusterMetadata metadata); - public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, + public AbstractWriteResponseHandler getWriteResponseHandler(CoordinationPlan.ForWrite coordinationPlan, + CoordinationPlan.ForWrite idealPlan, Runnable callback, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) - { - return getWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, - requestTime, DatabaseDescriptor.getIdealConsistencyLevel()); - } - - public AbstractWriteResponseHandler getWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, - Runnable callback, - WriteType writeType, - Supplier hintOnFailure, - Dispatcher.RequestTime requestTime, - ConsistencyLevel idealConsistencyLevel) { AbstractWriteResponseHandler resultResponseHandler; - if (replicaPlan.consistencyLevel().isDatacenterLocal()) + if (coordinationPlan.consistencyLevel().isDatacenterLocal()) { // block for in this context will be localnodes block. - resultResponseHandler = new DatacenterWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, requestTime); + resultResponseHandler = new DatacenterWriteResponseHandler(coordinationPlan, callback, writeType, hintOnFailure, requestTime); } - else if (replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy)) + else if (coordinationPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM && (this instanceof NetworkTopologyStrategy)) { - resultResponseHandler = new DatacenterSyncWriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, requestTime); + resultResponseHandler = new DatacenterSyncWriteResponseHandler(coordinationPlan, callback, writeType, hintOnFailure, requestTime); } else { - resultResponseHandler = new WriteResponseHandler(replicaPlan, callback, writeType, hintOnFailure, requestTime); + resultResponseHandler = new WriteResponseHandler(coordinationPlan, callback, writeType, hintOnFailure, requestTime); } //Check if tracking the ideal consistency level is configured - if (idealConsistencyLevel != null) + if (idealPlan != null) { //If ideal and requested are the same just use this handler to track the ideal consistency level //This is also used so that the ideal consistency level handler when constructed knows it is the ideal //one for tracking purposes - if (idealConsistencyLevel == replicaPlan.consistencyLevel()) + if (coordinationPlan.consistencyLevel() == idealPlan.consistencyLevel()) { resultResponseHandler.setIdealCLResponseHandler(resultResponseHandler); } else { - //Construct a delegate response handler to use to track the ideal consistency level - AbstractWriteResponseHandler idealHandler = getWriteResponseHandler(replicaPlan.withConsistencyLevel(idealConsistencyLevel), + // Construct a delegate response handler to track the ideal consistency level. + // We pass idealPlan twice so that the recursive call sees coordinationPlan == idealPlan, + // causing the ideal handler to set itself as its own idealCLDelegate. This is required + // for the idealCLWriteLatency metric to be recorded (only fires when idealCLDelegate == this). + AbstractWriteResponseHandler idealHandler = getWriteResponseHandler(idealPlan, idealPlan, callback, writeType, hintOnFailure, - requestTime, - idealConsistencyLevel); + requestTime); resultResponseHandler.setIdealCLResponseHandler(idealHandler); } } @@ -150,6 +162,16 @@ public abstract class AbstractReplicationStrategy return resultResponseHandler; } + + public AbstractWriteResponseHandler getWriteResponseHandler(CoordinationPlan.ForWriteWithIdeal forWritePlan, + Runnable callback, + WriteType writeType, + Supplier hintOnFailure, + Dispatcher.RequestTime requestTime) + { + return getWriteResponseHandler(forWritePlan, forWritePlan.ideal, callback, writeType, hintOnFailure, requestTime); + } + /** * calculate the RF based on strategy_options. When overwriting, ensure that this get() * is FAST, as this is called often. @@ -437,4 +459,354 @@ public abstract class AbstractReplicationStrategy return newRanges; } } + + protected CoordinationPlan.ForWrite planForWriteInternal(ClusterMetadata metadata, + Keyspace keyspace, + ConsistencyLevel consistencyLevel, + Function liveAndDown, + ReplicaPlans.Selector selector) + { + ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(metadata, keyspace, consistencyLevel, liveAndDown, selector); + ResponseTracker tracker = createTrackerForWrite(consistencyLevel, plan, plan.pending, metadata); + return new CoordinationPlan.ForWrite(plan, tracker); + } + + protected ReplicaPlan.ForWrite createReplicaPlanForWrite(ClusterMetadata metadata, + Keyspace keyspace, + ConsistencyLevel consistencyLevel, + Function liveAndDown, + ReplicaPlans.Selector selector) + { + return ReplicaPlans.forWrite(metadata, keyspace, consistencyLevel, liveAndDown, selector); + } + + public CoordinationPlan.ForWriteWithIdeal planForWrite(ClusterMetadata metadata, + Keyspace keyspace, + ConsistencyLevel consistencyLevel, + Function liveAndDown, + ReplicaPlans.Selector selector) + { + ReplicaPlan.ForWrite plan = createReplicaPlanForWrite(metadata, keyspace, consistencyLevel, liveAndDown, selector); + ResponseTracker tracker = createTrackerForWrite(consistencyLevel, plan, plan.pending, metadata); + + CoordinationPlan.ForWrite ideal = null; + ConsistencyLevel idealCL = DatabaseDescriptor.getIdealConsistencyLevel(); + if (idealCL != null) + { + if (idealCL == consistencyLevel) + { + ideal = new CoordinationPlan.ForWrite(plan, tracker); + } + else + { + ideal = new CoordinationPlan.ForWrite(createReplicaPlanForWrite(metadata, keyspace, idealCL, liveAndDown, selector), + createTrackerForWrite(idealCL, plan, plan.pending, metadata)); + } + } + + return new CoordinationPlan.ForWriteWithIdeal(plan, tracker, ideal); + } + + public CoordinationPlan.ForWriteWithIdeal planForWrite(ClusterMetadata metadata, + Keyspace keyspace, + ConsistencyLevel consistencyLevel, + Token token, + ReplicaPlans.Selector selector) + { + return planForWrite(metadata, keyspace, consistencyLevel, + (newClusterMetadata) -> ReplicaLayout.forTokenWriteLiveAndDown(newClusterMetadata, keyspace, token), selector); + } + + /** + * Create coordination plan for forwarding a counter write to the leader replica. + * + * In cases where the original coordinator is not a replica of the counter key, the counter + * mutation is forwarded to a leader replica that will coordinate the actual counter update. + */ + public CoordinationPlan.ForWrite planForForwardingCounterWrite(ClusterMetadata metadata, + Keyspace keyspace, + Token token, + Function replicaSupplier) + { + ReplicaPlan.ForWrite plan = ReplicaPlans.forSingleReplicaWrite(metadata, keyspace, token, replicaSupplier); + ResponseTracker tracker = createTrackerForWrite(plan.consistencyLevel(), plan, plan.pending, metadata); + + return new CoordinationPlan.ForWriteWithIdeal(plan, tracker, null); + } + + /** + * Create coordination plan for replaying a mutation from the batchlog. + * + * When recovering failed batches, mutations are replayed to remote replicas only + * (local replica is handled separately). This method creates a replica plan + * targeting live remote replicas with CL.ONE, and a response tracker that waits on + * all contacts + */ + public CoordinationPlan.ForWriteWithIdeal planForReplayMutation(ClusterMetadata metadata, + Keyspace keyspace, + Token token) + { + Preconditions.checkState(!replicationType.isTracked(), "Batch replay not supported with tracked keyspaces"); + + ReplicaPlan.ForWrite plan = ReplicaPlans.forReplayMutation(metadata, keyspace, token); + + // wait until all contacts respond + int blockFor = plan.contacts().size(); + ResponseTracker tracker = new SimpleResponseTracker(blockFor, blockFor); + + return new CoordinationPlan.ForWriteWithIdeal(plan, tracker, null); + } + + /** + * Create coordination plan for a single-partition token read. + */ + public CoordinationPlan.ForTokenRead planForTokenRead(ClusterMetadata metadata, + Keyspace keyspace, + TableId tableId, + Token token, + @Nullable Index.QueryPlan indexQueryPlan, + ConsistencyLevel consistencyLevel, + SpeculativeRetryPolicy retry, + ReadCoordinator coordinator) + { + ReplicaPlan.ForTokenRead plan = ReplicaPlans.forRead(metadata, keyspace, tableId, token, indexQueryPlan, consistencyLevel, retry, coordinator); + ReplicaPlan.SharedForTokenRead shared = ReplicaPlan.shared(plan); + ResponseTracker tracker = createTrackerForRead(plan); + return new CoordinationPlan.ForTokenRead(shared, tracker); + } + + /** + * Create coordination plan for a range read. + */ + public CoordinationPlan.ForRangeRead planForRangeRead(ClusterMetadata metadata, + Keyspace keyspace, + TableId tableId, + @Nullable Index.QueryPlan indexQueryPlan, + ConsistencyLevel consistencyLevel, + AbstractBounds range, + int vnodeCount) + { + ReplicaPlan.ForRangeRead plan = ReplicaPlans.forRangeRead(metadata, keyspace, tableId, indexQueryPlan, consistencyLevel, range, vnodeCount, true); + ReplicaPlan.SharedForRangeRead shared = ReplicaPlan.shared(plan); + ResponseTracker tracker = createTrackerForRead(plan); + return new CoordinationPlan.ForRangeRead(shared, tracker); + } + + /** + * Attempt to merge two adjacent range read coordination plans into one. + * + * If the two plans share enough live endpoints to satisfy the consistency level + * and the merge is worthwhile returns a merged plan otherwise returns null. + */ + public CoordinationPlan.ForRangeRead maybeMergeRangeReads(ClusterMetadata metadata, + Keyspace keyspace, + TableId tableId, + ConsistencyLevel consistencyLevel, + ReplicaPlan.ForRangeRead left, + ReplicaPlan.ForRangeRead right) + { + ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(metadata, keyspace, tableId, consistencyLevel, left, right); + if (merged == null) + return null; + + ReplicaPlan.SharedForRangeRead shared = ReplicaPlan.shared(merged); + ResponseTracker tracker = createTrackerForRead(merged); + return new CoordinationPlan.ForRangeRead(shared, tracker); + } + + /** + * Create coordination plan for a full range read + */ + public CoordinationPlan.ForRangeRead planForFullRangeRead(Keyspace keyspace, + ConsistencyLevel consistencyLevel, + AbstractBounds range, + Set endpointsToContact, + int vnodeCount) + { + ReplicaPlan.ForRangeRead plan = ReplicaPlans.forFullRangeRead(keyspace, consistencyLevel, range, endpointsToContact, vnodeCount); + ReplicaPlan.SharedForRangeRead shared = ReplicaPlan.shared(plan); + ResponseTracker tracker = createTrackerForRead(plan); + return new CoordinationPlan.ForRangeRead(shared, tracker); + } + + /** + * Create coordination plan for a single-replica token read. + */ + public CoordinationPlan.ForTokenRead planForSingleReplicaTokenRead(Keyspace keyspace, Token token, Replica replica) + { + ReplicaPlan.ForTokenRead plan = ReplicaPlans.forSingleReplicaRead(keyspace, token, replica); + ReplicaPlan.SharedForTokenRead shared = ReplicaPlan.shared(plan); + ResponseTracker tracker = createTrackerForRead(plan); + return new CoordinationPlan.ForTokenRead(shared, tracker); + } + + /** + * Create coordination plan for a single-replica range read. + * + * Used by short read protection to fetch additional partitions from a + * specific replica. blockFor=1, totalReplicas=1. + */ + public CoordinationPlan.ForRangeRead planForSingleReplicaRangeRead(Keyspace keyspace, + AbstractBounds range, + Replica replica, + int vnodeCount) + { + ReplicaPlan.ForRangeRead plan = ReplicaPlans.forSingleReplicaRead(keyspace, range, replica, vnodeCount); + ReplicaPlan.SharedForRangeRead shared = ReplicaPlan.shared(plan); + ResponseTracker tracker = createTrackerForRead(plan); + return new CoordinationPlan.ForRangeRead(shared, tracker); + } + + /** + * Create ResponseTracker for read operation. + */ + private , P extends ReplicaPlan.ForRead> ResponseTracker createTrackerForRead(P plan) + { + int blockFor = plan.readQuorum(); + + // Use candidates.size() for totalReplicas to allow for speculation + // (speculation can contact additional candidates beyond initial contacts) + int totalReplicas = plan.readCandidates().size(); + + return new SimpleResponseTracker(blockFor, totalReplicas); + } + + public Paxos.Participants paxosParticipants(ClusterMetadata metadata, + TableMetadata table, + Token token, + ConsistencyLevel consistencyForConsensus, + Predicate isReplicaAlive) + { + + KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaceMetadata(table.keyspace); + // MetaStrategy distributes the entire keyspace to all replicas. In addition, its tables (currently only + // the dist log table) don't use the globally configured partitioner. For these reasons we don't lookup the + // replicas using the supplied token as this can actually be of the incorrect type (for example when + // performing Paxos repair). + final Token actualToken = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : token; + ReplicaLayout.ForTokenWrite all = forTokenWriteLiveAndDown(metadata, keyspaceMetadata, actualToken); + ReplicaLayout.ForTokenWrite electorate = consistencyForConsensus.isDatacenterLocal() + ? all.filter(InOurDc.replicas()) : all; + + EndpointsForToken live = all.all().filter(isReplicaAlive); + return new Paxos.Participants(metadata.epoch, Keyspace.open(table.keyspace), consistencyForConsensus, all, electorate, live, + (cm) -> Paxos.Participants.get(cm, table, actualToken, consistencyForConsensus)); + } + + /** + * Create ResponseTracker for write operation based on consistency level. + */ + @VisibleForTesting + public ResponseTracker createTrackerForWrite(ConsistencyLevel cl, ReplicaPlan.ForWrite plan, Endpoints pending, ClusterMetadata metadata) + { + switch (cl) + { + case ANY: + case ONE: + case TWO: + case THREE: + case QUORUM: + case ALL: + { + int totalContacts = plan.contacts().size(); + int baseBlockFor = cl.blockFor(this); + int totalBlockFor = cl.blockForWrite(this, pending); + + // Check if double count model applies (some CLs like ANY don't add pending) + // If totalBlockFor == baseBlockFor, no double-count needed (e.g., ANY) + if (totalBlockFor == baseBlockFor) + return new SimpleResponseTracker(baseBlockFor, totalContacts); + + // Double count model: natural must satisfy base CL, total must include pending + int pendingReplicas = pending.size(); + // contacts() includes both natural and pending replicas + int naturalReplicas = totalContacts - pendingReplicas; + return new WriteResponseTracker(baseBlockFor, totalBlockFor, + naturalReplicas, pendingReplicas, + endpoint -> pending.endpoints().contains(endpoint)); + } + + case LOCAL_ONE: + case LOCAL_QUORUM: + { + int localContacts = plan.contacts().filter(InOurDc.replicas()).size(); + // Check if double count model applies (depends on local pending) + int baseBlockFor = cl.blockFor(this); + int totalBlockFor = cl.blockForWrite(this, pending); + + // If totalBlockFor == baseBlockFor, no local pending so no double-count needed + if (totalBlockFor == baseBlockFor) + return new SimpleResponseTracker(baseBlockFor, localContacts, InOurDc.endpoints()); + + // Double count model for local DC + int localPending = pending.count(InOurDc.replicas()); + // localContacts includes both natural and pending in local DC + int localNatural = localContacts - localPending; + return new WriteResponseTracker(baseBlockFor, totalBlockFor, + localNatural, localPending, + endpoint -> pending.endpoints().contains(endpoint), + InOurDc.endpoints()); + } + + case EACH_QUORUM: + return createPerDcTracker(plan, pending, metadata); + + default: + throw new UnsupportedOperationException("Unsupported consistency level for writes: " + cl); + } + } + + /** + * Create per-datacenter tracker for EACH_QUORUM. + */ + private ResponseTracker createPerDcTracker(ReplicaPlan.ForWrite plan, Endpoints pending, ClusterMetadata metadata) + { + Map trackerPerDc = new HashMap<>(); + Locator locator = metadata.locator; + + // Group replicas by datacenter + Map> replicasByDc = new HashMap<>(); + for (Replica replica : plan.contacts()) + { + String dc = locator.location(replica.endpoint()).datacenter; + replicasByDc.computeIfAbsent(dc, k -> new ArrayList<>()).add(replica); + } + + // Group pending replicas by datacenter + Map> pendingByDc = new HashMap<>(); + for (Replica replica : pending) + { + String dc = locator.location(replica.endpoint()).datacenter; + pendingByDc.computeIfAbsent(dc, k -> new ArrayList<>()).add(replica); + } + + // Create tracker for each DC + for (Map.Entry> entry : replicasByDc.entrySet()) + { + String dc = entry.getKey(); + int dcContacts = entry.getValue().size(); + List dcPending = pendingByDc.getOrDefault(dc, Collections.emptyList()); + int dcPendingCount = dcPending.size(); + int dcNatural = dcContacts - dcPendingCount; + int dcBlockFor = dcNatural / 2 + 1; + + // Each sub-tracker must filter by DC since CompositeTracker broadcasts to all children + Predicate dcFilter = endpoint -> dc.equals(locator.location(endpoint).datacenter); + + if (dcPending.isEmpty()) + { + trackerPerDc.put(dc, new SimpleResponseTracker(dcBlockFor, dcContacts, dcFilter)); + } + else + { + int totalBlockFor = dcBlockFor + dcPendingCount; + trackerPerDc.put(dc, new WriteResponseTracker(dcBlockFor, totalBlockFor, + dcNatural, dcPendingCount, + endpoint -> pending.endpoints().contains(endpoint), + dcFilter)); + } + } + + return new PerDcResponseTracker(trackerPerDc, locator); + } } diff --git a/src/java/org/apache/cassandra/locator/CoordinationPlan.java b/src/java/org/apache/cassandra/locator/CoordinationPlan.java new file mode 100644 index 0000000000..7260b65a24 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/CoordinationPlan.java @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; + +import javax.annotation.Nullable; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; +import org.apache.cassandra.tcm.ClusterMetadata; + +/** + * Ties together replica selection and response tracking for a single operation. + * + * This immutable container ensures that the replica plan (who to contact) and + * the response tracker (how to determine success) are created atomically by the + * replication strategy, consulting the same state. This is particularly important + * for strategies that have per-range state (e.g., failover state in SRS) where + * the replica selection and quorum requirements must be consistent. + * + * The separation between ReplicaPlan and ResponseTracker allows: + *
    + *
  • ReplicaPlan to focus on replica topology and selection
    • + *
  • ResponseTracker to encapsulate completion logic
    • + *
  • Replication strategies to customize both consistently
    • + *
+ * + * No polymorphism is needed at this level - the variation is captured in the + * ResponseTracker implementations. This is just a typed tuple ensuring the + * two pieces travel together. + * + * @param

the type of ReplicaPlan (ForRead, ForWrite, ForPaxosWrite, etc.) + */ +public abstract class CoordinationPlan, P extends ReplicaPlan> +{ + // TODO (now): consolidate callback Condition instances into this. Replace all condition await calls with calls to this. + private final ResponseTracker responses; + + /** + * Create a coordination plan. + * + * @param responses the response tracker for determining completion/success + */ + public CoordinationPlan(ResponseTracker responses) + { + Preconditions.checkNotNull(responses); + this.responses = responses; + } + + public ConsistencyLevel consistencyLevel() + { + return replicas().consistencyLevel(); + } + + public AbstractReplicationStrategy replicationStrategy() + { + return replicas().replicationStrategy(); + } + + public abstract P replicas(); + + /** + * The response tracker for determining completion/success. + * + * The tracker encapsulates the logic for: + * - Recording responses and failures + * - Determining when the operation is complete + * - Checking if the operation succeeded + * + * @return the response tracker + */ + public ResponseTracker responses() + { + return responses; + } + + @Override + public String toString() + { + return String.format("CoordinationPlan[replicaPlan=%s, tracker=%s]", replicas(), responses.getClass().getSimpleName()); + } + + /** + * Extended plan including source cluster metadata and ideal coordination plan. + */ + public static class ForWrite extends CoordinationPlan + { + private final ReplicaPlan.ForWrite replicas; + + public ForWrite(ReplicaPlan.ForWrite replicas, ResponseTracker responses) + { + super(responses); + this.replicas = replicas; + } + + @Override + public ReplicaPlan.ForWrite replicas() + { + return replicas; + } + } + + public static class ForWriteWithIdeal extends CoordinationPlan.ForWrite + { + public final CoordinationPlan.ForWrite ideal; + + public ForWriteWithIdeal(ReplicaPlan.ForWrite replicas, ResponseTracker responses, CoordinationPlan.ForWrite ideal) + { + super(replicas, responses); + this.ideal = ideal; + } + + /** + * Create coordination plan for batchlog write. + * + * The batchlog is a system-level durability mechanism independent of keyspace replication: + * - Stored in system.batches regardless of which keyspace(s) the mutations target + * - Replica selection is DC-local based on rack diversity and liveness + * - Uses simple ack counting (ONE or TWO based on available replicas) + * + * @param metadata the cluster metadata + * @param isAny whether to allow any node (for legacy batch compatibility) + * @return coordination plan for batchlog write + * @throws UnavailableException if insufficient replicas are available + */ + public static ForWriteWithIdeal forBatchlogWrite(ClusterMetadata metadata, boolean isAny) + throws UnavailableException + { + ReplicaPlan.ForWrite plan = ReplicaPlans.forBatchlogWrite(metadata, isAny); + int blockFor = plan.consistencyLevel().blockFor(plan.replicationStrategy()); + ResponseTracker tracker = new SimpleResponseTracker(blockFor, plan.contacts().size()); + return new ForWriteWithIdeal(plan, tracker, null); + } + } + + public abstract static class ForRead, P extends ReplicaPlan> extends CoordinationPlan implements Supplier

+ { + final ReplicaPlan.Shared replicas; + + public ForRead(ReplicaPlan.Shared replicas, ResponseTracker responses) + { + super(responses); + this.replicas = replicas; + } + + public abstract ForRead copyWithResetTracker(); + + @Override + public P get() + { + return replicas.get(); + } + + @Override + public P replicas() + { + return replicas.get(); + } + + public void addToContacts(Replica replica) + { + replicas.addToContacts(replica); + } + } + + public static class ForTokenRead extends ForRead + { + public ForTokenRead(ReplicaPlan.Shared replicas, ResponseTracker responses) + { + super(replicas, responses); + } + + @Override + public ForTokenRead copyWithResetTracker() + { + return new ForTokenRead(replicas, responses().resetCopy()); + } + } + + public static class ForRangeRead extends ForRead + { + public ForRangeRead(ReplicaPlan.Shared replicas, ResponseTracker responses) + { + super(replicas, responses); + } + + @Override + public ForRangeRead copyWithResetTracker() + { + return new ForRangeRead(replicas, responses().resetCopy()); + } + } + + // ---- Static convenience methods that look up the replication strategy internally ---- + + private static AbstractReplicationStrategy getStrategy(ClusterMetadata metadata, Keyspace keyspace) + { + if (SchemaConstants.isLocalSystemKeyspace(keyspace.getName())) + return keyspace.getReplicationStrategy(); + + return metadata.schema.getKeyspaceMetadata(keyspace.getName()).replicationStrategy; + } + + public static ForWriteWithIdeal forWrite(ClusterMetadata metadata, + Keyspace keyspace, + ConsistencyLevel consistencyLevel, + Function liveAndDown, + ReplicaPlans.Selector selector) + { + return getStrategy(metadata, keyspace).planForWrite(metadata, keyspace, consistencyLevel, liveAndDown, selector); + } + + public static ForWriteWithIdeal forWrite(ClusterMetadata metadata, + Keyspace keyspace, + ConsistencyLevel consistencyLevel, + Token token, + ReplicaPlans.Selector selector) + { + return getStrategy(metadata, keyspace).planForWrite(metadata, keyspace, consistencyLevel, token, selector); + } + + public static ForWrite forForwardingCounterWrite(ClusterMetadata metadata, + Keyspace keyspace, + Token token, + Function replicaSupplier) + { + return getStrategy(metadata, keyspace).planForForwardingCounterWrite(metadata, keyspace, token, replicaSupplier); + } + + public static ForWriteWithIdeal forReplayMutation(ClusterMetadata metadata, + Keyspace keyspace, + Token token) + { + return getStrategy(metadata, keyspace).planForReplayMutation(metadata, keyspace, token); + } + + public static ForTokenRead forTokenRead(ClusterMetadata metadata, + Keyspace keyspace, + TableId tableId, + Token token, + @Nullable Index.QueryPlan indexQueryPlan, + ConsistencyLevel consistencyLevel, + SpeculativeRetryPolicy retry, + ReadCoordinator coordinator) + { + return getStrategy(metadata, keyspace).planForTokenRead(metadata, keyspace, tableId, token, indexQueryPlan, consistencyLevel, retry, coordinator); + } + + public static ForRangeRead forRangeRead(ClusterMetadata metadata, + Keyspace keyspace, + TableId tableId, + @Nullable Index.QueryPlan indexQueryPlan, + ConsistencyLevel consistencyLevel, + AbstractBounds range, + int vnodeCount) + { + return getStrategy(metadata, keyspace).planForRangeRead(metadata, keyspace, tableId, indexQueryPlan, consistencyLevel, range, vnodeCount); + } + + public static ForRangeRead maybeMergeRangeReads(ClusterMetadata metadata, + Keyspace keyspace, + TableId tableId, + ConsistencyLevel consistencyLevel, + ForRangeRead left, + ForRangeRead right) + { + return getStrategy(metadata, keyspace).maybeMergeRangeReads(metadata, keyspace, tableId, consistencyLevel, left.replicas(), right.replicas()); + } + + public static ForRangeRead forFullRangeRead(Keyspace keyspace, + ConsistencyLevel consistencyLevel, + AbstractBounds range, + Set endpointsToContact, + int vnodeCount) + { + return keyspace.getReplicationStrategy().planForFullRangeRead(keyspace, consistencyLevel, range, endpointsToContact, vnodeCount); + } + + public static ForTokenRead forSingleReplicaTokenRead(Keyspace keyspace, Token token, Replica replica) + { + return keyspace.getReplicationStrategy().planForSingleReplicaTokenRead(keyspace, token, replica); + } + + public static ForRangeRead forSingleReplicaRangeRead(Keyspace keyspace, + AbstractBounds range, + Replica replica, + int vnodeCount) + { + return keyspace.getReplicationStrategy().planForSingleReplicaRangeRead(keyspace, range, replica, vnodeCount); + } +} diff --git a/src/java/org/apache/cassandra/locator/PerDcResponseTracker.java b/src/java/org/apache/cassandra/locator/PerDcResponseTracker.java new file mode 100644 index 0000000000..7b0466e2ae --- /dev/null +++ b/src/java/org/apache/cassandra/locator/PerDcResponseTracker.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.Map; +import java.util.function.ToIntFunction; + +import org.apache.cassandra.exceptions.RequestFailureReason; + +/** + * Per-datacenter response tracker that requires ALL datacenters to independently reach quorum. + *

+ * Composes multiple ResponseTrackers (one per datacenter) and delegates operations + * to the appropriate tracker based on endpoint datacenter. Used for EACH_QUORUM consistency. + *

+ * The composed trackers can be any ResponseTracker implementation: + *

    + *
  • {@link SimpleResponseTracker} for basic quorum tracking
    • + *
  • {@link WriteResponseTracker} for writes with pending replicas (double-count model)
    • + *
+ *

+ * Thread-safe through delegation to thread-safe ResponseTracker implementations. + */ +public class PerDcResponseTracker implements ResponseTracker +{ + private final Map trackerPerDc; + private final Locator locator; + + /** + * Create per-DC tracker with pre-built trackers for each datacenter. + * + * @param trackerPerDc map of datacenter name to tracker (must be non-empty) + * @param locator for looking up datacenter from endpoint + */ + public PerDcResponseTracker(Map trackerPerDc, Locator locator) + { + if (trackerPerDc == null || trackerPerDc.isEmpty()) + throw new IllegalArgumentException("trackerPerDc cannot be null or empty"); + if (locator == null) + throw new IllegalArgumentException("locator cannot be null"); + + this.locator = locator; + this.trackerPerDc = trackerPerDc; + } + + private int count(ToIntFunction getter) + { + int total = 0; + for (ResponseTracker tracker : trackerPerDc.values()) + total += getter.applyAsInt(tracker); + return total; + } + + @Override + public void onResponse(InetAddressAndPort from) + { + ResponseTracker tracker = getTrackerForEndpoint(from); + if (tracker != null) + tracker.onResponse(from); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason reason) + { + ResponseTracker tracker = getTrackerForEndpoint(from); + if (tracker != null) + tracker.onFailure(from, reason); + } + + @Override + public boolean isComplete() + { + // Complete when ALL DCs are complete (either success or definite failure) + for (ResponseTracker tracker : trackerPerDc.values()) + { + if (!tracker.isComplete()) + return false; + } + return true; + } + + @Override + public boolean isSuccessful() + { + // Successful only if ALL DCs are successful + for (ResponseTracker tracker : trackerPerDc.values()) + { + if (!tracker.isSuccessful()) + return false; + } + return true; + } + + @Override + public int required() + { + return count(ResponseTracker::required); + } + + @Override + public int received() + { + return count(ResponseTracker::received); + } + + @Override + public int failures() + { + return count(ResponseTracker::failures); + } + + @Override + public boolean countsTowardQuorum(InetAddressAndPort from) + { + String dc = locator.location(from).datacenter; + if (dc == null) + return false; + ResponseTracker tracker = trackerPerDc.get(dc); + return tracker != null && tracker.countsTowardQuorum(from); + } + + private ResponseTracker getTrackerForEndpoint(InetAddressAndPort from) + { + String dc = locator.location(from).datacenter; + return trackerPerDc.get(dc); + } + + /** + * @return the tracker for the specified datacenter, or null if not tracked + */ + public ResponseTracker getTrackerForDc(String datacenter) + { + return trackerPerDc.get(datacenter); + } + + @Override + public String toString() + { + return String.format("PerDcResponseTracker[datacenters=%s, trackerPerDc=%s]", + trackerPerDc.keySet(), trackerPerDc); + } + + @Override + public boolean isPending(InetAddressAndPort from) + { + String dc = locator.location(from).datacenter; + if (dc == null) + return false; + ResponseTracker tracker = trackerPerDc.get(dc); + return tracker != null && tracker.isPending(from); + } + + @Override + public int totalRequired() + { + return count(ResponseTracker::totalRequired); + } + + @Override + public int totalContacts() + { + return count(ResponseTracker::totalContacts); + } + + @Override + public int pendingContacts() + { + return count(ResponseTracker::pendingContacts); + } +} diff --git a/src/java/org/apache/cassandra/locator/ReplicaPlans.java b/src/java/org/apache/cassandra/locator/ReplicaPlans.java index 01de6d4a5d..c5c78623c1 100644 --- a/src/java/org/apache/cassandra/locator/ReplicaPlans.java +++ b/src/java/org/apache/cassandra/locator/ReplicaPlans.java @@ -264,15 +264,6 @@ public class ReplicaPlans return localReplicas.get(ThreadLocalRandom.current().nextInt(localReplicas.size())); } - /** - * A forwarding counter write is always sent to a single owning coordinator for the range, by the original coordinator - * (if it is not itself an owner) - */ - public static ReplicaPlan.ForWrite forForwardingCounterWrite(ClusterMetadata metadata, Keyspace keyspace, Token token, Function replica) - { - return forSingleReplicaWrite(metadata, keyspace, token, replica); - } - public static ReplicaPlan.ForWrite forLocalBatchlogWrite() { Token token = DatabaseDescriptor.getPartitioner().getMinimumToken(); @@ -287,6 +278,36 @@ public class ReplicaPlans return forWrite(systemKeyspace, ConsistencyLevel.ONE, (cm) -> liveAndDown, (cm) -> true, writeAll); } + /** + * Create a replica plan for replaying a mutation from the batchlog. + * + * When recovering failed batches, mutations are replayed to live remote replicas only + * (local replica is handled separately by the caller). + * + * @param metadata the cluster metadata + * @param keyspace the keyspace + * @param token the token for the mutation + * @return replica plan targeting live remote replicas with CL.ONE + */ + public static ReplicaPlan.ForWrite forReplayMutation(ClusterMetadata metadata, Keyspace keyspace, Token token) + { + ReplicaLayout.ForTokenWrite liveAndDown = ReplicaLayout.forTokenWriteLiveAndDown(metadata, keyspace.getMetadata(), token); + Replicas.temporaryAssertFull(liveAndDown.all()); // TODO in CASSANDRA-14549 + + Replica selfReplica = liveAndDown.all().selfIfPresent(); + ReplicaLayout.ForTokenWrite liveRemoteOnly = liveAndDown.filter(r -> FailureDetector.isReplicaAlive.test(r) && r != selfReplica); + + EndpointsForToken allLiveRemoteOnly = liveRemoteOnly.all(); + return new ReplicaPlan.ForWrite(keyspace, keyspace.getReplicationStrategy(), + ConsistencyLevel.ONE, + liveRemoteOnly.pending(), + allLiveRemoteOnly, + allLiveRemoteOnly, + allLiveRemoteOnly, + (cm) -> forReplayMutation(cm, keyspace, token), + metadata.epoch); + } + /** * Requires that the provided endpoints are alive. Converts them to their relevant system replicas. * Note that the liveAndDown collection and live are equal to the provided endpoints. diff --git a/src/java/org/apache/cassandra/locator/ResponseTracker.java b/src/java/org/apache/cassandra/locator/ResponseTracker.java new file mode 100644 index 0000000000..28747ca9af --- /dev/null +++ b/src/java/org/apache/cassandra/locator/ResponseTracker.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Black-box response tracker encapsulating coordination completion logic. + * + * Created by replication strategy for each operation, this interface allows + * strategies to customize how quorum requirements are calculated and enforced. + * Different implementations can provide different semantics. + * + * The tracker is responsible for: + * 1. Recording responses and failures from replicas + * 2. Determining when the operation has completed (success or definite failure) + * 3. Providing metrics for error messages and monitoring + * + * Thread safety: Implementations must be thread-safe as onResponse/onFailure + * can be called concurrently from multiple network threads. + */ +public interface ResponseTracker +{ + // TODO: review replica plan members and move here as appropriate + + /** + * Record a successful response from a replica. + * + * @param from endpoint that responded successfully + */ + void onResponse(InetAddressAndPort from); + + /** + * Record a failed response from a replica. + * + * @param from endpoint that failed + */ + void onFailure(InetAddressAndPort from); + + // TODO: consider having an outcome method that returns an enum (PENDING, SUCCESS, FAILURE) + /** + * Has the operation completed (either success or definite failure)? + * + * An operation is complete when: + * - Success: Required quorum has been achieved + * - Definite failure: Not enough replicas remain to achieve quorum + * + * @return true if no more responses are needed to make a decision + */ + boolean isComplete(); + + /** + * Did the operation succeed (quorum achieved)? + * + * Only meaningful if isComplete() returns true. + * + * @return true if required quorum was met + */ + boolean isSuccessful(); + + /** + * How many responses are required for success? + * + * Used for error messages, metrics, and UnavailableException construction. + * For complex trackers (e.g., EACH_QUORUM), this may be a sum or other + * aggregate value rather than the actual completion criteria. + * + * @return number of responses required + */ + int required(); + + /** + * How many successful responses have been received so far? + * + * @return number of successful responses + */ + int received(); + + /** + * How many failures have been recorded so far? + * + * @return number of failures + */ + int failures(); + + /** + * Should responses from this endpoint be counted toward quorum? + * + * Allows filtering of responses based on datacenter, state, or other criteria. + * For example, LOCAL_QUORUM trackers would return false for remote DC replicas. + * + * @param from endpoint to check + * @return true if responses from this endpoint count toward quorum + */ + @VisibleForTesting + boolean countsTowardQuorum(InetAddressAndPort from); + + /** + * Indicates that the given address is a pending replica. Accepting writes but not reads + */ + boolean isPending(InetAddressAndPort from); + + int totalContacts(); + + int pendingContacts(); + + /** + * creates a copy of the tracker will all response counts reset + */ + ResponseTracker resetCopy(); +} diff --git a/src/java/org/apache/cassandra/locator/SimpleResponseTracker.java b/src/java/org/apache/cassandra/locator/SimpleResponseTracker.java new file mode 100644 index 0000000000..d9fca0e925 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/SimpleResponseTracker.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.function.Predicate; + +/** + * Simple response tracker that counts responses against a single threshold. + *

+ * Supports optional predicate filtering to selectively count responses + * (e.g., LOCAL_* consistency levels that only count local DC responses). + */ +public class SimpleResponseTracker implements ResponseTracker +{ + private static final AtomicIntegerFieldUpdater RESPONSES_UPDATER + = AtomicIntegerFieldUpdater.newUpdater(SimpleResponseTracker.class, "responses"); + private static final AtomicIntegerFieldUpdater FAILURES_UPDATER + = AtomicIntegerFieldUpdater.newUpdater(SimpleResponseTracker.class, "failures"); + private static final Predicate NO_FILTER = address -> true; + + private final int blockFor; + private final int totalReplicas; + private final Predicate filter; + private volatile int responses = 0; + private volatile int failures = 0; + + /** + * Create unfiltered tracker + * + * @param blockFor number of responses required for quorum + * @param totalReplicas total replicas available (for early failure detection) + */ + public SimpleResponseTracker(int blockFor, int totalReplicas) + { + this(blockFor, totalReplicas, NO_FILTER); + } + + /** + * Create filtered tracker + * + * @param blockFor number of responses required for quorum + * @param totalReplicas total replicas available (for early failure detection) + * @param filter predicate to test if response counts (null = all count) + */ + public SimpleResponseTracker(int blockFor, int totalReplicas, + Predicate filter) + { + if (blockFor < 0) + throw new IllegalArgumentException("blockFor must be non-negative: " + blockFor); + if (totalReplicas < 0) + throw new IllegalArgumentException("totalReplicas must be non-negative: " + totalReplicas); + + this.blockFor = blockFor; + this.totalReplicas = totalReplicas; + this.filter = filter != null ? filter : NO_FILTER; + } + + @Override + public void onResponse(InetAddressAndPort from) + { + if (countsTowardQuorum(from)) + RESPONSES_UPDATER.incrementAndGet(this); + } + + @Override + public void onFailure(InetAddressAndPort from) + { + if (countsTowardQuorum(from)) + FAILURES_UPDATER.incrementAndGet(this); + } + + @Override + public boolean isComplete() + { + int r = responses; + int f = failures; + + if (r >= blockFor) + return true; + + // failure: can't reach blockFor + int needed = blockFor - r; + int remaining = totalReplicas - (r + f); + return needed > remaining; + } + + @Override + public boolean isSuccessful() + { + return responses >= blockFor; + } + + @Override + public int required() + { + return blockFor; + } + + @Override + public int received() + { + return responses; + } + + @Override + public int failures() + { + return failures; + } + + @Override + public boolean countsTowardQuorum(InetAddressAndPort from) + { + return filter.test(from); + } + + @Override + public String toString() + { + return String.format("SimpleResponseTracker[blockFor=%d, totalReplicas=%d, responses=%d, failures=%d, filtered=%s]", + blockFor, totalReplicas, responses, failures, filter != NO_FILTER); + } + + @Override + public boolean isPending(InetAddressAndPort from) + { + return false; + } + + @Override + public int totalContacts() + { + return totalReplicas; + } + + @Override + public int pendingContacts() + { + return 0; + } + + @Override + public SimpleResponseTracker resetCopy() + { + return new SimpleResponseTracker(blockFor, totalReplicas, filter); + } +} diff --git a/src/java/org/apache/cassandra/locator/WriteResponseTracker.java b/src/java/org/apache/cassandra/locator/WriteResponseTracker.java new file mode 100644 index 0000000000..15bfbcedb6 --- /dev/null +++ b/src/java/org/apache/cassandra/locator/WriteResponseTracker.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.util.function.Predicate; + +/** + * Response tracker for writes with pending replicas using the double count model. + *

+ * Two requirements must be satisfied for success: + *

    + *
  1. Committed replicas must satisfy base CL: {@code naturalSuccesses >= baseBlockFor}
    2. + *
  2. Total replicas (natural + pending) must satisfy: {@code totalSuccesses >= totalBlockFor}
    3. + *
+ *

+ * This ensures both consistency (natural replicas have the data) and bootstrap safety + * (pending replicas also receive the write). + *

+ * Implemented by composing two {@link SimpleResponseTracker}s: + *

    + *
  • naturalTracker: tracks responses from natural replicas only
    • + *
  • totalTracker: tracks responses from all replicas (natural + pending)
    • + *
+ *

+ * Thread-safe through delegation to thread-safe SimpleResponseTrackers. + */ +public class WriteResponseTracker implements ResponseTracker +{ + private final SimpleResponseTracker natural; + private final SimpleResponseTracker total; + + /** + * Create a write response tracker with the double count model. + * + * @param baseBlockFor number of natural replica responses required for CL + * @param totalBlockFor total responses required (baseBlockFor + pending count) + * @param naturalReplicas number of natural replicas available + * @param pendingReplicas number of pending replicas available + * @param isPending predicate to determine if an endpoint is pending + */ + public WriteResponseTracker(int baseBlockFor, + int totalBlockFor, + int naturalReplicas, + int pendingReplicas, + Predicate isPending) + { + this(baseBlockFor, totalBlockFor, naturalReplicas, pendingReplicas, isPending, null); + } + + /** + * Create a write response tracker with the double count model and a filter. + * + * @param naturalBlockFor number of natural replica responses required for CL + * @param totalBlockFor total responses required (naturalBlockFor + pending count) + * @param naturalReplicas number of natural replicas available + * @param pendingReplicas number of pending replicas available + * @param isPending predicate to determine if an endpoint is pending + * @param filter predicate to filter which responses count (e.g., InOurDc for LOCAL_QUORUM) + */ + public WriteResponseTracker(int naturalBlockFor, + int totalBlockFor, + int naturalReplicas, + int pendingReplicas, + Predicate isPending, + Predicate filter) + { + if (naturalBlockFor < 0) + throw new IllegalArgumentException("naturalBlockFor must be non-negative: " + naturalBlockFor); + if (totalBlockFor < naturalBlockFor) + throw new IllegalArgumentException("totalBlockFor (" + totalBlockFor + ") must be >= naturalBlockFor (" + naturalBlockFor + ")"); + if (naturalReplicas < 0) + throw new IllegalArgumentException("naturalReplicas must be non-negative: " + naturalReplicas); + if (pendingReplicas < 0) + throw new IllegalArgumentException("pendingReplicas must be non-negative: " + pendingReplicas); + if (naturalBlockFor > naturalReplicas) + throw new IllegalArgumentException("naturalBlockFor (" + naturalBlockFor + ") cannot exceed naturalReplicas (" + naturalReplicas + ")"); + if (totalBlockFor > naturalReplicas + pendingReplicas) + throw new IllegalArgumentException("totalBlockFor (" + totalBlockFor + ") cannot exceed total replicas (" + (naturalReplicas + pendingReplicas) + ")"); + if (isPending == null) + throw new IllegalArgumentException("isPending predicate cannot be null"); + + Predicate naturalFilter = isPending.negate(); + if (filter != null) + naturalFilter = naturalFilter.and(filter); + + this.natural = new SimpleResponseTracker(naturalBlockFor, naturalReplicas, naturalFilter); + this.total = new SimpleResponseTracker(totalBlockFor, naturalReplicas + pendingReplicas, filter); + } + + private WriteResponseTracker(SimpleResponseTracker natural, SimpleResponseTracker total) + { + this.natural = natural; + this.total = total; + } + + @Override + public void onResponse(InetAddressAndPort from) + { + natural.onResponse(from); + total.onResponse(from); + } + + @Override + public void onFailure(InetAddressAndPort from) + { + natural.onFailure(from); + total.onFailure(from); + } + + @Override + public boolean isComplete() + { + // Early failure if either tracker fails + if (natural.isComplete() && !natural.isSuccessful()) + return true; + if (total.isComplete() && !total.isSuccessful()) + return true; + + // Success requires both to succeed + return natural.isSuccessful() && total.isSuccessful(); + } + + @Override + public boolean isSuccessful() + { + return natural.isSuccessful() && total.isSuccessful(); + } + + @Override + public int required() + { + // Return total requirement for error messages + return total.required(); + } + + @Override + public int received() + { + // Return total successes for error messages + return total.received(); + } + + @Override + public int failures() + { + // Return total failures for error messages + return total.failures(); + } + + @Override + public boolean countsTowardQuorum(InetAddressAndPort from) + { + return total.countsTowardQuorum(from); + } + + public int naturalReceived() + { + return natural.received(); + } + + public int pendingReceived() + { + return total.received() - natural.received(); + } + + public int naturalFailures() + { + return natural.failures(); + } + + public int pendingFailures() + { + return total.failures() - natural.failures(); + } + + public int baseBlockFor() + { + return natural.required(); + } + + @Override + public String toString() + { + return String.format("WriteResponseTracker[baseBlockFor=%d, totalBlockFor=%d, " + + "naturalTracker=%s, totalTracker=%s]", + baseBlockFor(), required(), + natural, total); + } + + @Override + public boolean isPending(InetAddressAndPort from) + { + return total.countsTowardQuorum(from) && !natural.countsTowardQuorum(from); + } + + @Override + public int totalContacts() + { + return total.totalContacts(); + } + + @Override + public int pendingContacts() + { + return total.totalContacts() - natural.totalContacts(); + } + + @Override + public ResponseTracker resetCopy() + { + return new WriteResponseTracker(natural.resetCopy(), total.resetCopy()); + } +} diff --git a/src/java/org/apache/cassandra/replication/ForwardedWrite.java b/src/java/org/apache/cassandra/replication/ForwardedWrite.java index d69129a906..0075c71c2a 100644 --- a/src/java/org/apache/cassandra/replication/ForwardedWrite.java +++ b/src/java/org/apache/cassandra/replication/ForwardedWrite.java @@ -46,6 +46,7 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.NodeProximity; @@ -296,7 +297,7 @@ public class ForwardedWrite } } - public static AbstractWriteResponseHandler forwardMutation(Mutation mutation, ReplicaPlan.ForWrite plan, AbstractReplicationStrategy strategy, Dispatcher.RequestTime requestTime) + public static AbstractWriteResponseHandler forwardMutation(Mutation mutation, CoordinationPlan.ForWriteWithIdeal plan, AbstractReplicationStrategy strategy, Dispatcher.RequestTime requestTime) { // find leader NodeProximity proximity = DatabaseDescriptor.getNodeProximity(); @@ -310,7 +311,7 @@ public class ForwardedWrite Replica leader = null; for (Replica replica : proximity.sortedByProximity(FBUtilities.getBroadcastAddressAndPort(), endpoints)) { - if (plan.isAlive(replica)) + if (plan.replicas().isAlive(replica)) leader = replica; } Preconditions.checkState(leader != null, "Could not find leader for %s", mutation); @@ -321,17 +322,17 @@ public class ForwardedWrite AbstractWriteResponseHandler handler = strategy.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime); // Add callbacks for replicas to respond directly to coordinator - Message toLeader = Message.outWithRequestTime(Verb.FORWARD_WRITE_REQ, new MutationRequest(mutation, plan), requestTime); + Message toLeader = Message.outWithRequestTime(Verb.FORWARD_WRITE_REQ, new MutationRequest(mutation, plan.replicas()), requestTime); for (Replica endpoint : endpoints) { - if (plan.isAlive(endpoint)) + if (plan.replicas().isAlive(endpoint)) { logger.trace("Adding forwarding callback for response from {} id {}", endpoint, toLeader.id()); MessagingService.instance().callbacks.addWithExpiration(handler, toLeader, endpoint); } else { - handler.expired(); + handler.expired(endpoint.endpoint()); } } @@ -345,7 +346,7 @@ public class ForwardedWrite * The leader will apply the counter mutation, assign a mutation ID, and replicate to other replicas. */ public static AbstractWriteResponseHandler forwardCounterMutation(CounterMutation counterMutation, - ReplicaPlan.ForWrite plan, + CoordinationPlan.ForWriteWithIdeal plan, AbstractReplicationStrategy strategy, Dispatcher.RequestTime requestTime) { @@ -375,18 +376,19 @@ public class ForwardedWrite // Create response handler for all replicas AbstractWriteResponseHandler handler = strategy.getWriteResponseHandler(plan, null, WriteType.COUNTER, null, requestTime); + ReplicaPlan.ForWrite replicas = plan.replicas(); // Add callbacks for all live replicas to respond directly to coordinator Message forwardMessage = Message.outWithRequestTime(Verb.COUNTER_MUTATION_REQ, counterMutation, requestTime); - for (Replica replica : plan.contacts()) + for (Replica replica : replicas.contacts()) { - if (plan.isAlive(replica)) + if (replicas.isAlive(replica)) { logger.trace("Adding forwarding callback for tracked counter response from {} id {}", replica, forwardMessage.id()); MessagingService.instance().callbacks.addWithExpiration(handler, forwardMessage, replica); } else { - handler.expired(); + handler.expired(replica.endpoint()); } } @@ -407,7 +409,7 @@ public class ForwardedWrite * @return the write response handler */ public static AbstractWriteResponseHandler forward(IMutation mutation, - ReplicaPlan.ForWrite plan, + CoordinationPlan.ForWriteWithIdeal plan, AbstractReplicationStrategy strategy, Dispatcher.RequestTime requestTime) { diff --git a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java index ed6975525f..7b5fe3fa72 100644 --- a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java +++ b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java @@ -43,6 +43,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.Replica; @@ -171,13 +172,14 @@ public class TrackedWriteRequest Preconditions.checkArgument(mutation.id().isNone()); String keyspaceName = mutation.getKeyspaceName(); + ClusterMetadata cm = ClusterMetadata.current(); Keyspace keyspace = Keyspace.open(keyspaceName); Token token = mutation.key().getToken(); - ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(keyspace, consistencyLevel, token, ReplicaPlans.writeAll); - AbstractReplicationStrategy rs = plan.replicationStrategy(); + AbstractReplicationStrategy rs = cm.schema.getKeyspaceMetadata(keyspaceName).replicationStrategy; + CoordinationPlan.ForWriteWithIdeal plan = CoordinationPlan.forWrite(cm, keyspace, consistencyLevel, token, ReplicaPlans.writeAll); - if (plan.lookup(FBUtilities.getBroadcastAddressAndPort()) == null) + if (plan.replicas().lookup(FBUtilities.getBroadcastAddressAndPort()) == null) { logger.trace("Remote tracked request {} {}", mutation, plan); writeMetrics.remoteRequests.mark(); @@ -193,19 +195,19 @@ public class TrackedWriteRequest if (logger.isTraceEnabled()) { logger.trace("Write replication plan for mutation {}: live={}, pending={}, all={}", - id, plan.live(), plan.pending(), plan.contacts()); + id, plan.replicas().live(), plan.replicas().pending(), plan.replicas().contacts()); } final TrackedWriteResponseHandler handler; if (mutation instanceof CounterMutation) { handler = TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.COUNTER, null, requestTime), id); - applyCounterMutationLocally((CounterMutation) mutation, plan, handler); + applyCounterMutationLocally((CounterMutation) mutation, plan.replicas(), handler); } else { handler = TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime), id); - applyLocallyAndSendToReplicas((Mutation) mutation, plan, handler); + applyLocallyAndSendToReplicas((Mutation) mutation, plan.replicas(), handler); } return handler; } @@ -264,7 +266,7 @@ public class TrackedWriteRequest logger.trace("Skipping dead replica {} for mutation {}", destination, mutation.id()); // Only call expired() for AbstractWriteResponseHandler (not for LeaderCallback) if (handler instanceof AbstractWriteResponseHandler) - ((AbstractWriteResponseHandler) handler).expired(); // immediately mark the response as expired since the request will not be sent + ((AbstractWriteResponseHandler) handler).expired(destination.endpoint()); // immediately mark the response as expired since the request will not be sent continue; } diff --git a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java index 73cec3b146..176124517f 100644 --- a/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/AbstractWriteResponseHandler.java @@ -29,6 +29,9 @@ import java.util.function.Supplier; import javax.annotation.Nullable; +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.locator.CoordinationPlan; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +48,6 @@ import org.apache.cassandra.exceptions.WriteFailureException; import org.apache.cassandra.exceptions.WriteTimeoutException; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.locator.ReplicaPlan.ForWrite; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.RequestCallback; @@ -81,17 +83,17 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback //Count down until all responses and expirations have occured before deciding whether the ideal CL was reached. private AtomicInteger responsesAndExpirations; private final Condition condition = newOneTimeCondition(); - protected final ReplicaPlan.ForWrite replicaPlan; + protected final CoordinationPlan.ForWrite plan; protected final Runnable callback; protected final WriteType writeType; - private static final AtomicIntegerFieldUpdater failuresUpdater = - AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "failures"); - private volatile int failures = 0; private static final AtomicIntegerFieldUpdater alreadyHintedForRetryOnDifferentSystemUpdater = AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "alreadyHintedForRetryOnDifferentSystem"); // Only write a hint to be applied as a transaction once private volatile int alreadyHintedForRetryOnDifferentSystem = 0; + private static final AtomicIntegerFieldUpdater signaledUpdater = + AtomicIntegerFieldUpdater.newUpdater(AbstractWriteResponseHandler.class, "signaled"); + private volatile int signaled = 0; private volatile Map failureReasonByEndpoint; private final Dispatcher.RequestTime requestTime; private @Nullable final Supplier hintOnFailure; @@ -117,16 +119,26 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback * @param hintOnFailure * @param requestTime */ - protected AbstractWriteResponseHandler(ForWrite replicaPlan, Runnable callback, WriteType writeType, + protected AbstractWriteResponseHandler(CoordinationPlan.ForWrite plan, Runnable callback, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) { - this.replicaPlan = replicaPlan; + this.plan = plan; this.callback = callback; this.writeType = writeType; this.hintOnFailure = hintOnFailure; this.requestTime = requestTime; } + public CoordinationPlan.ForWrite coordinationPlan() + { + return plan; + } + + public ForWrite replicaPlan() + { + return plan.replicas(); + } + public void get() throws WriteTimeoutException, WriteFailureException, RetryOnDifferentSystemException { long timeoutNanos = currentTimeoutNanos(); @@ -144,9 +156,9 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback if (!signaled) throwTimeout(); - int candidateReplicaCount = candidateReplicaCount(); - if (blockFor() + failures > candidateReplicaCount) + if (!plan.responses().isSuccessful()) { + int candidateReplicaCount = candidateReplicaCount(); // failures keeps incrementing, and this.failureReasonByEndpoint keeps getting new entries after signaling. // Simpler to reason about what happened by copying this.failureReasonByEndpoint and then inferring // failures from it @@ -179,10 +191,10 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback throw new CoordinatorBehindException("Write request failed due to coordinator behind"); } - throw new WriteFailureException(replicaPlan.consistencyLevel(), ackCount(), blockFor(), writeType, getFailureReasonByEndpointMap()); + throw new WriteFailureException(replicaPlan().consistencyLevel(), ackCount(), blockFor(), writeType, getFailureReasonByEndpointMap()); } - if (replicaPlan.stillAppliesTo(ClusterMetadata.current())) + if (replicaPlan().stillAppliesTo(ClusterMetadata.current())) { if (warningContext != null) { @@ -217,7 +229,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback // avoid sending confusing info to the user (see CASSANDRA-6491). if (acks >= blockedFor) acks = blockedFor - 1; - throw new WriteTimeoutException(writeType, replicaPlan.consistencyLevel(), acks, blockedFor); + throw new WriteTimeoutException(writeType, replicaPlan().consistencyLevel(), acks, blockedFor); } public final long currentTimeoutNanos() @@ -236,7 +248,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback public void setIdealCLResponseHandler(AbstractWriteResponseHandler handler) { this.idealCLDelegate = handler; - idealCLDelegate.responsesAndExpirations = new AtomicInteger(replicaPlan.contacts().size()); + idealCLDelegate.responsesAndExpirations = new AtomicInteger(replicaPlan().contacts().size()); } /** @@ -287,9 +299,12 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback } } - public final void expired() + public final void expired(InetAddressAndPort from) { + plan.responses().onFailure(from); logFailureOrTimeoutToIdealCLDelegate(); + if (plan.responses().isComplete() && !plan.responses().isSuccessful()) + signal(); } /** @@ -299,7 +314,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback { // During bootstrap, we have to include the pending endpoints or we may fail the consistency level // guarantees (see #833) - return replicaPlan.writeQuorum(); + return replicaPlan().writeQuorum(); } /** @@ -309,15 +324,15 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback */ protected int candidateReplicaCount() { - if (replicaPlan.consistencyLevel().isDatacenterLocal()) - return countInOurDc(replicaPlan.liveAndDown()).allReplicas(); + if (replicaPlan().consistencyLevel().isDatacenterLocal()) + return countInOurDc(replicaPlan().liveAndDown()).allReplicas(); - return replicaPlan.liveAndDown().size(); + return replicaPlan().liveAndDown().size(); } public ConsistencyLevel consistencyLevel() { - return replicaPlan.consistencyLevel(); + return replicaPlan().consistencyLevel(); } /** @@ -345,14 +360,17 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback protected void signal() { + if (!signaledUpdater.compareAndSet(this, 0, 1)) + return; + //The ideal CL should only count as a strike if the requested CL was achieved. //If the requested CL is not achieved it's fine for the ideal CL to also not be achieved. - if (idealCLDelegate != null && blockFor() + failures <= candidateReplicaCount()) + if (idealCLDelegate != null && plan.responses().isSuccessful()) { idealCLDelegate.requestedCLAchieved = true; if (idealCLDelegate == this) { - replicaPlan.keyspace().metric.idealCLWriteLatency.addNano(nanoTime() - requestTime.startedAtNanos()); + replicaPlan().keyspace().metric.idealCLWriteLatency.addNano(nanoTime() - requestTime.startedAtNanos()); } } @@ -361,9 +379,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback callback.run(); } - /** - * Check if the handler has completed (for testing purposes). - */ + @VisibleForTesting public boolean isComplete() { return condition.isSignalled(); @@ -374,10 +390,6 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback { logger.trace("Got failure {} from {}", failure, from); - int n = waitingFor(from) - ? failuresUpdater.incrementAndGet(this) - : failures; - if (failureReasonByEndpoint == null) synchronized (this) { @@ -386,14 +398,16 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback } failureReasonByEndpoint.put(from, failure.reason); + plan.responses().onFailure(from); + logFailureOrTimeoutToIdealCLDelegate(); - if (blockFor() + n > candidateReplicaCount()) + if (plan.responses().isComplete() && !plan.responses().isSuccessful()) signal(); // If the failure was RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM then we only want to hint once // and not for each instance since odds are it will be applied as a transaction at all replicas - if (hintOnFailure != null && StorageProxy.shouldHint(replicaPlan.lookup(from)) ) + if (hintOnFailure != null && StorageProxy.shouldHint(replicaPlan().lookup(from))) { if (failure.reason == RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM) { @@ -402,7 +416,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback } else { - StorageProxy.submitHint(hintOnFailure.get(), replicaPlan.lookup(from), null); + StorageProxy.submitHint(hintOnFailure.get(), replicaPlan().lookup(from), null); } } } @@ -427,7 +441,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback // Only mark it as failed if the requested CL was achieved. if (!condition.isSignalled() && requestedCLAchieved) { - replicaPlan.keyspace().metric.writeFailedIdealCL.inc(); + replicaPlan().keyspace().metric.writeFailedIdealCL.inc(); } } } @@ -437,7 +451,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback */ public void maybeTryAdditionalReplicas(IMutation mutation, WritePerformer writePerformer, String localDC) { - EndpointsForToken uncontacted = replicaPlan.liveUncontacted(); + EndpointsForToken uncontacted = replicaPlan().liveUncontacted(); if (uncontacted.isEmpty()) return; @@ -459,7 +473,7 @@ public abstract class AbstractWriteResponseHandler implements RequestCallback for (ColumnFamilyStore cf : cfs) cf.metric.additionalWrites.inc(); - writePerformer.apply(mutation, replicaPlan.withContacts(uncontacted), + writePerformer.apply(mutation, replicaPlan().withContacts(uncontacted), (AbstractWriteResponseHandler) this, localDC, requestTime); diff --git a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java index 41b68de395..84cdbe26cd 100644 --- a/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java +++ b/src/java/org/apache/cassandra/service/BatchlogResponseHandler.java @@ -38,7 +38,7 @@ public class BatchlogResponseHandler extends AbstractWriteResponseHandler public BatchlogResponseHandler(AbstractWriteResponseHandler wrapped, int requiredBeforeFinish, BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime) { - super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, requestTime); + super(wrapped.plan, wrapped.callback, wrapped.writeType, null, requestTime); this.wrapped = wrapped; this.requiredBeforeFinish = requiredBeforeFinish; this.cleanup = cleanup; diff --git a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java index b897298dbb..aef1c1ea7b 100644 --- a/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterSyncWriteResponseHandler.java @@ -17,109 +17,65 @@ */ package org.apache.cassandra.service; -import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.MessageParams; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.locator.Locator; -import org.apache.cassandra.locator.NetworkTopologyStrategy; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.ParamType; import org.apache.cassandra.service.writes.thresholds.WriteWarningContext; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.FBUtilities; /** * This class blocks for a quorum of responses _in all datacenters_ (CL.EACH_QUORUM). + * + * The PerDcResponseTracker handles per-datacenter counting. */ public class DatacenterSyncWriteResponseHandler extends AbstractWriteResponseHandler { - private static final Locator locator = DatabaseDescriptor.getLocator(); - - private final Map responses = new HashMap(); - private final AtomicInteger acks = new AtomicInteger(0); - - public DatacenterSyncWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, + public DatacenterSyncWriteResponseHandler(CoordinationPlan.ForWrite coordinationPlan, Runnable callback, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) { - // Response is been managed by the map so make it 1 for the superclass. - super(replicaPlan, callback, writeType, hintOnFailure, requestTime); - assert replicaPlan.consistencyLevel() == ConsistencyLevel.EACH_QUORUM; - - if (replicaPlan.replicationStrategy() instanceof NetworkTopologyStrategy) - { - NetworkTopologyStrategy strategy = (NetworkTopologyStrategy) replicaPlan.replicationStrategy(); - for (String dc : strategy.getDatacenters()) - { - int rf = strategy.getReplicationFactor(dc).allReplicas; - responses.put(dc, new AtomicInteger((rf / 2) + 1)); - } - } - else - { - responses.put(locator.local().datacenter, new AtomicInteger(ConsistencyLevel.quorumFor(replicaPlan.replicationStrategy()))); - } - - // During bootstrap, we have to include the pending endpoints or we may fail the consistency level - // guarantees (see #833) - for (Replica pending : replicaPlan.pending()) - { - responses.get(locator.location(pending.endpoint()).datacenter).incrementAndGet(); - } + super(coordinationPlan, callback, writeType, hintOnFailure, requestTime); + assert replicaPlan().consistencyLevel() == ConsistencyLevel.EACH_QUORUM; } public void onResponse(Message message) { try { - Map params; - String dataCenter; - - if (message != null) - { - params = message.header.params(); - dataCenter = locator.location(message.from()).datacenter; - } - else - { - params = MessageParams.capture(); - dataCenter = locator.local().datacenter; - } + Map params = message != null + ? message.header.params() + : MessageParams.capture(); if (WriteWarningContext.isSupported(params.keySet())) getWarningContext().updateCounters(params); - responses.get(dataCenter).getAndDecrement(); - acks.incrementAndGet(); + InetAddressAndPort from = message == null ? FBUtilities.getBroadcastAddressAndPort() : message.from(); - for (AtomicInteger i : responses.values()) - { - if (i.get() > 0) - return; - } + plan.responses().onResponse(from); - // all the quorum conditions are met - signal(); + if (plan.responses().isComplete()) + signal(); } finally { - //Must be last after all subclass processing + // Must be last - forward to ideal CL delegate logResponseToIdealCLDelegate(message); } } protected int ackCount() { - return acks.get(); + return plan.responses().received(); } } diff --git a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java index dc9f38a99e..b29f8c6453 100644 --- a/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/DatacenterWriteResponseHandler.java @@ -17,47 +17,63 @@ */ package org.apache.cassandra.service; +import java.util.Map; import java.util.function.Predicate; import java.util.function.Supplier; +import org.apache.cassandra.db.MessageParams; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.InOurDc; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.ParamType; +import org.apache.cassandra.service.writes.thresholds.WriteWarningContext; import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.FBUtilities; /** * This class blocks for a quorum of responses _in the local datacenter only_ (CL.LOCAL_QUORUM). + * + * The response tracker handles DC filtering via countsTowardQuorum(). + * This handler still uses waitingFor to filter collectSuccess() calls. */ public class DatacenterWriteResponseHandler extends WriteResponseHandler { private final Predicate waitingFor = InOurDc.endpoints(); - public DatacenterWriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, + public DatacenterWriteResponseHandler(CoordinationPlan.ForWrite coordinationPlan, Runnable callback, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) { - super(replicaPlan, callback, writeType, hintOnFailure, requestTime); - assert replicaPlan.consistencyLevel().isDatacenterLocal(); + super(coordinationPlan, callback, writeType, hintOnFailure, requestTime); + assert coordinationPlan.consistencyLevel().isDatacenterLocal(); } @Override public void onResponse(Message message) { - if (message == null || waitingFor(message.from())) + InetAddressAndPort from = message == null ? FBUtilities.getBroadcastAddressAndPort() : message.from(); + Map params = message != null ? message.header.params() : MessageParams.capture(); + + if (WriteWarningContext.isSupported(params.keySet())) + getWarningContext().updateCounters(params); + + plan.responses().onResponse(from); + + if (message == null || waitingFor(from)) { - super.onResponse(message); - } - else - { - //WriteResponseHandler.response will call logResonseToIdealCLDelegate so only do it if not calling WriteResponseHandler.response. - //Must be last after all subclass processing - logResponseToIdealCLDelegate(message); + replicaPlan().collectSuccess(from); } + + if (plan.responses().isComplete()) + signal(); + + // Must be last (see comment on AbstractWriteResponseHandler.logResponseToIdealCLDelegate for why) - forward to ideal CL delegate + logResponseToIdealCLDelegate(message); } @Override diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index eaf511fd9a..ba75b80453 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -120,6 +120,7 @@ import org.apache.cassandra.gms.Gossiper; import org.apache.cassandra.hints.Hint; import org.apache.cassandra.hints.HintsService; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.DynamicEndpointSnitch; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; @@ -1088,11 +1089,11 @@ public class StorageProxy implements StorageProxyMBean } // NOTE: this ReplicaPlan is a lie, this usage of ReplicaPlan could do with being clarified - the selected() collection is essentially (I think) never used - ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); - AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); + CoordinationPlan.ForWriteWithIdeal plan = CoordinationPlan.forWrite(ClusterMetadata.current(), keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); + AbstractReplicationStrategy rs = plan.replicationStrategy(); // If we are coordinating a new mutation id for the first time then create a TrackedWriteResponseHandler - AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, null, requestTime); + AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime); responseHandler = TrackedWriteResponseHandler.wrap(responseHandler, mutationId); // For tracked keyspaces, the local commit MUST execute synchronously BEFORE sending to remote replicas. @@ -1100,7 +1101,7 @@ public class StorageProxy implements StorageProxyMBean // reconciliation via ActiveLogReconciler. Message message = Message.outWithFlag(PAXOS_COMMIT_REQ, proposal, MessageFlag.CALL_BACK_ON_FAILURE); Replica localReplica = null; - for (Replica replica : replicaPlan.liveAndDown()) + for (Replica replica : plan.replicas().liveAndDown()) { if (replica.isSelf()) { @@ -1129,7 +1130,7 @@ public class StorageProxy implements StorageProxyMBean // Now send to remote replicas IntHashSet remoteReplicas = new IntHashSet(); - for (Replica replica : replicaPlan.liveAndDown()) + for (Replica replica : plan.replicas().liveAndDown()) { if (replica.isSelf()) continue; // Already executed locally above @@ -1158,22 +1159,22 @@ public class StorageProxy implements StorageProxyMBean Token tk = update.partitionKey().getToken(); + ClusterMetadata cm = ClusterMetadata.current(); + + CoordinationPlan.ForWriteWithIdeal plan = CoordinationPlan.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); + AbstractWriteResponseHandler responseHandler = null; // NOTE: this ReplicaPlan is a lie, this usage of ReplicaPlan could do with being clarified - the selected() collection is essentially (I think) never used - ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); if (shouldBlock) - { - AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); - responseHandler = rs.getWriteResponseHandler(replicaPlan, null, WriteType.SIMPLE, proposal::makeMutation, requestTime); - } + responseHandler = plan.replicationStrategy().getWriteResponseHandler(plan, null, WriteType.SIMPLE, proposal::makeMutation, requestTime); Message message = Message.outWithFlag(PAXOS_COMMIT_REQ, proposal, MessageFlag.CALL_BACK_ON_FAILURE); - for (Replica replica : replicaPlan.liveAndDown()) + for (Replica replica : plan.replicas().liveAndDown()) { InetAddressAndPort destination = replica.endpoint(); checkHintOverload(replica); - if (replicaPlan.isAlive(replica)) + if (plan.replicas().isAlive(replica)) { if (shouldBlock) { @@ -1191,7 +1192,7 @@ public class StorageProxy implements StorageProxyMBean { if (responseHandler != null) { - responseHandler.expired(); + responseHandler.expired(destination); } if (allowHints && shouldHint(replica)) { @@ -1616,11 +1617,11 @@ public class StorageProxy implements StorageProxyMBean pending.get()); }; - ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(metadata, Keyspace.open(keyspaceName), consistencyLevel, computeReplicas, ReplicaPlans.writeAll); + CoordinationPlan.ForWriteWithIdeal plan = CoordinationPlan.forWrite(metadata, Keyspace.open(keyspaceName), consistencyLevel, computeReplicas, ReplicaPlans.writeAll); wrappers.add(wrapViewBatchResponseHandler(mutation, consistencyLevel, - replicaPlan, + plan, baseComplete, WriteType.BATCH, cleanup, @@ -1911,7 +1912,7 @@ public class StorageProxy implements StorageProxyMBean { ConsistencyLevel batchConsistencyLevel = consistencyLevelForBatchLog(consistencyLevel, requireQuorumForRemove); // This can't be updated for each iteration because cleanup has to go to the correct replicas which is where the batchlog is originally written - ReplicaPlan.ForWrite batchlogReplicaPlan = ReplicaPlans.forBatchlogWrite(ClusterMetadata.current(), batchConsistencyLevel == ConsistencyLevel.ANY); + CoordinationPlan.ForWrite coordinationPlan = CoordinationPlan.ForWriteWithIdeal.forBatchlogWrite(ClusterMetadata.current(), batchConsistencyLevel == ConsistencyLevel.ANY); final TimeUUID batchUUID = nextTimeUUID(); boolean wroteToBatchLog = false; while (true) @@ -1921,7 +1922,7 @@ public class StorageProxy implements StorageProxyMBean attributeNonAccordLatency = true; List wrappers = new ArrayList<>(mutations.size()); List accordMutations = new ArrayList<>(mutations.size()); - BatchlogCleanup cleanup = new BatchlogCleanup(() -> asyncRemoveFromBatchlog(batchlogReplicaPlan, batchUUID, requestTime)); + BatchlogCleanup cleanup = new BatchlogCleanup(() -> asyncRemoveFromBatchlog(coordinationPlan.replicas(), batchUUID, requestTime)); // add a handler for each mutation that will not be written on Accord - includes checking availability, but doesn't initiate any writes, yet SplitConsumer splitConsumer = (accordMutation, untrackedMutation, trackedMutation, originalMutations, mutationIndex) -> { @@ -1940,15 +1941,15 @@ public class StorageProxy implements StorageProxyMBean // Always construct the replica plan to check availability - ReplicaPlan.ForWrite dataReplicaPlan = ReplicaPlans.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); + CoordinationPlan.ForWriteWithIdeal plan = CoordinationPlan.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); - if (dataReplicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null) + if (plan.replicas().lookup(FBUtilities.getBroadcastAddressAndPort()) != null) writeMetrics.localRequests.mark(); else writeMetrics.remoteRequests.mark(); WriteResponseHandlerWrapper wrapper = wrapBatchResponseHandler(untrackedMutation, - dataReplicaPlan, + plan, batchConsistencyLevel, WriteType.BATCH, cleanup, @@ -1971,7 +1972,7 @@ public class StorageProxy implements StorageProxyMBean // with the mutations delivered by the batch log since an unacknowledged Accord txn won't be retried // unless those mutations are also written to the batch log // Only write to the log once and reuse the batchUUID for every attempt to route the mutations correctly - doFallibleWriteWithMetricTracking(() -> syncWriteToBatchlog(mutations, batchlogReplicaPlan, batchUUID, requestTime), consistencyLevel); + doFallibleWriteWithMetricTracking(() -> syncWriteToBatchlog(mutations, coordinationPlan, batchUUID, requestTime), consistencyLevel); Tracing.trace("Successfully wrote to batchlog"); wroteToBatchLog = true; } @@ -2104,17 +2105,17 @@ public class StorageProxy implements StorageProxyMBean } } - private static void syncWriteToBatchlog(Collection mutations, ReplicaPlan.ForWrite replicaPlan, TimeUUID uuid, Dispatcher.RequestTime requestTime) + private static void syncWriteToBatchlog(Collection mutations, CoordinationPlan.ForWrite coordinationPlan, TimeUUID uuid, Dispatcher.RequestTime requestTime) throws WriteTimeoutException, WriteFailureException { - WriteResponseHandler handler = new WriteResponseHandler<>(replicaPlan, + WriteResponseHandler handler = new WriteResponseHandler<>(coordinationPlan, WriteType.BATCH_LOG, null, requestTime); Batch batch = Batch.createLocal(uuid, FBUtilities.timestampMicros(), mutations); Message message = Message.out(BATCH_STORE_REQ, batch); - for (Replica replica : replicaPlan.liveAndDown()) + for (Replica replica : coordinationPlan.replicas().liveAndDown()) { if (logger.isTraceEnabled()) logger.trace("Sending batchlog store request {} to {} for {} mutations", batch.id, replica, batch.size()); @@ -2146,8 +2147,8 @@ public class StorageProxy implements StorageProxyMBean { for (WriteResponseHandlerWrapper wrapper : wrappers) { - Replicas.temporaryAssertFull(wrapper.handler.replicaPlan.liveAndDown()); // TODO: CASSANDRA-14549 - ReplicaPlan.ForWrite replicas = wrapper.handler.replicaPlan.withContacts(wrapper.handler.replicaPlan.liveAndDown()); + Replicas.temporaryAssertFull(wrapper.handler.replicaPlan().liveAndDown()); // TODO: CASSANDRA-14549 + ReplicaPlan.ForWrite replicas = wrapper.handler.replicaPlan().withContacts(wrapper.handler.replicaPlan().liveAndDown()); try { @@ -2167,9 +2168,9 @@ public class StorageProxy implements StorageProxyMBean for (WriteResponseHandlerWrapper wrapper : wrappers) { - EndpointsForToken sendTo = wrapper.handler.replicaPlan.liveAndDown(); + EndpointsForToken sendTo = wrapper.handler.replicaPlan().liveAndDown(); Replicas.temporaryAssertFull(sendTo); // TODO: CASSANDRA-14549 - sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan.withContacts(sendTo), wrapper.handler, localDataCenter, stage, requestTime); + sendToHintedReplicas(wrapper.mutation, wrapper.handler.replicaPlan().withContacts(sendTo), wrapper.handler, localDataCenter, stage, requestTime); } for (WriteResponseHandlerWrapper wrapper : wrappers) @@ -2202,30 +2203,32 @@ public class StorageProxy implements StorageProxyMBean Keyspace keyspace = Keyspace.open(keyspaceName); Token tk = mutation.key().getToken(); - ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); - if (replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null) + ClusterMetadata cm = ClusterMetadata.current(); + + CoordinationPlan.ForWriteWithIdeal plan = CoordinationPlan.forWrite(cm, keyspace, consistencyLevel, tk, ReplicaPlans.writeAll); + + if (plan.replicas().lookup(FBUtilities.getBroadcastAddressAndPort()) != null) writeMetrics.localRequests.mark(); else writeMetrics.remoteRequests.mark(); - AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); - AbstractWriteResponseHandler responseHandler = rs.getWriteResponseHandler(replicaPlan, callback, writeType, mutation.hintOnFailure(), requestTime); + AbstractWriteResponseHandler responseHandler = plan.replicationStrategy().getWriteResponseHandler(plan, callback, writeType, mutation.hintOnFailure(), requestTime); - performer.apply(mutation, replicaPlan, responseHandler, localDataCenter, requestTime); + performer.apply(mutation, plan.replicas(), responseHandler, localDataCenter, requestTime); return responseHandler; } // same as performWrites except does not initiate writes (but does perform availability checks). private static WriteResponseHandlerWrapper wrapBatchResponseHandler(Mutation mutation, - ReplicaPlan.ForWrite replicaPlan, + CoordinationPlan.ForWriteWithIdeal plan, ConsistencyLevel batchConsistencyLevel, WriteType writeType, BatchlogResponseHandler.BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime) { - AbstractReplicationStrategy rs = replicaPlan.replicationStrategy(); - AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(replicaPlan, null, writeType, mutation, requestTime); + AbstractReplicationStrategy rs = plan.replicationStrategy(); + AbstractWriteResponseHandler writeHandler = rs.getWriteResponseHandler(plan, null, writeType, mutation, requestTime); BatchlogResponseHandler batchHandler = new BatchlogResponseHandler<>(writeHandler, batchConsistencyLevel.blockFor(rs), cleanup, requestTime); return new WriteResponseHandlerWrapper(batchHandler, mutation); } @@ -2236,14 +2239,14 @@ public class StorageProxy implements StorageProxyMBean */ private static WriteResponseHandlerWrapper wrapViewBatchResponseHandler(Mutation mutation, ConsistencyLevel batchConsistencyLevel, - ReplicaPlan.ForWrite replicaPlan, + CoordinationPlan.ForWriteWithIdeal plan, AtomicLong baseComplete, WriteType writeType, BatchlogResponseHandler.BatchlogCleanup cleanup, Dispatcher.RequestTime requestTime) { - AbstractReplicationStrategy replicationStrategy = replicaPlan.replicationStrategy(); - AbstractWriteResponseHandler writeHandler = replicationStrategy.getWriteResponseHandler(replicaPlan, () -> { + AbstractReplicationStrategy replicationStrategy = plan.replicationStrategy(); + AbstractWriteResponseHandler writeHandler = replicationStrategy.getWriteResponseHandler(plan, () -> { long delay = Math.max(0, currentTimeMillis() - baseComplete.get()); viewWriteMetrics.viewWriteLatency.update(delay, MILLISECONDS); }, writeType, mutation, requestTime); @@ -2362,7 +2365,7 @@ public class StorageProxy implements StorageProxyMBean else { //Immediately mark the response as expired since the request will not be sent - responseHandler.expired(); + responseHandler.expired(destination.endpoint()); if (shouldHint(destination)) { if (endpointsToHint == null) @@ -2567,10 +2570,10 @@ public class StorageProxy implements StorageProxyMBean // there we'll mark a local request against the metrics. writeMetrics.remoteRequests.mark(); - ReplicaPlan.ForWrite forWrite = ReplicaPlans.forForwardingCounterWrite(metadata, keyspace, tk, - clm -> ReplicaPlans.findCounterLeaderReplica(clm, cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency())); + CoordinationPlan.ForWrite plan = CoordinationPlan.forForwardingCounterWrite(metadata, keyspace, tk, + clm -> ReplicaPlans.findCounterLeaderReplica(clm, cm.getKeyspaceName(), cm.key(), localDataCenter, cm.consistency())); // Forward the actual update to the chosen leader replica - AbstractWriteResponseHandler responseHandler = new WriteResponseHandler<>(forWrite, + AbstractWriteResponseHandler responseHandler = new WriteResponseHandler<>(plan, WriteType.COUNTER, null, requestTime); Tracing.trace("Enqueuing counter update to {}", replica); @@ -3831,7 +3834,7 @@ public class StorageProxy implements StorageProxyMBean HintsService.instance.write(hostIds, Hint.create(mutation, creationTime)); validTargets.forEach(HintsService.instance.metrics::incrCreatedHints); // Notify the handler only for CL == ANY - if (responseHandler != null && responseHandler.replicaPlan.consistencyLevel() == ConsistencyLevel.ANY) + if (responseHandler != null && responseHandler.replicaPlan().consistencyLevel() == ConsistencyLevel.ANY) responseHandler.onResponse(null); } }; diff --git a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java index e98212772b..d85eae227a 100644 --- a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java @@ -38,7 +38,7 @@ public class TrackedWriteResponseHandler extends AbstractWriteResponseHandler private TrackedWriteResponseHandler(AbstractWriteResponseHandler wrapped, MutationId mutationId) { - super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, wrapped.getRequestTime()); + super(wrapped.plan, wrapped.callback, wrapped.writeType, null, wrapped.getRequestTime()); this.wrapped = wrapped; this.mutationId = mutationId; } diff --git a/src/java/org/apache/cassandra/service/WriteResponseHandler.java b/src/java/org/apache/cassandra/service/WriteResponseHandler.java index d99aa3989f..505ee62b37 100644 --- a/src/java/org/apache/cassandra/service/WriteResponseHandler.java +++ b/src/java/org/apache/cassandra/service/WriteResponseHandler.java @@ -18,7 +18,6 @@ package org.apache.cassandra.service; import java.util.Map; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.function.Supplier; import org.slf4j.Logger; @@ -27,8 +26,8 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.MessageParams; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.ParamType; import org.apache.cassandra.service.writes.thresholds.WriteWarningContext; @@ -37,28 +36,25 @@ import org.apache.cassandra.utils.FBUtilities; /** * Handles blocking writes for ONE, ANY, TWO, THREE, QUORUM, and ALL consistency levels. + * + * Response tracking is delegated to coordinationPlan.tracker(). */ public class WriteResponseHandler extends AbstractWriteResponseHandler { protected static final Logger logger = LoggerFactory.getLogger(WriteResponseHandler.class); - protected volatile int responses; - private static final AtomicIntegerFieldUpdater responsesUpdater - = AtomicIntegerFieldUpdater.newUpdater(WriteResponseHandler.class, "responses"); - - public WriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, + public WriteResponseHandler(CoordinationPlan.ForWrite coordinationPlan, Runnable callback, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) { - super(replicaPlan, callback, writeType, hintOnFailure, requestTime); - responses = blockFor(); + super(coordinationPlan, callback, writeType, hintOnFailure, requestTime); } - public WriteResponseHandler(ReplicaPlan.ForWrite replicaPlan, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) + public WriteResponseHandler(CoordinationPlan.ForWrite coordinationPlan, WriteType writeType, Supplier hintOnFailure, Dispatcher.RequestTime requestTime) { - this(replicaPlan, null, writeType, hintOnFailure, requestTime); + this(coordinationPlan, null, writeType, hintOnFailure, requestTime); } public void onResponse(Message m) @@ -69,17 +65,19 @@ public class WriteResponseHandler extends AbstractWriteResponseHandler if (WriteWarningContext.isSupported(params.keySet())) getWarningContext().updateCounters(params); - replicaPlan.collectSuccess(from); - if (responsesUpdater.decrementAndGet(this) == 0) + replicaPlan().collectSuccess(from); + + plan.responses().onResponse(from); + + if (plan.responses().isComplete()) signal(); - //Must be last after all subclass processing - //The two current subclasses both assume logResponseToIdealCLDelegate is called - //here. + + // Must be last (see comment on AbstractWriteResponseHandler.logResponseToIdealCLDelegate for why) - forward to ideal CL delegate logResponseToIdealCLDelegate(m); } protected int ackCount() { - return blockFor() - responses; + return plan.responses().received(); } } diff --git a/src/java/org/apache/cassandra/service/paxos/Paxos.java b/src/java/org/apache/cassandra/service/paxos/Paxos.java index f2f60f27d3..3f6fcc42fc 100644 --- a/src/java/org/apache/cassandra/service/paxos/Paxos.java +++ b/src/java/org/apache/cassandra/service/paxos/Paxos.java @@ -28,7 +28,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Predicate; -import java.util.function.Supplier; import javax.annotation.Nullable; @@ -88,7 +87,6 @@ import org.apache.cassandra.locator.ReplicaPlan.ForRead; import org.apache.cassandra.metrics.ClientRequestMetrics; import org.apache.cassandra.metrics.ClientRequestSizeMetrics; import org.apache.cassandra.net.Message; -import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; @@ -404,7 +402,7 @@ public class Paxos final Epoch epoch; final Function recompute; - Participants(Epoch epoch, Keyspace keyspace, ConsistencyLevel consistencyForConsensus, ReplicaLayout.ForTokenWrite all, ReplicaLayout.ForTokenWrite electorate, EndpointsForToken live, + public Participants(Epoch epoch, Keyspace keyspace, ConsistencyLevel consistencyForConsensus, ReplicaLayout.ForTokenWrite all, ReplicaLayout.ForTokenWrite electorate, EndpointsForToken live, Function recompute) { this.epoch = epoch; @@ -464,26 +462,15 @@ public class Paxos } - static Participants get(ClusterMetadata metadata, TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus) + public static Participants get(ClusterMetadata metadata, TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus) { return get(metadata, table, token, consistencyForConsensus, FailureDetector.isReplicaAlive); } static Participants get(ClusterMetadata metadata, TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus, Predicate isReplicaAlive) { - KeyspaceMetadata keyspaceMetadata = metadata.schema.getKeyspaceMetadata(table.keyspace); - // MetaStrategy distributes the entire keyspace to all replicas. In addition, its tables (currently only - // the dist log table) don't use the globally configured partitioner. For these reasons we don't lookup the - // replicas using the supplied token as this can actually be of the incorrect type (for example when - // performing Paxos repair). - final Token actualToken = table.partitioner == MetaStrategy.partitioner ? MetaStrategy.entireRange.right : token; - ReplicaLayout.ForTokenWrite all = forTokenWriteLiveAndDown(keyspaceMetadata, actualToken); - ReplicaLayout.ForTokenWrite electorate = consistencyForConsensus.isDatacenterLocal() - ? all.filter(InOurDc.replicas()) : all; - - EndpointsForToken live = all.all().filter(isReplicaAlive); - return new Participants(metadata.epoch, Keyspace.open(table.keyspace), consistencyForConsensus, all, electorate, live, - (cm) -> get(cm, table, actualToken, consistencyForConsensus)); + AbstractReplicationStrategy strategy = metadata.schema.getKeyspaceMetadata(table.keyspace).replicationStrategy; + return strategy.paxosParticipants(metadata, table, token, consistencyForConsensus, isReplicaAlive); } static Participants get(TableMetadata table, Token token, ConsistencyLevel consistencyForConsensus) @@ -1139,7 +1126,6 @@ public class Paxos // round's proposal (if any). PaxosPrepare.Success success = prepare.success(); - Supplier plan = () -> success.participants; List> responses = success.responses; // There should be only a single response from the coordinator that was selected to do the tracked read @@ -1161,7 +1147,7 @@ public class Paxos } else { - DataResolver resolver = new DataResolver<>(ReadCoordinator.DEFAULT, query, plan, NoopReadRepair.instance, requestTime); + DataResolver resolver = new DataResolver<>(ReadCoordinator.DEFAULT, query, () -> success.participants, NoopReadRepair.instance, requestTime); for (int i = 0 ; i < responses.size() ; ++i) { diff --git a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java index 36315bf06c..b88cdd0528 100644 --- a/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java +++ b/src/java/org/apache/cassandra/service/paxos/PaxosCommit.java @@ -323,11 +323,22 @@ public class PaxosCommit> ex } } } + if (localIsReplica) { executeOnSelf(); localExecutedSynchronously = true; } + + // Set up additional commit work from the replication strategy (e.g., satellite writes for SRS). + // This needs to happen before executeOnSelf() can trigger onPaxosDecision(), so that + // additionalCommitFuture is set before it's read. The base strategy returns an + // already-completed future, so this is a no-op for non-SRS keyspaces. + // For SRS, satellite messages are sent here (in parallel with local execution below). + // MutationTrackingService.retryFailedWrite for down satellite endpoints schedules async retries, + // which will find the mutation in the journal after executeOnSelf() completes below. + AbstractReplicationStrategy strategy = Keyspace.open(commit.metadata().keyspace).getReplicationStrategy(); + setAugmentedCommitFuture(strategy.sendPaxosCommitMutations(commit, isUrgent)); } // Now send to remote replicas (and record local execution for non-tracked keyspaces) diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java index df694b0477..75d13d38d7 100644 --- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java @@ -34,12 +34,12 @@ import org.apache.cassandra.db.transform.DuplicateRowChecker; import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaCollection; import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageProxy.LocalReadRunnable; @@ -68,7 +68,7 @@ public abstract class AbstractReadExecutor implements ReadExecutor protected final ReadCoordinator coordinator; protected final ReadCommand command; - private final ReplicaPlan.SharedForTokenRead replicaPlan; + private final CoordinationPlan.ForTokenRead plan; protected final ReadRepair readRepair; protected final DigestResolver digestResolver; protected final ReadCallback handler; @@ -79,16 +79,16 @@ public abstract class AbstractReadExecutor implements ReadExecutor private final int initialDataRequestCount; protected volatile PartitionIterator result = null; - AbstractReadExecutor(ReadCoordinator coordinator, ColumnFamilyStore cfs, ReadCommand command, ReplicaPlan.ForTokenRead replicaPlan, int initialDataRequestCount, Dispatcher.RequestTime requestTime) + AbstractReadExecutor(ReadCoordinator coordinator, ColumnFamilyStore cfs, ReadCommand command, CoordinationPlan.ForTokenRead plan, int initialDataRequestCount, Dispatcher.RequestTime requestTime) { this.coordinator = coordinator; this.command = command; - this.replicaPlan = ReplicaPlan.shared(replicaPlan); + this.plan = plan; this.initialDataRequestCount = initialDataRequestCount; // the ReadRepair and DigestResolver both need to see our updated - this.readRepair = ReadRepair.create(coordinator, command, this.replicaPlan, requestTime); - this.digestResolver = new DigestResolver<>(coordinator, command, this.replicaPlan, requestTime); - this.handler = new ReadCallback<>(digestResolver, command, this.replicaPlan, requestTime); + this.readRepair = ReadRepair.create(coordinator, command, plan, requestTime); + this.digestResolver = new DigestResolver<>(coordinator, command, plan, requestTime); + this.handler = new ReadCallback<>(digestResolver, command, plan, requestTime); this.cfs = cfs; this.traceState = Tracing.instance.get(); this.requestTime = requestTime; @@ -99,7 +99,7 @@ public abstract class AbstractReadExecutor implements ReadExecutor // TODO: we need this when talking with pre-3.0 nodes. So if we preserve the digest format moving forward, we can get rid of this once // we stop being compatible with pre-3.0 nodes. int digestVersion = MessagingService.current_version; - for (Replica replica : replicaPlan.contacts()) + for (Replica replica : plan.replicas().contacts()) digestVersion = Math.min(digestVersion, MessagingService.instance().versions.get(replica.endpoint())); command.setDigestVersion(digestVersion); } @@ -195,32 +195,32 @@ public abstract class AbstractReadExecutor implements ReadExecutor ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id); SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry; - ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(metadata, - keyspace, - command.metadata().id, - command.partitionKey().getToken(), - command.indexQueryPlan(), - consistencyLevel, - retry, - coordinator); + CoordinationPlan.ForTokenRead plan = CoordinationPlan.forTokenRead(metadata, + keyspace, + command.metadata().id, + command.partitionKey().getToken(), + command.indexQueryPlan(), + consistencyLevel, + retry, + coordinator); // Speculative retry is disabled *OR* // 11980: Disable speculative retry if using EACH_QUORUM in order to prevent miscounting DC responses if (retry.equals(NeverSpeculativeRetryPolicy.INSTANCE) || consistencyLevel == ConsistencyLevel.EACH_QUORUM) - return new NeverSpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime, false); + return new NeverSpeculatingReadExecutor(coordinator, cfs, command, plan, requestTime, false); if (retry.equals(AlwaysSpeculativeRetryPolicy.INSTANCE)) - return new AlwaysSpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime); + return new AlwaysSpeculatingReadExecutor(coordinator, cfs, command, plan, requestTime); // There are simply no extra replicas to speculate. // Handle this separately so it can record failed attempts to speculate due to lack of replicas - if (replicaPlan.contacts().size() == replicaPlan.readCandidates().size()) + if (plan.replicas().contacts().size() == plan.replicas().readCandidates().size()) { boolean recordFailedSpeculation = consistencyLevel != ConsistencyLevel.ALL; - return new NeverSpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime, recordFailedSpeculation); + return new NeverSpeculatingReadExecutor(coordinator, cfs, command, plan, requestTime, recordFailedSpeculation); } else // PERCENTILE or CUSTOM. - return new SpeculatingReadExecutor(coordinator, cfs, command, replicaPlan, requestTime); + return new SpeculatingReadExecutor(coordinator, cfs, command, plan, requestTime); } /** @@ -256,7 +256,7 @@ public abstract class AbstractReadExecutor implements ReadExecutor @Override public ReplicaPlan.ForTokenRead replicaPlan() { - return replicaPlan.get(); + return plan.replicas(); } void onReadTimeout() {} @@ -273,11 +273,11 @@ public abstract class AbstractReadExecutor implements ReadExecutor public NeverSpeculatingReadExecutor(ReadCoordinator coordinator, ColumnFamilyStore cfs, ReadCommand command, - ReplicaPlan.ForTokenRead replicaPlan, + CoordinationPlan.ForTokenRead plan, Dispatcher.RequestTime requestTime, boolean logFailedSpeculation) { - super(coordinator, cfs, command, replicaPlan, 1, requestTime); + super(coordinator, cfs, command, plan, 1, requestTime); this.logFailedSpeculation = logFailedSpeculation; } @@ -297,13 +297,13 @@ public abstract class AbstractReadExecutor implements ReadExecutor public SpeculatingReadExecutor(ReadCoordinator coordinator, ColumnFamilyStore cfs, ReadCommand command, - ReplicaPlan.ForTokenRead replicaPlan, + CoordinationPlan.ForTokenRead plan, Dispatcher.RequestTime requestTime) { // We're hitting additional targets for read repair (??). Since our "extra" replica is the least- // preferred by the snitch, we do an extra data read to start with against a replica more // likely to respond; better to let RR fail than the entire query. - super(coordinator, cfs, command, replicaPlan, replicaPlan.readQuorum() < replicaPlan.contacts().size() ? 2 : 1, requestTime); + super(coordinator, cfs, command, plan, plan.replicas().readQuorum() < plan.replicas().contacts().size() ? 2 : 1, requestTime); } public void maybeTryAdditionalReplicas() @@ -342,7 +342,7 @@ public abstract class AbstractReadExecutor implements ReadExecutor // we must update the plan to include this new node, else when we come to read-repair, we may not include this // speculated response in the data requests we make again, and we will not be able to 'speculate' an extra repair read, // nor would we be able to speculate a new 'write' if the repair writes are insufficient - super.replicaPlan.addToContacts(extraReplica); + super.plan.addToContacts(extraReplica); if (traceState != null) traceState.trace("speculating read retry on {}", extraReplica); @@ -367,12 +367,12 @@ public abstract class AbstractReadExecutor implements ReadExecutor public AlwaysSpeculatingReadExecutor(ReadCoordinator coordinator, ColumnFamilyStore cfs, ReadCommand command, - ReplicaPlan.ForTokenRead replicaPlan, + CoordinationPlan.ForTokenRead plan, Dispatcher.RequestTime requestTime) { // presumably, we speculate an extra data request here in case it is our data request that fails to respond, // and there are no more nodes to consult - super(coordinator, cfs, command, replicaPlan, replicaPlan.contacts().size() > 1 ? 2 : 1, requestTime); + super(coordinator, cfs, command, plan, plan.replicas().contacts().size() > 1 ? 2 : 1, requestTime); } public void maybeTryAdditionalReplicas() @@ -397,7 +397,7 @@ public abstract class AbstractReadExecutor implements ReadExecutor public void setResult(PartitionIterator result) { Preconditions.checkState(this.result == null, "Result can only be set once"); - this.result = DuplicateRowChecker.duringRead(result, this.replicaPlan.get().readCandidates().endpointList()); + this.result = DuplicateRowChecker.duringRead(result, this.plan.replicas().readCandidates().endpointList()); } public void awaitResponses() throws ReadTimeoutException diff --git a/src/java/org/apache/cassandra/service/reads/DigestResolver.java b/src/java/org/apache/cassandra/service/reads/DigestResolver.java index 59c3df383d..fcd07fcc16 100644 --- a/src/java/org/apache/cassandra/service/reads/DigestResolver.java +++ b/src/java/org/apache/cassandra/service/reads/DigestResolver.java @@ -20,6 +20,7 @@ package org.apache.cassandra.service.reads; import java.nio.ByteBuffer; import java.util.Collection; import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -45,7 +46,7 @@ public class DigestResolver, P extends ReplicaPlan.ForRea { private volatile Message dataResponse; - public DigestResolver(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public DigestResolver(ReadCoordinator coordinator, ReadCommand command, Supplier

replicaPlan, Dispatcher.RequestTime requestTime) { super(coordinator, command, replicaPlan, requestTime); Preconditions.checkArgument(command instanceof SinglePartitionReadCommand, diff --git a/src/java/org/apache/cassandra/service/reads/ReadCallback.java b/src/java/org/apache/cassandra/service/reads/ReadCallback.java index 7f68b3e4d5..9696080863 100644 --- a/src/java/org/apache/cassandra/service/reads/ReadCallback.java +++ b/src/java/org/apache/cassandra/service/reads/ReadCallback.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import com.google.common.collect.ImmutableMap; @@ -40,6 +39,7 @@ import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.exceptions.RetryOnDifferentSystemException; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.ReplicaPlan; @@ -57,7 +57,6 @@ import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.atomic.AtomicIntegerFieldUpdater.newUpdater; import static org.apache.cassandra.exceptions.RequestFailureReason.COORDINATOR_BEHIND; import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM; import static org.apache.cassandra.tracing.Tracing.isTracing; @@ -72,33 +71,30 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< private final Dispatcher.RequestTime requestTime; // this uses a plain reference, but is initialised before handoff to any other threads; the later updates // may not be visible to the threads immediately, but ReplicaPlan only contains final fields, so they will never see an uninitialised object - final ReplicaPlan.Shared replicaPlan; + final CoordinationPlan.ForRead plan; private final ReadCommand command; - private static final AtomicIntegerFieldUpdater failuresUpdater - = newUpdater(ReadCallback.class, "failures"); - private volatile int failures = 0; private final Map failureReasonByEndpoint; private volatile WarningContext warningContext; private static final AtomicReferenceFieldUpdater warningsUpdater = AtomicReferenceFieldUpdater.newUpdater(ReadCallback.class, WarningContext.class, "warningContext"); - public ReadCallback(ResponseResolver resolver, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public ReadCallback(ResponseResolver resolver, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { this.command = command; this.resolver = resolver; this.requestTime = requestTime; - this.replicaPlan = replicaPlan; + this.plan = plan; this.failureReasonByEndpoint = new ConcurrentHashMap<>(); // we don't support read repair (or rapid read protection) for range scans yet (CASSANDRA-6897) assert !(command instanceof PartitionRangeReadCommand) || replicaPlan().readQuorum() >= replicaPlan().contacts().size(); if (logger.isTraceEnabled()) - logger.trace("Blockfor is {}; setting up requests to {}", replicaPlan().readQuorum(), this.replicaPlan); + logger.trace("Blockfor is {}; setting up requests to {}", replicaPlan().readQuorum(), this.plan.replicas()); } protected P replicaPlan() { - return replicaPlan.get(); + return plan.replicas(); } public boolean await(long commandTimeout, TimeUnit unit) @@ -141,8 +137,8 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< * See {@link DigestResolver#preprocess(Message)} * CASSANDRA-16097 */ - int received = resolver.responses.size(); - boolean failed = failures > 0 && (replicaPlan().readQuorum() > received || !resolver.isDataPresent()); + int received = plan.responses().received(); + boolean failed = plan.responses().failures() > 0 && (!plan.responses().isSuccessful() || !resolver.isDataPresent()); // If all messages came back as a TIMEOUT then signaled=true and failed=true. // Need to distinguish between a timeout and a failure (network, bad data, etc.), so store an extra field. // see CASSANDRA-17828 @@ -168,16 +164,16 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< if (isTracing()) { String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : ""; - Tracing.trace("{}; received {} of {} responses{}", !timedout ? "Failed" : "Timed out", received, replicaPlan().readQuorum(), gotData); + Tracing.trace("{}; received {} of {} responses{}", !timedout ? "Failed" : "Timed out", received, plan.responses().required(), gotData); } else if (logger.isDebugEnabled()) { String gotData = received > 0 ? (resolver.isDataPresent() ? " (including data)" : " (only digests)") : ""; - logger.debug("{}; received {} of {} responses{}", !timedout ? "Failed" : "Timed out", received, replicaPlan().readQuorum(), gotData); + logger.debug("{}; received {} of {} responses{}", !timedout ? "Failed" : "Timed out", received, plan.responses().required(), gotData); } if (snapshot != null) - snapshot.maybeAbort(command, replicaPlan().consistencyLevel(), received, replicaPlan().readQuorum(), resolver.isDataPresent(), failureReasonByEndpoint); + snapshot.maybeAbort(command, replicaPlan().consistencyLevel(), received, plan.responses().required(), resolver.isDataPresent(), failureReasonByEndpoint); // failures keeps incrementing, and this.failureReasonByEndpoint keeps getting new entries after signaling. // Simpler to reason about what happened by copying this.failureReasonByEndpoint and then inferring @@ -209,8 +205,8 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< // Same as for writes, see AbstractWriteResponseHandler throw !timedout - ? new ReadFailureException(replicaPlan().consistencyLevel(), received, replicaPlan().readQuorum(), resolver.isDataPresent(), failureReasonByEndpoint) - : new ReadTimeoutException(replicaPlan().consistencyLevel(), received, replicaPlan().readQuorum(), resolver.isDataPresent()); + ? new ReadFailureException(replicaPlan().consistencyLevel(), received, plan.responses().required(), resolver.isDataPresent(), failureReasonByEndpoint) + : new ReadTimeoutException(replicaPlan().consistencyLevel(), received, plan.responses().required(), resolver.isDataPresent()); } @Override @@ -231,6 +227,7 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< } resolver.preprocess(message); replicaPlan().collectSuccess(message.from()); + plan.responses().onResponse(from); /* * Ensure that data is present and the response accumulator has properly published the @@ -238,7 +235,7 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< * the minimum number of required results, but it guarantees at least the minimum will * be accessible when we do signal. (see CASSANDRA-16807) */ - if (resolver.isDataPresent() && resolver.responses.size() >= replicaPlan().readQuorum()) + if (resolver.isDataPresent() && plan.responses().isSuccessful()) condition.signalAll(); } @@ -274,10 +271,11 @@ public class ReadCallback, P extends ReplicaPlan.ForRead< public void onFailure(InetAddressAndPort from, RequestFailure failure) { assertWaitingFor(from); - - failureReasonByEndpoint.put(from, failure.reason); - if (replicaPlan().readQuorum() + failuresUpdater.incrementAndGet(this) > replicaPlan().contacts().size()) + failureReasonByEndpoint.put(from, failure.reason); + plan.responses().onFailure(from); + + if (plan.responses().isComplete() && !plan.responses().isSuccessful()) condition.signalAll(); } diff --git a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java index 12a23af2bd..50c72e689c 100644 --- a/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ReplicaFilteringProtection.java @@ -60,11 +60,11 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.exceptions.OverloadedException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.UnavailableException; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.schema.ColumnMetadata; @@ -164,13 +164,13 @@ public class ReplicaFilteringProtection> mergeListener = new QueryMergeListener(); } - private UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, Replica source, ReplicaPlan.Shared replicaPlan) + private UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, Replica source, CoordinationPlan.ForTokenRead plan) { @SuppressWarnings("unchecked") DataResolver resolver = - new DataResolver<>(coordinator, cmd, replicaPlan, (NoopReadRepair) NoopReadRepair.instance, requestTime); + new DataResolver<>(coordinator, cmd, plan, (NoopReadRepair) NoopReadRepair.instance, requestTime); - ReadCallback handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime); + ReadCallback handler = new ReadCallback<>(resolver, cmd, plan, requestTime); // TODO No tracked path here yet so assert it doesn't handle transient replication correctly checkState(!source.isTransient()); if (source.isSelf() && coordinator.localReadSupported()) @@ -636,20 +636,20 @@ public class ReplicaFilteringProtection> key, filter); - ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forSingleReplicaRead(keyspace, key.getToken(), source); + CoordinationPlan.ForTokenRead plan = CoordinationPlan.forSingleReplicaTokenRead(keyspace, key.getToken(), source); try { - return executeReadCommand(cmd, source, ReplicaPlan.shared(replicaPlan)); + return executeReadCommand(cmd, source, plan); } catch (ReadTimeoutException e) { - int blockFor = consistency.blockFor(replicaPlan.replicationStrategy()); + int blockFor = consistency.blockFor(plan.replicationStrategy()); throw new ReadTimeoutException(consistency, blockFor - 1, blockFor, true); } catch (UnavailableException e) { - int blockFor = consistency.blockFor(replicaPlan.replicationStrategy()); + int blockFor = consistency.blockFor(plan.replicationStrategy()); throw UnavailableException.create(consistency, blockFor, blockFor - 1); } } diff --git a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java index 1dc8d9c87e..67079acf47 100644 --- a/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java +++ b/src/java/org/apache/cassandra/service/reads/ShortReadPartitionsProtection.java @@ -38,10 +38,10 @@ import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.ExcludingBounds; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.reads.repair.NoopReadRepair; import org.apache.cassandra.tracing.Tracing; @@ -93,17 +93,17 @@ public class ShortReadPartitionsProtection extends Transformation executeReadCommand(cmd, sharedReplicaPlan), + (cmd) -> executeReadCommand(cmd, plan), singleResultCounter, mergedResultCounter); return Transformation.apply(MoreRows.extend(partition, protection), protection); @@ -177,16 +177,17 @@ public class ShortReadPartitionsProtection extends Transformation(lastPartitionKey, bounds.right); DataRange newDataRange = cmd.dataRange().forSubRange(newBounds); - ReplicaPlan.ForRangeRead replicaPlan = ReplicaPlans.forSingleReplicaRead(Keyspace.open(command.metadata().keyspace), cmd.dataRange().keyRange(), source, 1); - return executeReadCommand(cmd.withUpdatedLimitsAndDataRange(newLimits, newDataRange), ReplicaPlan.shared(replicaPlan)); + Keyspace keyspace = Keyspace.open(command.metadata().keyspace); + CoordinationPlan.ForRangeRead plan = CoordinationPlan.forSingleReplicaRangeRead(keyspace, cmd.dataRange().keyRange(), source, 1); + return executeReadCommand(cmd.withUpdatedLimitsAndDataRange(newLimits, newDataRange), plan); } private , P extends ReplicaPlan.ForRead> - UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, ReplicaPlan.Shared replicaPlan) + UnfilteredPartitionIterator executeReadCommand(ReadCommand cmd, CoordinationPlan.ForRead plan) { - cmd = coordinator.maybeAllowOutOfRangeReads(cmd, replicaPlan.get().consistencyLevel()); - DataResolver resolver = new DataResolver<>(coordinator, cmd, replicaPlan, (NoopReadRepair)NoopReadRepair.instance, requestTime); - ReadCallback handler = new ReadCallback<>(resolver, cmd, replicaPlan, requestTime); + cmd = coordinator.maybeAllowOutOfRangeReads(cmd, plan.consistencyLevel()); + DataResolver resolver = new DataResolver<>(coordinator, cmd, plan, (NoopReadRepair)NoopReadRepair.instance, requestTime); + ReadCallback handler = new ReadCallback<>(resolver, cmd, plan, requestTime); if (source.isSelf() && coordinator.localReadSupported()) { diff --git a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java b/src/java/org/apache/cassandra/service/reads/range/CoordinationPlanIterator.java similarity index 89% rename from src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java rename to src/java/org/apache/cassandra/service/reads/range/CoordinationPlanIterator.java index e138fab4f1..4875b704ca 100644 --- a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/CoordinationPlanIterator.java @@ -34,8 +34,8 @@ import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Bounds; import org.apache.cassandra.dht.Token; import org.apache.cassandra.index.Index; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.tcm.ClusterMetadata; @@ -43,7 +43,7 @@ import org.apache.cassandra.tcm.compatibility.TokenRingUtils; import org.apache.cassandra.utils.AbstractIterator; import org.apache.cassandra.utils.Pair; -class ReplicaPlanIterator extends AbstractIterator +class CoordinationPlanIterator extends AbstractIterator { private final Keyspace keyspace; private final ConsistencyLevel consistency; @@ -53,11 +53,11 @@ class ReplicaPlanIterator extends AbstractIterator final Iterator> ranges; private final int rangeCount; - ReplicaPlanIterator(AbstractBounds keyRange, - @Nullable Index.QueryPlan indexQueryPlan, - Keyspace keyspace, - TableId tableId, - ConsistencyLevel consistency) + CoordinationPlanIterator(AbstractBounds keyRange, + @Nullable Index.QueryPlan indexQueryPlan, + Keyspace keyspace, + TableId tableId, + ConsistencyLevel consistency) { this.indexQueryPlan = indexQueryPlan; this.keyspace = keyspace; @@ -81,12 +81,12 @@ class ReplicaPlanIterator extends AbstractIterator } @Override - protected ReplicaPlan.ForRangeRead computeNext() + protected CoordinationPlan.ForRangeRead computeNext() { if (!ranges.hasNext()) return endOfData(); - return ReplicaPlans.forRangeRead(keyspace, tableId, indexQueryPlan, consistency, ranges.next(), 1); + return CoordinationPlan.forRangeRead(ClusterMetadata.current(), keyspace, tableId, indexQueryPlan, consistency, ranges.next(), 1); } /** diff --git a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java b/src/java/org/apache/cassandra/service/reads/range/CoordinationPlanMerger.java similarity index 78% rename from src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java rename to src/java/org/apache/cassandra/service/reads/range/CoordinationPlanMerger.java index 743ac8d8e6..028b89c3f0 100644 --- a/src/java/org/apache/cassandra/service/reads/range/ReplicaPlanMerger.java +++ b/src/java/org/apache/cassandra/service/reads/range/CoordinationPlanMerger.java @@ -25,20 +25,19 @@ import com.google.common.collect.PeekingIterator; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.AbstractIterator; -class ReplicaPlanMerger extends AbstractIterator +class CoordinationPlanMerger extends AbstractIterator { private final Keyspace keyspace; private final ConsistencyLevel consistency; private final TableId tableId; - private final PeekingIterator ranges; + private final PeekingIterator ranges; - ReplicaPlanMerger(Iterator iterator, Keyspace keyspace, TableId tableId, ConsistencyLevel consistency) + CoordinationPlanMerger(Iterator iterator, Keyspace keyspace, TableId tableId, ConsistencyLevel consistency) { this.keyspace = keyspace; this.tableId = tableId; @@ -47,12 +46,12 @@ class ReplicaPlanMerger extends AbstractIterator } @Override - protected ReplicaPlan.ForRangeRead computeNext() + protected CoordinationPlan.ForRangeRead computeNext() { if (!ranges.hasNext()) return endOfData(); - ReplicaPlan.ForRangeRead current = ranges.next(); + CoordinationPlan.ForRangeRead current = ranges.next(); ClusterMetadata metadata = ClusterMetadata.current(); // getRestrictedRange has broken the queried range into per-[vnode] token ranges, but this doesn't take @@ -65,11 +64,11 @@ class ReplicaPlanMerger extends AbstractIterator // Note: it would be slightly more efficient to have CFS.getRangeSlice on the destination nodes unwraps // the range if necessary and deal with it. However, we can't start sending wrapped range without breaking // wire compatibility, so it's likely easier not to bother; - if (current.range().right.isMinimum()) + if (current.replicas().range().right.isMinimum()) break; - ReplicaPlan.ForRangeRead next = ranges.peek(); - ReplicaPlan.ForRangeRead merged = ReplicaPlans.maybeMerge(metadata, keyspace, tableId, consistency, current, next); + CoordinationPlan.ForRangeRead next = ranges.peek(); + CoordinationPlan.ForRangeRead merged = CoordinationPlan.maybeMergeRangeReads(metadata, keyspace, tableId, consistency, current, next); if (merged == null) break; diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java index bae9953a50..4f4dd16fd1 100644 --- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java @@ -26,6 +26,9 @@ import java.util.function.Function; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.db.*; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.service.reads.DataResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +59,6 @@ import org.apache.cassandra.service.accord.txn.TxnResult; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadTarget; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter.RangeReadWithTarget; -import org.apache.cassandra.service.reads.DataResolver; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.repair.ReadRepair; @@ -82,7 +84,7 @@ public class RangeCommandIterator extends AbstractIterator implemen public static final ClientRangeRequestMetrics rangeMetrics = new ClientRangeRequestMetrics("RangeSlice"); - final CloseableIterator replicaPlans; + final CloseableIterator coordinatorPlans; final int totalRangeCount; final PartitionRangeReadCommand command; final boolean enforceStrictLiveness; @@ -101,7 +103,7 @@ public class RangeCommandIterator extends AbstractIterator implemen // when it was not good enough initially. private int liveReturned; - RangeCommandIterator(CloseableIterator replicaPlans, + RangeCommandIterator(CloseableIterator coordinatorPlans, PartitionRangeReadCommand command, ReadCoordinator readCoordinator, int concurrencyFactor, @@ -109,7 +111,7 @@ public class RangeCommandIterator extends AbstractIterator implemen int totalRangeCount, Dispatcher.RequestTime requestTime) { - this.replicaPlans = replicaPlans; + this.coordinatorPlans = coordinatorPlans; this.command = command; this.readCoordinator = readCoordinator; this.concurrencyFactor = concurrencyFactor; @@ -127,7 +129,7 @@ public class RangeCommandIterator extends AbstractIterator implemen while (sentQueryIterator == null || !sentQueryIterator.hasNext()) { // If we don't have more range to handle, we're done - if (!replicaPlans.hasNext()) + if (!coordinatorPlans.hasNext()) return endOfData(); // else, sends the next batch of concurrent queries (after having close the previous iterator) @@ -205,31 +207,30 @@ public class RangeCommandIterator extends AbstractIterator implemen return new AccordRangeResponse(result, rangeCommand.isReversed()); } - private SingleRangeResponse executeNormal(ReplicaPlan.ForRangeRead replicaPlan, PartitionRangeReadCommand rangeCommand, ReadCoordinator readCoordinator) + private SingleRangeResponse executeNormal(CoordinationPlan.ForRangeRead plan, PartitionRangeReadCommand rangeCommand, ReadCoordinator readCoordinator) { - rangeCommand = (PartitionRangeReadCommand) readCoordinator.maybeAllowOutOfRangeReads(rangeCommand, replicaPlan.consistencyLevel()); + rangeCommand = (PartitionRangeReadCommand) readCoordinator.maybeAllowOutOfRangeReads(rangeCommand, plan.consistencyLevel()); // If enabled, request repaired data tracking info from full replicas, but // only if there are multiple full replicas to compare results from. boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled() - && replicaPlan.contacts().filter(Replica::isFull).size() > 1 + && plan.replicas().contacts().filter(Replica::isFull).size() > 1 && !command.metadata().replicationType().isTracked(); - ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan); ReadRepair readRepair = - ReadRepair.create(readCoordinator, command, sharedReplicaPlan, requestTime); + ReadRepair.create(readCoordinator, command, plan, requestTime); DataResolver resolver = - new DataResolver<>(readCoordinator, rangeCommand, sharedReplicaPlan, readRepair, requestTime, trackRepairedStatus); + new DataResolver<>(readCoordinator, rangeCommand, plan, readRepair, requestTime, trackRepairedStatus); ReadCallback handler = - new ReadCallback<>(resolver, rangeCommand, sharedReplicaPlan, requestTime); - checkState(!replicaPlan.contacts().anyMatch(Replica::isTransient), "Transient replication requires mutation tracking"); + new ReadCallback<>(resolver, rangeCommand, plan, requestTime); + checkState(!plan.replicas().contacts().anyMatch(Replica::isTransient), "Transient replication requires mutation tracking"); - if (replicaPlan.contacts().size() == 1 && replicaPlan.contacts().get(0).isSelf() && readCoordinator.localReadSupported()) + if (plan.replicas().contacts().size() == 1 && plan.replicas().contacts().get(0).isSelf() && readCoordinator.localReadSupported()) { Stage.READ.execute(new StorageProxy.LocalReadRunnable(rangeCommand, handler, requestTime, trackRepairedStatus)); } else { - for (Replica replica : replicaPlan.contacts()) + for (Replica replica : plan.replicas().contacts()) { Tracing.trace("Enqueuing request to {}", replica); Message message = rangeCommand.createMessage(trackRepairedStatus && replica.isFull(), requestTime); @@ -243,20 +244,20 @@ public class RangeCommandIterator extends AbstractIterator implemen /** * Queries the provided sub-range. * - * @param replicaPlan the subRange to query. + * @param plan the subRange to query. * @param isFirst in the case where multiple queries are sent in parallel, whether that's the first query on * that batch or not. The reason it matters is that whe paging queries, the command (more specifically the * {@code DataLimits}) may have "state" information and that state may only be valid for the first query (in * that it's the query that "continues" whatever we're previously queried). */ - private PartitionIterator query(ClusterMetadata cm, ReplicaPlan.ForRangeRead replicaPlan, ReadCoordinator readCoordinator, List> readRepairs, boolean isFirst) + private PartitionIterator query(ClusterMetadata cm, CoordinationPlan.ForRangeRead plan, ReadCoordinator readCoordinator, List> readRepairs, boolean isFirst) { - PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), isFirst); + PartitionRangeReadCommand rangeCommand = command.forSubRange(plan.replicas().range(), isFirst); // Accord interop execution should always be coordinated through the C* plumbing if (!readCoordinator.isEventuallyConsistent()) { - SingleRangeResponse response = executeNormal(replicaPlan, rangeCommand, readCoordinator); + SingleRangeResponse response = executeNormal(plan, rangeCommand, readCoordinator); readRepairs.add(response.getReadRepair()); return response; } @@ -271,11 +272,11 @@ public class RangeCommandIterator extends AbstractIterator implemen if (accordSplit.target == RangeReadTarget.accord && readCoordinator.isEventuallyConsistent()) { - return executeAccord(cm, accordSplit.read, replicaPlan.consistencyLevel()); + return executeAccord(cm, accordSplit.read, plan.consistencyLevel()); } else { - return executeNormalWithMigrationSplit(cm, replicaPlan, readCoordinator, readRepairs, accordSplit.read); + return executeNormalWithMigrationSplit(cm, plan, readCoordinator, readRepairs, accordSplit.read); } } @@ -310,11 +311,11 @@ public class RangeCommandIterator extends AbstractIterator implemen { if (accordSplit.target == RangeReadTarget.accord && readCoordinator.isEventuallyConsistent()) { - responses.add(executeAccord(cm, accordSplit.read, replicaPlan.consistencyLevel())); + responses.add(executeAccord(cm, accordSplit.read, plan.consistencyLevel())); } else { - responses.add(executeNormalWithMigrationSplit(cm, replicaPlan, readCoordinator, readRepairs, accordSplit.read)); + responses.add(executeNormalWithMigrationSplit(cm, plan, readCoordinator, readRepairs, accordSplit.read)); } } @@ -325,7 +326,7 @@ public class RangeCommandIterator extends AbstractIterator implemen command.metadata().enforceStrictLiveness()); } - private PartitionIterator executeSplit(RangeReadWithReplication split, ReplicaPlan.ForRangeRead replicaPlan, List> readRepairs) + private PartitionIterator executeSplit(RangeReadWithReplication split, CoordinationPlan.ForRangeRead replicaPlan, List> readRepairs) { if (split.useTracked) { @@ -345,20 +346,20 @@ public class RangeCommandIterator extends AbstractIterator implemen * Execute a normal C* range, splitting for migration if needed. */ private PartitionIterator executeNormalWithMigrationSplit(ClusterMetadata cm, - ReplicaPlan.ForRangeRead replicaPlan, - ReadCoordinator readCoordinator, - List> readRepairs, - PartitionRangeReadCommand rangeCommand) + CoordinationPlan.ForRangeRead plan, + ReadCoordinator readCoordinator, + List> readRepairs, + PartitionRangeReadCommand rangeCommand) { List migrationSplits = MigrationRouter.splitRangeRead(cm, rangeCommand); if (migrationSplits.size() == 1) - return executeSplit(migrationSplits.get(0), replicaPlan, readRepairs); + return executeSplit(migrationSplits.get(0), plan, readRepairs); List responses = new ArrayList<>(migrationSplits.size()); for (RangeReadWithReplication split : migrationSplits) - responses.add(executeSplit(split, replicaPlan, readRepairs)); + responses.add(executeSplit(split, plan, readRepairs)); // Apply limits since migration splits may have gaps in results return rangeCommand.limits().filter(PartitionIterators.concat(responses), @@ -375,26 +376,26 @@ public class RangeCommandIterator extends AbstractIterator implemen ClusterMetadata cm = ClusterMetadata.current(); try { - for (int i = 0; i < concurrencyFactor && replicaPlans.hasNext(); ) + for (int i = 0; i < concurrencyFactor && coordinatorPlans.hasNext(); ) { - ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next(); + CoordinationPlan.ForRangeRead plan = coordinatorPlans.next(); boolean isFirst = i == 0; PartitionIterator response; // Only add the retry wrapper to reroute for the top level coordinator execution // not Accord's interop execution if (readCoordinator.isEventuallyConsistent()) { - Function querySupplier = clusterMetadata -> query(clusterMetadata, replicaPlan, readCoordinator, readRepairs, isFirst); - response = retryingPartitionIterator(querySupplier, replicaPlan.consistencyLevel()); + Function querySupplier = clusterMetadata -> query(clusterMetadata, plan, readCoordinator, readRepairs, isFirst); + response = retryingPartitionIterator(querySupplier, plan.consistencyLevel()); } else { - response = query(cm, replicaPlan, readCoordinator, readRepairs, isFirst); + response = query(cm, plan, readCoordinator, readRepairs, isFirst); } concurrentQueries.add(response); // due to RangeMerger, coordinator may fetch more ranges than required by concurrency factor. - rangesQueried += replicaPlan.vnodeCount(); - i += replicaPlan.vnodeCount(); + rangesQueried += plan.replicas().vnodeCount(); + i += plan.replicas().vnodeCount(); } batchesRequested++; } @@ -480,7 +481,7 @@ public class RangeCommandIterator extends AbstractIterator implemen if (sentQueryIterator != null) sentQueryIterator.close(); - replicaPlans.close(); + coordinatorPlans.close(); } finally { diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java index 298793d721..fe46324922 100644 --- a/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java +++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommands.java @@ -77,11 +77,11 @@ public class RangeCommands Tracing.trace("Computing ranges to query"); Keyspace keyspace = Keyspace.open(command.metadata().keyspace); - ReplicaPlanIterator replicaPlans = new ReplicaPlanIterator(command.dataRange().keyRange(), - command.indexQueryPlan(), - keyspace, - command.metadata().id(), - consistencyLevel); + CoordinationPlanIterator replicaPlans = new CoordinationPlanIterator(command.dataRange().keyRange(), + command.indexQueryPlan(), + keyspace, + command.metadata().id(), + consistencyLevel); if (command.isTopK()) return new ScanAllRangesCommandIterator(keyspace, replicaPlans, command, readCoordinator, replicaPlans.size(), requestTime); @@ -112,7 +112,7 @@ public class RangeCommands Tracing.trace("Submitting range requests on {} ranges with a concurrency of {}", replicaPlans.size(), concurrencyFactor); } - ReplicaPlanMerger mergedReplicaPlans = new ReplicaPlanMerger(replicaPlans, keyspace, command.metadata().id(), consistencyLevel); + CoordinationPlanMerger mergedReplicaPlans = new CoordinationPlanMerger(replicaPlans, keyspace, command.metadata().id(), consistencyLevel); return new RangeCommandIterator(mergedReplicaPlans, command, readCoordinator, @@ -150,15 +150,15 @@ public class RangeCommands try { Keyspace keyspace = Keyspace.open(metadata.keyspace); - ReplicaPlanIterator rangeIterator = new ReplicaPlanIterator(DataRange.allData(metadata.partitioner).keyRange(), - null, - keyspace, - metadata.id, - consistency); + CoordinationPlanIterator rangeIterator = new CoordinationPlanIterator(DataRange.allData(metadata.partitioner).keyRange(), + null, + keyspace, + metadata.id, + consistency); // Called for the side effect of running assureSufficientLiveReplicasForRead. // Deliberately called with an invalid vnode count in case it is used elsewhere in the future.. - rangeIterator.forEachRemaining(r -> ReplicaPlans.forRangeRead(keyspace, metadata.id, null, consistency, r.range(), -1)); + rangeIterator.forEachRemaining(r -> ReplicaPlans.forRangeRead(keyspace, metadata.id, null, consistency, r.replicas().range(), -1)); return true; } catch (UnavailableException e) diff --git a/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java index 8ebc6a6697..0eab36b62c 100644 --- a/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/ScanAllRangesCommandIterator.java @@ -30,10 +30,10 @@ import org.apache.cassandra.db.PartitionRangeReadCommand; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.index.Index; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.reads.DataResolver; @@ -61,13 +61,13 @@ public class ScanAllRangesCommandIterator extends RangeCommandIterator { private final Keyspace keyspace; - ScanAllRangesCommandIterator(Keyspace keyspace, CloseableIterator replicaPlans, + ScanAllRangesCommandIterator(Keyspace keyspace, CloseableIterator coordinationPlans, PartitionRangeReadCommand command, ReadCoordinator readCoordinator, int totalRangeCount, Dispatcher.RequestTime requestTime) { - super(replicaPlans, command, readCoordinator, totalRangeCount, totalRangeCount, totalRangeCount, requestTime); + super(coordinationPlans, command, readCoordinator, totalRangeCount, totalRangeCount, totalRangeCount, requestTime); Preconditions.checkState(command.isTopK()); this.keyspace = keyspace; @@ -79,23 +79,22 @@ public class ScanAllRangesCommandIterator extends RangeCommandIterator // get all replicas to contact Set replicasToQuery = null; ConsistencyLevel consistencyLevel = null; - while (replicaPlans.hasNext()) + while (coordinatorPlans.hasNext()) { if (replicasToQuery == null) replicasToQuery = new HashSet<>(); - ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next(); - replicasToQuery.addAll(replicaPlan.contacts().endpoints()); + CoordinationPlan.ForRangeRead replicaPlan = coordinatorPlans.next(); + replicasToQuery.addAll(replicaPlan.replicas().contacts().endpoints()); consistencyLevel = replicaPlan.consistencyLevel(); } if (replicasToQuery == null || replicasToQuery.isEmpty()) return EmptyIterators.partition(); - ReplicaPlan.ForRangeRead plan = ReplicaPlans.forFullRangeRead(keyspace, consistencyLevel, command.dataRange().keyRange(), replicasToQuery, totalRangeCount); - ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(plan); - DataResolver resolver = new DataResolver<>(ReadCoordinator.DEFAULT, command, sharedReplicaPlan, NoopReadRepair.instance, requestTime, false); - ReadCallback handler = new ReadCallback<>(resolver, command, sharedReplicaPlan, requestTime); + CoordinationPlan.ForRangeRead plan = CoordinationPlan.forFullRangeRead(keyspace, consistencyLevel, command.dataRange().keyRange(), replicasToQuery, totalRangeCount); + DataResolver resolver = new DataResolver<>(ReadCoordinator.DEFAULT, command, plan, NoopReadRepair.instance, requestTime, false); + ReadCallback handler = new ReadCallback<>(resolver, command, plan, requestTime); int nodes = 0; for (InetAddressAndPort endpoint : replicasToQuery) @@ -106,7 +105,7 @@ public class ScanAllRangesCommandIterator extends RangeCommandIterator nodes++; } - rangesQueried += plan.vnodeCount(); + rangesQueried += plan.replicas().vnodeCount(); batchesRequested++; Tracing.trace("Submitted scanning all ranges requests to {} nodes", nodes); diff --git a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java index c8f11df0cb..d228f94038 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/AbstractReadRepair.java @@ -34,6 +34,7 @@ import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; @@ -58,7 +59,7 @@ public abstract class AbstractReadRepair, P extends Repli protected final ReadCoordinator coordinator; protected final ReadCommand command; protected final Dispatcher.RequestTime requestTime; - protected final ReplicaPlan.Shared replicaPlan; + protected final CoordinationPlan.ForRead plan; protected final ColumnFamilyStore cfs; private volatile DigestRepair digestRepair = null; @@ -78,19 +79,19 @@ public abstract class AbstractReadRepair, P extends Repli } public AbstractReadRepair(ReadCoordinator coordinator, ReadCommand command, - ReplicaPlan.Shared replicaPlan, + CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { this.coordinator = coordinator; this.command = command; this.requestTime = requestTime; - this.replicaPlan = replicaPlan; + this.plan = plan; this.cfs = Keyspace.openAndGetStore(command.metadata()); } protected P replicaPlan() { - return replicaPlan.get(); + return plan.replicas(); } void sendReadCommand(Replica to, ReadCallback readCallback, boolean speculative, boolean trackRepairedStatus) @@ -135,9 +136,10 @@ public abstract class AbstractReadRepair, P extends Repli */ boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForPartitionReadsEnabled(); + CoordinationPlan.ForRead repairPlan = plan.copyWithResetTracker(); // Do a full data read to resolve the correct response (and repair node that need be) - DataResolver resolver = new DataResolver<>(coordinator, command, replicaPlan, this, requestTime, trackRepairedStatus); - ReadCallback readCallback = new ReadCallback<>(resolver, command, replicaPlan, requestTime); + DataResolver resolver = new DataResolver<>(coordinator, command, repairPlan, this, requestTime, trackRepairedStatus); + ReadCallback readCallback = new ReadCallback<>(resolver, command, repairPlan, requestTime); digestRepair = new DigestRepair<>(resolver, readCallback, resultConsumer); @@ -175,7 +177,7 @@ public abstract class AbstractReadRepair, P extends Repli ConsistencyLevel consistency = replicaPlan().consistencyLevel(); ConsistencyLevel speculativeCL = consistency.isDatacenterLocal() ? ConsistencyLevel.LOCAL_QUORUM : ConsistencyLevel.QUORUM; return consistency != ConsistencyLevel.EACH_QUORUM - && consistency.satisfies(speculativeCL, replicaPlan.get().replicationStrategy()) + && consistency.satisfies(speculativeCL, plan.replicationStrategy()) && cfs.sampleReadLatencyMicros <= command.getTimeout(MICROSECONDS); } @@ -193,7 +195,7 @@ public abstract class AbstractReadRepair, P extends Repli if (uncontacted == null) return; - replicaPlan.addToContacts(uncontacted); + plan.addToContacts(uncontacted); sendReadCommand(uncontacted, repair.readCallback, true, false); ReadRepairMetrics.speculatedRead.mark(); ReadRepairDiagnostics.speculatedRead(this, uncontacted.endpoint(), replicaPlan()); diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java index 3535f6ed50..812fda25cb 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepair.java @@ -43,6 +43,7 @@ import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; @@ -135,9 +136,9 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo ForWrite repairPlan(); } - BlockingReadRepair(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + BlockingReadRepair(ReadCoordinator coordinator, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - super(coordinator, command, replicaPlan, requestTime); + super(coordinator, command, plan, requestTime); } @Override @@ -293,7 +294,7 @@ public class BlockingReadRepair, P extends ReplicaPlan.Fo public void repairPartitionDirectly(ReadCoordinator readCoordinator, DecoratedKey dk, Map mutations, ForWrite writePlan) { - ReadRepair delegateRR = ReadRepairStrategy.BLOCKING.create(readCoordinator, command, replicaPlan, requestTime); + ReadRepair delegateRR = ReadRepairStrategy.BLOCKING.create(readCoordinator, command, plan, requestTime); delegateRR.repairPartition(dk, mutations, writePlan, ReadRepairSource.REPAIR_VIA_ACCORD); delegateRR.maybeSendAdditionalWrites(); delegateRR.awaitWrites(); diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java index 92a8d87ec7..e53cbcab2d 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepair.java @@ -26,6 +26,7 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; @@ -41,9 +42,9 @@ import org.apache.cassandra.transport.Dispatcher; public class ReadOnlyReadRepair, P extends ReplicaPlan.ForRead> extends AbstractReadRepair { - ReadOnlyReadRepair(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadOnlyReadRepair(ReadCoordinator coordinator, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - super(coordinator, command, replicaPlan, requestTime); + super(coordinator, command, plan, requestTime); } @Override diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java index a51dfaeb21..5239618676 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepair.java @@ -29,6 +29,7 @@ import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts; import org.apache.cassandra.db.partitions.PartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; @@ -41,13 +42,13 @@ public interface ReadRepair, P extends ReplicaPlan.ForRea public interface Factory { , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime); + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime); } static , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - return command.metadata().params.readRepair.create(coordinator, command, replicaPlan, requestTime); + return command.metadata().params.readRepair.create(coordinator, command, plan, requestTime); } /** diff --git a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java index f1272f6c57..74e1901731 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java +++ b/src/java/org/apache/cassandra/service/reads/repair/ReadRepairStrategy.java @@ -19,6 +19,7 @@ package org.apache.cassandra.service.reads.repair; import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.service.reads.ReadCoordinator; @@ -31,18 +32,18 @@ public enum ReadRepairStrategy implements ReadRepair.Factory NONE { public , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - return new ReadOnlyReadRepair<>(coordinator, command, replicaPlan, requestTime); + return new ReadOnlyReadRepair<>(coordinator, command, plan, requestTime); } }, BLOCKING { public , P extends ReplicaPlan.ForRead> - ReadRepair create(ReadCoordinator coordinator, ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + ReadRepair create(ReadCoordinator coordinator, ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - return new BlockingReadRepair<>(coordinator, command, replicaPlan, requestTime); + return new BlockingReadRepair<>(coordinator, command, plan, requestTime); } }; diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java index 5525217273..0da69146e5 100644 --- a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java +++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java @@ -53,9 +53,9 @@ import org.apache.cassandra.dht.ExcludingBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.index.Index; import org.apache.cassandra.index.transactions.UpdateTransaction; -import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; import org.apache.cassandra.utils.concurrent.Future; @@ -333,14 +333,15 @@ public abstract class PartialTrackedRangeRead extends PartialTrackedRead Keyspace keyspace = Keyspace.open(command.metadata().keyspace); PartitionRangeReadCommand followUpCmd = command.withUpdatedLimitsAndDataRange(newLimits, newDataRange); - ReplicaPlan.ForRangeRead replicaPlan = ReplicaPlans.forRangeRead(keyspace, - command.metadata().id, - followUpCmd.indexQueryPlan(), - consistencyLevel, - followUpCmd.dataRange().keyRange(), - 1); + CoordinationPlan.ForRangeRead plan = CoordinationPlan.forRangeRead(ClusterMetadata.current(), + keyspace, + command.metadata().id, + followUpCmd.indexQueryPlan(), + consistencyLevel, + followUpCmd.dataRange().keyRange(), + 1); - TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, replicaPlan, requestTime); + TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, plan, requestTime); logger.trace("Short read detected, starting followup read {}", read); return read; } diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java index d693fd4767..e3993c42b3 100644 --- a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java +++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java @@ -57,13 +57,13 @@ import org.apache.cassandra.gms.FailureDetector; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; -import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageFlag; @@ -167,7 +167,7 @@ public abstract class TrackedRead, P extends ReplicaPlan. private final Id readId = Id.nextId(); private final ReadCommand command; - private final ReplicaPlan.AbstractForRead replicaPlan; + private final CoordinationPlan.ForRead plan; private final ConsistencyLevel consistencyLevel; private final Dispatcher.RequestTime requestTime; @@ -188,17 +188,17 @@ public abstract class TrackedRead, P extends ReplicaPlan. } } - public TrackedRead(ReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + public TrackedRead(ReadCommand command, CoordinationPlan.ForRead plan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { this.command = command; - this.replicaPlan = replicaPlan; + this.plan = plan; this.consistencyLevel = consistencyLevel; this.requestTime = requestTime; } - public ReplicaPlan.AbstractForRead replicaPlan() + public ReplicaPlan.ForRead replicaPlan() { - return replicaPlan; + return plan.replicas(); } @Override @@ -216,9 +216,9 @@ public abstract class TrackedRead, P extends ReplicaPlan. public static class Partition extends TrackedRead { - private Partition(SinglePartitionReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + private Partition(SinglePartitionReadCommand command, CoordinationPlan.ForTokenRead plan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, consistencyLevel, requestTime); + super(command, plan, consistencyLevel, requestTime); } public static Partition create(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) @@ -227,15 +227,15 @@ public abstract class TrackedRead, P extends ReplicaPlan. Keyspace keyspace = Keyspace.open(command.metadata().keyspace); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id); SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry; - ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(metadata, - keyspace, - cfs.getTableId(), - command.partitionKey().getToken(), - command.indexQueryPlan(), - consistencyLevel, - retry, - ReadCoordinator.DEFAULT); - return new Partition(command, replicaPlan, consistencyLevel, requestTime); + CoordinationPlan.ForTokenRead plan = CoordinationPlan.forTokenRead(metadata, + keyspace, + cfs.getTableId(), + command.partitionKey().getToken(), + command.indexQueryPlan(), + consistencyLevel, + retry, + ReadCoordinator.DEFAULT); + return new Partition(command, plan, consistencyLevel, requestTime); } @Override @@ -247,15 +247,15 @@ public abstract class TrackedRead, P extends ReplicaPlan. public static class Range extends TrackedRead { - private Range(PartitionRangeReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + private Range(PartitionRangeReadCommand command, CoordinationPlan.ForRangeRead plan, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) { - super(command, replicaPlan, consistencyLevel, requestTime); + super(command, plan, consistencyLevel, requestTime); } - public static TrackedRead.Range create(PartitionRangeReadCommand command, ReplicaPlan.ForRangeRead replicaPlan, Dispatcher.RequestTime requestTime) + public static TrackedRead.Range create(PartitionRangeReadCommand command, CoordinationPlan.ForRangeRead plan, Dispatcher.RequestTime requestTime) { Preconditions.checkArgument(command.metadata().replicationType().isTracked()); - return new Range(command, replicaPlan, replicaPlan.consistencyLevel(), requestTime); + return new Range(command, plan, plan.consistencyLevel(), requestTime); } @Override @@ -285,7 +285,7 @@ public abstract class TrackedRead, P extends ReplicaPlan. { // TODO: skip local coordination if this node knows its recovering from an outage // TODO: read speculation - Replica localReplica = replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()); + Replica localReplica = plan.replicas().lookup(FBUtilities.getBroadcastAddressAndPort()); if (localReplica != null) readMetrics.localRequests.mark(); else @@ -294,10 +294,10 @@ public abstract class TrackedRead, P extends ReplicaPlan. // create an id // select data node // select summary nodes - E selected = replicaPlan.contacts().filter(r -> FailureDetector.instance.isAlive(r.endpoint())); - if (selected.size() < replicaPlan.readQuorum()) - throw new UnavailableException(String.format("Insufficient replicas available for read (%d < %d)", selected.size(), replicaPlan.readQuorum()), - replicaPlan.consistencyLevel(), selected.size(), replicaPlan.readQuorum()); + E selected = plan.replicas().contacts().filter(r -> FailureDetector.instance.isAlive(r.endpoint())); + if (selected.size() < plan.replicas().readQuorum()) + throw new UnavailableException(String.format("Insufficient replicas available for read (%d < %d)", selected.size(), plan.replicas().readQuorum()), + plan.consistencyLevel(), selected.size(), plan.replicas().readQuorum()); Replica dataReplica = localReplica != null && localReplica.isFull() ? localReplica : Iterables.getOnlyElement(selected.filter(Replica::isFull, 1)); @@ -413,17 +413,17 @@ public abstract class TrackedRead, P extends ReplicaPlan. RequestFailure failure = (RequestFailure) ex; if (failure.reason == RequestFailureReason.TIMEOUT) { - throw new ReadTimeoutException(replicaPlan.consistencyLevel(), 0, replicaPlan.readQuorum(), false); + throw new ReadTimeoutException(plan.consistencyLevel(), 0, plan.replicas().readQuorum(), false); } reasons = failure.reasonByEndpoint(); } - throw new ReadFailureException(replicaPlan.consistencyLevel(), 0, replicaPlan.readQuorum(), false, reasons); + throw new ReadFailureException(plan.consistencyLevel(), 0, plan.replicas().readQuorum(), false, reasons); } catch (TimeoutException e) { - throw new ReadTimeoutException(replicaPlan.consistencyLevel(), 0, replicaPlan.readQuorum(), false); + throw new ReadTimeoutException(plan.consistencyLevel(), 0, plan.replicas().readQuorum(), false); } } diff --git a/test/unit/org/apache/cassandra/locator/CoordinationPlans.java b/test/unit/org/apache/cassandra/locator/CoordinationPlans.java new file mode 100644 index 0000000000..ecaa84e6dd --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/CoordinationPlans.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.tcm.ClusterMetadata; + +/** + * Creates coordination plans with arbitrary replica plans + */ +public class CoordinationPlans +{ + private static ResponseTracker createTrackerForWrite(ReplicaPlan.ForWrite plan) + { + return plan.replicationStrategy().createTrackerForWrite(plan.consistencyLevel(), plan, plan.pending, ClusterMetadata.current()); + } + + public static CoordinationPlan.ForWriteWithIdeal create(ReplicaPlan.ForWrite plan, ConsistencyLevel idealCL) + { + ResponseTracker tracker = createTrackerForWrite(plan); + + CoordinationPlan.ForWrite idealPlan = null; + if (idealCL != null && idealCL != plan.consistencyLevel()) + { + ReplicaPlan.ForWrite idealReplicaPlan = plan.withConsistencyLevel(idealCL); + ResponseTracker idealTracker = createTrackerForWrite(idealReplicaPlan); + idealPlan = new CoordinationPlan.ForWrite(idealReplicaPlan, idealTracker); + } + + return new CoordinationPlan.ForWriteWithIdeal(plan, tracker, idealPlan); + } + + public static CoordinationPlan.ForTokenRead create(ReplicaPlan.ForTokenRead plan) + { + return new CoordinationPlan.ForTokenRead(ReplicaPlan.shared(plan), trackerForRead(plan)); + } + + public static CoordinationPlan.ForTokenRead create(ReplicaPlan.SharedForTokenRead shared) + { + return new CoordinationPlan.ForTokenRead(shared, trackerForRead(shared.get())); + } + + public static CoordinationPlan.ForRangeRead create(ReplicaPlan.ForRangeRead plan) + { + return new CoordinationPlan.ForRangeRead(ReplicaPlan.shared(plan), trackerForRead(plan)); + } + + public static CoordinationPlan.ForRangeRead create(ReplicaPlan.SharedForRangeRead shared) + { + return new CoordinationPlan.ForRangeRead(shared, trackerForRead(shared.get())); + } + + private static , P extends ReplicaPlan.ForRead> ResponseTracker trackerForRead(P plan) + { + return new SimpleResponseTracker(plan.readQuorum(), plan.readCandidates().size()); + } +} diff --git a/test/unit/org/apache/cassandra/locator/PerDcResponseTrackerTest.java b/test/unit/org/apache/cassandra/locator/PerDcResponseTrackerTest.java new file mode 100644 index 0000000000..0af71cdcea --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/PerDcResponseTrackerTest.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; + +import org.junit.Test; + +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.tcm.membership.Location; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +public class PerDcResponseTrackerTest +{ + private static final RequestFailureReason TIMEOUT = RequestFailureReason.TIMEOUT; + + private InetAddressAndPort endpoint(String ip) throws UnknownHostException + { + return InetAddressAndPort.getByName(ip); + } + + private Locator mockLocator(Map endpointToDc) throws Exception + { + Locator locator = mock(Locator.class); + for (Map.Entry entry : endpointToDc.entrySet()) + { + when(locator.location(entry.getKey())).thenReturn(new Location(entry.getValue(), "rack1")); + } + return locator; + } + + /** + * Helper to create a map of SimpleResponseTrackers from blockFor/totalReplicas config. + */ + private Map simpleTrackers(Object... dcBlockForTotal) + { + Map trackers = new HashMap<>(); + for (int i = 0; i < dcBlockForTotal.length; i += 3) + { + String dc = (String) dcBlockForTotal[i]; + int blockFor = (Integer) dcBlockForTotal[i + 1]; + int totalReplicas = (Integer) dcBlockForTotal[i + 2]; + trackers.put(dc, new SimpleResponseTracker(blockFor, totalReplicas)); + } + return trackers; + } + + @Test + public void testAllDcsReachQuorum() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3, + "DC2", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("127.0.0.2"), "DC1"); + endpointToDc.put(endpoint("127.0.0.3"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + endpointToDc.put(endpoint("192.168.1.2"), "DC2"); + endpointToDc.put(endpoint("192.168.1.3"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + // DC1 reaches quorum + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertFalse(tracker.isComplete()); // DC2 not done yet + + // DC2 reaches quorum + tracker.onResponse(endpoint("192.168.1.1")); + tracker.onResponse(endpoint("192.168.1.2")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(4, tracker.received()); + assertEquals(4, tracker.required()); // 2 + 2 + } + + @Test + public void testOneDcFails() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3, + "DC2", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("127.0.0.2"), "DC1"); + endpointToDc.put(endpoint("127.0.0.3"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + endpointToDc.put(endpoint("192.168.1.2"), "DC2"); + endpointToDc.put(endpoint("192.168.1.3"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + // DC1 reaches quorum + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + + // DC2 fails (all fail) + tracker.onFailure(endpoint("192.168.1.1"), TIMEOUT); + tracker.onFailure(endpoint("192.168.1.2"), TIMEOUT); + tracker.onFailure(endpoint("192.168.1.3"), TIMEOUT); + + assertTrue(tracker.isComplete()); // DC2 reached definite failure + assertFalse(tracker.isSuccessful()); // DC2 failed + assertEquals(2, tracker.received()); + assertEquals(3, tracker.failures()); + } + + @Test + public void testPartialDcProgress() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3, + "DC2", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + // DC1 gets one response + tracker.onResponse(endpoint("127.0.0.1")); + + assertFalse(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(1, tracker.received()); + assertEquals(4, tracker.required()); // 2 + 2 + } + + @Test + public void testIgnoresUnknownDc() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("127.0.0.2"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); // DC2 not in config + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + // DC2 response is ignored + tracker.onResponse(endpoint("192.168.1.1")); + assertFalse(tracker.countsTowardQuorum(endpoint("192.168.1.1"))); + assertEquals(0, tracker.received()); + + // DC1 responses count + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.received()); + } + + @Test + public void testAsymmetricRequirements() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 3, 5, // Need 3 of 5 + "DC2", 2, 3 // Need 2 of 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("127.0.0.2"), "DC1"); + endpointToDc.put(endpoint("127.0.0.3"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + endpointToDc.put(endpoint("192.168.1.2"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + // DC2 reaches quorum (2 of 3) + tracker.onResponse(endpoint("192.168.1.1")); + tracker.onResponse(endpoint("192.168.1.2")); + assertFalse(tracker.isComplete()); // DC1 not done + + // DC1 reaches quorum (3 of 5) + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.3")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(5, tracker.received()); + assertEquals(5, tracker.required()); // 3 + 2 + } + + // Aggregation tests + + @Test + public void testRequiredSum() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 3, 5, + "DC2", 2, 3, + "DC3", 1, 2 + ); + + Map endpointToDc = new HashMap<>(); + Locator locator = mockLocator(endpointToDc); + + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + assertEquals(6, tracker.required()); // 3 + 2 + 1 + } + + @Test + public void testReceivedSum() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3, + "DC2", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("127.0.0.2"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("192.168.1.1")); + + assertEquals(3, tracker.received()); // 2 from DC1 + 1 from DC2 + } + + @Test + public void testFailuresSum() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3, + "DC2", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + endpointToDc.put(endpoint("192.168.1.2"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + tracker.onFailure(endpoint("127.0.0.1"), TIMEOUT); + tracker.onFailure(endpoint("192.168.1.1"), TIMEOUT); + tracker.onFailure(endpoint("192.168.1.2"), TIMEOUT); + + assertEquals(3, tracker.failures()); // 1 from DC1 + 2 from DC2 + } + + // Composition tests + + @Test + public void testCountsTowardQuorum() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3 + ); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + assertTrue(tracker.countsTowardQuorum(endpoint("127.0.0.1"))); // DC1 tracked + assertFalse(tracker.countsTowardQuorum(endpoint("192.168.1.1"))); // DC2 not tracked + } + + @Test + public void testGetTrackerForDc() throws Exception + { + Map trackers = simpleTrackers( + "DC1", 2, 3, + "DC2", 1, 2 + ); + + Locator locator = mock(Locator.class); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + assertNotNull(tracker.getTrackerForDc("DC1")); + assertNotNull(tracker.getTrackerForDc("DC2")); + assertNull(tracker.getTrackerForDc("DC3")); + assertEquals(2, tracker.getTrackerForDc("DC1").required()); + assertEquals(1, tracker.getTrackerForDc("DC2").required()); + } + + @Test + public void testWithWriteResponseTrackers() throws Exception + { + // Test that PerDcResponseTracker works with WriteResponseTrackers (double-count model) + Map trackers = new HashMap<>(); + // DC1: baseBlockFor=2, totalBlockFor=3, committed=3, pending=1 + trackers.put("DC1", new WriteResponseTracker(2, 3, 3, 1, + addr -> addr.getHostAddress(false).equals("127.0.0.4"))); // .4 is pending + // DC2: no pending, degenerates to simple case + trackers.put("DC2", new SimpleResponseTracker(2, 3)); + + Map endpointToDc = new HashMap<>(); + endpointToDc.put(endpoint("127.0.0.1"), "DC1"); + endpointToDc.put(endpoint("127.0.0.2"), "DC1"); + endpointToDc.put(endpoint("127.0.0.3"), "DC1"); + endpointToDc.put(endpoint("127.0.0.4"), "DC1"); // pending + endpointToDc.put(endpoint("192.168.1.1"), "DC2"); + endpointToDc.put(endpoint("192.168.1.2"), "DC2"); + endpointToDc.put(endpoint("192.168.1.3"), "DC2"); + + Locator locator = mockLocator(endpointToDc); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + // DC1: 2 committed successes (meets base requirement but not total) + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertFalse(tracker.isComplete()); + + // DC2: 2 successes (meets requirement) + tracker.onResponse(endpoint("192.168.1.1")); + tracker.onResponse(endpoint("192.168.1.2")); + assertFalse(tracker.isComplete()); // DC1 still needs pending + + // DC1: 1 pending success (now meets total requirement) + tracker.onResponse(endpoint("127.0.0.4")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + } + + // Validation tests + + @Test(expected = IllegalArgumentException.class) + public void testNullTrackers() + { + Locator locator = mock(Locator.class); + new PerDcResponseTracker(null, locator); + } + + @Test(expected = IllegalArgumentException.class) + public void testEmptyTrackers() + { + Locator locator = mock(Locator.class); + new PerDcResponseTracker(new HashMap<>(), locator); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullLocator() + { + Map trackers = simpleTrackers("DC1", 2, 3); + new PerDcResponseTracker(trackers, null); + } + + @Test + public void testToString() throws Exception + { + Map trackers = simpleTrackers("DC1", 2, 3); + + Locator locator = mock(Locator.class); + PerDcResponseTracker tracker = new PerDcResponseTracker(trackers, locator); + + String str = tracker.toString(); + assertTrue(str.contains("PerDcResponseTracker")); + assertTrue(str.contains("DC1")); + } +} diff --git a/test/unit/org/apache/cassandra/locator/SimpleResponseTrackerTest.java b/test/unit/org/apache/cassandra/locator/SimpleResponseTrackerTest.java new file mode 100644 index 0000000000..af4f289aef --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/SimpleResponseTrackerTest.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.net.UnknownHostException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SimpleResponseTrackerTest +{ + private InetAddressAndPort endpoint(String ip) throws UnknownHostException + { + return InetAddressAndPort.getByName(ip); + } + + @Test + public void testQuorumReached() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 3); + + assertFalse(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(0, tracker.received()); + + tracker.onResponse(endpoint("127.0.0.1")); + assertFalse(tracker.isComplete()); + assertEquals(1, tracker.received()); + + tracker.onResponse(endpoint("127.0.0.2")); + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.received()); + } + + @Test + public void testPartialProgress() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(3, 5); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + + assertFalse(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(2, tracker.received()); + assertEquals(3, tracker.required()); + } + + @Test + public void testEarlyFailure() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(3, 5); + + // Need 3, have 5 total + tracker.onResponse(endpoint("127.0.0.1")); // 1 success + tracker.onFailure(endpoint("127.0.0.2")); // 1 failure + tracker.onFailure(endpoint("127.0.0.3")); // 2 failures + tracker.onFailure(endpoint("127.0.0.4")); // 3 failures + + // Have 1 success, 3 failures, 1 remaining + // Need 2 more but only 1 remaining -> impossible + assertTrue(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(1, tracker.received()); + assertEquals(3, tracker.failures()); + } + + @Test + public void testAllSucceed() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(3, 3); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.3")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(3, tracker.received()); + assertEquals(0, tracker.failures()); + } + + @Test + public void testAllFail() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 3); + + tracker.onFailure(endpoint("127.0.0.1")); + tracker.onFailure(endpoint("127.0.0.2")); + tracker.onFailure(endpoint("127.0.0.3")); + + assertTrue(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(0, tracker.received()); + assertEquals(3, tracker.failures()); + } + + @Test + public void testBlockForOne() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(1, 3); + + tracker.onResponse(endpoint("127.0.0.1")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(1, tracker.received()); + } + + @Test + public void testBlockForAll() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(3, 3); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertFalse(tracker.isComplete()); + + tracker.onResponse(endpoint("127.0.0.3")); + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + } + + @Test + public void testZeroResponses() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 3); + + assertFalse(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(0, tracker.received()); + assertEquals(0, tracker.failures()); + assertEquals(2, tracker.required()); + } + + @Test + public void testConcurrentResponses() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(50, 100); + ExecutorService executor = Executors.newFixedThreadPool(10); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(50); + + try + { + // Launch 50 threads to call onResponse concurrently + for (int i = 0; i < 50; i++) + { + final int index = i; + executor.submit(() -> { + try + { + startLatch.await(); + tracker.onResponse(endpoint("127.0.0." + index)); + doneLatch.countDown(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + }); + } + + // Start all threads at once + startLatch.countDown(); + + // Wait for completion + assertTrue(doneLatch.await(10, TimeUnit.SECONDS)); + + // Verify no lost updates + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(50, tracker.received()); + assertEquals(0, tracker.failures()); + } + finally + { + executor.shutdownNow(); + } + } + + // Filtering tests + + @Test + public void testUnfilteredTracker() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 4); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("192.168.1.1")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.received()); + } + + @Test + public void testFilteredTracker() throws Exception + { + // Filter that only accepts local endpoints (127.0.0.*) + Predicate localFilter = endpoint -> + endpoint.getHostAddress(false).startsWith("127.0.0."); + + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 3, localFilter); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.received()); + assertTrue(tracker.countsTowardQuorum(endpoint("127.0.0.1"))); + } + + @Test + public void testFilteredIgnoresNonMatching() throws Exception + { + // Filter that only accepts local endpoints + Predicate localFilter = endpoint -> + endpoint.getHostAddress(false).startsWith("127.0.0."); + + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 2, localFilter); + + // Remote endpoint response is ignored + tracker.onResponse(endpoint("192.168.1.1")); + assertFalse(tracker.isComplete()); + assertEquals(0, tracker.received()); + assertFalse(tracker.countsTowardQuorum(endpoint("192.168.1.1"))); + + // Local endpoints count + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.received()); + } + + @Test + public void testCountsTowardQuorum() throws Exception + { + Predicate filter = endpoint -> + endpoint.getHostAddress(false).startsWith("127.0.0."); + + SimpleResponseTracker unfilteredTracker = new SimpleResponseTracker(2, 3); + assertTrue(unfilteredTracker.countsTowardQuorum(endpoint("127.0.0.1"))); + assertTrue(unfilteredTracker.countsTowardQuorum(endpoint("192.168.1.1"))); + + SimpleResponseTracker filteredTracker = new SimpleResponseTracker(2, 3, filter); + assertTrue(filteredTracker.countsTowardQuorum(endpoint("127.0.0.1"))); + assertFalse(filteredTracker.countsTowardQuorum(endpoint("192.168.1.1"))); + } + + // Usage pattern tests + + @Test + public void testQuorumUsage() throws Exception + { + // Simulates QUORUM with RF=5 + int rf = 5; + int blockFor = rf / 2 + 1; // 3 + SimpleResponseTracker tracker = new SimpleResponseTracker(blockFor, rf); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.3")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(3, tracker.required()); + } + + @Test + public void testLocalQuorumUsage() throws Exception + { + // Simulates LOCAL_QUORUM with localRF=3 + int localRf = 3; + int blockFor = localRf / 2 + 1; // 2 + Predicate localFilter = endpoint -> + endpoint.getHostAddress(false).startsWith("127.0.0."); + + SimpleResponseTracker tracker = new SimpleResponseTracker(blockFor, localRf, localFilter); + + // Remote response ignored + tracker.onResponse(endpoint("192.168.1.1")); + assertFalse(tracker.isComplete()); + + // Local responses count + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.required()); + } + + @Test + public void testSerialUsage() throws Exception + { + // Simulates SERIAL paxos with participants=5 (RF=4 + 1 pending) + int participants = 5; + int blockFor = participants / 2 + 1; // 3 + SimpleResponseTracker tracker = new SimpleResponseTracker(blockFor, participants); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.3")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(3, tracker.required()); + } + + // Validation tests + + @Test(expected = IllegalArgumentException.class) + public void testNegativeBlockFor() + { + new SimpleResponseTracker(-1, 3); + } + + @Test(expected = IllegalArgumentException.class) + public void testNegativeTotalReplicas() + { + new SimpleResponseTracker(2, -1); + } + + @Test + public void testToString() throws Exception + { + SimpleResponseTracker tracker = new SimpleResponseTracker(2, 3); + String str = tracker.toString(); + + assertTrue(str.contains("SimpleResponseTracker")); + assertTrue(str.contains("blockFor=2")); + assertTrue(str.contains("totalReplicas=3")); + } +} diff --git a/test/unit/org/apache/cassandra/locator/WriteResponseTrackerTest.java b/test/unit/org/apache/cassandra/locator/WriteResponseTrackerTest.java new file mode 100644 index 0000000000..e1d9d077ce --- /dev/null +++ b/test/unit/org/apache/cassandra/locator/WriteResponseTrackerTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.locator; + +import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Predicate; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Tests for WriteResponseTracker implementing the double count model. + */ +public class WriteResponseTrackerTest +{ + private InetAddressAndPort endpoint(String ip) throws UnknownHostException + { + return InetAddressAndPort.getByName(ip); + } + + @Test + public void testBothRequirementsMet() throws Exception + { + // RF=3, pending=1: baseBlockFor=2, totalBlockFor=3 + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + // 2 committed successes + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertFalse("Should not be complete - need 3 total", tracker.isComplete()); + + // 1 pending success -> 3 total + tracker.onResponse(endpoint("127.0.0.4")); + assertTrue("Should be complete", tracker.isComplete()); + assertTrue("Should be successful", tracker.isSuccessful()); + assertEquals(2, tracker.naturalReceived()); + assertEquals(1, tracker.pendingReceived()); + assertEquals(3, tracker.received()); + } + + @Test + public void testCommittedRequirementNotMet() throws Exception + { + // RF=3, pending=1: baseBlockFor=2, totalBlockFor=3 + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + // 1 committed success, 1 pending success + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.4")); + assertFalse("Should not be complete - need 2 committed", tracker.isComplete()); + assertEquals(1, tracker.naturalReceived()); + assertEquals(1, tracker.pendingReceived()); + + // 2 committed failures -> can't reach 2 committed + tracker.onFailure(endpoint("127.0.0.2")); + tracker.onFailure(endpoint("127.0.0.3")); + assertTrue("Should be complete - impossible to reach committed requirement", tracker.isComplete()); + assertFalse("Should not be successful", tracker.isSuccessful()); + } + + @Test + public void testTotalRequirementNotMet() throws Exception + { + // RF=3, pending=2: baseBlockFor=2, totalBlockFor=4 + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + pending.add(endpoint("127.0.0.5")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 4, 3, 2, isPending); + + // 2 committed successes (meets base requirement) + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertFalse("Should not be complete - need 4 total", tracker.isComplete()); + assertEquals(2, tracker.naturalReceived()); + + // 1 committed failure, 2 pending failures -> only 3 total possible, need 4 + tracker.onFailure(endpoint("127.0.0.3")); + tracker.onFailure(endpoint("127.0.0.4")); + tracker.onFailure(endpoint("127.0.0.5")); + assertTrue("Should be complete - impossible to reach total requirement", tracker.isComplete()); + assertFalse("Should not be successful", tracker.isSuccessful()); + } + + @Test + public void testNoPendingReplicas() throws Exception + { + // RF=3, pending=0: baseBlockFor=2, totalBlockFor=2 (degenerates to simple case) + Predicate isPending = addr -> false; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 2, 3, 0, isPending); + + tracker.onResponse(endpoint("127.0.0.1")); + assertFalse(tracker.isComplete()); + + tracker.onResponse(endpoint("127.0.0.2")); + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(2, tracker.naturalReceived()); + assertEquals(0, tracker.pendingReceived()); + } + + @Test + public void testAllFail() throws Exception + { + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + tracker.onFailure(endpoint("127.0.0.1")); + tracker.onFailure(endpoint("127.0.0.2")); + // After 2 committed failures, can't reach baseBlockFor=2 with only 1 remaining + assertTrue(tracker.isComplete()); + assertFalse(tracker.isSuccessful()); + assertEquals(0, tracker.received()); + assertEquals(2, tracker.naturalFailures()); + } + + @Test + public void testAllSucceed() throws Exception + { + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.3")); + tracker.onResponse(endpoint("127.0.0.4")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(3, tracker.naturalReceived()); + assertEquals(1, tracker.pendingReceived()); + assertEquals(4, tracker.received()); + assertEquals(0, tracker.failures()); + } + + @Test + public void testPendingSuccessBeforeCommitted() throws Exception + { + // Pending responses arrive first + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + // Pending arrives first + tracker.onResponse(endpoint("127.0.0.4")); + assertFalse(tracker.isComplete()); + assertEquals(0, tracker.naturalReceived()); + assertEquals(1, tracker.pendingReceived()); + + // Then committed + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + } + + @Test + public void testExactlyMeetsRequirements() throws Exception + { + // RF=2, pending=1: baseBlockFor=2, totalBlockFor=3 + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.3")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 2, 1, isPending); + + // Exactly 2 committed (all of them) + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + assertFalse("Need 3 total", tracker.isComplete()); + + // Exactly 1 pending (all of them) + tracker.onResponse(endpoint("127.0.0.3")); + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + } + + @Test + public void testMixedSuccessesAndFailures() throws Exception + { + // RF=5, pending=2: baseBlockFor=3, totalBlockFor=5 + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.6")); + pending.add(endpoint("127.0.0.7")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(3, 5, 5, 2, isPending); + + // 3 committed successes, 2 committed failures + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onResponse(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.3")); + tracker.onFailure(endpoint("127.0.0.4")); + tracker.onFailure(endpoint("127.0.0.5")); + + assertFalse("Need 5 total, only have 3", tracker.isComplete()); + assertEquals(3, tracker.naturalReceived()); + assertEquals(2, tracker.naturalFailures()); + + // 2 pending successes + tracker.onResponse(endpoint("127.0.0.6")); + tracker.onResponse(endpoint("127.0.0.7")); + + assertTrue(tracker.isComplete()); + assertTrue(tracker.isSuccessful()); + assertEquals(3, tracker.naturalReceived()); + assertEquals(2, tracker.pendingReceived()); + assertEquals(5, tracker.received()); + } + + @Test(expected = IllegalArgumentException.class) + public void testNegativeBaseBlockFor() + { + new WriteResponseTracker(-1, 2, 3, 1, addr -> false); + } + + @Test(expected = IllegalArgumentException.class) + public void testTotalRequiredLessThanBase() + { + new WriteResponseTracker(3, 2, 3, 1, addr -> false); + } + + @Test(expected = IllegalArgumentException.class) + public void testBaseBlockForExceedsCommitted() + { + new WriteResponseTracker(4, 5, 3, 2, addr -> false); + } + + @Test(expected = IllegalArgumentException.class) + public void testTotalBlockForExceedsTotalReplicas() + { + new WriteResponseTracker(2, 6, 3, 2, addr -> false); + } + + @Test(expected = IllegalArgumentException.class) + public void testNullPredicate() + { + new WriteResponseTracker(2, 3, 3, 1, null); + } + + @Test + public void testAccessors() throws Exception + { + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + assertEquals(2, tracker.baseBlockFor()); + assertEquals(3, tracker.required()); // Returns totalBlockFor for error messages + + tracker.onResponse(endpoint("127.0.0.1")); + tracker.onFailure(endpoint("127.0.0.2")); + tracker.onResponse(endpoint("127.0.0.4")); + + assertEquals(1, tracker.naturalReceived()); + assertEquals(1, tracker.pendingReceived()); + assertEquals(1, tracker.naturalFailures()); + assertEquals(0, tracker.pendingFailures()); + assertEquals(2, tracker.received()); + assertEquals(1, tracker.failures()); + } + + @Test + public void testCountsTowardQuorum() throws Exception + { + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + + // All endpoints count toward quorum in writes + assertTrue(tracker.countsTowardQuorum(endpoint("127.0.0.1"))); + assertTrue(tracker.countsTowardQuorum(endpoint("127.0.0.4"))); + assertTrue(tracker.countsTowardQuorum(endpoint("192.168.1.1"))); + } + + @Test + public void testToString() throws Exception + { + Set pending = new HashSet<>(); + pending.add(endpoint("127.0.0.4")); + Predicate isPending = pending::contains; + + WriteResponseTracker tracker = new WriteResponseTracker(2, 3, 3, 1, isPending); + String str = tracker.toString(); + + assertTrue(str.contains("WriteResponseTracker")); + assertTrue(str.contains("baseBlockFor=2")); + assertTrue(str.contains("totalBlockFor=3")); + } +} diff --git a/test/unit/org/apache/cassandra/service/ReadCallbackPropertyTest.java b/test/unit/org/apache/cassandra/service/ReadCallbackPropertyTest.java index d5511218d5..8425d37a86 100644 --- a/test/unit/org/apache/cassandra/service/ReadCallbackPropertyTest.java +++ b/test/unit/org/apache/cassandra/service/ReadCallbackPropertyTest.java @@ -24,7 +24,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; import com.google.common.collect.ImmutableList; import org.junit.Assert; @@ -34,6 +33,8 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.ReadResponse; import org.apache.cassandra.exceptions.RequestFailure; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -179,9 +180,9 @@ public class ReadCallbackPropertyTest extends ResponseHandlerPropertyTestBase { private volatile boolean dataPresent = false; - public TestResponseResolver(Supplier replicaPlan, Dispatcher.RequestTime requestTime) + public TestResponseResolver(CoordinationPlan.ForTokenRead plan, Dispatcher.RequestTime requestTime) { - super(null, null, replicaPlan, requestTime); + super(null, null, plan, requestTime); } @Override @@ -216,9 +217,9 @@ public class ReadCallbackPropertyTest extends ResponseHandlerPropertyTestBase /** * Creates a ReplicaPlan for testing with the given parameters. */ - private static ReplicaPlan.SharedForTokenRead createReplicaPlan(Keyspace ks, - ConsistencyLevel cl, - EndpointsForToken contacts) + private static CoordinationPlan.ForTokenRead createReplicaPlan(Keyspace ks, + ConsistencyLevel cl, + EndpointsForToken contacts) { ReplicaPlan.ForTokenRead plan = new ReplicaPlan.ForTokenRead( ks, @@ -231,7 +232,7 @@ public class ReadCallbackPropertyTest extends ResponseHandlerPropertyTestBase (self) -> null, // repair plan function Epoch.EMPTY ); - return ReplicaPlan.shared(plan); + return CoordinationPlans.create(plan); } /** @@ -313,11 +314,11 @@ public class ReadCallbackPropertyTest extends ResponseHandlerPropertyTestBase contactedIndices.add(i); } - ReplicaPlan.SharedForTokenRead sharedPlan = createReplicaPlan(ks, cl, contacts); + CoordinationPlan.ForTokenRead plan = createReplicaPlan(ks, cl, contacts); Dispatcher.RequestTime requestTime = new Dispatcher.RequestTime(System.nanoTime(), System.nanoTime()); - TestResponseResolver resolver = new TestResponseResolver(sharedPlan, requestTime); - ReadCallback handler = new ReadCallback<>(resolver, null, sharedPlan, requestTime); + TestResponseResolver resolver = new TestResponseResolver(plan, requestTime); + ReadCallback handler = new ReadCallback<>(resolver, null, plan, requestTime); return new HandlerWithContacts(handler, contactedIndices); } diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerPropertyTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerPropertyTest.java index 92eb0399a0..67022ed3f1 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerPropertyTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerPropertyTest.java @@ -18,33 +18,24 @@ package org.apache.cassandra.service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; -import org.junit.Test; - import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.WriteType; -import org.apache.cassandra.exceptions.CoordinatorBehindException; -import org.apache.cassandra.exceptions.RequestFailure; -import org.apache.cassandra.exceptions.RetryOnDifferentSystemException; -import org.apache.cassandra.exceptions.WriteFailureException; -import org.apache.cassandra.exceptions.WriteTimeoutException; -import org.apache.cassandra.locator.EndpointsForToken; -import org.apache.cassandra.locator.InOurDc; -import org.apache.cassandra.locator.InetAddressAndPort; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.exceptions.*; +import org.apache.cassandra.locator.*; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.Verb; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.transport.Dispatcher; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.apache.cassandra.net.NoPayload.noPayload; import static org.quicktheories.QuickTheory.qt; @@ -286,13 +277,15 @@ public class WriteResponseHandlerPropertyTest extends ResponseHandlerPropertyTes EndpointsForToken targets, EndpointsForToken pending) throws Exception { - return ks.getReplicationStrategy().getWriteResponseHandler( - ReplicaPlans.forWrite(ks, cl, (cm) -> targets, (cm) -> pending, ClusterMetadata.current().epoch, - Predicates.alwaysTrue(), ReplicaPlans.writeAll), - null, - WriteType.SIMPLE, - null, - Dispatcher.RequestTime.forImmediateExecution()); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(ks, cl, + (cm) -> targets, + (cm) -> pending, + ClusterMetadata.current().epoch, + Predicates.alwaysTrue(), + ReplicaPlans.writeAll); + CoordinationPlan.ForWriteWithIdeal coordinationPlan = CoordinationPlans.create(replicaPlan, null); + return ks.getReplicationStrategy().getWriteResponseHandler(coordinationPlan, null, WriteType.SIMPLE, null, + Dispatcher.RequestTime.forImmediateExecution()); } /** @@ -496,8 +489,8 @@ public class WriteResponseHandlerPropertyTest extends ResponseHandlerPropertyTes // Build endpoint lookup for replica indices List allEndpoints = new ArrayList<>(); - handler.replicaPlan.contacts().forEach(r -> allEndpoints.add(r.endpoint())); - handler.replicaPlan.pending().forEach(r -> allEndpoints.add(r.endpoint())); + handler.replicaPlan().contacts().forEach(r -> allEndpoints.add(r.endpoint())); + handler.replicaPlan().pending().forEach(r -> allEndpoints.add(r.endpoint())); for (int i = 0; i < Math.min(responses.size(), 10); i++) { @@ -529,8 +522,8 @@ public class WriteResponseHandlerPropertyTest extends ResponseHandlerPropertyTes } // Debug: show what the handler thinks about pending replicas - diagnostic.append(String.format("Handler's replicaPlan.pending() size: %d\n", handler.replicaPlan.pending().size())); - for (Replica replica : handler.replicaPlan.pending()) + diagnostic.append(String.format("Handler's replicaPlan.pending() size: %d\n", handler.replicaPlan().pending().size())); + for (Replica replica : handler.replicaPlan().pending()) { diagnostic.append(String.format(" Pending replica: %s\n", replica.endpoint())); } diff --git a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java index 77cc452efc..5c062ff966 100644 --- a/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java +++ b/test/unit/org/apache/cassandra/service/WriteResponseHandlerTest.java @@ -25,6 +25,8 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Predicates; +import org.apache.cassandra.locator.*; +import org.apache.cassandra.locator.CoordinationPlans; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -57,6 +59,7 @@ import static java.util.concurrent.TimeUnit.DAYS; import static org.apache.cassandra.net.NoPayload.noPayload; import static org.apache.cassandra.utils.Clock.Global.nanoTime; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class WriteResponseHandlerTest @@ -148,8 +151,8 @@ public class WriteResponseHandlerTest assertEquals(startingCount + 1, ks.metric.idealCLWriteLatency.latency.getCount()); //Don't need the others - awr.expired(); - awr.expired(); + awr.expired(targets.get(2).endpoint()); + awr.expired(targets.get(3).endpoint()); assertEquals(0, ks.metric.writeFailedIdealCL.getCount()); } @@ -216,10 +219,9 @@ public class WriteResponseHandlerTest awr.onResponse(createDummyMessage(2)); //Fail in remote DC - awr.expired(); - awr.expired(); - awr.expired(); - assertEquals(1, ks.metric.writeFailedIdealCL.getCount()); + awr.expired(targets.get(3).endpoint()); + awr.expired(targets.get(4).endpoint()); + awr.expired(targets.get(5).endpoint()); assertEquals(0, ks.metric.idealCLWriteLatency.totalLatency.getCount()); } @@ -260,14 +262,14 @@ public class WriteResponseHandlerTest // Failure in local DC awr.onResponse(createDummyMessage(0)); - - awr.expired(); - awr.expired(); + + awr.expired(targets.get(1).endpoint()); + awr.expired(targets.get(2).endpoint()); //Fail in remote DC - awr.expired(); - awr.expired(); - awr.expired(); + awr.expired(targets.get(3).endpoint()); + awr.expired(targets.get(4).endpoint()); + awr.expired(targets.get(5).endpoint()); assertEquals(startingCount, ks.metric.writeFailedIdealCL.getCount()); } @@ -297,6 +299,28 @@ public class WriteResponseHandlerTest } + /** + * expired(from) must notify the ResponseTracker so that when enough down-node expirations + * accumulate to make quorum mathematically impossible, the handler signals failure immediately + * rather than blocking until the full RPC timeout. + */ + @Test + public void expiredUpdatesResponseTrackerAndSignalsFailureWhenQuorumImpossible() + { + // QUORUM on 6 replicas (3 DC1 + 3 DC2) requires 4 acks. + // If 3 replicas are expired (down at dispatch time), only 3 remain — quorum is impossible. + AbstractWriteResponseHandler awr = createWriteResponseHandler(ConsistencyLevel.QUORUM, null); + + awr.expired(targets.get(0).endpoint()); + awr.expired(targets.get(1).endpoint()); + awr.expired(targets.get(2).endpoint()); // 3 remaining, blockFor=4: impossible to succeed + + assertTrue("handler must be complete (failed) after quorum becomes impossible via expired()", + awr.isComplete()); + assertFalse("handler must not report success", + awr.coordinationPlan().responses().isSuccessful()); + } + private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal) { return createWriteResponseHandler(cl, ideal, Dispatcher.RequestTime.forImmediateExecution()); @@ -304,8 +328,9 @@ public class WriteResponseHandlerTest private static AbstractWriteResponseHandler createWriteResponseHandler(ConsistencyLevel cl, ConsistencyLevel ideal, Dispatcher.RequestTime requestTime) { - return ks.getReplicationStrategy().getWriteResponseHandler(ReplicaPlans.forWrite(ks, cl, (cm) -> targets, (cm) -> pending, Epoch.FIRST, Predicates.alwaysTrue(), ReplicaPlans.writeAll), - null, WriteType.SIMPLE, null, requestTime, ideal); + ReplicaPlan.ForWrite replicaPlan = ReplicaPlans.forWrite(ks, cl, (cm) -> targets, (cm) -> pending, Epoch.FIRST, Predicates.alwaysTrue(), ReplicaPlans.writeAll); + CoordinationPlan.ForWriteWithIdeal coordinationPlan = CoordinationPlans.create(replicaPlan, ideal); + return ks.getReplicationStrategy().getWriteResponseHandler(coordinationPlan, null, WriteType.SIMPLE, null, requestTime); } private static Message createDummyMessage(int target) diff --git a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java index 40f5afeec8..2f241797d7 100644 --- a/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java +++ b/test/unit/org/apache/cassandra/service/reads/DataResolverTest.java @@ -58,6 +58,8 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -1250,7 +1252,7 @@ public class DataResolverTest extends AbstractReadResponseTest } private DataResolver resolverWithVerifier(final ReadCommand command, - final ReplicaPlan.SharedForRangeRead plan, + final CoordinationPlan.ForRangeRead plan, final ReadRepair readRepair, final Dispatcher.RequestTime requestTime, final RepairedDataVerifier verifier) @@ -1258,7 +1260,7 @@ public class DataResolverTest extends AbstractReadResponseTest class TestableDataResolver extends DataResolver { - public TestableDataResolver(ReadCommand command, ReplicaPlan.SharedForRangeRead plan, ReadRepair readRepair, Dispatcher.RequestTime requestTime) + public TestableDataResolver(ReadCommand command, CoordinationPlan.ForRangeRead plan, ReadRepair readRepair, Dispatcher.RequestTime requestTime) { super(ReadCoordinator.DEFAULT, command, plan, readRepair, requestTime, true); } @@ -1324,17 +1326,17 @@ public class DataResolverTest extends AbstractReadResponseTest assertEquals(update.metadata().name, cfm.name); } - private ReplicaPlan.SharedForRangeRead plan(EndpointsForRange replicas, ConsistencyLevel consistencyLevel) + private CoordinationPlan.ForRangeRead plan(EndpointsForRange replicas, ConsistencyLevel consistencyLevel) { BiFunction, Token, ReplicaPlan.ForWrite> repairPlan = (self, t) -> ReplicaPlans.forReadRepair(self, ClusterMetadata.current(), ks, null, consistencyLevel, t, (i) -> true, ReadCoordinator.DEFAULT); - return ReplicaPlan.shared(new ReplicaPlan.ForRangeRead(ks, - ks.getReplicationStrategy(), - consistencyLevel, - ReplicaUtils.FULL_BOUNDS, - replicas, replicas, replicas, - 1, null, - repairPlan, - Epoch.EMPTY)); + return CoordinationPlans.create(ReplicaPlan.shared(new ReplicaPlan.ForRangeRead(ks, + ks.getReplicationStrategy(), + consistencyLevel, + ReplicaUtils.FULL_BOUNDS, + replicas, replicas, replicas, + 1, null, + repairPlan, + Epoch.EMPTY))); } private static void resolveAndConsume(DataResolver resolver) diff --git a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java index 0286066638..dea69b86fb 100644 --- a/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java +++ b/test/unit/org/apache/cassandra/service/reads/DigestResolverTest.java @@ -23,6 +23,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; +import org.apache.cassandra.locator.ReplicaPlan; import org.junit.Assert; import org.junit.Test; @@ -32,7 +35,6 @@ import org.apache.cassandra.db.SinglePartitionReadCommand; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.locator.EndpointsForToken; -import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.transport.Dispatcher; @@ -91,7 +93,7 @@ public class DigestResolverTest extends AbstractReadResponseTest SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(cfm, nowInSec, dk); EndpointsForToken targetReplicas = EndpointsForToken.of(dk.getToken(), full(EP1), full(EP2)); PartitionUpdate response = update(row(1000, 4, 4), row(1000, 5, 5)).build(); - ReplicaPlan.SharedForTokenRead plan = plan(ConsistencyLevel.ONE, targetReplicas); + CoordinationPlan.ForTokenRead plan = plan(ConsistencyLevel.ONE, targetReplicas); ExecutorService pool = Executors.newFixedThreadPool(2); long endTime = System.nanoTime() + TimeUnit.MINUTES.toNanos(2); @@ -213,9 +215,9 @@ public class DigestResolverTest extends AbstractReadResponseTest resolver.getData()); } - private ReplicaPlan.SharedForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken replicas) + private CoordinationPlan.ForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken replicas) { - return ReplicaPlan.shared(new ReplicaPlan.ForTokenRead(ks, ks.getReplicationStrategy(), consistencyLevel, replicas, replicas, replicas, null, (self) -> null, Epoch.EMPTY)); + return CoordinationPlans.create(new ReplicaPlan.ForTokenRead(ks, ks.getReplicationStrategy(), consistencyLevel, replicas, replicas, replicas, null, (self) -> null, Epoch.EMPTY)); } private void waitForLatch(CountDownLatch startlatch) diff --git a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java index 2429ba9282..274fafd379 100644 --- a/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/ReadExecutorTest.java @@ -39,6 +39,8 @@ import org.apache.cassandra.exceptions.ReadFailureException; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.exceptions.RequestFailure; import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.ReplicaPlan; @@ -207,7 +209,7 @@ public class ReadExecutorTest public void testRaceWithNonSpeculativeFailure() { MockSinglePartitionReadCommand command = new MockSinglePartitionReadCommand(TimeUnit.DAYS.toMillis(365)); - ReplicaPlan.ForTokenRead plan = plan(ConsistencyLevel.LOCAL_ONE, targets, targets.subList(0, 1)); + CoordinationPlan.ForTokenRead plan = plan(ConsistencyLevel.LOCAL_ONE, targets, targets.subList(0, 1)); AbstractReadExecutor executor = new AbstractReadExecutor.SpeculatingReadExecutor(ReadCoordinator.DEFAULT, cfs, command, plan, Dispatcher.RequestTime.forImmediateExecution()); // Issue an initial request against the first endpoint... @@ -271,13 +273,13 @@ public class ReadExecutorTest } } - private ReplicaPlan.ForTokenRead plan(EndpointsForToken targets, ConsistencyLevel consistencyLevel) + private CoordinationPlan.ForTokenRead plan(EndpointsForToken targets, ConsistencyLevel consistencyLevel) { return plan(consistencyLevel, targets, targets); } - private ReplicaPlan.ForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken selected) + private CoordinationPlan.ForTokenRead plan(ConsistencyLevel consistencyLevel, EndpointsForToken natural, EndpointsForToken selected) { - return new ReplicaPlan.ForTokenRead(ks, ks.getReplicationStrategy(), consistencyLevel, natural, selected, natural, (cm) -> null, (self) -> null, Epoch.EMPTY); + return CoordinationPlans.create(new ReplicaPlan.ForTokenRead(ks, ks.getReplicationStrategy(), consistencyLevel, natural, selected, natural, (cm) -> null, (self) -> null, Epoch.EMPTY)); } } diff --git a/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanIteratorTest.java b/test/unit/org/apache/cassandra/service/reads/range/CoordinationPlanIteratorTest.java similarity index 97% rename from test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanIteratorTest.java rename to test/unit/org/apache/cassandra/service/reads/range/CoordinationPlanIteratorTest.java index 6a687d57e0..79dbaeed15 100644 --- a/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanIteratorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/range/CoordinationPlanIteratorTest.java @@ -42,7 +42,7 @@ import static org.apache.cassandra.Util.rp; import static org.apache.cassandra.Util.token; import static org.junit.Assert.assertEquals; -public class ReplicaPlanIteratorTest +public class CoordinationPlanIteratorTest { private static final String KEYSPACE = "ReplicaPlanIteratorTest"; private static final TableId TABLE_ID = TableId.generate(); @@ -165,11 +165,11 @@ public class ReplicaPlanIteratorTest @SafeVarargs private final void testRanges(Keyspace keyspace, AbstractBounds queryRange, AbstractBounds... expected) { - try (ReplicaPlanIterator iterator = new ReplicaPlanIterator(queryRange, null, keyspace, TABLE_ID, ConsistencyLevel.ANY)) + try (CoordinationPlanIterator iterator = new CoordinationPlanIterator(queryRange, null, keyspace, TABLE_ID, ConsistencyLevel.ANY)) { List> restrictedRanges = new ArrayList<>(expected.length); while (iterator.hasNext()) - restrictedRanges.add(iterator.next().range()); + restrictedRanges.add(iterator.next().replicas().range()); // verify range counts assertEquals(expected.length, restrictedRanges.size()); diff --git a/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java b/test/unit/org/apache/cassandra/service/reads/range/CoordinationPlanMergerTest.java similarity index 97% rename from test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java rename to test/unit/org/apache/cassandra/service/reads/range/CoordinationPlanMergerTest.java index 46b71d532e..301e789b45 100644 --- a/test/unit/org/apache/cassandra/service/reads/range/ReplicaPlanMergerTest.java +++ b/test/unit/org/apache/cassandra/service/reads/range/CoordinationPlanMergerTest.java @@ -63,9 +63,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; /** - * Tests for {@link ReplicaPlanMerger}. + * Tests for {@link CoordinationPlanMerger}. */ -public class ReplicaPlanMergerTest +public class CoordinationPlanMergerTest { private static final String KEYSPACE = "ReplicaPlanMergerTest"; private static Keyspace keyspace; @@ -416,13 +416,13 @@ public class ReplicaPlanMergerTest AbstractBounds queryRange, AbstractBounds... expected) { - try (ReplicaPlanIterator originals = new ReplicaPlanIterator(queryRange, null, keyspace, null, ANY); // ANY avoids endpoint erros - ReplicaPlanMerger merger = new ReplicaPlanMerger(originals, keyspace, null, consistencyLevel)) + try (CoordinationPlanIterator originals = new CoordinationPlanIterator(queryRange, null, keyspace, null, ANY); // ANY avoids endpoint erros + CoordinationPlanMerger merger = new CoordinationPlanMerger(originals, keyspace, null, consistencyLevel)) { // collect the merged ranges List> mergedRanges = new ArrayList<>(expected.length); while (merger.hasNext()) - mergedRanges.add(merger.next().range()); + mergedRanges.add(merger.next().replicas().range()); assertFalse("The number of merged ranges should never be greater than the number of original ranges", mergedRanges.size() > originals.size()); diff --git a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java index 78095cb9d1..ed1a9861e0 100644 --- a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java +++ b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandIteratorTest.java @@ -38,7 +38,8 @@ import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; import org.apache.cassandra.locator.ReplicaPlans; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.TableId; @@ -70,18 +71,18 @@ public class RangeCommandIteratorTest int vnodeCount = 0; Keyspace keyspace = Keyspace.open(KEYSPACE1); - List ranges = new ArrayList<>(); + List ranges = new ArrayList<>(); for (int i = 0; i + 1 < tokens.size(); i++) { Range range = Range.makeRowRange(tokens.get(i), tokens.get(i + 1)); - ranges.add(ReplicaPlans.forRangeRead(keyspace, TABLE_ID, null, ConsistencyLevel.ONE, range, 1)); + ranges.add(CoordinationPlans.create(ReplicaPlans.forRangeRead(keyspace, TABLE_ID, null, ConsistencyLevel.ONE, range, 1))); vnodeCount++; } - ReplicaPlanMerger merge = new ReplicaPlanMerger(ranges.iterator(), keyspace, TABLE_ID, ConsistencyLevel.ONE); - ReplicaPlan.ForRangeRead mergedRange = Iterators.getOnlyElement(merge); + CoordinationPlanMerger merge = new CoordinationPlanMerger(ranges.iterator(), keyspace, TABLE_ID, ConsistencyLevel.ONE); + CoordinationPlan.ForRangeRead mergedRange = Iterators.getOnlyElement(merge); // all ranges are merged as test has only one node. - assertEquals(vnodeCount, mergedRange.vnodeCount()); + assertEquals(vnodeCount, mergedRange.replicas().vnodeCount()); } @Test @@ -108,27 +109,27 @@ public class RangeCommandIteratorTest AbstractBounds keyRange = command.dataRange().keyRange(); // without range merger, there will be 2 batches requested: 1st batch with 1 range and 2nd batch with remaining ranges - CloseableIterator replicaPlans = replicaPlanIterator(keyRange, keyspace, false); + CloseableIterator replicaPlans = coordinatorPlanIterator(keyRange, keyspace, false); RangeCommandIterator data = new RangeCommandIterator(replicaPlans, command, ReadCoordinator.DEFAULT, 1, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution()); verifyRangeCommandIterator(data, rows, 2, vnodeCount); // without range merger and initial cf=5, there will be 1 batches requested: 5 vnode ranges for 1st batch - replicaPlans = replicaPlanIterator(keyRange, keyspace, false); + replicaPlans = coordinatorPlanIterator(keyRange, keyspace, false); data = new RangeCommandIterator(replicaPlans, command, ReadCoordinator.DEFAULT, vnodeCount, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution()); verifyRangeCommandIterator(data, rows, 1, vnodeCount); // without range merger and max cf=1, there will be 5 batches requested: 1 vnode range per batch - replicaPlans = replicaPlanIterator(keyRange, keyspace, false); + replicaPlans = coordinatorPlanIterator(keyRange, keyspace, false); data = new RangeCommandIterator(replicaPlans, command, ReadCoordinator.DEFAULT, 1, 1, vnodeCount, Dispatcher.RequestTime.forImmediateExecution()); verifyRangeCommandIterator(data, rows, vnodeCount, vnodeCount); // with range merger, there will be only 1 batch requested, as all ranges share the same replica - localhost - replicaPlans = replicaPlanIterator(keyRange, keyspace, true); + replicaPlans = coordinatorPlanIterator(keyRange, keyspace, true); data = new RangeCommandIterator(replicaPlans, command, ReadCoordinator.DEFAULT, 1, 1000, vnodeCount, Dispatcher.RequestTime.forImmediateExecution()); verifyRangeCommandIterator(data, rows, 1, vnodeCount); // with range merger and max cf=1, there will be only 1 batch requested, as all ranges share the same replica - localhost - replicaPlans = replicaPlanIterator(keyRange, keyspace, true); + replicaPlans = coordinatorPlanIterator(keyRange, keyspace, true); data = new RangeCommandIterator(replicaPlans, command, ReadCoordinator.DEFAULT, 1, 1, vnodeCount, Dispatcher.RequestTime.forImmediateExecution()); verifyRangeCommandIterator(data, rows, 1, vnodeCount); } @@ -164,13 +165,13 @@ public class RangeCommandIteratorTest return new TokenUpdater().withKeys(values).update().getTokens(); } - private static CloseableIterator replicaPlanIterator(AbstractBounds keyRange, - Keyspace keyspace, - boolean withRangeMerger) + private static CloseableIterator coordinatorPlanIterator(AbstractBounds keyRange, + Keyspace keyspace, + boolean withRangeMerger) { - CloseableIterator replicaPlans = new ReplicaPlanIterator(keyRange, null, keyspace, null, ConsistencyLevel.ONE); + CloseableIterator replicaPlans = new CoordinationPlanIterator(keyRange, null, keyspace, null, ConsistencyLevel.ONE); if (withRangeMerger) - replicaPlans = new ReplicaPlanMerger(replicaPlans, keyspace, null, ConsistencyLevel.ONE); + replicaPlans = new CoordinationPlanMerger(replicaPlans, keyspace, null, ConsistencyLevel.ONE); return replicaPlans; } diff --git a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java index 42ea24173e..b2927ddc2b 100644 --- a/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java +++ b/test/unit/org/apache/cassandra/service/reads/range/RangeCommandsTest.java @@ -80,7 +80,7 @@ public class RangeCommandsTest extends CQLTester // verify that a low concurrency factor is not capped by the max concurrency factor PartitionRangeReadCommand command = command(cfs, 50, 50); try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, ReadCoordinator.DEFAULT, Dispatcher.RequestTime.forImmediateExecution()); - ReplicaPlanIterator ranges = new ReplicaPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, command.metadata().id, ONE)) + CoordinationPlanIterator ranges = new CoordinationPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, command.metadata().id, ONE)) { assertEquals(2, partitions.concurrencyFactor()); assertEquals(MAX_CONCURRENCY_FACTOR, partitions.maxConcurrencyFactor()); @@ -90,7 +90,7 @@ public class RangeCommandsTest extends CQLTester // verify that a high concurrency factor is capped by the max concurrency factor command = command(cfs, 1000, 50); try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, ReadCoordinator.DEFAULT, Dispatcher.RequestTime.forImmediateExecution()); - ReplicaPlanIterator ranges = new ReplicaPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, command.metadata().id, ONE)) + CoordinationPlanIterator ranges = new CoordinationPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, command.metadata().id, ONE)) { assertEquals(MAX_CONCURRENCY_FACTOR, partitions.concurrencyFactor()); assertEquals(MAX_CONCURRENCY_FACTOR, partitions.maxConcurrencyFactor()); @@ -100,7 +100,7 @@ public class RangeCommandsTest extends CQLTester // with 0 estimated results per range the concurrency factor should be 1 command = command(cfs, 1000, 0); try (RangeCommandIterator partitions = RangeCommands.rangeCommandIterator(command, ONE, ReadCoordinator.DEFAULT, Dispatcher.RequestTime.forImmediateExecution()); - ReplicaPlanIterator ranges = new ReplicaPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, command.metadata().id, ONE)) + CoordinationPlanIterator ranges = new CoordinationPlanIterator(command.dataRange().keyRange(), command.indexQueryPlan(), keyspace, command.metadata().id, ONE)) { assertEquals(1, partitions.concurrencyFactor()); assertEquals(MAX_CONCURRENCY_FACTOR, partitions.maxConcurrencyFactor()); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java index 6707602fec..9019e44fda 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/AbstractReadRepairTest.java @@ -61,6 +61,8 @@ import org.apache.cassandra.dht.ByteOrderedPartitioner.BytesToken; import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.EndpointsForToken; import org.apache.cassandra.locator.InetAddressAndPort; @@ -366,11 +368,11 @@ public abstract class AbstractReadRepairTest Epoch.EMPTY); } - public abstract InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime); + public abstract InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime); - public InstrumentedReadRepair createInstrumentedReadRepair(ReplicaPlan.Shared replicaPlan) + public InstrumentedReadRepair createInstrumentedReadRepair(CoordinationPlan.ForRead plan) { - return createInstrumentedReadRepair(command, replicaPlan, Dispatcher.RequestTime.forImmediateExecution()); + return createInstrumentedReadRepair(command, plan, Dispatcher.RequestTime.forImmediateExecution()); } @@ -381,7 +383,7 @@ public abstract class AbstractReadRepairTest @Test public void readSpeculationCycle() { - InstrumentedReadRepair repair = createInstrumentedReadRepair(ReplicaPlan.shared(replicaPlan(replicas, EndpointsForRange.of(replica1, replica2)))); + InstrumentedReadRepair repair = createInstrumentedReadRepair(CoordinationPlans.create(replicaPlan(replicas, EndpointsForRange.of(replica1, replica2)))); ResultConsumer consumer = new ResultConsumer(); Assert.assertEquals(epSet(), repair.getReadRecipients()); @@ -400,7 +402,7 @@ public abstract class AbstractReadRepairTest @Test public void noSpeculationRequired() { - InstrumentedReadRepair repair = createInstrumentedReadRepair(ReplicaPlan.shared(replicaPlan(replicas, EndpointsForRange.of(replica1, replica2)))); + InstrumentedReadRepair repair = createInstrumentedReadRepair(CoordinationPlans.create(replicaPlan(replicas, EndpointsForRange.of(replica1, replica2)))); ResultConsumer consumer = new ResultConsumer(); Assert.assertEquals(epSet(), repair.getReadRecipients()); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java index 01644e0122..a07bc658c1 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/BlockingReadRepairTest.java @@ -35,6 +35,7 @@ import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.CoordinationPlan; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; @@ -86,9 +87,9 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest private static class InstrumentedBlockingReadRepair, P extends ReplicaPlan.ForRead> extends BlockingReadRepair implements InstrumentedReadRepair { - public InstrumentedBlockingReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public InstrumentedBlockingReadRepair(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - super(ReadCoordinator.DEFAULT, command, replicaPlan, requestTime); + super(ReadCoordinator.DEFAULT, command, plan, requestTime); } Set readCommandRecipients = new HashSet<>(); @@ -116,9 +117,9 @@ public class BlockingReadRepairTest extends AbstractReadRepairTest } @Override - public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - return new InstrumentedBlockingReadRepair(command, replicaPlan, requestTime); + return new InstrumentedBlockingReadRepair(command, plan, requestTime); } @Test diff --git a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java index 5391fdb3ad..2c40718dd4 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/DiagEventsBlockingReadRepairTest.java @@ -29,6 +29,8 @@ import java.util.function.Predicate; import com.google.common.collect.Lists; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.ReplicaPlan; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; @@ -44,7 +46,6 @@ import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.EndpointsForRange; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.Message; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.service.reads.ReadCoordinator; @@ -119,9 +120,9 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest } @Override - public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - return new DiagnosticBlockingRepairHandler(command, replicaPlan, requestTime); + return new DiagnosticBlockingRepairHandler(command, plan, requestTime); } private static DiagnosticPartitionReadRepairHandler createRepairHandler(Map repairs, ReplicaPlan.ForWrite writePlan) @@ -134,9 +135,9 @@ public class DiagEventsBlockingReadRepairTest extends AbstractReadRepairTest private Set recipients = Collections.emptySet(); private ReadCallback readCallback = null; - DiagnosticBlockingRepairHandler(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + DiagnosticBlockingRepairHandler(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - super(ReadCoordinator.DEFAULT, command, replicaPlan, requestTime); + super(ReadCoordinator.DEFAULT, command, plan, requestTime); DiagnosticEventService.instance().subscribe(ReadRepairEvent.class, this::onRepairEvent); } diff --git a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java index 311b8265c4..8046a260c4 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/ReadOnlyReadRepairTest.java @@ -28,6 +28,8 @@ import org.junit.Test; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.locator.CoordinationPlan; +import org.apache.cassandra.locator.CoordinationPlans; import org.apache.cassandra.locator.Endpoints; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; @@ -42,9 +44,9 @@ public class ReadOnlyReadRepairTest extends AbstractReadRepairTest private static class InstrumentedReadOnlyReadRepair, P extends ReplicaPlan.ForRead> extends ReadOnlyReadRepair implements InstrumentedReadRepair { - public InstrumentedReadOnlyReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public InstrumentedReadOnlyReadRepair(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - super(ReadCoordinator.DEFAULT, command, replicaPlan, requestTime); + super(ReadCoordinator.DEFAULT, command, plan, requestTime); } Set readCommandRecipients = new HashSet<>(); @@ -78,25 +80,25 @@ public class ReadOnlyReadRepairTest extends AbstractReadRepairTest } @Override - public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, ReplicaPlan.Shared replicaPlan, Dispatcher.RequestTime requestTime) + public InstrumentedReadRepair createInstrumentedReadRepair(ReadCommand command, CoordinationPlan.ForRead plan, Dispatcher.RequestTime requestTime) { - return new InstrumentedReadOnlyReadRepair(command, replicaPlan, requestTime); + return new InstrumentedReadOnlyReadRepair(command, plan, requestTime); } @Test public void getMergeListener() { - ReplicaPlan.SharedForRangeRead replicaPlan = ReplicaPlan.shared(replicaPlan(replicas, replicas)); - InstrumentedReadRepair repair = createInstrumentedReadRepair(replicaPlan); - Assert.assertSame(UnfilteredPartitionIterators.MergeListener.NOOP, repair.getMergeListener(replicaPlan.get())); + CoordinationPlan.ForRangeRead plan = CoordinationPlans.create(replicaPlan(replicas, replicas)); + InstrumentedReadRepair repair = createInstrumentedReadRepair(plan); + Assert.assertSame(UnfilteredPartitionIterators.MergeListener.NOOP, repair.getMergeListener(plan.replicas())); } @Test(expected = UnsupportedOperationException.class) public void repairPartitionFailure() { - ReplicaPlan.SharedForRangeRead readPlan = ReplicaPlan.shared(replicaPlan(replicas, replicas)); + CoordinationPlan.ForRangeRead plan = CoordinationPlans.create(replicaPlan(replicas, replicas)); ReplicaPlan.ForWrite writePlan = repairPlan(replicas, replicas); - InstrumentedReadRepair repair = createInstrumentedReadRepair(readPlan); + InstrumentedReadRepair repair = createInstrumentedReadRepair(plan); repair.repairPartition(null, Collections.emptyMap(), writePlan, ReadRepairSource.OTHER); } }