diff --git a/CHANGES.txt b/CHANGES.txt index 83b3bb7d67..4ca05b0360 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,9 @@ +cep-45-mutation-tracking + * Fix mutation tracking startup (CASSANDRA-20540) + * Mutation tracking journal integration, read, and write path (CASSANDRA-20304, CASSANDRA-20305, CASSANDRA-20308) + * Introduce MutationJournal for coordinator logs (CASSANDRA-20353) + * Copy over Journal and dependencies from cep-15-accord (CASSANDRA-20321) + 6.0-alpha2 * Fix a removed TTLed row re-appearance in a materialized view after a cursor compaction (CASSANDRA-21152) * Rework ZSTD dictionary compression logic to create a trainer per training (CASSANDRA-21209) @@ -7,7 +13,6 @@ Merged from 4.1: Merged from 4.0: * Rate limit password changes (CASSANDRA-21202) - 6.0-alpha1 * Improve performance when calculating settled placements during range movements (CASSANDRA-21144) * Make shadow gossip round parameters configurable for testing (CASSANDRA-21149) @@ -107,6 +112,8 @@ Merged from 4.0: * Optimize DataPlacement lookup by ReplicationParams (CASSANDRA-20804) * Fix ShortPaxosSimulationTest and AccordSimulationRunner do not execute from the cli (CASSANDRA-20805) * Allow overriding arbitrary settings via environment variables (CASSANDRA-20749) + +5.1 * Optimize MessagingService.getVersionOrdinal (CASSANDRA-20816) * Optimize TrieMemtable#getFlushSet (CASSANDRA-20760) * Support manual secondary index selection at the CQL level (CASSANDRA-18112) diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index c32690b424..580e302c7a 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -794,7 +794,7 @@ class TestCqlshOutput(BaseTestCase): self.assertNoHasColors(output) # Since CASSANDRA-7622 'DESC FULL SCHEMA' also shows all VIRTUAL keyspaces self.assertIn('VIRTUAL KEYSPACE system_virtual_schema', output) - self.assertIn("\nCREATE KEYSPACE system_auth WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true AND fast_path = 'simple';\n", + self.assertIn("\nCREATE KEYSPACE system_auth WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true AND fast_path = 'simple' AND replication_type = 'untracked';\n", output) self.assertRegex(output, r'.*\s*$') diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index c777d0fc89..2269cb2eda 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -694,6 +694,8 @@ public class Config public boolean dynamic_data_masking_enabled = false; + public boolean mutation_tracking_enabled = false; + /** * Time in milliseconds after a warning will be emitted to the log and to the client that a UDF runs too long. * (Only valid, if user_defined_functions_threads_enabled==true) diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 59ffb9f02a..d7c997ba1e 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -6025,6 +6025,20 @@ public class DatabaseDescriptor } } + public static boolean getMutationTrackingEnabled() + { + return conf.mutation_tracking_enabled; + } + + public static void setMutationTrackingEnabled(boolean enabled) + { + if (enabled != conf.mutation_tracking_enabled) + { + logger.info("Setting mutation_tracking_enabled to {}", enabled); + conf.mutation_tracking_enabled = enabled; + } + } + public static OptionalDouble getSeverityDuringDecommission() { return conf.severity_during_decommission > 0 ? diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java index 8a38387d88..6fd03204ce 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchStatement.java @@ -542,7 +542,7 @@ public class BatchStatement implements CQLStatement.CompositeCQLStatement updatePartitionsPerBatchMetrics(mutations.size()); boolean mutateAtomic = (isLogged() && mutations.size() > 1); - StorageProxy.mutateWithTriggers(mutations, cl, mutateAtomic, requestTime, preserveTimestamp); + StorageProxy.mutateWithoutConditions(mutations, cl, mutateAtomic, requestTime, preserveTimestamp); ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(mutations); } diff --git a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java index 4bc0d909d2..1db782f1c5 100644 --- a/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java +++ b/src/java/org/apache/cassandra/cql3/statements/BatchUpdatesCollector.java @@ -38,6 +38,7 @@ import org.apache.cassandra.db.commitlog.CommitLogSegment; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.virtual.VirtualMutation; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; @@ -224,7 +225,7 @@ final class BatchUpdatesCollector implements UpdatesCollector PartitionUpdate update = updateEntry.getValue().build(); updates.put(updateEntry.getKey(), update); } - return new Mutation(keyspaceName, key, updates.build(), createdAt, potentialTxnConflicts); + return new Mutation(MutationId.none(), keyspaceName, key, updates.build(), createdAt, potentialTxnConflicts); } public PartitionUpdate.Builder get(TableId tableId) diff --git a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java index fd853004be..9e98c43c66 100644 --- a/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java @@ -72,9 +72,11 @@ import org.apache.cassandra.cql3.transactions.ReferenceOperation; import org.apache.cassandra.db.CBuilder; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.CounterMutation; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.IMutation; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts; import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.RegularAndStaticColumns; @@ -107,6 +109,8 @@ import org.apache.cassandra.locator.NetworkTopologyStrategy; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaLayout; import org.apache.cassandra.metrics.ClientRequestSizeMetrics; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; @@ -676,11 +680,11 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa false, options.getTimestamp(queryState), options.getNowInSeconds(queryState), - requestTime - ); + requestTime); + if (!mutations.isEmpty()) { - StorageProxy.mutateWithTriggers(mutations, cl, false, requestTime, attrs.isTimestampSet() ? PreserveTimestamp.yes : PreserveTimestamp.no); + StorageProxy.mutateWithoutConditions(mutations, cl, false, requestTime, attrs.isTimestampSet() ? PreserveTimestamp.yes : PreserveTimestamp.no); if (!SchemaConstants.isSystemKeyspace(metadata.keyspace)) ClientRequestSizeMetrics.recordRowAndColumnCountMetrics(mutations); @@ -842,8 +846,28 @@ public abstract class ModificationStatement implements CQLStatement.SingleKeyspa { long timestamp = options.getTimestamp(queryState); long nowInSeconds = options.getNowInSeconds(queryState); - for (IMutation mutation : getMutations(queryState.getClientState(), options, true, timestamp, nowInSeconds, requestTime)) + List mutations = getMutations(queryState.getClientState(), options, true, timestamp, nowInSeconds, requestTime); + boolean isTracked = !mutations.isEmpty() && Schema.instance.getKeyspaceMetadata(mutations.get(0).getKeyspaceName()).params.replicationType.isTracked(); + if (isTracked) + { + if (mutations.stream().anyMatch(m -> m instanceof CounterMutation)) + throw new InvalidRequestException("Mutation tracking is currently unsupported with counters"); + if (mutations.size() > 1) + throw new InvalidRequestException("Mutation tracking is currently unsupported with unlogged batches"); + + Mutation mutation = (Mutation) mutations.get(0); + + String keyspaceName = mutation.getKeyspaceName(); + Token token = mutation.key().getToken(); + MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, token); + mutation = mutation.withMutationId(id); mutation.apply(); + } + else + { + for (IMutation mutation : mutations) + mutation.apply(); + } return null; } diff --git a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java index d4ca44cdcf..27ee037ab3 100644 --- a/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java +++ b/src/java/org/apache/cassandra/cql3/statements/schema/KeyspaceAttributes.java @@ -29,6 +29,7 @@ import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.KeyspaceParams.Option; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.schema.ReplicationParams; import org.apache.cassandra.service.accord.fastpath.FastPathStrategy; @@ -103,18 +104,26 @@ public final class KeyspaceAttributes extends PropertyDefinitions { boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), KeyspaceParams.DEFAULT_DURABLE_WRITES); FastPathStrategy fastPath = getFastPathStrategy(); - return KeyspaceParams.create(durableWrites, getAllReplicationOptions(), fastPath != null ? fastPath : FastPathStrategy.simple()); + + String rtypeName = getString(Option.REPLICATION_TYPE.toString()); + ReplicationType replicationType = rtypeName != null ? ReplicationType.valueOf(rtypeName) : KeyspaceParams.DEFAULT_REPLICATION_TYPE; + + return KeyspaceParams.create(durableWrites, getAllReplicationOptions(), fastPath != null ? fastPath : FastPathStrategy.simple(), replicationType); } KeyspaceParams asAlteredKeyspaceParams(KeyspaceParams previous) { boolean durableWrites = getBoolean(Option.DURABLE_WRITES.toString(), previous.durableWrites); + String rtypeName = getString(Option.REPLICATION_TYPE.toString()); + ReplicationType replicationType = rtypeName != null ? ReplicationType.valueOf(rtypeName) : previous.replicationType; + + Map previousOptions = previous.replication.options; ReplicationParams replication = getReplicationStrategyClass() == null ? previous.replication : ReplicationParams.fromMapWithDefaults(getAllReplicationOptions(), previousOptions); FastPathStrategy fastPath = getFastPathStrategy(); - return new KeyspaceParams(durableWrites, replication, fastPath != null ? fastPath : previous.fastPath); + return new KeyspaceParams(durableWrites, replication, fastPath != null ? fastPath : previous.fastPath, replicationType); } public boolean hasOption(Option option) diff --git a/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java new file mode 100644 index 0000000000..9647891f8a --- /dev/null +++ b/src/java/org/apache/cassandra/db/AbstractReadCommandVerbHandler.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.CoordinatorBehindException; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.InvalidRoutingException; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.TCMMetrics; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.ClusterMetadataService; +import org.apache.cassandra.tcm.Epoch; +import org.apache.cassandra.utils.FBUtilities; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +public abstract class AbstractReadCommandVerbHandler implements IVerbHandler +{ + protected abstract void performRead(Message message, ClusterMetadata metadata); + + protected abstract ReadCommand getCommand(T payload); + + public final void doVerb(Message message) + { + ClusterMetadata metadata = ClusterMetadata.current(); + if (message.epoch().isAfter(Epoch.EMPTY)) + { + metadata = checkTokenOwnership(metadata, message); + metadata = checkSchemaVersion(metadata, message); + } + + MessageParams.reset(); + + long timeout = message.expiresAtNanos() - message.createdAtNanos(); + ReadCommand command = getCommand(message.payload); + command.setMonitoringTime(message.createdAtNanos(), message.isCrossNode(), timeout, DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS)); + + if (message.trackWarnings()) + command.trackWarnings(); + + performRead(message, metadata); + } + + private ClusterMetadata checkSchemaVersion(ClusterMetadata metadata, Message message) + { + ReadCommand readCommand = getCommand(message.payload); + + if (SchemaConstants.isSystemKeyspace(readCommand.metadata().keyspace) || + readCommand.serializedAtEpoch() == null) // don't try to catch up with pre-5.0 nodes + return metadata; + + Keyspace ks = metadata.schema.getKeyspace(readCommand.metadata().keyspace); + ColumnFamilyStore cfs = ks != null ? ks.getColumnFamilyStore(readCommand.metadata().id) : null; + Epoch localComparisonEpoch = metadata.epoch; + if (cfs != null) + localComparisonEpoch = cfs.metadata().epoch; + + if (localComparisonEpoch.isBefore(readCommand.serializedAtEpoch())) + metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch()); + else if (localComparisonEpoch.isAfter(readCommand.serializedAtEpoch())) + { + TCMMetrics.instance.coordinatorBehindSchema.mark(); + throw new CoordinatorBehindException(String.format("Coordinator schema for %s.%s with epoch %s is behind our schema %s", + readCommand.metadata().keyspace, + readCommand.metadata().name, + readCommand.serializedAtEpoch(), + localComparisonEpoch)); + } + ks = metadata.schema.getKeyspace(readCommand.metadata().keyspace); + if (ks == null || ks.getColumnFamilyStore(readCommand.metadata().id) == null) + throw new IllegalStateException("Unknown table " + readCommand.metadata().id +" after fetching remote log entries"); + return metadata; + } + + private ClusterMetadata checkTokenOwnership(ClusterMetadata metadata, Message message) + { + ReadCommand command = getCommand(message.payload); + + if (command.metadata().isVirtual()) + return metadata; + + // Some read commands may be sent using an older Epoch intentionally so validating using the current Epoch + // doesn't work + if (command.potentialTxnConflicts().allowed) + return metadata; + + if (command.isTopK()) + return metadata; + + if (command instanceof SinglePartitionReadCommand) + { + Token token = ((SinglePartitionReadCommand) command).partitionKey().getToken(); + Replica localReplica = getLocalReplica(metadata, token, command.metadata().keyspace); + if (localReplica == null) + { + metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch()); + localReplica = getLocalReplica(metadata, token, command.metadata().keyspace); + } + if (localReplica == null) + { + StorageService.instance.incOutOfRangeOperationCount(); + Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc(); + throw InvalidRoutingException.forTokenRead(message.from(), token, metadata.epoch, command); + } + + if (!command.acceptsTransient() && localReplica.isTransient()) + { + MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS); + throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s", + command.acceptsTransient() ? "transient" : "full", + localReplica.isTransient() ? "transient" : "full", + this)); + } + } + else + { + AbstractBounds range = command.dataRange().keyRange(); + + // TODO: preexisting issue: for the range queries or queries that span multiple replicas, we can only make requests where the right token is owned, but not the left one + Replica maxTokenLocalReplica = getLocalReplica(metadata, range.right.getToken(), command.metadata().keyspace); + if (maxTokenLocalReplica == null) + { + metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch()); + maxTokenLocalReplica = getLocalReplica(metadata, range.right.getToken(), command.metadata().keyspace); + } + if (maxTokenLocalReplica == null) + { + StorageService.instance.incOutOfRangeOperationCount(); + Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc(); + throw InvalidRoutingException.forRangeRead(message.from(), range, metadata.epoch, command); + } + + // TODO: preexisting issue: we should change the whole range for transient-ness, not just the right token + if (command.acceptsTransient() != maxTokenLocalReplica.isTransient()) + { + MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS); + throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s", + command.acceptsTransient() ? "transient" : "full", + maxTokenLocalReplica.isTransient() ? "transient" : "full", + this)); + } + } + return metadata; + } + + private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace) + { + return metadata.placements + .get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication) + .reads + .forToken(token) + .get() + .lookup(FBUtilities.getBroadcastAddressAndPort()); + } +} diff --git a/src/java/org/apache/cassandra/db/CounterMutation.java b/src/java/org/apache/cassandra/db/CounterMutation.java index 52068473e5..be4b9951e7 100644 --- a/src/java/org/apache/cassandra/db/CounterMutation.java +++ b/src/java/org/apache/cassandra/db/CounterMutation.java @@ -53,6 +53,7 @@ import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.tracing.Tracing; @@ -64,6 +65,7 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.net.MessagingService.VERSION_40; import static org.apache.cassandra.net.MessagingService.VERSION_50; import static org.apache.cassandra.net.MessagingService.VERSION_60; +import static org.apache.cassandra.net.MessagingService.VERSION_61; import static org.apache.cassandra.utils.Clock.Global.nanoTime; public class CounterMutation implements IMutation @@ -81,6 +83,18 @@ public class CounterMutation implements IMutation this.consistency = consistency; } + @Override + public MutationId id() + { + return mutation.id(); + } + + @Override + public CounterMutation withMutationId(MutationId mutationId) + { + return new CounterMutation(mutation.withMutationId(mutationId), consistency); + } + public String getKeyspaceName() { return mutation.getKeyspaceName(); @@ -148,7 +162,7 @@ public class CounterMutation implements IMutation */ public Mutation applyCounterMutation() throws WriteTimeoutException { - Mutation.PartitionUpdateCollector resultBuilder = new Mutation.PartitionUpdateCollector(getKeyspaceName(), key()); + Mutation.PartitionUpdateCollector resultBuilder = new Mutation.PartitionUpdateCollector(id(), getKeyspaceName(), key()); Keyspace keyspace = Keyspace.open(getKeyspaceName()); List locks = new ArrayList<>(); @@ -374,6 +388,7 @@ public class CounterMutation implements IMutation private int serializedSize40; private int serializedSize50; private int serializedSize51; + private int serializedSize52; public int serializedSize(int version) { @@ -391,6 +406,10 @@ public class CounterMutation implements IMutation if (serializedSize51 == 0) serializedSize51 = (int) serializer.serializedSize(this, VERSION_60); return serializedSize51; + case VERSION_61: + if (serializedSize52 == 0) + serializedSize52 = (int) serializer.serializedSize(this, VERSION_61); + return serializedSize52; default: throw new IllegalStateException("Unknown serialization version: " + version); } diff --git a/src/java/org/apache/cassandra/db/IMutation.java b/src/java/org/apache/cassandra/db/IMutation.java index f497a2aea5..f0116b58ff 100644 --- a/src/java/org/apache/cassandra/db/IMutation.java +++ b/src/java/org/apache/cassandra/db/IMutation.java @@ -27,6 +27,7 @@ import javax.annotation.Nullable; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ReadCommand.PotentialTxnConflicts; import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ClientState; @@ -34,6 +35,8 @@ public interface IMutation { long MAX_MUTATION_SIZE = DatabaseDescriptor.getMaxMutationSize(); + MutationId id(); + IMutation withMutationId(MutationId mutationId); void apply(); String getKeyspaceName(); Collection getTableIds(); diff --git a/src/java/org/apache/cassandra/db/Keyspace.java b/src/java/org/apache/cassandra/db/Keyspace.java index e0cdf8b5c2..ff9a52fefe 100644 --- a/src/java/org/apache/cassandra/db/Keyspace.java +++ b/src/java/org/apache/cassandra/db/Keyspace.java @@ -33,6 +33,7 @@ import java.util.concurrent.locks.Lock; import java.util.stream.Stream; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import org.slf4j.Logger; @@ -44,6 +45,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.lifecycle.SSTableSet; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.repair.CassandraKeyspaceRepairManager; +import org.apache.cassandra.db.tracked.TrackedKeyspaceWriteHandler; import org.apache.cassandra.db.view.ViewManager; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.exceptions.WriteTimeoutException; @@ -54,6 +56,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.metrics.KeyspaceMetrics; import org.apache.cassandra.repair.KeyspaceRepairManager; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; @@ -106,6 +109,7 @@ public class Keyspace public final ViewManager viewManager; private final KeyspaceWriteHandler writeHandler; + private final TrackedKeyspaceWriteHandler trackedWriteHandler; private final KeyspaceRepairManager repairManager; private final SchemaProvider schema; private final String name; @@ -285,6 +289,7 @@ public class Keyspace this.repairManager = new CassandraKeyspaceRepairManager(this); this.writeHandler = new CassandraKeyspaceWriteHandler(this); + this.trackedWriteHandler = new TrackedKeyspaceWriteHandler(); } public Keyspace(KeyspaceMetadata metadata) @@ -296,6 +301,7 @@ public class Keyspace this.viewManager = new ViewManager(this); this.repairManager = new CassandraKeyspaceRepairManager(this); this.writeHandler = new CassandraKeyspaceWriteHandler(this); + this.trackedWriteHandler = new TrackedKeyspaceWriteHandler(); this.metadataRef = new KeyspaceMetadataRef(metadata, schema); } @@ -395,13 +401,9 @@ public class Keyspace public Future applyFuture(Mutation mutation, boolean writeCommitLog, boolean updateIndexes) { - return applyInternal(mutation, writeCommitLog, updateIndexes, true, true, new AsyncPromise<>()); - } - - public Future applyFuture(Mutation mutation, boolean writeCommitLog, boolean updateIndexes, boolean isDroppable, - boolean isDeferrable) - { - return applyInternal(mutation, writeCommitLog, updateIndexes, isDroppable, isDeferrable, new AsyncPromise<>()); + return getMetadata().useMutationTracking() + ? applyInternalTracked(mutation, new AsyncPromise<>()) + : applyInternal(mutation, writeCommitLog, updateIndexes, true, true, new AsyncPromise<>()); } public void apply(Mutation mutation, boolean writeCommitLog, boolean updateIndexes) @@ -431,7 +433,10 @@ public class Keyspace boolean updateIndexes, boolean isDroppable) { - applyInternal(mutation, makeDurable, updateIndexes, isDroppable, false, null); + if (getMetadata().useMutationTracking()) + applyInternalTracked(mutation, null); + else + applyInternal(mutation, makeDurable, updateIndexes, isDroppable, false, null); } /** @@ -451,6 +456,8 @@ public class Keyspace boolean isDeferrable, Promise future) { + Preconditions.checkState(!getMetadata().useMutationTracking() && mutation.id().isNone()); + if (TEST_FAIL_WRITES && getMetadata().name.equals(TEST_FAIL_WRITES_KS)) throw new RuntimeException("Testing write failures"); @@ -598,6 +605,41 @@ public class Keyspace } } + /** + * Append the mutation to the mutation journal, then update memtables and indexes. + */ + private Future applyInternalTracked(Mutation mutation, Promise future) + { + Preconditions.checkState(getMetadata().useMutationTracking() && !mutation.id().isNone()); + + if (TEST_FAIL_WRITES && getMetadata().name.equals(TEST_FAIL_WRITES_KS)) + throw new RuntimeException("Testing write failures"); + + try (WriteContext ctx = trackedWriteHandler.beginWrite(mutation, true)) + { + MutationTrackingService.instance.startWriting(mutation); + + for (PartitionUpdate upd : mutation.getPartitionUpdates()) + { + ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(upd.metadata().id); + if (cfs == null) + { + logger.error("Attempting to mutate non-existant table {} ({}.{})", upd.metadata().id, upd.metadata().keyspace, upd.metadata().name); + continue; + } + + cfs.getWriteHandler().write(upd, ctx, true); + } + } + + MutationTrackingService.instance.finishWriting(mutation); + + if (future != null) + future.trySuccess(null); + + return future; + } + public AbstractReplicationStrategy getReplicationStrategy() { return getMetadata().replicationStrategy; diff --git a/src/java/org/apache/cassandra/db/Mutation.java b/src/java/org/apache/cassandra/db/Mutation.java index 9fc49ae15e..e1a01c117f 100644 --- a/src/java/org/apache/cassandra/db/Mutation.java +++ b/src/java/org/apache/cassandra/db/Mutation.java @@ -52,6 +52,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.TeeDataInputPlus; import org.apache.cassandra.locator.ReplicaPlan; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; @@ -63,6 +64,7 @@ import static com.google.common.base.Preconditions.checkState; import static org.apache.cassandra.net.MessagingService.VERSION_40; import static org.apache.cassandra.net.MessagingService.VERSION_50; import static org.apache.cassandra.net.MessagingService.VERSION_60; +import static org.apache.cassandra.net.MessagingService.VERSION_61; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; public class Mutation implements IMutation, Supplier @@ -71,6 +73,7 @@ public class Mutation implements IMutation, Supplier public static final int ALLOW_POTENTIAL_TRANSACTION_CONFLICTS = 0x01; + private final MutationId id; // todo this is redundant // when we remove it, also restore SerializationsTest.testMutationRead to not regenerate new Mutations each test private final String keyspaceName; @@ -103,23 +106,29 @@ public class Mutation implements IMutation, Supplier // because it is being applied by one or in a context where transaction conflicts don't occur private PotentialTxnConflicts potentialTxnConflicts; + public Mutation(MutationId id, PartitionUpdate update) + { + this(id, update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc, PotentialTxnConflicts.DISALLOW); + } + public Mutation(PartitionUpdate update) { - this(update, PotentialTxnConflicts.DISALLOW); + this(MutationId.none(), update, PotentialTxnConflicts.DISALLOW); } - public Mutation(PartitionUpdate update, PotentialTxnConflicts potentialTxnConflicts) + public Mutation(MutationId id, PartitionUpdate update, PotentialTxnConflicts potentialTxnConflicts) { - this(update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc, potentialTxnConflicts); + this(id, update.metadata().keyspace, update.partitionKey(), ImmutableMap.of(update.metadata().id, update), approxTime.now(), update.metadata().params.cdc, potentialTxnConflicts); } - public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, PotentialTxnConflicts potentialTxnConflicts) + public Mutation(MutationId id, String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, PotentialTxnConflicts potentialTxnConflicts) { - this(keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), potentialTxnConflicts); + this(id, keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled(modifications.values()), potentialTxnConflicts); } - public Mutation(String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, boolean cdcEnabled, PotentialTxnConflicts potentialTxnConflicts) + public Mutation(MutationId id, String keyspaceName, DecoratedKey key, ImmutableMap modifications, long approxCreatedAtNanos, boolean cdcEnabled, PotentialTxnConflicts potentialTxnConflicts) { + this.id = id; this.keyspaceName = keyspaceName; this.key = key; this.modifications = modifications; @@ -128,6 +137,18 @@ public class Mutation implements IMutation, Supplier this.potentialTxnConflicts = potentialTxnConflicts; } + @Override + public MutationId id() + { + return id; + } + + @Override + public Mutation withMutationId(MutationId mutationId) + { + return new Mutation(mutationId, keyspaceName, key, modifications, approxCreatedAtNanos, cdcEnabled, potentialTxnConflicts); + } + private static boolean cdcEnabled(Iterable modifications) { boolean cdc = false; @@ -159,7 +180,7 @@ public class Mutation implements IMutation, Supplier Map updates = builder.build(); checkState(!updates.isEmpty(), "Updates should not be empty"); - return new Mutation(keyspaceName, key, builder.build(), approxCreatedAtNanos, potentialTxnConflicts); + return new Mutation(id, keyspaceName, key, builder.build(), approxCreatedAtNanos, potentialTxnConflicts); } public @Nullable Mutation without(TableId tableId) @@ -258,6 +279,8 @@ public class Mutation implements IMutation, Supplier throw new IllegalArgumentException("Can't merge mutations with differing policies on allowing potential transaction conflicts"); potentialTxnConflicts = mutation.potentialTxnConflicts; updatedTables.addAll(mutation.modifications.keySet()); + if (!mutation.id().isNone()) + throw new IllegalArgumentException(); if (ks != null && !ks.equals(mutation.keyspaceName)) throw new IllegalArgumentException(); if (key != null && !key.equals(mutation.key)) @@ -283,7 +306,7 @@ public class Mutation implements IMutation, Supplier modifications.put(table, updates.size() == 1 ? updates.get(0) : PartitionUpdate.merge(updates)); updates.clear(); } - return new Mutation(ks, key, modifications.build(), approxTime.now(), potentialTxnConflicts); + return new Mutation(MutationId.none(), ks, key, modifications.build(), approxTime.now(), potentialTxnConflicts); } public Future applyFuture() @@ -399,6 +422,7 @@ public class Mutation implements IMutation, Supplier private int serializedSize40; private int serializedSize50; private int serializedSize51; + private int serializedSize52; public int serializedSize(int version) { @@ -416,6 +440,10 @@ public class Mutation implements IMutation, Supplier if (serializedSize51 == 0) serializedSize51 = (int) serializer.serializedSize(this, VERSION_60); return serializedSize51; + case VERSION_61: + if (serializedSize52 == 0) + serializedSize52 = (int) serializer.serializedSize(this, VERSION_61); + return serializedSize52; default: throw new IllegalStateException("Unknown serialization version: " + version); } @@ -428,9 +456,9 @@ public class Mutation implements IMutation, Supplier * @param partitionKey the key of partition this if a mutation for. * @return a newly created builder. */ - public static SimpleBuilder simpleBuilder(String keyspaceName, DecoratedKey partitionKey) + public static SimpleBuilder simpleBuilder(MutationId mutationId, String keyspaceName, DecoratedKey partitionKey) { - return new SimpleBuilders.MutationBuilder(keyspaceName, partitionKey); + return new SimpleBuilders.MutationBuilder(mutationId, keyspaceName, partitionKey); } /** @@ -554,9 +582,9 @@ public class Mutation implements IMutation, Supplier } static void serializeInternal(PartitionUpdate.PartitionUpdateSerializer serializer, - Mutation mutation, - DataOutputPlus out, - int version) throws IOException + Mutation mutation, + DataOutputPlus out, + int version) throws IOException { Map modifications = mutation.modifications; @@ -567,6 +595,9 @@ public class Mutation implements IMutation, Supplier out.write(flags); } + if (version >= MessagingService.VERSION_61) + MutationId.serializer.serialize(mutation.id, out, version); + /* serialize the modifications in the mutation */ int size = modifications.size(); out.writeUnsignedVInt32(size); @@ -592,13 +623,18 @@ public class Mutation implements IMutation, Supplier int flags = teeIn.readByte(); potentialTxnConflicts = potentialTxnConflicts(flags); } + + MutationId id = version >= MessagingService.VERSION_61 + ? MutationId.serializer.deserialize(teeIn, version) + : MutationId.none(); + int size = teeIn.readUnsignedVInt32(); assert size > 0; PartitionUpdate update = PartitionUpdate.serializer.deserialize(teeIn, version, flag); if (size == 1) { - m = new Mutation(update, potentialTxnConflicts); + m = new Mutation(id, update, potentialTxnConflicts); } else { @@ -611,7 +647,7 @@ public class Mutation implements IMutation, Supplier update = PartitionUpdate.serializer.deserialize(teeIn, version, flag); modifications.put(update.metadata().id, update); } - m = new Mutation(update.metadata().keyspace, dk, modifications.build(), approxTime.now(), potentialTxnConflicts); + m = new Mutation(id, update.metadata().keyspace, dk, modifications.build(), approxTime.now(), potentialTxnConflicts); } //Only cache serializations that don't hit the limit @@ -691,6 +727,8 @@ public class Mutation implements IMutation, Supplier { if (version >= VERSION_60) size += TypeSizes.sizeof((byte)ALLOW_POTENTIAL_TRANSACTION_CONFLICTS); // flags + if (version >= MessagingService.VERSION_61) + size += MutationId.serializer.serializedSize(mutation.id, version); size += TypeSizes.sizeofUnsignedVInt(mutation.modifications.size()); for (PartitionUpdate partitionUpdate : mutation.modifications.values()) size += serializer.serializedSize(partitionUpdate, version); @@ -706,20 +744,22 @@ public class Mutation implements IMutation, Supplier public static class PartitionUpdateCollector { private final ImmutableMap.Builder modifications = new ImmutableMap.Builder<>(); + private final MutationId mutationId; private final String keyspaceName; private final DecoratedKey key; private final long approxCreatedAtNanos = approxTime.now(); private boolean empty = true; - private PotentialTxnConflicts potentialTxnConflicts; + private final PotentialTxnConflicts potentialTxnConflicts; - public PartitionUpdateCollector(String keyspaceName, DecoratedKey key) + public PartitionUpdateCollector(MutationId mutationId, String keyspaceName, DecoratedKey key) { - this(keyspaceName, key, PotentialTxnConflicts.DISALLOW); + this(mutationId, keyspaceName, key, PotentialTxnConflicts.DISALLOW); } - public PartitionUpdateCollector(String keyspaceName, DecoratedKey key, PotentialTxnConflicts potentialTxnConflicts) + public PartitionUpdateCollector(MutationId mutationId, String keyspaceName, DecoratedKey key, PotentialTxnConflicts potentialTxnConflicts) { + this.mutationId = mutationId; this.keyspaceName = keyspaceName; this.key = key; this.potentialTxnConflicts = potentialTxnConflicts; @@ -756,7 +796,7 @@ public class Mutation implements IMutation, Supplier public Mutation build() { - return new Mutation(keyspaceName, key, modifications.build(), approxCreatedAtNanos, potentialTxnConflicts); + return new Mutation(mutationId, keyspaceName, key, modifications.build(), approxCreatedAtNanos, potentialTxnConflicts); } } } diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index d49cf986cb..43bb60d0d4 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -51,10 +51,14 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.service.reads.tracked.PartialTrackedRangeRead; +import org.apache.cassandra.service.reads.tracked.PartialTrackedRead; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; @@ -89,6 +93,11 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata()); } + public Slices requestedSlices() + { + return requestedSlices; + } + private static PartitionRangeReadCommand create(Epoch serializedAtEpoch, boolean isDigest, int digestVersion, @@ -469,6 +478,22 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR } } + @Override + protected PartialTrackedRead createInProgressRead(UnfilteredPartitionIterator iterator, + ReadExecutionController executionController, + Index.Searcher searcher, + ColumnFamilyStore cfs, + long startTimeNanos) + { + return PartialTrackedRangeRead.create(executionController, searcher, cfs, startTimeNanos, this, iterator); + } + + @Override + protected MutationSummary createMutationSummaryInternal(boolean includePending) + { + return MutationTrackingService.instance.createSummaryForRange(dataRange.keyRange, metadata().id, includePending); + } + @Override protected boolean intersects(SSTableReader sstable) { diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index f5f9105602..edd5aad08e 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -89,6 +89,7 @@ import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.ParamType; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationSummary; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; @@ -99,6 +100,7 @@ import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.ClientWarn; import org.apache.cassandra.service.accord.serializers.TableMetadatas; import org.apache.cassandra.service.consensus.migration.ConsensusRequestRouter; +import org.apache.cassandra.service.reads.tracked.PartialTrackedRead; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; @@ -122,10 +124,15 @@ import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; * General interface for storage-engine read commands (common to both range and * single partition commands). *

- * This contains all the informations needed to do a local read. + * This contains all the information needed to do a local read. */ public abstract class ReadCommand extends AbstractReadQuery { + private interface ReadCompleter + { + T complete(UnfilteredPartitionIterator iterator, ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos); + } + private static final int TEST_ITERATION_DELAY_MILLIS = CassandraRelevantProperties.TEST_READ_ITERATION_DELAY_MS.getInt(); protected static final Logger logger = LoggerFactory.getLogger(ReadCommand.class); @@ -428,6 +435,13 @@ public abstract class ReadCommand extends AbstractReadQuery protected abstract UnfilteredPartitionIterator queryStorage(ColumnFamilyStore cfs, ReadExecutionController executionController); + protected abstract MutationSummary createMutationSummaryInternal(boolean includePending); + + public MutationSummary createMutationSummary(boolean includePending) + { + return createMutationSummaryInternal(includePending); + } + /** * Whether the underlying {@code ClusteringIndexFilter} is reversed or not. * @@ -489,24 +503,10 @@ public abstract class ReadCommand extends AbstractReadQuery selectOptions.validate(metadata(), IndexRegistry.obtain(metadata()), indexQueryPlan()); } - /** - * Executes this command on the local host. - * - * @param executionController the execution controller spanning this command - * - * @return an iterator over the result of executing this command locally. - */ - // iterators created inside the try as long as we do close the original resultIterator), or by closing the result. - public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController) - { - return executeLocally(executionController, null); - } - // ClusterMetadata is null on startup when there are local reads from system tables before it's initialized - public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController, @Nullable ClusterMetadata cm) + private T beginRead(ReadExecutionController executionController, @Nullable ClusterMetadata cm, ReadCompleter completer) { long startTimeNanos = nanoTime(); - COMMAND.set(this); try { @@ -531,57 +531,12 @@ public abstract class ReadCommand extends AbstractReadQuery .collect(Collectors.joining(","))); } + if (searcher != null && metadata().replicationType().isTracked()) + throw new UnsupportedOperationException("TODO: support tracked index reads"); + UnfilteredPartitionIterator iterator = (null == searcher) ? queryStorage(cfs, executionController) : searcher.search(executionController); - iterator = RTBoundValidator.validate(iterator, Stage.MERGED, false); - try - { - iterator = withQuerySizeTracking(iterator); - iterator = maybeSlowDownForTesting(iterator); - iterator = withQueryCancellation(iterator); - iterator = maybeRecordPurgeableTombstones(iterator, cfs); - iterator = RTBoundValidator.validate(withoutPurgeableTombstones(iterator, cfs, executionController), Stage.PURGED, false); - iterator = withMetricsRecording(iterator, cfs.metric, startTimeNanos); - - // If we've used a 2ndary index, we know the result already satisfy the primary expression used, so - // no point in checking it again. - RowFilter filter = (null == searcher) ? rowFilter() : indexQueryPlan.postIndexQueryFilter(); - - /* - * TODO: We'll currently do filtering by the rowFilter here because it's convenient. However, - * we'll probably want to optimize by pushing it down the layer (like for dropped columns) as it - * would be more efficient (the sooner we discard stuff we know we don't care, the less useless - * processing we do on it). - */ - iterator = filter.filter(iterator, nowInSec()); - - // apply the limits/row counter; this transformation is stopping and would close the iterator as soon - // as the count is observed; if that happens in the middle of an open RT, its end bound will not be included. - // If tracking repaired data, the counter is needed for overreading repaired data, otherwise we can - // optimise the case where this.limit = DataLimits.NONE which skips an unnecessary transform - if (executionController.isTrackingRepairedStatus()) - { - DataLimits.Counter limit = - limits().newCounter(nowInSec(), false, selectsFullPartition(), metadata().enforceStrictLiveness()); - iterator = limit.applyTo(iterator); - // ensure that a consistent amount of repaired data is read on each replica. This causes silent - // overreading from the repaired data set, up to limits(). The extra data is not visible to - // the caller, only iterated to produce the repaired data digest. - iterator = executionController.getRepairedDataInfo().extend(iterator, limit); - } - else - { - iterator = limits().filter(iterator, nowInSec(), selectsFullPartition()); - } - - // because of the above, we need to append an aritifical end bound if the source iterator was stopped short by a counter. - return RTBoundCloser.close(iterator); - } - catch (RuntimeException | Error e) - { - iterator.close(); - throw e; - } + return completer.complete(iterator, executionController, searcher, cfs, startTimeNanos); } finally { @@ -589,6 +544,103 @@ public abstract class ReadCommand extends AbstractReadQuery } } + private UnfilteredPartitionIterator completeRead(UnfilteredPartitionIterator iterator, ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos) + { + COMMAND.set(this); + try + { + iterator = RTBoundValidator.validate(iterator, Stage.MERGED, false); + iterator = withQuerySizeTracking(iterator); + iterator = maybeSlowDownForTesting(iterator); + iterator = withQueryCancellation(iterator); + iterator = maybeRecordPurgeableTombstones(iterator, cfs); + iterator = RTBoundValidator.validate(withoutPurgeableTombstones(iterator, cfs, executionController), Stage.PURGED, false); + iterator = withMetricsRecording(iterator, cfs.metric, startTimeNanos); + + // If we've used a 2ndary index, we know the result already satisfy the primary expression used, so + // no point in checking it again. + RowFilter filter = (null == searcher) ? rowFilter() : indexQueryPlan.postIndexQueryFilter(); + + /* + * TODO: We'll currently do filtering by the rowFilter here because it's convenient. However, + * we'll probably want to optimize by pushing it down the layer (like for dropped columns) as it + * would be more efficient (the sooner we discard stuff we know we don't care, the less useless + * processing we do on it). + */ + iterator = filter.filter(iterator, nowInSec()); + + // apply the limits/row counter; this transformation is stopping and would close the iterator as soon + // as the count is observed; if that happens in the middle of an open RT, its end bound will not be included. + // If tracking repaired data, the counter is needed for overreading repaired data, otherwise we can + // optimise the case where this.limit = DataLimits.NONE which skips an unnecessary transform + if (executionController.isTrackingRepairedStatus()) + { + DataLimits.Counter limit = + limits().newCounter(nowInSec(), false, selectsFullPartition(), metadata().enforceStrictLiveness()); + iterator = limit.applyTo(iterator); + // ensure that a consistent amount of repaired data is read on each replica. This causes silent + // overreading from the repaired data set, up to limits(). The extra data is not visible to + // the caller, only iterated to produce the repaired data digest. + iterator = executionController.getRepairedDataInfo().extend(iterator, limit); + } + else + { + iterator = limits().filter(iterator, nowInSec(), selectsFullPartition()); + } + + // because of the above, we need to append an aritifical end bound if the source iterator was stopped short by a counter. + return RTBoundCloser.close(iterator); + } + catch (RuntimeException | Error e) + { + iterator.close(); + throw e; + } + finally + { + COMMAND.set(null); + } + } + + /** + * Convert the initial iterator into a snapshot of data, either by capturing immutable references or materializing + * the contents of the iterator into memory. Once this method returns, subsequent writes against this table should + * not be reflected in the result of the InProgressRead + */ + protected abstract PartialTrackedRead createInProgressRead(UnfilteredPartitionIterator iterator, + ReadExecutionController executionController, + Index.Searcher searcher, + ColumnFamilyStore cfs, + long startTimeNanos); + + public PartialTrackedRead beginTrackedRead(ReadExecutionController executionController) + { + return beginRead(executionController, null, this::createInProgressRead); + } + + public UnfilteredPartitionIterator completeTrackedRead(UnfilteredPartitionIterator iterator, PartialTrackedRead read) + { + return completeRead(iterator, read.executionController(), read.searcher(), read.cfs(), read.startTimeNanos()); + } + + /** + * Executes this command on the local host. + * + * @param executionController the execution controller spanning this command + * + * @return an iterator over the result of executing this command locally. + */ + // iterators created inside the try as long as we do close the original resultIterator), or by closing the result. + public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController) + { + return beginRead(executionController, null, this::completeRead); + } + + public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController, @Nullable ClusterMetadata cm) + { + return beginRead(executionController, cm, this::completeRead); + } + protected abstract void recordLatency(TableMetrics metric, long latencyNanos); public ReadExecutionController executionController(boolean trackRepairedStatus) diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index 8ff89f4e9b..6fe8e063b2 100644 --- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -16,41 +16,32 @@ * limitations under the License. */ package org.apache.cassandra.db; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; -import org.apache.cassandra.dht.AbstractBounds; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.exceptions.CoordinatorBehindException; -import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.exceptions.InvalidRoutingException; import org.apache.cassandra.exceptions.QueryCancelledException; import org.apache.cassandra.exceptions.RetryOnDifferentSystemException; -import org.apache.cassandra.locator.Replica; -import org.apache.cassandra.metrics.TCMMetrics; -import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.schema.SchemaConstants; -import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tcm.ClusterMetadata; -import org.apache.cassandra.tcm.ClusterMetadataService; -import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; -import org.apache.cassandra.utils.FBUtilities; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.exceptions.RequestFailureReason.RETRY_ON_DIFFERENT_TRANSACTION_SYSTEM; -public class ReadCommandVerbHandler implements IVerbHandler +public class ReadCommandVerbHandler extends AbstractReadCommandVerbHandler { public static final ReadCommandVerbHandler instance = new ReadCommandVerbHandler(); private static final Logger logger = LoggerFactory.getLogger(ReadCommandVerbHandler.class); + @Override + protected ReadCommand getCommand(ReadCommand payload) + { + return payload; + } + public ReadResponse doRead(ReadCommand command, boolean trackRepairedData) { ReadResponse response; @@ -63,25 +54,11 @@ public class ReadCommandVerbHandler implements IVerbHandler return response; } - public void doVerb(Message message) + @Override + protected void performRead(Message message, ClusterMetadata metadata) { - if (message.epoch().isAfter(Epoch.EMPTY)) - { - ClusterMetadata metadata = ClusterMetadata.current(); - metadata = checkTokenOwnership(metadata, message); - metadata = checkSchemaVersion(metadata, message); - } - - MessageParams.reset(); - - long timeout = message.expiresAtNanos() - message.createdAtNanos(); - ReadCommand command = message.payload; - command.setMonitoringTime(message.createdAtNanos(), message.isCrossNode(), timeout, DatabaseDescriptor.getSlowQueryTimeout(NANOSECONDS)); - - if (message.trackWarnings()) - command.trackWarnings(); - ReadResponse response; + ReadCommand command = message.payload; try { response = doRead(command, message.trackRepairedData()); @@ -132,117 +109,4 @@ public class ReadCommandVerbHandler implements IVerbHandler MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS); } } - - private ClusterMetadata checkSchemaVersion(ClusterMetadata metadata, Message message) - { - ReadCommand readCommand = message.payload; - - if (SchemaConstants.isSystemKeyspace(readCommand.metadata().keyspace) || - readCommand.serializedAtEpoch() == null) // don't try to catch up with pre-5.0 nodes - return metadata; - - Keyspace ks = metadata.schema.getKeyspace(readCommand.metadata().keyspace); - ColumnFamilyStore cfs = ks != null ? ks.getColumnFamilyStore(readCommand.metadata().id) : null; - Epoch localComparisonEpoch = metadata.epoch; - if (cfs != null) - localComparisonEpoch = cfs.metadata().epoch; - - if (localComparisonEpoch.isBefore(readCommand.serializedAtEpoch())) - metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch()); - else if (localComparisonEpoch.isAfter(readCommand.serializedAtEpoch())) - { - TCMMetrics.instance.coordinatorBehindSchema.mark(); - throw new CoordinatorBehindException(String.format("Coordinator schema for %s.%s with epoch %s is behind our schema %s", - message.payload.metadata().keyspace, - message.payload.metadata().name, - readCommand.serializedAtEpoch(), - localComparisonEpoch)); - } - ks = metadata.schema.getKeyspace(readCommand.metadata().keyspace); - if (ks == null || ks.getColumnFamilyStore(readCommand.metadata().id) == null) - throw new IllegalStateException("Unknown table " + readCommand.metadata().id +" after fetching remote log entries"); - return metadata; - } - - private ClusterMetadata checkTokenOwnership(ClusterMetadata metadata, Message message) - { - ReadCommand command = message.payload; - - if (command.metadata().isVirtual()) - return metadata; - - // Some read commands may be sent using an older Epoch intentionally so validating using the current Epoch - // doesn't work - if (command.potentialTxnConflicts().allowed) - return metadata; - - if (command.isTopK()) - return metadata; - - if (command instanceof SinglePartitionReadCommand) - { - Token token = ((SinglePartitionReadCommand)command).partitionKey().getToken(); - Replica localReplica = getLocalReplica(metadata, token, command.metadata().keyspace); - if (localReplica == null) - { - metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch()); - localReplica = getLocalReplica(metadata, token, command.metadata().keyspace); - } - if (localReplica == null) - { - StorageService.instance.incOutOfRangeOperationCount(); - Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc(); - throw InvalidRoutingException.forTokenRead(message.from(), token, metadata.epoch, message.payload); - } - - if (!command.acceptsTransient() && localReplica.isTransient()) - { - MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS); - throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s", - command.acceptsTransient() ? "transient" : "full", - localReplica.isTransient() ? "transient" : "full", - this)); - } - } - else - { - AbstractBounds range = ((PartitionRangeReadCommand) command).dataRange().keyRange(); - - // TODO: preexisting issue: for the range queries or queries that span multiple replicas, we can only make requests where the right token is owned, but not the left one - Replica maxTokenLocalReplica = getLocalReplica(metadata, range.right.getToken(), command.metadata().keyspace); - if (maxTokenLocalReplica == null) - { - metadata = ClusterMetadataService.instance().fetchLogFromPeerOrCMS(metadata, message.from(), message.epoch()); - maxTokenLocalReplica = getLocalReplica(metadata, range.right.getToken(), command.metadata().keyspace); - } - if (maxTokenLocalReplica == null) - { - StorageService.instance.incOutOfRangeOperationCount(); - Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc(); - throw InvalidRoutingException.forRangeRead(message.from(), range, metadata.epoch, message.payload); - } - - - // TODO: preexisting issue: we should change the whole range for transient-ness, not just the right token - if (command.acceptsTransient() != maxTokenLocalReplica.isTransient()) - { - MessagingService.instance().metrics.recordDroppedMessage(message, message.elapsedSinceCreated(NANOSECONDS), NANOSECONDS); - throw new InvalidRequestException(String.format("Attempted to serve %s data request from %s node in %s", - command.acceptsTransient() ? "transient" : "full", - maxTokenLocalReplica.isTransient() ? "transient" : "full", - this)); - } - } - return metadata; - } - - private static Replica getLocalReplica(ClusterMetadata metadata, Token token, String keyspace) - { - return metadata.placements - .get(metadata.schema.getKeyspaces().getNullable(keyspace).params.replication) - .reads - .forToken(token) - .get() - .lookup(FBUtilities.getBroadcastAddressAndPort()); - } } diff --git a/src/java/org/apache/cassandra/db/ReadExecutionController.java b/src/java/org/apache/cassandra/db/ReadExecutionController.java index 8a62ea390d..85b17d4690 100644 --- a/src/java/org/apache/cassandra/db/ReadExecutionController.java +++ b/src/java/org/apache/cassandra/db/ReadExecutionController.java @@ -21,6 +21,7 @@ import java.nio.ByteBuffer; import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.index.Index; @@ -69,6 +70,7 @@ public class ReadExecutionController implements AutoCloseable if (trackRepairedStatus) { + Preconditions.checkArgument(!command.metadata().replicationType().isTracked(), "Tracking repaired status is not supported for tracked reads"); DataLimits.Counter repairedReadCount = command.limits().newCounter(command.nowInSec(), false, command.selectsFullPartition(), diff --git a/src/java/org/apache/cassandra/db/SimpleBuilders.java b/src/java/org/apache/cassandra/db/SimpleBuilders.java index 7139b6c8b2..cc8d38ec32 100644 --- a/src/java/org/apache/cassandra/db/SimpleBuilders.java +++ b/src/java/org/apache/cassandra/db/SimpleBuilders.java @@ -39,6 +39,7 @@ import org.apache.cassandra.db.rows.BufferCell; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.CellPath; import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; @@ -116,6 +117,7 @@ public abstract class SimpleBuilders public static class MutationBuilder extends AbstractBuilder implements Mutation.SimpleBuilder { + private final MutationId mutationId; private final String keyspaceName; private final DecoratedKey key; @@ -123,8 +125,9 @@ public abstract class SimpleBuilders private PotentialTxnConflicts potentialTxnConflicts = PotentialTxnConflicts.DISALLOW; - public MutationBuilder(String keyspaceName, DecoratedKey key) + public MutationBuilder(MutationId mutationId, String keyspaceName, DecoratedKey key) { + this.mutationId = mutationId; this.keyspaceName = keyspaceName; this.key = key; } @@ -163,9 +166,9 @@ public abstract class SimpleBuilders assert !updateBuilders.isEmpty() : "Cannot create empty mutation"; if (updateBuilders.size() == 1) - return new Mutation(updateBuilders.values().iterator().next().build(), potentialTxnConflicts); + return new Mutation(mutationId, updateBuilders.values().iterator().next().build(), potentialTxnConflicts); - Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(keyspaceName, key, potentialTxnConflicts); + Mutation.PartitionUpdateCollector mutationBuilder = new Mutation.PartitionUpdateCollector(mutationId, keyspaceName, key, potentialTxnConflicts); for (PartitionUpdateBuilder builder : updateBuilders.values()) mutationBuilder.add(builder.build()); return mutationBuilder.build(); diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index b487d011b2..667c8349f9 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -57,6 +57,7 @@ import org.apache.cassandra.db.partitions.PartitionIterators; import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.db.rows.BaseRowIterator; +import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.db.rows.Rows; @@ -78,12 +79,16 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.metrics.TableMetrics; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.CacheService; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.accord.api.PartitionKey; +import org.apache.cassandra.service.reads.tracked.PartialTrackedRead; +import org.apache.cassandra.service.reads.tracked.PartialTrackedSinglePartitionRead; import org.apache.cassandra.tcm.Epoch; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; @@ -561,6 +566,16 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar return new SingletonUnfilteredPartitionIterator(partition); } + @Override + protected PartialTrackedRead createInProgressRead(UnfilteredPartitionIterator iterator, + ReadExecutionController executionController, + Index.Searcher searcher, + ColumnFamilyStore cfs, + long startTimeNanos) + { + return PartialTrackedSinglePartitionRead.create(executionController, searcher, cfs, startTimeNanos, this, iterator); + } + /** * Fetch the rows requested if in cache; if not, read it from disk and cache it. *

@@ -926,6 +941,11 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar } } + protected MutationSummary createMutationSummaryInternal(boolean includePending) + { + return MutationTrackingService.instance.createSummaryForKey(partitionKey, metadata().id, includePending); + } + @Override protected boolean intersects(SSTableReader sstable) { diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java index c7d3d3f16c..a00ba2094d 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogDescriptor.java @@ -64,13 +64,14 @@ public class CommitLogDescriptor public static final int VERSION_40 = 7; public static final int VERSION_50 = 8; public static final int VERSION_60 = 9; + public static final int VERSION_61 = 10; /** * Increment this number if there is a changes in the commit log disc layout or MessagingVersion changes. * Note: make sure to handle {@link #getMessagingVersion()} */ @VisibleForTesting - public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_60; + public static final int current_version = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_61; final int version; public final long id; @@ -228,6 +229,8 @@ public class CommitLogDescriptor return MessagingService.VERSION_50; case VERSION_60: return MessagingService.VERSION_60; + case VERSION_61: + return MessagingService.VERSION_61; default: throw new IllegalStateException("Unknown commitlog version " + version); } diff --git a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java index 427510258c..e6762aee5c 100644 --- a/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java +++ b/src/java/org/apache/cassandra/db/commitlog/CommitLogReplayer.java @@ -313,7 +313,7 @@ public class CommitLogReplayer implements CommitLogReadHandler if (commitLogReplayer.shouldReplay(update.metadata().id, new CommitLogPosition(segmentId, entryLocation))) { if (newPUCollector == null) - newPUCollector = new Mutation.PartitionUpdateCollector(mutation.getKeyspaceName(), mutation.key()); + newPUCollector = new Mutation.PartitionUpdateCollector(mutation.id(), mutation.getKeyspaceName(), mutation.key()); newPUCollector.add(update); commitLogReplayer.replayedCount.incrementAndGet(); } diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractPartitionIterator.java b/src/java/org/apache/cassandra/db/partitions/AbstractPartitionIterator.java new file mode 100644 index 0000000000..5dc4663bbb --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/AbstractPartitionIterator.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.utils.AbstractIterator; + +public abstract class AbstractPartitionIterator extends AbstractIterator implements PartitionIterator +{ +} diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java index bf65b3392a..569cee34d4 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionIterators.java @@ -17,14 +17,20 @@ */ package org.apache.cassandra.db.partitions; +import java.io.IOError; +import java.io.IOException; import java.util.List; import org.apache.cassandra.db.EmptyIterators; import org.apache.cassandra.db.SinglePartitionReadQuery; +import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.RowIterators; import org.apache.cassandra.db.transform.MorePartitions; import org.apache.cassandra.db.transform.Transformation; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.AbstractIterator; public abstract class PartitionIterators @@ -167,4 +173,41 @@ public abstract class PartitionIterators iterator.close(); } } + + public static class Serializer + { + public static void serialize(PartitionIterator iterator, ColumnFilter selection, DataOutputPlus out, int version) throws IOException + { + while (iterator.hasNext()) + { + try (RowIterator partition = iterator.next()) + { + out.writeBoolean(true); + RowIterators.Serializer.serialize(partition, selection, out, version); + } + } + out.writeBoolean(false); + } + + public static PartitionIterator deserialize(TableMetadata metadata, ColumnFilter selection, DataInputPlus in, int version) throws IOException + { + return new AbstractPartitionIterator() + { + @Override + protected RowIterator computeNext() + { + try + { + if (in.readBoolean()) + return RowIterators.Serializer.deserialize(metadata, selection, in, version); + return endOfData(); + } + catch (IOException e) + { + throw new IOError(e); + } + } + }; + } + }; } diff --git a/src/java/org/apache/cassandra/db/partitions/SimpleBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/SimpleBTreePartition.java new file mode 100644 index 0000000000..1d04c2c592 --- /dev/null +++ b/src/java/org/apache/cassandra/db/partitions/SimpleBTreePartition.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.partitions; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionInfo; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.index.transactions.UpdateTransaction; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.btree.BTree; +import org.apache.cassandra.utils.btree.UpdateFunction; +import org.apache.cassandra.utils.memory.HeapCloner; + +/** + * Single threaded btree partition, no waste or allocation tracking + */ +public class SimpleBTreePartition extends AbstractBTreePartition implements UpdateFunction +{ + private final TableMetadata metadata; + private final UpdateTransaction indexer; + private BTreePartitionData data = BTreePartitionData.EMPTY; + + public SimpleBTreePartition(DecoratedKey partitionKey, TableMetadata metadata, UpdateTransaction indexer) + { + super(partitionKey); + this.metadata = metadata; + this.indexer = indexer; + } + + @Override + protected BTreePartitionData holder() + { + return data; + } + + @Override + public TableMetadata metadata() + { + return metadata; + } + + @Override + protected boolean canHaveShadowedData() + { + return true; + } + + @Override + public void onAllocatedOnHeap(long heapSize) + { + + } + + protected BTreePartitionData makeMergedPartition(BTreePartitionData current, PartitionUpdate update) + { + DeletionInfo newDeletionInfo = merge(current.deletionInfo, update.deletionInfo()); + + RegularAndStaticColumns columns = current.columns; + RegularAndStaticColumns newColumns = update.columns().mergeTo(columns); + Row newStatic = mergeStatic(current.staticRow, update.staticRow()); + + Object[] tree = BTree.update(current.tree, update.holder().tree, update.metadata().comparator, this); + EncodingStats newStats = current.stats.mergeWith(update.stats()); + + return new BTreePartitionData(newColumns, tree, newDeletionInfo, newStatic, newStats); + } + + private Row mergeStatic(Row current, Row update) + { + if (update.isEmpty()) + return current; + if (current.isEmpty()) + return insert(update); + + return merge(current, update); + } + + private DeletionInfo merge(DeletionInfo existing, DeletionInfo update) + { + if (update.isLive() || !update.mayModify(existing)) + return existing; + + if (!update.getPartitionDeletion().isLive()) + indexer.onPartitionDeletion(update.getPartitionDeletion()); + + if (update.hasRanges()) + update.rangeIterator(false).forEachRemaining(indexer::onRangeTombstone); + + // Like for rows, we have to clone the update in case internal buffers (when it has range tombstones) reference + // memory we shouldn't hold into. But we don't ever store this off-heap currently so we just default to the + // HeapAllocator (rather than using 'allocator'). + DeletionInfo newInfo = existing.mutableCopy().add(update.clone(HeapCloner.instance)); + return newInfo; + } + + public Row insert(Row insert) + { + indexer.onInserted(insert); + return insert; + } + + public Row merge(Row existing, Row update) + { + Row reconciled = Rows.merge(existing, update, ColumnData.noOp); + indexer.onUpdated(existing, reconciled); + + return reconciled; + } + + public void update(PartitionUpdate update) + { + data = makeMergedPartition(data, update); + } +} diff --git a/src/java/org/apache/cassandra/db/rows/AbstractRowIterator.java b/src/java/org/apache/cassandra/db/rows/AbstractRowIterator.java new file mode 100644 index 0000000000..3481063596 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/AbstractRowIterator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.rows; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.AbstractIterator; + +public abstract class AbstractRowIterator extends AbstractIterator implements RowIterator +{ + protected final TableMetadata metadata; + protected final DecoratedKey key; + protected final RegularAndStaticColumns columns; + protected final boolean isReversedOrder; + protected final Row staticRow; + + public AbstractRowIterator(TableMetadata metadata, DecoratedKey key, RegularAndStaticColumns columns, boolean isReversedOrder, Row staticRow) + { + this.metadata = metadata; + this.key = key; + this.columns = columns; + this.isReversedOrder = isReversedOrder; + this.staticRow = staticRow; + } + + @Override + public TableMetadata metadata() + { + return metadata; + } + + @Override + public DecoratedKey partitionKey() + { + return key; + } + + @Override + public RegularAndStaticColumns columns() + { + return columns; + } + + @Override + public Row staticRow() + { + return staticRow; + } + + @Override + public boolean isReverseOrder() + { + return isReversedOrder; + } +} diff --git a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java index 062b389ad8..c9543a308b 100644 --- a/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java +++ b/src/java/org/apache/cassandra/db/rows/ComplexColumnData.java @@ -220,7 +220,7 @@ public class ComplexColumnData extends ColumnData implements Iterable> public ComplexColumnData withOnlyQueriedData(ColumnFilter filter) { - return transformAndFilter(complexDeletion, (cell) -> filter.fetchedCellIsQueried(column, cell.path()) ? null : cell); + return transformAndFilter(complexDeletion, (cell) -> filter.fetchedCellIsQueried(column, cell.path()) ? cell : null); } public ComplexColumnData purgeDataOlderThan(long timestamp) diff --git a/src/java/org/apache/cassandra/db/rows/RowIterators.java b/src/java/org/apache/cassandra/db/rows/RowIterators.java index 640cbc8d5b..b398b27522 100644 --- a/src/java/org/apache/cassandra/db/rows/RowIterators.java +++ b/src/java/org/apache/cassandra/db/rows/RowIterators.java @@ -17,14 +17,25 @@ */ package org.apache.cassandra.db.rows; +import com.google.common.base.Preconditions; +import net.nicoulaj.compilecommand.annotations.Inline; +import org.apache.cassandra.db.*; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.SearchIterator; +import org.apache.cassandra.utils.WrappedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.Digest; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.transform.Transformation; import org.apache.cassandra.schema.TableMetadata; +import java.io.IOError; +import java.io.IOException; + /** * Static methods to work with row iterators. */ @@ -101,4 +112,272 @@ public abstract class RowIterators } return Transformation.apply(iterator, new Log()); } + + private static class RowSerializer + { + /* + * Row flags constants. + */ + private final static int END_OF_PARTITION = 0x01; // Signal the end of the partition. Nothing follows a field with that flag. + private final static int HAS_ALL_COLUMNS = 0x02; // Whether the encoded row has all of the columns from the header present. + private final static int IS_STATIC = 0x04; // Whether the encoded row is a static. If there is no extended flag, the row is assumed not static. + + + public static void writeEndOfPartition(DataOutputPlus out) throws IOException + { + out.writeByte((byte) END_OF_PARTITION); + } + + public static void serializeRow(Row row, SerializationHelper helper, DataOutputPlus out, int version) + throws IOException + { + int flags = 0; + + boolean isStatic = row.isStatic(); + SerializationHeader header = helper.header; + Preconditions.checkArgument(!header.isForSSTable()); + boolean hasAllColumns = row.columnCount() == header.columns(isStatic).size(); + + if (isStatic) + flags |= IS_STATIC; + + if (hasAllColumns) + flags |= HAS_ALL_COLUMNS; + + out.writeByte((byte)flags); + + if (!isStatic) + Clustering.serializer.serialize(row.clustering(), out, version, header.clusteringTypes()); + + serializeRowBody(row, flags, helper, out); + } + + public static void serializeStaticRow(Row row, SerializationHelper helper, DataOutputPlus out, int version) throws IOException + { + Preconditions.checkArgument(row.isStatic()); + serializeRow(row, helper, out, version); + } + + @Inline + private static void serializeRowBody(Row row, int flags, SerializationHelper helper, DataOutputPlus out) + throws IOException + { + boolean isStatic = row.isStatic(); + + SerializationHeader header = helper.header; + Preconditions.checkState(!header.isForSSTable()); + Columns headerColumns = header.columns(isStatic); + + if ((flags & HAS_ALL_COLUMNS) == 0) + Columns.serializer.serializeSubset(row.columns(), headerColumns, out); + + SearchIterator si = helper.iterator(isStatic); + + try + { + row.apply(cd -> { + // We can obtain the column for data directly from data.column(). However, if the cell/complex data + // originates from a sstable, the column we'll get will have the type used when the sstable was serialized, + // and if that type have been recently altered, that may not be the type we want to serialize the column + // with. So we use the ColumnMetadata from the "header" which is "current". Also see #11810 for what + // happens if we don't do that. + ColumnMetadata column = si.next(cd.column()); + assert column != null : cd.column.toString(); + + try + { + if (cd.column.isSimple()) + Cell.serializer.serialize((Cell) cd, column, out, LivenessInfo.EMPTY, header); + else + UnfilteredSerializer.writeComplexColumn((ComplexColumnData) cd, column, false, LivenessInfo.EMPTY, header, out); + } + catch (IOException e) + { + throw new WrappedException(e); + } + }); + } + catch (WrappedException e) + { + if (e.getCause() instanceof IOException) + throw (IOException) e.getCause(); + + throw e; + } + } + + public static Row deserializeStaticRow(DataInputPlus in, SerializationHeader header, DeserializationHelper helper) + throws IOException + { + int flags = in.readUnsignedByte(); + Preconditions.checkState(!isEndOfPartition(flags)); + Preconditions.checkState(isStatic(flags)); + Row.Builder builder = BTreeRow.sortedBuilder(); + builder.newRow(Clustering.STATIC_CLUSTERING); + return deserializeRowBody(in, header, helper, flags, builder); + } + + public static Row deserializeRow(DataInputPlus in, SerializationHeader header, DeserializationHelper helper, Row.Builder builder) throws IOException + { + int flags = in.readUnsignedByte(); + if (isEndOfPartition(flags)) + return null; + builder.newRow(Clustering.serializer.deserialize(in, helper.version, header.clusteringTypes())); + return deserializeRowBody(in, header, helper, flags, builder); + } + + public static Row deserializeRowBody(DataInputPlus in, + SerializationHeader header, + DeserializationHelper helper, + int flags, + Row.Builder builder) + throws IOException + { + try + { + boolean isStatic = isStatic(flags); + boolean hasAllColumns = (flags & HAS_ALL_COLUMNS) != 0; + Columns headerColumns = header.columns(isStatic); + + Preconditions.checkState(!header.isForSSTable()); + + Columns columns = hasAllColumns ? headerColumns : Columns.serializer.deserializeSubset(headerColumns, in); + + try + { + DataInputPlus finalIn = in; + columns.apply(column -> { + try + { + if (column.isSimple()) + UnfilteredSerializer.readSimpleColumn(column, finalIn, header, helper, builder, LivenessInfo.EMPTY); + else + UnfilteredSerializer.readComplexColumn(column, finalIn, header, helper, false, builder, LivenessInfo.EMPTY); + } + catch (IOException e) + { + throw new WrappedException(e); + } + }); + } + catch (WrappedException e) + { + if (e.getCause() instanceof IOException) + throw (IOException) e.getCause(); + + throw e; + } + + return builder.build(); + } + catch (RuntimeException | AssertionError e) + { + // Corrupted data could be such that it triggers an assertion in the row Builder, or break one of its assumption. + // Of course, a bug in said builder could also trigger this, but it's impossible a priori to always make the distinction + // between a real bug and data corrupted in just the bad way. Besides, re-throwing as an IOException doesn't hide the + // exception, it just make we catch it properly and mark the sstable as corrupted. + throw new IOException("Error building row with data deserialized from " + in, e); + } + } + + public static boolean isEndOfPartition(int flags) + { + return (flags & END_OF_PARTITION) != 0; + } + + public static boolean isStatic(int extendedFlags) + { + return (extendedFlags & IS_STATIC) != 0; + } + } + + public static class Serializer + { + public static final int IS_EMPTY = 0x01; + private static final int IS_REVERSED = 0x02; + private static final int HAS_STATIC_ROW = 0x04; + + public static void serialize(RowIterator iterator, ColumnFilter selection, DataOutputPlus out, int version) throws IOException + { + ByteBufferUtil.writeWithVIntLength(iterator.partitionKey().getKey(), out); + + SerializationHeader header = new SerializationHeader(false, + iterator.metadata(), + iterator.columns(), + EncodingStats.NO_STATS); + + int flags = 0; + if (iterator.isReverseOrder()) + flags |= IS_REVERSED; + + if (iterator.isEmpty()) + { + out.writeByte(flags | IS_EMPTY); + return; + } + + Row staticRow = iterator.staticRow(); + boolean hasStatic = staticRow != Rows.EMPTY_STATIC_ROW; + if (hasStatic) + flags |= HAS_STATIC_ROW; + + out.writeByte((byte)flags); + + SerializationHeader.serializer.serializeForMessaging(header, selection, out, hasStatic); + SerializationHelper helper = new SerializationHelper(header); + + if (hasStatic) + RowSerializer.serializeStaticRow(staticRow, helper, out, version); + + while (iterator.hasNext()) + RowSerializer.serializeRow(iterator.next(), helper, out, version); + RowSerializer.writeEndOfPartition(out); + } + + public static RowIterator deserialize(TableMetadata metadata, ColumnFilter selection, DataInputPlus in, int version) throws IOException + { + DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithVIntLength(in)); + int flags = in.readUnsignedByte(); + boolean isReversed = (flags & IS_REVERSED) != 0; + if ((flags & IS_EMPTY) != 0) + { + return EmptyIterators.row(metadata, key, isReversed); + } + + boolean hasStatic = (flags & HAS_STATIC_ROW) != 0; + + SerializationHeader header = SerializationHeader.serializer.deserializeForMessaging(in, metadata, selection, hasStatic); + + DeserializationHelper helper = new DeserializationHelper(metadata, version, DeserializationHelper.Flag.FROM_REMOTE); + Row staticRow = hasStatic ? RowSerializer.deserializeStaticRow(in, header, helper) : Rows.EMPTY_STATIC_ROW; + + return new AbstractRowIterator(metadata, key, header.columns(), isReversed, staticRow) + { + private final Row.Builder builder = BTreeRow.sortedBuilder(); + + @Override + protected Row computeNext() + { + try + { + Row row = RowSerializer.deserializeRow(in, header, helper, builder); + return row != null ? row : endOfData(); + } + catch (IOException e) + { + throw new IOError(e); + } + } + + @Override + public void close() + { + while (hasNext()) + next(); + + super.close(); + } + }; + } + }; } diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java index 112c1e4c6b..7c80c463cf 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java @@ -279,7 +279,7 @@ public class UnfilteredSerializer } } - private static void writeComplexColumn(ComplexColumnData data, ColumnMetadata column, boolean hasComplexDeletion, LivenessInfo rowLiveness, SerializationHeader header, DataOutputPlus out) + static void writeComplexColumn(ComplexColumnData data, ColumnMetadata column, boolean hasComplexDeletion, LivenessInfo rowLiveness, SerializationHeader header, DataOutputPlus out) throws IOException { if (hasComplexDeletion) @@ -661,7 +661,7 @@ public class UnfilteredSerializer } } - private void readSimpleColumn(ColumnMetadata column, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, Row.Builder builder, LivenessInfo rowLiveness) + static void readSimpleColumn(ColumnMetadata column, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, Row.Builder builder, LivenessInfo rowLiveness) throws IOException { if (helper.includes(column)) @@ -676,7 +676,7 @@ public class UnfilteredSerializer } } - private void readComplexColumn(ColumnMetadata column, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, boolean hasComplexDeletion, Row.Builder builder, LivenessInfo rowLiveness) + static void readComplexColumn(ColumnMetadata column, DataInputPlus in, SerializationHeader header, DeserializationHelper helper, boolean hasComplexDeletion, Row.Builder builder, LivenessInfo rowLiveness) throws IOException { if (helper.includes(column)) @@ -733,7 +733,7 @@ public class UnfilteredSerializer in.skipBytesFully(markerSize); } - private void skipComplexColumn(DataInputPlus in, ColumnMetadata column, SerializationHeader header, boolean hasComplexDeletion) + private static void skipComplexColumn(DataInputPlus in, ColumnMetadata column, SerializationHeader header, boolean hasComplexDeletion) throws IOException { if (hasComplexDeletion) diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java index 40931e4295..c448d5edbe 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java @@ -48,6 +48,7 @@ import org.apache.cassandra.dht.Token; import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.SSTableTxnSingleStreamWriter; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.service.accord.AccordService; import org.apache.cassandra.service.accord.AccordTopology; import org.apache.cassandra.service.accord.IAccordService; @@ -200,7 +201,7 @@ public class CassandraStreamReceiver implements StreamReceiver // // If the CFS has CDC, however, these updates need to be written to the CommitLog // so they get archived into the cdc_raw folder - ks.apply(new Mutation(PartitionUpdate.fromIterator(throttledPartitions.next(), filter)), + ks.apply(new Mutation(MutationId.fixme(), PartitionUpdate.fromIterator(throttledPartitions.next(), filter)), writeCDCCommitLog, true, false); diff --git a/src/java/org/apache/cassandra/db/tracked/TrackedKeyspaceWriteHandler.java b/src/java/org/apache/cassandra/db/tracked/TrackedKeyspaceWriteHandler.java new file mode 100644 index 0000000000..bfd6d593c6 --- /dev/null +++ b/src/java/org/apache/cassandra/db/tracked/TrackedKeyspaceWriteHandler.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.tracked; + +import org.apache.cassandra.db.CassandraWriteContext; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.KeyspaceWriteHandler; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.WriteContext; +import org.apache.cassandra.db.commitlog.CommitLogPosition; +import org.apache.cassandra.exceptions.RequestExecutionException; +import org.apache.cassandra.journal.RecordPointer; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.concurrent.OpOrder; + +public class TrackedKeyspaceWriteHandler implements KeyspaceWriteHandler +{ + @Override + public WriteContext beginWrite(Mutation mutation, boolean makeDurable) throws RequestExecutionException + { + OpOrder.Group group = null; + try + { + group = Keyspace.writeOrder.start(); + + Tracing.trace("Appending to mutation journal"); + RecordPointer pointer = MutationJournal.instance.write(mutation.id(), mutation); + + // TODO (preferred): update journal to return CommitLogPosition or otherwise remove requirement to allocate second object here + return new CassandraWriteContext(group, new CommitLogPosition(pointer.segment, pointer.position)); + } + catch (Throwable t) + { + if (group != null) + group.close(); + throw t; + } + } + + @Override + public WriteContext createContextForIndexing() + { + return createEmptyContext(); + } + + @Override + public WriteContext createContextForRead() + { + return createEmptyContext(); + } + + private WriteContext createEmptyContext() + { + OpOrder.Group group = Keyspace.writeOrder.start(); + try + { + return new CassandraWriteContext(group, null); + } + catch (Throwable t) + { + if (group != null) + group.close(); + throw t; + } + } +} diff --git a/src/java/org/apache/cassandra/db/view/TableViews.java b/src/java/org/apache/cassandra/db/view/TableViews.java index 0f6d877e73..d7b31053e1 100644 --- a/src/java/org/apache/cassandra/db/view/TableViews.java +++ b/src/java/org/apache/cassandra/db/view/TableViews.java @@ -41,6 +41,7 @@ import org.apache.cassandra.db.DeletionInfo; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.db.RangeTombstone; import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.ReadQuery; @@ -555,7 +556,7 @@ public class TableViews extends AbstractCollection Collection updates = generator.generateViewUpdates(); List mutations = new ArrayList<>(updates.size()); for (PartitionUpdate update : updates) - mutations.add(new Mutation(update)); + mutations.add(new Mutation(MutationId.fixme(), update)); generator.clear(); return mutations; @@ -570,7 +571,7 @@ public class TableViews extends AbstractCollection Mutation.PartitionUpdateCollector collector = mutations.get(key); if (collector == null) { - collector = new Mutation.PartitionUpdateCollector(baseTableMetadata.keyspace, key); + collector = new Mutation.PartitionUpdateCollector(MutationId.none(), baseTableMetadata.keyspace, key); mutations.put(key, collector); } collector.add(update); diff --git a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java index 929ead029f..4aa718462c 100644 --- a/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java +++ b/src/java/org/apache/cassandra/db/virtual/VirtualMutation.java @@ -43,6 +43,7 @@ import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.NoPayload; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ClientState; @@ -98,6 +99,18 @@ public final class VirtualMutation implements IMutation this.modifications = modifications; } + @Override + public MutationId id() + { + return MutationId.none(); + } + + @Override + public IMutation withMutationId(MutationId mutationId) + { + throw new IllegalArgumentException("MutationId cannot be used for virtual mutations"); + } + @Override public void apply() { diff --git a/src/java/org/apache/cassandra/hints/HintsDescriptor.java b/src/java/org/apache/cassandra/hints/HintsDescriptor.java index 5b8db34035..a049ec6cdc 100644 --- a/src/java/org/apache/cassandra/hints/HintsDescriptor.java +++ b/src/java/org/apache/cassandra/hints/HintsDescriptor.java @@ -71,7 +71,8 @@ final class HintsDescriptor static final int VERSION_40 = 2; static final int VERSION_50 = 3; static final int VERSION_60 = 4; - static final int CURRENT_VERSION = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_60; + static final int VERSION_61 = 5; + static final int CURRENT_VERSION = DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? VERSION_40 : VERSION_61; static final String COMPRESSION = "compression"; static final String ENCRYPTION = "encryption"; @@ -261,6 +262,8 @@ final class HintsDescriptor return MessagingService.VERSION_50; case VERSION_60: return MessagingService.VERSION_60; + case VERSION_61: + return MessagingService.VERSION_61; default: throw new AssertionError(); } diff --git a/src/java/org/apache/cassandra/journal/Journal.java b/src/java/org/apache/cassandra/journal/Journal.java index 4db2f7bfd8..29972bb20d 100644 --- a/src/java/org/apache/cassandra/journal/Journal.java +++ b/src/java/org/apache/cassandra/journal/Journal.java @@ -206,8 +206,11 @@ public class Journal implements Shutdownable public void start() { + if (state.get() == State.NORMAL) + return; + Invariants.require(state.compareAndSet(State.UNINITIALIZED, State.INITIALIZING), - "Unexpected journal state during initialization", state); + "Unexpected journal state during initialization: %s", state); metrics.register(flusher); deleteTmpFiles(); @@ -301,7 +304,7 @@ public class Journal implements Shutdownable releaser.awaitTermination(1, TimeUnit.MINUTES); metrics.deregister(); Invariants.require(state.compareAndSet(State.SHUTDOWN, State.TERMINATED), - "Unexpected journal state while trying to shut down", state); + "Unexpected journal state while trying to shut down: %s", state); } catch (InterruptedException e) { diff --git a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java index 67d3a56ca7..28d2a91f82 100644 --- a/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ReadRepairMetrics.java @@ -49,6 +49,7 @@ public class ReadRepairMetrics */ public static final Meter repairedBlockingFromAccord = Metrics.meter(factory.createMetricName("RepairedBlockingFromAccord")); public static final Meter reconcileRead = Metrics.meter(factory.createMetricName("ReconcileRead")); + public static final Meter trackedReconcile = Metrics.meter(factory.createMetricName("TrackedReconcile")); /** @deprecated See CASSANDRA-13910 */ @Deprecated(since = "4.0") diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 9e00988da7..356e9c8e92 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -59,6 +59,7 @@ import org.apache.cassandra.io.sstable.metadata.MetadataCollector; import org.apache.cassandra.metrics.Sampler.SamplerType; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; +import org.apache.cassandra.service.reads.ReplicaFilteringProtection; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.ExpMovingAverage; import org.apache.cassandra.utils.MovingAverage; @@ -377,7 +378,7 @@ public class TableMetrics public final Meter replicaFilteringProtectionRequests; /** - * This histogram records the maximum number of rows {@link org.apache.cassandra.service.reads.ReplicaFilteringProtection} + * This histogram records the maximum number of rows {@link ReplicaFilteringProtection} * caches at a point in time per query. With no replica divergence, this is equivalent to the maximum number of * cached rows in a single partition during a query. It can be helpful when choosing appropriate values for the * replica_filtering_protection thresholds in cassandra.yaml. diff --git a/src/java/org/apache/cassandra/net/Message.java b/src/java/org/apache/cassandra/net/Message.java index 98ecca9ef5..81001fcf59 100644 --- a/src/java/org/apache/cassandra/net/Message.java +++ b/src/java/org/apache/cassandra/net/Message.java @@ -61,6 +61,7 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; import static org.apache.cassandra.net.MessagingService.VERSION_40; import static org.apache.cassandra.net.MessagingService.VERSION_50; import static org.apache.cassandra.net.MessagingService.VERSION_60; +import static org.apache.cassandra.net.MessagingService.VERSION_61; import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; import static org.apache.cassandra.utils.MonotonicClock.Global.approxTime; import static org.apache.cassandra.utils.vint.VIntCoding.computeUnsignedVIntSize; @@ -246,6 +247,12 @@ public class Message implements ResponseContext return outWithParam(nextId(), verb, 0, payload, flag.addTo(0), null, null); } + public static Message outWithParam(Verb verb, T payload, ParamType paramType, Object paramValue) + { + assert !verb.isResponse() : verb; + return outWithParam(nextId(), verb, payload, paramType, paramValue); + } + public static Message outWithFlags(Verb verb, T payload, MessageFlag flag1, MessageFlag flag2) { assert !verb.isResponse(); @@ -1263,6 +1270,7 @@ public class Message implements ResponseContext private int serializedSize40; private int serializedSize50; private int serializedSize51; + private int serializedSize52; /** * Serialized size of the entire message, for the provided messaging version. Caches the calculated value. @@ -1283,6 +1291,10 @@ public class Message implements ResponseContext if (serializedSize51 == 0) serializedSize51 = serializer.serializedSize(this, VERSION_60); return serializedSize51; + case VERSION_61: + if (serializedSize52 == 0) + serializedSize52 = serializer.serializedSize(this, VERSION_61); + return serializedSize52; default: throw new IllegalStateException("Unknown serialization version " + version); } @@ -1290,7 +1302,8 @@ public class Message implements ResponseContext private int payloadSize40 = -1; private int payloadSize50 = -1; - private int payloadSize51 = -1; + private int payloadSize60 = -1; + private int payloadSize61 = -1; private int payloadSize(int version) { @@ -1305,9 +1318,13 @@ public class Message implements ResponseContext payloadSize50 = serializer.payloadSize(this, VERSION_50); return payloadSize50; case VERSION_60: - if (payloadSize51 < 0) - payloadSize51 = serializer.payloadSize(this, VERSION_60); - return payloadSize51; + if (payloadSize60 < 0) + payloadSize60 = serializer.payloadSize(this, VERSION_60); + return payloadSize60; + case VERSION_61: + if (payloadSize61 < 0) + payloadSize61 = serializer.payloadSize(this, VERSION_61); + return payloadSize61; default: throw new IllegalStateException("Unkown serialization version " + version); diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index f7208f694e..4d567d4d11 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -225,7 +225,9 @@ public class MessagingService extends MessagingServiceMBeanImpl implements Messa // c14227 TTL overflow, 'uint' timestamps VERSION_50(13), // TCM, index hints - VERSION_60(14); + VERSION_60(14), + // Mutation Tracking + VERSION_61(15); public static final Version MIN_ACCORD_VERSION = Version.VERSION_60; @@ -266,8 +268,9 @@ public class MessagingService extends MessagingServiceMBeanImpl implements Messa public static final int VERSION_40 = 12; public static final int VERSION_50 = 13; // c14227 TTL overflow, 'uint' timestamps public static final int VERSION_60 = 14; // TCM, index hints + public static final int VERSION_61 = 15; // Mutation Tracking public static final int minimum_version = VERSION_40; - public static final int maximum_version = VERSION_60; + public static final int maximum_version = VERSION_61; // we want to use a modified behavior for the tools and clients - that is, since they are not running a server, they // should not need to run in a compatibility mode. They should be able to connect to the server regardless whether // it uses messaving version 4 or 5 @@ -309,7 +312,7 @@ public class MessagingService extends MessagingServiceMBeanImpl implements Messa private static Version currentVersion() { - return DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? Version.VERSION_40 : Version.VERSION_60; + return DatabaseDescriptor.getStorageCompatibilityMode().isBefore(5) ? Version.VERSION_40 : Version.VERSION_61; } private static class MSHandle diff --git a/src/java/org/apache/cassandra/net/Verb.java b/src/java/org/apache/cassandra/net/Verb.java index ad2317e5ab..3969a27de7 100644 --- a/src/java/org/apache/cassandra/net/Verb.java +++ b/src/java/org/apache/cassandra/net/Verb.java @@ -131,6 +131,12 @@ import org.apache.cassandra.service.paxos.cleanup.PaxosStartPrepareCleanup; import org.apache.cassandra.service.paxos.cleanup.PaxosUpdateLowBallot; import org.apache.cassandra.service.paxos.v1.PrepareVerbHandler; import org.apache.cassandra.service.paxos.v1.ProposeVerbHandler; +import org.apache.cassandra.service.reads.tracked.ReadReconcileNotify; +import org.apache.cassandra.service.reads.tracked.ReadReconcileReceive; +import org.apache.cassandra.service.reads.tracked.ReadReconcileSend; +import org.apache.cassandra.service.reads.tracked.TrackedDataResponse; +import org.apache.cassandra.service.reads.tracked.TrackedRead; +import org.apache.cassandra.service.reads.tracked.TrackedSummaryResponse; import org.apache.cassandra.streaming.DataMovement; import org.apache.cassandra.streaming.DataMovementVerbHandler; import org.apache.cassandra.streaming.ReplicationDoneVerbHandler; @@ -314,6 +320,18 @@ public enum Verb TCM_FETCH_PEER_LOG_RSP (818, P0, shortTimeout, FETCH_METADATA, MessageSerializers::logStateSerializer, RESPONSE_HANDLER ), TCM_FETCH_PEER_LOG_REQ (819, P0, rpcTimeout, FETCH_METADATA, () -> FetchPeerLog.serializer, () -> FetchPeerLog.Handler.instance, TCM_FETCH_PEER_LOG_RSP ), + // tracked replication + READ_RECONCILE_SEND (901, P0, rpcTimeout, READ, () -> ReadReconcileSend.serializer, () -> ReadReconcileSend.verbHandler ), + READ_RECONCILE_RCV (902, P0, rpcTimeout, MUTATION, () -> ReadReconcileReceive.serializer, () -> ReadReconcileReceive.verbHandler ), + READ_RECONCILE_NOTIFY (903, P0, rpcTimeout, REQUEST_RESPONSE, () -> ReadReconcileNotify.serializer, () -> ReadReconcileNotify.verbHandler ), + + TRACKED_PARTITION_READ_RSP (906, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer, RESPONSE_HANDLER ), + TRACKED_PARTITION_READ_REQ (907, P3, readTimeout, READ, () -> TrackedRead.DataRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_PARTITION_READ_RSP), + TRACKED_RANGE_READ_RSP (908, P2, rangeTimeout, REQUEST_RESPONSE, () -> TrackedDataResponse.serializer, RESPONSE_HANDLER ), + TRACKED_RANGE_READ_REQ (909, P3, rangeTimeout, READ, () -> TrackedRead.DataRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_RANGE_READ_RSP ), + TRACKED_SUMMARY_RSP (910, P2, readTimeout, REQUEST_RESPONSE, () -> TrackedSummaryResponse.serializer, () -> TrackedSummaryResponse.verbHandler ), + TRACKED_SUMMARY_REQ (911, P3, readTimeout, READ, () -> TrackedRead.SummaryRequest.serializer, () -> TrackedRead.verbHandler, TRACKED_SUMMARY_RSP ), + INITIATE_DATA_MOVEMENTS_RSP (814, P1, rpcTimeout, MISC, () -> NoPayload.serializer, RESPONSE_HANDLER ), INITIATE_DATA_MOVEMENTS_REQ (815, P1, rpcTimeout, MISC, () -> DataMovement.serializer, () -> DataMovementVerbHandler.instance, INITIATE_DATA_MOVEMENTS_RSP ), DATA_MOVEMENT_EXECUTED_RSP (816, P1, rpcTimeout, MISC, () -> NoPayload.serializer, RESPONSE_HANDLER ), diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLog.java b/src/java/org/apache/cassandra/replication/CoordinatorLog.java new file mode 100644 index 0000000000..0abd0db93e --- /dev/null +++ b/src/java/org/apache/cassandra/replication/CoordinatorLog.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.schema.TableId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; + +public abstract class CoordinatorLog +{ + private static final Logger logger = LoggerFactory.getLogger(CoordinatorLog.class); + + protected final int localHostId; + protected final CoordinatorLogId logId; + protected final Participants participants; + + /** + * State machines and an Id <-> token index for unreconciled mutation ids that exist oh this host. + */ + private final LocalMutationStates unreconciledMutations; + + protected final Offsets.Mutable[] witnessedIds; + protected final Offsets.Mutable reconciledIds; + protected final ReadWriteLock lock; + + CoordinatorLog(int localHostId, CoordinatorLogId logId, Participants participants) + { + this.localHostId = localHostId; + this.logId = logId; + this.participants = participants; + this.unreconciledMutations = new LocalMutationStates(); + this.lock = new ReentrantReadWriteLock(); + + Offsets.Mutable[] ids = new Offsets.Mutable[participants.size()]; + for (int i = 0; i < participants.size(); i++) + ids[i] = new Offsets.Mutable(logId); + + witnessedIds = ids; + reconciledIds = new Offsets.Mutable(logId); + } + + static CoordinatorLog create(int localHostId, CoordinatorLogId id, Participants participants) + { + return id.hostId == localHostId ? new CoordinatorLogPrimary(localHostId, id, participants) + : new CoordinatorLogReplica(localHostId, id, participants); + } + + void witnessedRemoteMutation(MutationId mutationId, int onHostId) + { + logger.trace("witnessed remote mutation {} from {}", mutationId, onHostId); + lock.writeLock().lock(); + try + { + if (!get(onHostId).add(mutationId.offset())) + return; // already witnessed + + if (!getLocal().contains(mutationId.offset())) + return; // local host hasn't witnessed -> no cleanup needed + + // see if any other replicas haven't witnessed the id yet + boolean allOtherReplicasWitnessed = true; + for (int i = 0; i < participants.size() && allOtherReplicasWitnessed; i++) + { + int hostId = participants.get(i); + if (hostId != onHostId && hostId != localHostId && !get(hostId).contains(mutationId.offset())) + allOtherReplicasWitnessed = false; + } + + if (allOtherReplicasWitnessed) + { + logger.trace("marking mutation {} as fully reconciled", mutationId); + // if all replicas have now witnessed the id, remove it from the index + unreconciledMutations.remove(mutationId.offset()); + reconciledIds.add(mutationId.offset()); + } + } + finally + { + lock.writeLock().unlock(); + } + } + + void startWriting(Mutation mutation) + { + lock.writeLock().lock(); + try + { + if (getLocal().contains(mutation.id().offset())) + return; // already witnessed; shouldn't get to this path often (duplicate mutation) + + unreconciledMutations.startWriting(mutation); + } + finally + { + lock.writeLock().unlock(); + } + } + + void finishWriting(Mutation mutation) + { + logger.trace("witnessed local mutation {}", mutation.id()); + lock.writeLock().lock(); + try + { + if (!getLocal().add(mutation.id().offset())) + throw new IllegalStateException("finishWriting() called on a reconciled mutation"); + + // see if any other replicas haven't witnessed the id yet + boolean allOtherReplicasWitnessed = true; + for (int i = 0; i < participants.size() && allOtherReplicasWitnessed; i++) + { + int hostId = participants.get(i); + if (hostId != localHostId && !get(hostId).contains(mutation.id().offset())) + allOtherReplicasWitnessed = false; + } + + // if some replicas also haven't witnessed the mutation yet, we should update local mutation state; + // otherwise we are the last node to witness this mutation, and can clean it up + if (allOtherReplicasWitnessed) + reconciledIds.add(mutation.id().offset()); + else + unreconciledMutations.finishWriting(mutation); + } + finally + { + lock.writeLock().unlock(); + } + } + + /** + * Look up unreconciled sequence ids of mutations witnessed by this host in this coordinataor log. + * Adds the ids to the supplied collection, so it can be reused to aggregate lookups for multiple logs. + */ + boolean collectOffsetsFor(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever unreconciledInto, Offsets.OffsetReciever reconciledInto) + { + lock.readLock().lock(); + try + { + reconciledInto.addAll(reconciledIds); + return unreconciledMutations.collect(token, tableId, includePending, unreconciledInto); + } + finally + { + lock.readLock().unlock(); + } + } + + /** + * Look up unreconciled sequence ids of mutations witnessed by this host in this coordinataor log. + * Adds the ids to the supplied collection, so it can be reused to aggregate lookups for multiple logs. + */ + boolean collectOffsetsFor(AbstractBounds range, TableId tableId, boolean includePending, Offsets.OffsetReciever unreconciledInto, Offsets.OffsetReciever reconciledInto) + { + lock.readLock().lock(); + try + { + reconciledInto.addAll(reconciledIds); + return unreconciledMutations.collect(range, tableId, includePending, unreconciledInto); + } + finally + { + lock.readLock().unlock(); + } + } + + protected Offsets.Mutable get(int hostId) + { + return witnessedIds[participants.indexOf(hostId)]; + } + + protected Offsets.Mutable getLocal() + { + return witnessedIds[participants.indexOf(localHostId)]; + } + + public static class CoordinatorLogPrimary extends CoordinatorLog + { + AtomicLong sequenceId = new AtomicLong(-1); + + CoordinatorLogPrimary(int localHostId, CoordinatorLogId logId, Participants participants) + { + super(localHostId, logId, participants); + } + + MutationId nextId() + { + return new MutationId(logId.asLong(), nextSequenceId()); + } + + private long nextSequenceId() + { + while (true) + { + long prev = sequenceId.get(); + int prevOffset = MutationId.offset(prev); + int prevTimestamp = MutationId.timestamp(prev); + + int nextOffset = prevOffset + 1; + int nextTimestamp = Math.max(prevTimestamp + 1, (int) (currentTimeMillis() / 1000L)); + long next = MutationId.sequenceId(nextOffset, nextTimestamp); + + if (sequenceId.compareAndSet(prev, next)) + return next; + } + } + } + + public static class CoordinatorLogReplica extends CoordinatorLog + { + CoordinatorLogReplica(int localHostId, CoordinatorLogId logId, Participants participants) + { + super(localHostId, logId, participants); + } + } +} diff --git a/src/java/org/apache/cassandra/replication/CoordinatorLogId.java b/src/java/org/apache/cassandra/replication/CoordinatorLogId.java new file mode 100644 index 0000000000..55c38485b0 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/CoordinatorLogId.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +import java.io.IOException; +import java.io.Serializable; +import java.util.Comparator; + +public class CoordinatorLogId implements Serializable +{ + /** TCM host ID */ + protected final int hostId; + + /** + * Host log ID (unique within the host). + * Allocated anew on host restart - one per token range replicated by the host. + * Persisted on allocation, unique within the host. + */ + protected final int hostLogId; + + CoordinatorLogId(long id) + { + this(hostId(id), hostLogId(id)); + } + + CoordinatorLogId(int hostId, int hostLogId) + { + this.hostId = hostId; + this.hostLogId = hostLogId; + } + + public int hostId() + { + return hostId; + } + + public int hostLogId() + { + return hostLogId; + } + + public long asLong() + { + return asLong(hostId, hostLogId); + } + + static long asLong(int hostId, int hostLogId) + { + return ((long) hostId << 32) | hostLogId; + } + + static int hostId(long coordinatorLogId) + { + return (int) (coordinatorLogId >>> 32); + } + + static int hostLogId(long coordinatorLogId) + { + return (int) coordinatorLogId; + } + + @Override + public String toString() + { + return "CoordinatorLogId{" + hostId + ", " + hostLogId + '}'; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + CoordinatorLogId logId = (CoordinatorLogId) o; + return hostId == logId.hostId && hostLogId == logId.hostLogId; + } + + @Override + public int hashCode() + { + return Integer.hashCode(hostLogId) + 31 * Integer.hashCode(hostId); + } + + public static final Comparator comparator = (l, r) -> Long.compareUnsigned(l.asLong(), r.asLong()); + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(CoordinatorLogId logId, DataOutputPlus out, int version) throws IOException + { + out.writeInt(logId.hostId); + out.writeInt(logId.hostLogId); + } + + @Override + public CoordinatorLogId deserialize(DataInputPlus in, int version) throws IOException + { + int hostId = in.readInt(); + int hostLogId = in.readInt(); + return new CoordinatorLogId(hostId, hostLogId); + } + + @Override + public long serializedSize(CoordinatorLogId logId, int version) + { + return TypeSizes.sizeof(logId.hostId) + TypeSizes.sizeof(logId.hostLogId); + } + }; +} diff --git a/src/java/org/apache/cassandra/replication/LocalMutationStates.java b/src/java/org/apache/cassandra/replication/LocalMutationStates.java new file mode 100644 index 0000000000..9bb1b4d834 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/LocalMutationStates.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.util.Collection; +import java.util.Comparator; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Sets; + +import org.agrona.collections.Int2ObjectHashMap; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.schema.TableId; + +/** + * Tracks unreconciled local mutations - the subset of all unreconciled mutations + * that have been witnessed, or are currently being written to, on the local node. + */ +class LocalMutationStates +{ + private final Int2ObjectHashMap statesMap = new Int2ObjectHashMap<>(); + private final SortedSet statesSet = new TreeSet<>(Entry.comparator); + + enum Visibility + { + PENDING, // written to the journal, but not yet to LSM + VISIBLE, // written to both the journal and LSM + } + + private static final class Entry + { + private static final Comparator comparator = (left, right) -> + { + int cmp = left.token.compareTo(right.token); + return (cmp != 0) ? cmp : Integer.compare(left.offset, right.offset); + }; + + final Token token; + final int offset; + final Object tableOrTables; + private Visibility visibility; + + Entry(Token token, int offset, Object tableOrTables, Visibility visibility) + { + this.token = token; + this.offset = offset; + this.tableOrTables = tableOrTables; + this.visibility = visibility; + } + + static Entry create(Mutation mutation) + { + Collection ids = mutation.getTableIds(); + Preconditions.checkArgument(!ids.isEmpty()); + return new Entry(mutation.key().getToken(), mutation.id().offset(), tableOrTables(mutation), Visibility.PENDING); + } + + private static Object tableOrTables(Mutation mutation) + { + Collection ids = mutation.getTableIds(); + Preconditions.checkArgument(!ids.isEmpty()); + return ids.size() == 1 ? ids.iterator().next() : Sets.newHashSet(mutation.getTableIds()); + } + + boolean contains(TableId tableId) + { + return tableOrTables instanceof Set + ? ((Set) tableOrTables).contains(tableId) + : tableId.equals(tableOrTables); + } + + boolean isVisible() + { + return visibility == Visibility.VISIBLE; + } + + static Entry start(Token token, boolean isInclusive) + { + return new Entry(token, isInclusive ? 0 : Integer.MAX_VALUE, null, null); + } + + static Entry end(Token token, boolean isInclusive) + { + return new Entry(token, isInclusive ? Integer.MAX_VALUE : 0, null, null); + } + + @Override + public boolean equals(Object o) + { + if (!(o instanceof Entry)) + return false; + Entry that = (Entry) o; + return this.offset == that.offset && this.token.equals(that.token); + } + } + + void startWriting(Mutation mutation) + { + Entry entry = Entry.create(mutation); + statesMap.put(entry.offset, entry); + statesSet.add(entry); + } + + void finishWriting(Mutation mutation) + { + Entry entry = statesMap.get(mutation.id().offset()); + Preconditions.checkNotNull(entry); + entry.visibility = Visibility.VISIBLE; + } + + void remove(int offset) + { + Entry state = statesMap.remove(offset); + Preconditions.checkNotNull(state); + statesSet.remove(state); + } + + boolean collect(Token token, TableId tableId, boolean includePending, Offsets.OffsetReciever into) + { + SortedSet subset = statesSet.subSet(Entry.start(token, true), Entry.end(token, true)); + return collect(subset, tableId, includePending, into); + } + + boolean collect(AbstractBounds range, TableId tableId, boolean includePending, Offsets.OffsetReciever into) + { + Entry start = Entry.start(range.left.getToken(), range.left.kind() != PartitionPosition.Kind.MAX_BOUND); + Entry end = Entry.end(range.right.getToken(), range.right.kind() != PartitionPosition.Kind.MIN_BOUND); + return collect(start, end, tableId, includePending, into); + } + + private boolean collect(SortedSet subset, TableId tableId, boolean includePending, Offsets.OffsetReciever into) + { + boolean found = false; + for (Entry entry : subset) + { + if (entry.contains(tableId) && (includePending || entry.isVisible())) + { + into.add(entry.offset); + found = true; + } + } + return found; + } + + private boolean collect(Entry start, Entry end, TableId tableId, boolean includePending, Offsets.OffsetReciever into) + { + int cmp = start.token.compareTo(end.token); + if (cmp == 0) + { + // full range + return collect(statesSet, tableId, includePending, into); + } + else if (cmp > 0) + { + // wrap around range + boolean lFound = collect(statesSet.headSet(end), tableId, includePending, into); + boolean rFound = collect(statesSet.tailSet(start), tableId, includePending, into); + return lFound || rFound; + } + else + { + // contiguous range + return collect(statesSet.subSet(start, end), tableId, includePending, into); + } + } +} diff --git a/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java b/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java new file mode 100644 index 0000000000..efcdd195ec --- /dev/null +++ b/src/java/org/apache/cassandra/replication/Log2OffsetsMap.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.Iterator; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; + +import org.agrona.collections.Long2ObjectHashMap; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +public abstract class Log2OffsetsMap implements Iterable +{ + abstract Long2ObjectHashMap offsetMap(); + + @Override + public Iterator iterator() + { + return Iterables.concat(offsetMap().values()).iterator(); + } + + public int idCount() + { + int count = 0; + for (T offsets : offsetMap().values()) + count += offsets.offsetCount(); + return count; + } + + public boolean isEmpty() + { + for (T offsets : offsetMap().values()) + if (!offsets.isEmpty()) + return false; + return true; + } + + private static abstract class AbstractMutable> extends Log2OffsetsMap + { + protected final Long2ObjectHashMap offsetMap = new Long2ObjectHashMap<>(); + + @Override + Long2ObjectHashMap offsetMap() + { + return offsetMap; + } + + protected abstract T createOrNull(Offsets.RangeIterator iterator); + protected abstract T create(CoordinatorLogId logId); + protected abstract T copy(Offsets offsets); + + protected T create(long logId) + { + return create(new CoordinatorLogId(logId)); + } + + public void add(ShortMutationId id) + { + T offsets = offsetMap.computeIfAbsent(id.logId(), this::create); + offsets.add(id.offset()); + } + + public void add(Offsets offsets) + { + if (offsets.isEmpty()) + return; + + T existing = offsetMap.get(offsets.logId().asLong()); + if (existing == null) + { + offsetMap.put(offsets.logId().asLong(), copy(offsets)); + return; + } + + existing.addAll(offsets); + } + + public void addAll(Log2OffsetsMap that) + { + for (Offsets offsets : that.offsetMap().values()) + add(offsets); + } + + public void remove(Offsets offsets) + { + T existing = offsetMap.get(offsets.logId().asLong()); + if (existing == null) + return; + + if (existing.isEmpty()) + { + offsetMap.remove(offsets.logId().asLong()); + return; + } + + T next = createOrNull(Offsets.difference(existing.rangeIterator(), offsets.rangeIterator())); + if (next == null) + offsetMap.remove(offsets.logId().asLong()); + else + offsetMap.put(offsets.logId().asLong(), next); + } + + public void removeAll(Log2OffsetsMap that) + { + for (Offsets offsets : that.offsetMap().values()) + remove(offsets); + } + } + + public static class Mutable extends AbstractMutable + { + @Override + protected Offsets.Mutable createOrNull(Offsets.RangeIterator iterator) + { + return Offsets.Mutable.createOrNull(iterator); + } + + @Override + protected Offsets.Mutable create(CoordinatorLogId logId) + { + return new Offsets.Mutable(logId); + } + + @Override + protected Offsets.Mutable copy(Offsets offsets) + { + return Offsets.Mutable.copy(offsets); + } + } + + public static class Immutable extends Log2OffsetsMap + { + private final Long2ObjectHashMap offsetMap; + + private Immutable(Long2ObjectHashMap offsetMap) + { + this.offsetMap = offsetMap; + } + + @Override + Long2ObjectHashMap offsetMap() + { + return offsetMap; + } + + public static class Builder extends AbstractMutable + { + @Override + protected Offsets.Immutable.Builder createOrNull(Offsets.RangeIterator iterator) + { + return Offsets.Immutable.Builder.createOrNull(iterator); + } + + @Override + protected Offsets.Immutable.Builder create(CoordinatorLogId logId) + { + return new Offsets.Immutable.Builder(logId); + } + + @Override + protected Offsets.Immutable.Builder copy(Offsets offsets) + { + return Offsets.Immutable.Builder.copy(offsets); + } + + public Log2OffsetsMap.Immutable build() + { + Long2ObjectHashMap result = new Long2ObjectHashMap<>(); + offsetMap.forEachLong((key, builder) -> result.put(key, builder.build())); + return new Immutable(result); + } + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(Log2OffsetsMap.Immutable mo, DataOutputPlus out, int version) throws IOException + { + out.writeInt(mo.offsetMap.size()); + for (Offsets.Immutable offsets : mo.offsetMap().values()) + Offsets.serializer.serialize(offsets, out, version); + } + + @Override + public Log2OffsetsMap.Immutable deserialize(DataInputPlus in, int version) throws IOException + { + Long2ObjectHashMap offsetMap = new Long2ObjectHashMap<>(); + int size = in.readInt(); + for (int i=0; i + * equals() and hashCode() are intentionally not overridden by this class, since log id and offset alone + * are meant to uniquely identify a mutation. + */ +public class MutationId extends ShortMutationId +{ + private static final long NONE_LOG_ID = Long.MIN_VALUE; + private static final long NONE_SEQUENCE_ID = Long.MIN_VALUE; + private static final int NONE_OFFSET = offset(NONE_SEQUENCE_ID); + private static final int NONE_TIMESTAMP = timestamp(NONE_SEQUENCE_ID); + private static final MutationId NONE = new MutationId(NONE_LOG_ID, NONE_SEQUENCE_ID); + + /** + * 4 byte timestamp. The timestamp is monotonically non-decreasing. + * The offset alone is sufficient to identify the entry within a coordinator + * log, the timestamp is added for correlation purposes. + */ + protected final int timestamp; + + public MutationId(long logId, long sequenceId) + { + super(logId, offset(sequenceId)); + this.timestamp = timestamp(sequenceId); + } + + public long sequenceId() + { + return sequenceId(offset, timestamp); + } + + public int timestamp() + { + return timestamp; + } + + public static long sequenceId(int offset, int timestamp) + { + return ((long) offset << 32) | timestamp; + } + + public static int offset(long sequenceId) + { + return (int) (0xffffffffL & (sequenceId >> 32)); + } + + public static int timestamp(long sequenceId) + { + return (int) (0xffffffffL & sequenceId); + } + + // FIXME: used in place of figuring out if we should use a mutation id or not + public static MutationId fixme() + { + return none(); + } + + public static MutationId none() + { + return NONE; + } + + public boolean isNone() + { + if (this == NONE) + return true; + return logId() == NONE_LOG_ID && offset() == NONE_OFFSET && timestamp() == NONE_TIMESTAMP; + } + + @Override + public String toString() + { + return "MutationId{" + hostId() + ", " + hostLogId() + ", " + offset() + ", " + timestamp() + '}'; + } + + /** + * The comparator is intentionally not overridden by this class, since log id and offset alone + * are meant to uniquely identify a mutation, and only offset determines the order within a log. + */ + public static final Comparator comparator = ShortMutationId.comparator::compare; + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(MutationId id, DataOutputPlus out, int version) throws IOException + { + out.writeLong(id.logId()); + out.writeLong(id.sequenceId()); + } + + @Override + public MutationId deserialize(DataInputPlus in, int version) throws IOException + { + long logId = in.readLong(); + long sequenceId = in.readLong(); + if (logId == NONE_LOG_ID && sequenceId == NONE_SEQUENCE_ID) + return none(); + return new MutationId(logId, sequenceId); + } + + @Override + public long serializedSize(MutationId id, int version) + { + return TypeSizes.sizeof(id.logId()) + TypeSizes.sizeof(id.sequenceId()); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/tracking/MutationJournal.java b/src/java/org/apache/cassandra/replication/MutationJournal.java similarity index 57% rename from src/java/org/apache/cassandra/service/tracking/MutationJournal.java rename to src/java/org/apache/cassandra/replication/MutationJournal.java index 3f00e4387c..59f87d5389 100644 --- a/src/java/org/apache/cassandra/service/tracking/MutationJournal.java +++ b/src/java/org/apache/cassandra/replication/MutationJournal.java @@ -15,38 +15,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.tracking; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.concurrent.TimeUnit; -import java.util.zip.Checksum; - -import javax.annotation.Nullable; +package org.apache.cassandra.replication; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; - +import org.apache.cassandra.config.Config; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.Mutation; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.io.util.File; -import org.apache.cassandra.journal.Journal; -import org.apache.cassandra.journal.KeySupport; -import org.apache.cassandra.journal.Params; -import org.apache.cassandra.journal.RecordConsumer; -import org.apache.cassandra.journal.RecordPointer; -import org.apache.cassandra.journal.SegmentCompactor; -import org.apache.cassandra.journal.ValueSerializer; +import org.apache.cassandra.journal.*; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.FBUtilities; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.concurrent.TimeUnit; +import java.util.zip.Checksum; + public class MutationJournal { - private final Journal journal; + public static final MutationJournal instance = new MutationJournal(); + + private final Journal journal; private MutationJournal() { @@ -69,25 +64,25 @@ public class MutationJournal journal.shutdown(); } - public RecordPointer write(MutationId id, Mutation mutation) + public RecordPointer write(ShortMutationId id, Mutation mutation) { return journal.blockingWrite(id, mutation); } @Nullable - public Mutation read(MutationId id) + public Mutation read(ShortMutationId id) { return journal.readLast(id); } - public boolean read(MutationId id, RecordConsumer consumer) + public boolean read(ShortMutationId id, RecordConsumer consumer) { return journal.readLast(id, consumer); } - public void readAll(Iterable ids, Collection into) + public void readAll(Iterable ids, Collection into) { - for (MutationId id : ids) + for (ShortMutationId id : ids) { Mutation mutation = read(id); Preconditions.checkState(mutation != null); @@ -112,7 +107,18 @@ public class MutationJournal @Override public FlushMode flushMode() { - return FlushMode.PERIODIC; + Config.CommitLogSync mode = DatabaseDescriptor.getCommitLogSync(); + switch (DatabaseDescriptor.getCommitLogSync()) + { + case batch: + return FlushMode.BATCH; + case periodic: + return FlushMode.PERIODIC; + case group: + return FlushMode.GROUP; + default: + throw new IllegalStateException("Unhandled flush mode: " + mode); + } } @Override @@ -146,88 +152,88 @@ public class MutationJournal } } - static class MutationIdSupport implements KeySupport + static class MutationIdSupport implements KeySupport { static final int LOG_ID_OFFSET = 0; - static final int SEQUENCE_ID_OFFSET = LOG_ID_OFFSET + TypeSizes.LONG_SIZE; + static final int OFFSET_OFFSET = LOG_ID_OFFSET + TypeSizes.LONG_SIZE; @Override public int serializedSize(int userVersion) { return TypeSizes.LONG_SIZE // logId - + TypeSizes.LONG_SIZE; // sequenceId + + TypeSizes.INT_SIZE; // offset } @Override - public void serialize(MutationId id, DataOutputPlus out, int userVersion) throws IOException + public void serialize(ShortMutationId id, DataOutputPlus out, int userVersion) throws IOException { - out.writeLong(id.logId); - out.writeLong(id.sequenceId); + out.writeLong(id.logId()); + out.writeInt(id.offset()); } @Override - public void serialize(MutationId id, ByteBuffer out, int userVersion) throws IOException + public void serialize(ShortMutationId id, ByteBuffer out, int userVersion) throws IOException { - out.putLong(id.logId); - out.putLong(id.sequenceId); + out.putLong(id.logId()); + out.putInt(id.offset()); } @Override - public MutationId deserialize(DataInputPlus in, int userVersion) throws IOException + public ShortMutationId deserialize(DataInputPlus in, int userVersion) throws IOException { long logId = in.readLong(); - long sequenceId = in.readLong(); - return new MutationId(logId, sequenceId); + int offset = in.readInt(); + return new ShortMutationId(logId, offset); } @Override - public MutationId deserialize(ByteBuffer buffer, int position, int userVersion) + public ShortMutationId deserialize(ByteBuffer buffer, int position, int userVersion) { long logId = buffer.getLong(position + LOG_ID_OFFSET); - long sequenceId = buffer.getLong(position + SEQUENCE_ID_OFFSET); - return new MutationId(logId, sequenceId); + int offset = buffer.getInt(position + OFFSET_OFFSET); + return new ShortMutationId(logId, offset); } @Override - public MutationId deserialize(ByteBuffer buffer, int userVersion) + public ShortMutationId deserialize(ByteBuffer buffer, int userVersion) { long logId = buffer.getLong(); - long sequenceId = buffer.getLong(); - return new MutationId(logId, sequenceId); + int offset = buffer.getInt(); + return new ShortMutationId(logId, offset); } @Override - public void updateChecksum(Checksum crc, MutationId id, int userVersion) + public void updateChecksum(Checksum crc, ShortMutationId id, int userVersion) { - FBUtilities.updateChecksumLong(crc, id.logId); - FBUtilities.updateChecksumLong(crc, id.sequenceId); + FBUtilities.updateChecksumLong(crc, id.logId()); + FBUtilities.updateChecksumInt(crc, id.offset()); } @Override - public int compareWithKeyAt(MutationId id, ByteBuffer buffer, int position, int userVersion) + public int compareWithKeyAt(ShortMutationId id, ByteBuffer buffer, int position, int userVersion) { - int cmp = Long.compare(id.logId, buffer.getLong(position + LOG_ID_OFFSET)); - return cmp != 0 ? cmp : Long.compare(id.sequenceId, buffer.getLong(position + SEQUENCE_ID_OFFSET)); + int cmp = Long.compare(id.logId(), buffer.getLong(position + LOG_ID_OFFSET)); + return cmp != 0 ? cmp : Integer.compare(id.offset(), buffer.getInt(position + OFFSET_OFFSET)); } @Override - public int compare(MutationId id1, MutationId id2) + public int compare(ShortMutationId id1, ShortMutationId id2) { - int cmp = Long.compare(id1.logId, id2.logId); - return cmp != 0 ? cmp : Long.compare(id1.sequenceId, id2.sequenceId); + int cmp = Long.compare(id1.logId(), id2.logId()); + return cmp != 0 ? cmp : Integer.compare(id1.offset(), id2.offset()); } } - static class MutationSerializer implements ValueSerializer + static class MutationSerializer implements ValueSerializer { @Override - public void serialize(MutationId id, Mutation mutation, DataOutputPlus out, int userVersion) throws IOException + public void serialize(ShortMutationId id, Mutation mutation, DataOutputPlus out, int userVersion) throws IOException { Mutation.serializer.serialize(mutation, out, userVersion); } @Override - public Mutation deserialize(MutationId id, DataInputPlus in, int userVersion) throws IOException + public Mutation deserialize(ShortMutationId id, DataInputPlus in, int userVersion) throws IOException { return Mutation.serializer.deserialize(in, userVersion); } diff --git a/src/java/org/apache/cassandra/replication/MutationSummary.java b/src/java/org/apache/cassandra/replication/MutationSummary.java new file mode 100644 index 0000000000..3483f710ec --- /dev/null +++ b/src/java/org/apache/cassandra/replication/MutationSummary.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.*; + +import com.google.common.base.Preconditions; +import org.agrona.collections.Long2ObjectHashMap; +import org.apache.cassandra.db.Digest; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableId; + +public class MutationSummary +{ + public static class CoordinatorSummary + { + private static final Comparator idComparator = + (l, r) -> CoordinatorLogId.comparator.compare(l.logId(), r.logId()); + + public final Offsets.Immutable reconciled; + public final Offsets.Immutable unreconciled; + + public CoordinatorSummary(Offsets.Immutable reconciled, Offsets.Immutable unreconciled) + { + Preconditions.checkArgument(reconciled.logId().equals(unreconciled.logId())); + this.reconciled = reconciled; + this.unreconciled = unreconciled; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + CoordinatorSummary summary = (CoordinatorSummary) o; + return reconciled.equals(summary.reconciled) && unreconciled.equals(summary.unreconciled); + } + + @Override + public int hashCode() + { + return Objects.hash(reconciled, unreconciled); + } + + @Override + public String toString() + { + return "CoordinatorSummary{" + + "logId=" + logId() + + ", reconciled=" + reconciled + + ", unreconciled=" + unreconciled + + '}'; + } + + public CoordinatorLogId logId() + { + return reconciled.logId(); + } + + boolean contains(int offset) + { + return reconciled.contains(offset) || unreconciled.contains(offset); + } + + /** + * Finds all elements that are contained by {@code left} and not contained by {@code right} + */ + static void difference(CoordinatorSummary left, CoordinatorSummary right, Collection into) + { + Offsets.RangeIterator leftIds = Offsets.union(left.reconciled.rangeIterator(), left.unreconciled.rangeIterator()); + Offsets.RangeIterator rightIds = Offsets.union(right.reconciled.rangeIterator(), right.unreconciled.rangeIterator()); + Offsets.RangeIterator missing = Offsets.difference(leftIds, rightIds); + Offsets.forEachOffset(missing, (logId, offset) -> into.add(new ShortMutationId(logId, offset))); + } + + void digest(Digest digest) + { + reconciled.digest(digest); + unreconciled.digest(digest); + } + + public static class Builder + { + public final CoordinatorLogId logId; + public final Offsets.Immutable.Builder reconciled; + public final Offsets.Immutable.Builder unreconciled; + + public Builder(CoordinatorLogId logId) + { + this.logId = logId; + reconciled = new Offsets.Immutable.Builder(logId); + unreconciled = new Offsets.Immutable.Builder(logId); + } + + boolean isEmpty() + { + return reconciled.isEmpty() && unreconciled.isEmpty(); + } + + public CoordinatorSummary build() + { + return new CoordinatorSummary(reconciled.build(), unreconciled.build()); + } + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(CoordinatorSummary t, DataOutputPlus out, int version) throws IOException + { + Offsets.serializer.serialize(t.reconciled, out, version); + Offsets.serializer.serialize(t.unreconciled, out, version); + } + + @Override + public CoordinatorSummary deserialize(DataInputPlus in, int version) throws IOException + { + return new CoordinatorSummary(Offsets.serializer.deserialize(in, version), + Offsets.serializer.deserialize(in, version)); + } + + @Override + public long serializedSize(CoordinatorSummary t, int version) + { + return Offsets.serializer.serializedSize(t.reconciled, version) + + Offsets.serializer.serializedSize(t.unreconciled, version); + } + }; + } + + public static class Builder + { + public final TableId tableId; + private final Long2ObjectHashMap builders = new Long2ObjectHashMap<>(); + + public Builder(TableId tableId) + { + this.tableId = tableId; + } + + public CoordinatorSummary.Builder builderForLog(CoordinatorLogId logId) + { + CoordinatorSummary.Builder builder = builders.get(logId.asLong()); + if (builder == null) + { + builder = new CoordinatorSummary.Builder(logId); + builders.put(logId.asLong(), builder); + } + + return builder; + } + + public MutationSummary build() + { + List summaries = new ArrayList<>(builders.size()); + for (CoordinatorSummary.Builder builder : builders.values()) + if (!builder.isEmpty()) + summaries.add(builder.build()); + + summaries.sort(CoordinatorSummary.idComparator); + return new MutationSummary(tableId, summaries); + } + } + + private final TableId tableId; + private final List summaries; + private transient final Long2ObjectHashMap coordinatorSummaryMap = new Long2ObjectHashMap<>(); + + private MutationSummary(TableId tableId, List summaries) + { + long lastId = 0; + for (int i=0, mi=summaries.size(); i 0 && thisId <= lastId) + throw new IllegalArgumentException("duplicated or unsorted log id found"); + + coordinatorSummaryMap.put(thisId, summary); + lastId = thisId; + } + + this.tableId = tableId; + this.summaries = summaries; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + MutationSummary summary = (MutationSummary) o; + return tableId.equals(summary.tableId) && summaries.equals(summary.summaries); + } + + @Override + public int hashCode() + { + return tableId.hashCode() + 31 * summaries.hashCode(); + } + + @Override + public String toString() + { + return "MutationSummary{tableId=" + tableId + ", summaries=" + summaries + '}'; + } + + public TableId tableId() + { + return tableId; + } + + public byte[] digest() + { + Digest digest = Digest.forReadResponse(); + digest.updateWithLong(tableId.asUUID().getMostSignificantBits()); + digest.updateWithLong(tableId.asUUID().getLeastSignificantBits()); + digest.updateWithInt(summaries.size()); + + for (CoordinatorSummary summary : summaries) + summary.digest(digest); + + return digest.digest(); + } + + public boolean contains(ShortMutationId id) + { + CoordinatorSummary summary = coordinatorSummaryMap.get(id.logId()); + return summary != null && summary.contains(id.offset()); + } + + public int unreconciledIds() + { + int count = 0; + for (CoordinatorSummary summary : summaries) + count += summary.unreconciled.offsetCount(); + return count; + } + + public int size() + { + return summaries.size(); + } + + boolean isEmpty() + { + return size() == 0; + } + + public CoordinatorSummary get(int i) + { + return summaries.get(i); + } + + public CoordinatorSummary get(CoordinatorLogId logId) + { + return coordinatorSummaryMap.get(logId.asLong()); + } + + /** + * Finds all elements that are contained by {@code left} and not contained by {@code right} + */ + public static void difference(MutationSummary left, MutationSummary right, Collection into) + { + int i = 0, j = 0, lsize = left.size(), rsize = right.size(); + + while (i < lsize && j < rsize) + { + CoordinatorSummary l = left.get(i); + CoordinatorSummary r = right.get(j); + + int cmp = CoordinatorSummary.idComparator.compare(l, r); + + if (cmp == 0) + { + CoordinatorSummary.difference(l, r, into); + ++i; + ++j; + } + else if (cmp < 0) + { + l.reconciled.collectIds(into); + l.unreconciled.collectIds(into); + ++i; + } + else + { + ++j; + } + } + + while (i < lsize) + { + CoordinatorSummary l = left.get(i); + l.reconciled.collectIds(into); + l.unreconciled.collectIds(into); + ++i; + } + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(MutationSummary summary, DataOutputPlus out, int version) throws IOException + { + summary.tableId.serialize(out); + out.writeInt(summary.summaries.size()); + for (int i=0,mi=summary.summaries.size(); i summaries = new ArrayList<>(size); + for (int i = 0; i < size; i++) + summaries.add(CoordinatorSummary.serializer.deserialize(in, version)); + + return new MutationSummary(tableId, summaries); + } + + @Override + public long serializedSize(MutationSummary summary, int version) + { + long size = summary.tableId.serializedSize(); + size += TypeSizes.sizeof(summary.summaries.size()); + for (int i=0,mi=summary.summaries.size(); i shards = new ConcurrentHashMap<>(); + + private volatile boolean started = false; + + private MutationTrackingService() {} + + // TODO (expected): implement a TCM ChangeListener + public synchronized void start(ClusterMetadata metadata) + { + if (started) + return; + + logger.info("Starting replication tracking service"); + + for (KeyspaceMetadata keyspace : metadata.schema.getKeyspaces()) + if (keyspace.useMutationTracking()) + shards.put(keyspace.name, KeyspaceShards.make(keyspace, metadata, this::nextHostLogId)); + started = true; + } + + public synchronized boolean isStarted() + { + return started; + } + + public void shutdownBlocking() throws InterruptedException + { + localReads.shutdownBlocking(); + } + + public TrackedLocalReads localReads() + { + return localReads; + } + + public MutationId nextMutationId(String keyspace, Token token) + { + MutationId id = getOrCreate(keyspace).nextMutationId(token); + logger.trace("Created new mutation id {}", id); + return id; + } + + public void witnessedRemoteMutation(String keyspace, Token token, MutationId mutationId, InetAddressAndPort onHost) + { + getOrCreate(keyspace).witnessedRemoteMutation(token, mutationId, onHost); + } + + public void startWriting(Mutation mutation) + { + getOrCreate(mutation.getKeyspaceName()).startWriting(mutation); + } + + public void finishWriting(Mutation mutation) + { + getOrCreate(mutation.getKeyspaceName()).finishWriting(mutation); + } + + public MutationSummary createSummaryForKey(DecoratedKey key, TableId tableId, boolean includePending) + { + return getOrCreate(tableId).createSummaryForKey(key, tableId, includePending); + } + + public MutationSummary createSummaryForRange(AbstractBounds range, TableId tableId, boolean includePending) + { + return getOrCreate(tableId).createSummaryForRange(range, tableId, includePending); + } + + public MutationSummary createSummaryForRange(Range range, TableId tableId, boolean includePending) + { + return createSummaryForRange(Range.makeRowRange(range), tableId, includePending); + } + + private KeyspaceShards getOrCreate(TableId tableId) + { + //noinspection DataFlowIssue + return getOrCreate(Schema.instance.getTableMetadata(tableId).keyspace); + } + + private KeyspaceShards getOrCreate(String keyspace) + { + KeyspaceShards ks = shards.get(keyspace); + if (ks != null) + return ks; + + ClusterMetadata csm = ClusterMetadata.current(); + KeyspaceMetadata ksm = csm.schema.getKeyspaceMetadata(keyspace); + return shards.computeIfAbsent(keyspace, ignore -> KeyspaceShards.make(ksm, csm, this::nextHostLogId)); + } + + // TODO (expected): durability + int nextHostLogId() + { + return nextHostLogId.incrementAndGet(); + } + private final AtomicInteger nextHostLogId = new AtomicInteger(); + + private static class KeyspaceShards + { + private final String keyspace; + private final Map, Shard> shards; + + private transient final Map, Shard> ppShards; + + static KeyspaceShards make(KeyspaceMetadata keyspace, ClusterMetadata cluster, IntSupplier logIdProvider) + { + Map, Shard> shards = new HashMap<>(); + cluster.placements.get(keyspace.params.replication).writes.forEach((tokenRange, forRange) -> { + IntArrayList participants = new IntArrayList(forRange.size(), IntArrayList.DEFAULT_NULL_VALUE); + for (InetAddressAndPort endpoint : forRange.endpoints()) + participants.add(cluster.directory.peerId(endpoint).id()); + Shard shard = new Shard(keyspace.name, tokenRange, cluster.myNodeId().id(), new Participants(participants), forRange.lastModified(), logIdProvider); + shards.put(tokenRange, shard); + }); + return new KeyspaceShards(keyspace.name, shards); + } + + KeyspaceShards(String keyspace, Map, Shard> shards) + { + this.keyspace = keyspace; + this.shards = shards; + + this.ppShards = new HashMap<>(); + shards.forEach((range, shard) -> ppShards.put(Range.makeRowRange(range), shard)); + } + + MutationId nextMutationId(Token token) + { + return lookUp(token).nextId(); + } + + void witnessedRemoteMutation(Token token, MutationId mutationId, InetAddressAndPort onHost) + { + lookUp(token).witnessedRemoteMutation(mutationId, onHost); + } + + void startWriting(Mutation mutation) + { + lookUp(mutation).startWriting(mutation); + } + + void finishWriting(Mutation mutation) + { + lookUp(mutation).finishWriting(mutation); + } + + MutationSummary createSummaryForKey(DecoratedKey key, TableId tableId, boolean includePending) + { + MutationSummary.Builder builder = new MutationSummary.Builder(tableId); + lookUp(key.getToken()).addSummaryForKey(key.getToken(), includePending, builder); + return builder.build(); + } + + MutationSummary createSummaryForRange(AbstractBounds range, TableId tableId, boolean includePending) + { + MutationSummary.Builder builder = new MutationSummary.Builder(tableId); + forEachIntersectingShard(range, shard -> shard.addSummaryForRange(range, includePending, builder)); + return builder.build(); + } + + private void forEachIntersectingShard(AbstractBounds bounds, Consumer consumer) + { + ppShards.forEach((range, shard) -> { + // TODO (expected): partial workaround - is there a better way to do this? + // SELECT * statements create Bounds[min,min], (PartitionKeyRestrictions.java:L174) not Range(min,min], + // which Ranges generally won't intersect with (Range.java:L148), so contains is used here to make it work + if (bounds.contains(range.right) || range.intersects(bounds)) + consumer.accept(shard); + }); + } + + Shard lookUp(Mutation mutation) + { + return lookUp(mutation.key()); + } + + Shard lookUp(DecoratedKey key) + { + return lookUp(key.getToken()); + } + + Shard lookUp(Token token) + { + ClusterMetadata csm = ClusterMetadata.current(); + KeyspaceMetadata ksm = csm.schema.getKeyspaceMetadata(keyspace); + Range range = ClusterMetadata.current().placements.get(ksm.params.replication).writes.forRange(token).range(); + return shards.get(range); + } + } +} diff --git a/src/java/org/apache/cassandra/replication/Offsets.java b/src/java/org/apache/cassandra/replication/Offsets.java new file mode 100644 index 0000000000..df1512617a --- /dev/null +++ b/src/java/org/apache/cassandra/replication/Offsets.java @@ -0,0 +1,1496 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import com.google.common.base.Function; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +import org.apache.cassandra.db.Digest; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.AbstractIterator; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; + +public abstract class Offsets implements Iterable +{ + private static final int INITIAL_CAPACITY = 16; + + protected final CoordinatorLogId logId; + // even index is range start, odd index is range end (inclusive) + protected int[] bounds; + protected int size; + + protected Offsets(CoordinatorLogId logId, int[] bounds) + { + this(logId, bounds, bounds.length); + } + + protected Offsets(CoordinatorLogId logId, int[] bounds, int size) + { + this.logId = logId; + this.bounds = bounds; + this.size = size; + } + + public static boolean contentsEqual(Offsets a, Offsets b) + { + if (a == null || b == null) + return a == null && b == null; + if (a.size != b.size) + return false; + if (!a.logId.equals(b.logId)) + return false; + return Arrays.equals(a.bounds, 0, a.size, b.bounds, 0, b.size); + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + Offsets that = (Offsets) o; + return size == that.size && Objects.equal(logId, that.logId) && Arrays.equals(bounds, 0, size, that.bounds, 0, size); + } + + @Override + public int hashCode() + { + int result = logId.hashCode(); + result = 31 * result + Integer.hashCode(size); + for (int i = 0; i < size; i++) + result = 31 * result + Integer.hashCode(bounds[i]); + return result; + } + + @Override + public Iterator iterator() + { + return new AbstractIterator<>() + { + int range = -1; + int lastOffset = Integer.MAX_VALUE; + + @Override + protected ShortMutationId computeNext() + { + if (range < 0 || lastOffset >= bounds[rangeEnd(range)]) + { + range++; + if (range >= rangeCount()) + return endOfData(); + + lastOffset = bounds[rangeStart(range)]; + } + else + { + lastOffset++; + } + + return new ShortMutationId(logId, lastOffset); + } + }; + } + + public CoordinatorLogId logId() + { + return logId; + } + + public int rangeCount() + { + return size / 2; + } + + public int offsetCount() + { + int count = 0, i = 0; + while (i < size) + { + int start = bounds[i++]; + int end = bounds[i++]; + count += end - start + 1; + } + return count; + } + + public boolean isEmpty() + { + return size == 0; + } + + private static void forEachOffsetInRange(CoordinatorLogId logId, int start, int end, OffsetConsumer consumer) + { + for (int offset = start; offset <= end; offset++) + consumer.accept(logId, offset); + } + + public static void forEachOffset(RangeIterator iter, OffsetConsumer consumer) + { + while (iter.tryAdvance()) + forEachOffsetInRange(iter.logId(), iter.start(), iter.end(), consumer); + } + + public void collectIds(Collection into) + { + for (int i = 0; i < size; i += 2) + { + int start = bounds[i]; + int end = bounds[i + 1]; + for (int offset = start; offset <= end; offset++) + into.add(new ShortMutationId(logId, offset)); + } + } + + @Override + public String toString() + { + StringBuilder builder = new StringBuilder("{"); + int i = 0; + while (i < size) + { + int start = bounds[i++]; + int end = bounds[i++]; + builder.append('[').append(start).append(',').append(end).append(']'); + if (i < size) builder.append(','); + } + return builder.append('}').toString(); + } + + public boolean contains(int offset) + { + if (size == 0) + return false; + + int pos = Arrays.binarySearch(bounds, 0, size, offset); + if (pos >= 0) return true; // matches one of the bounds + + pos = -pos - 1; + return (pos - 1) % 2 == 0; // offset falls within bounds of an existing range if the bound to the left is an open one + } + + public void digest(Digest digest) + { + digest.updateWithLong(logId.asLong()); + digest.updateWithInt(size); + for (int i = 0; i < size; i++) + digest.updateWithInt(bounds[i]); + } + + static abstract class AbstractMutable> extends Offsets implements OffsetReciever + { + AbstractMutable(RangeIterator rangeIterator) + { + this(rangeIterator.logId()); + while (rangeIterator.tryAdvance()) + append(rangeIterator.start(), rangeIterator.end()); + } + + AbstractMutable(CoordinatorLogId logId) + { + this(logId, INITIAL_CAPACITY); + } + + AbstractMutable(CoordinatorLogId logId, int capacity) + { + super(logId, new int[capacity], 0); + } + AbstractMutable(CoordinatorLogId logId, int[] bounds) + { + this(logId, bounds, bounds.length); + } + + AbstractMutable(CoordinatorLogId logId, int[] bounds, int size) + { + super(logId, bounds, size); + } + + protected static > T createOrNull(Offsets.RangeIterator rangeIterator, Function ctor) + { + T result = null; + + while (rangeIterator.tryAdvance()) + { + if (result == null) + result = ctor.apply(rangeIterator.logId()); + result.append(rangeIterator.start(), rangeIterator.end()); + } + + return result; + } + + public void addAll(Offsets other, RangeConsumer onAdded) + { + for (int i = 0; i < other.size; i += 2) + add(other.bounds[i], other.bounds[i + 1], onAdded); + } + + public void addAll(Offsets other) + { + addAll(other, RangeConsumer.NONE); + } + + public boolean add(int offset, RangeConsumer onAdded) + { + boolean added = add(offset); + if (added) onAdded.consume(logId, offset, offset); + return added; + } + + public boolean add(int offset) + { + if (size == 0) + { + append(offset, offset); + return true; + } + + int pos = Arrays.binarySearch(bounds, 0, size, offset); + if (pos >= 0) return false; // matches one of the bounds + + pos = -pos - 1; + if (pos == size) // after all existing ranges + { + if (bounds[size - 1] == offset - 1) + bounds[size - 1] = offset; // extend the last range + else + append(offset, offset); // append a new single-offset range + + return true; + } + else if (pos == 0) // before all existing ranges + { + if (bounds[0] == offset + 1) + bounds[0] = offset; // extend the first range + else + insert(0, offset, offset); // prepend a new single-offset range + + return true; + } + else if ((pos - 1) % 2 == 0) // offset falls within bounds of an existing range (bound to the left is an open bound) + { + return false; + } + + // between two existing ranges + boolean extendsPrev = bounds[pos - 1] == offset - 1; + boolean extendsNext = bounds[pos] == offset + 1; + + if (extendsPrev && extendsNext) // closes the gap between two adjacent ranges + { + bounds[pos - 1] = bounds[pos + 1]; + System.arraycopy(bounds, pos + 2, bounds, pos, size - pos - 2); + bounds[--size] = 0; + bounds[--size] = 0; + } + else if (extendsPrev) + { + bounds[pos - 1] = offset; + } + else if (extendsNext) + { + bounds[pos] = offset; + } + else + { + insert(pos, offset, offset); + } + + return true; + } + + private enum AddAction + { + INSERT, MOVE, INCLUDE; + + boolean isMove() + { + return this == MOVE; + } + + boolean isInclude() + { + return this == INCLUDE; + } + + boolean isInsert() + { + return this == INSERT; + } + + boolean isMoveOrInclude() + { + return this == MOVE || this == INCLUDE; + } + } + + public boolean add(final int start, final int end, RangeConsumer onAdded) + { + Preconditions.checkArgument(start <= end); + + if (size == 0) + { + append(start, end); + return true; + } + + if (start == end) + return add(start, onAdded); + + int spos = Arrays.binarySearch(bounds, 0, size, start); + int epos = Arrays.binarySearch(bounds, 0, size, end); + + if (spos >= 0 && spos % 2 == 0 && epos == spos + 1) return false; // matches an existing bound + + if (spos < 0) spos = -spos - 1; + if (epos < 0) epos = -epos - 1; + + int numRanges = rangeCount(); + int sRange = Math.min(spos/2, numRanges - 1); + int eRange = Math.min(epos/2, numRanges - 1); + + AddAction sMerge; + { + int rStart = bounds[rangeStart(sRange)]; + int rEnd = bounds[rangeEnd(sRange)]; + if (start >= rStart) + { + // already included in the range or adjacent to range end + sMerge = start <= rEnd + 1 + ? AddAction.INCLUDE // included in the range + : AddAction.INSERT; // past the end of the range + } + else if (sRange > 0 && start == bounds[rangeEnd(sRange - 1)] + 1) + { + // adjacent to the previous range, so say we're included in it to merge + sRange--; + sMerge = AddAction.INCLUDE; + } + else + { + sMerge = AddAction.MOVE; + } + } + + AddAction eMerge; + { + int rStart = bounds[rangeStart(eRange)]; + int rEnd = bounds[rangeEnd(eRange)]; + + if (end <= rEnd) + { + if (end >= rStart - 1) + { + // included in the range or adjacent to range start + eMerge = AddAction.INCLUDE; + } + else if (sRange == eRange - 1) + { + // if we're before the start of this range, and the start is assigned to + // the previous range, then we should just extend the previous range + eRange--; + eMerge = AddAction.MOVE; + } + else + { + // before the start of the range + eMerge = AddAction.INSERT; + } + } + else if (eRange < numRanges - 1 && end == bounds[rangeStart(eRange + 1)] - 1) + { + // adjacent to the next range, so say we're included in it to merge + eRange++; + eMerge = AddAction.INCLUDE; + } + + else + { + eMerge = AddAction.MOVE; + } + } + + // this range isn't adjacent and doesn't intersect any existing, so create a new range + if (sMerge.isMove() && eMerge.isInsert()) + { + Preconditions.checkState(sRange == eRange); + onAdded.consume(logId, start, end); + insert(rangeStart(sRange), start, end); + return true; + } + + // this should only happen if we're adding a range to the very end of the set + if (sMerge.isInsert() && eMerge.isMove()) + { + Preconditions.checkState(sRange == eRange); + Preconditions.checkState(sRange == numRanges - 1); + onAdded.consume(logId, start, end); + append(start, end); + return true; + } + + boolean adjusted = false; + if (sMerge.isMove()) + { + onAdded.consume(logId, start, bounds[rangeStart(sRange)] - 1); + bounds[rangeStart(sRange)] = start; + adjusted = true; + } + + // combine existing ranges + if (sRange != eRange) + { + Preconditions.checkState(sMerge.isMoveOrInclude()); + Preconditions.checkState(eMerge.isMoveOrInclude()); + + adjusted = true; + // report merged ranges + for (int i = sRange; i < eRange; i++) + { + int sEnd = bounds[rangeEnd(i)]; + int eStart = bounds[rangeStart(i + 1)]; + onAdded.consume(logId, sEnd + 1, eStart - 1); + } + + // move array back - + int dstIdx = rangeEnd(sRange); + int srcIdx = rangeEnd(eRange); + System.arraycopy(bounds, srcIdx, bounds, dstIdx, size - srcIdx); + while (eRange > sRange) + { + eRange--; + bounds[--size] = 0; + bounds[--size] = 0; + } + } + + if (eMerge.isMove()) + { + onAdded.consume(logId, bounds[rangeEnd(eRange)] + 1, end); + bounds[rangeEnd(eRange)] = end; + adjusted = true; + } + + return adjusted; + } + + public boolean add(int start, int end) + { + return add(start, end, RangeConsumer.NONE); + } + + private void insert(int pos, int start, int end) + { + if (bounds.length == size) + { + int[] newBounds = new int[bounds.length * 2]; + System.arraycopy(bounds, 0, newBounds, 0, pos); + System.arraycopy(bounds, pos, newBounds, pos + 2, size - pos); + bounds = newBounds; + } + else + { + System.arraycopy(bounds, pos, bounds, pos + 2, size - pos); + } + bounds[pos] = start; + bounds[pos + 1] = end; + size += 2; + } + + protected void append(int start, int end) + { + if (bounds.length == size) + { + int[] newBounds = new int[Math.max(bounds.length * 2, INITIAL_CAPACITY)]; + System.arraycopy(bounds, 0, newBounds, 0, bounds.length); + bounds = newBounds; + } + Preconditions.checkState(size == 0 || start > bounds[size - 1]); + bounds[size++] = start; + bounds[size++] = end; + } + + public void append(int offset) + { + if (size == 0) + { + append(offset, offset); + return; + } + + int tail = bounds[size - 1]; + if (offset <= tail) + throw new IllegalArgumentException("Can't append " + offset + " to " + tail); + + if (offset == tail + 1) + bounds[size-1] = offset; + else + append(offset, offset); + } + } + + public static class Mutable extends AbstractMutable + { + private static final SetSupport.Adaptor adaptor = new SetSupport.Adaptor() + { + @Override + public Mutable empty(CoordinatorLogId logId) + { + return new Mutable(logId); + } + + @Override + public Mutable passThrough(Offsets offsets) + { + if (offsets instanceof Mutable) + return (Mutable) offsets; + return copy(offsets); + } + + @Override + public Mutable create(CoordinatorLogId logId, int[] bounds, int size) + { + return new Mutable(logId, bounds, size); + } + }; + + public Mutable(RangeIterator rangeIterator) + { + super(rangeIterator); + } + + public Mutable(CoordinatorLogId logId) + { + super(logId); + } + + public Mutable(CoordinatorLogId logId, int[] bounds) + { + super(logId, bounds); + } + + public Mutable(CoordinatorLogId logId, int[] bounds, int size) + { + super(logId, bounds, size); + } + + public static Mutable createOrNull(RangeIterator rangeIterator) + { + return AbstractMutable.createOrNull(rangeIterator, Mutable::new); + } + + public static Mutable copy(Offsets offsets) + { + return new Mutable(offsets.logId, Arrays.copyOf(offsets.bounds, offsets.size)); + } + + public static Mutable union(Offsets a, Offsets b) + { + return Offsets.union(a, b, adaptor); + } + + public static Mutable difference(Offsets a, Offsets b) + { + return Offsets.difference(a, b, adaptor); + } + + public static Mutable intersection(Offsets a, Offsets b) + { + return Offsets.intersection(a, b, adaptor); + } + } + + public static class Immutable extends Offsets + { + private static final SetSupport.Adaptor adaptor = new SetSupport.Adaptor() + { + @Override + public Immutable empty(CoordinatorLogId logId) + { + return new Immutable(logId); + } + + @Override + public Immutable passThrough(Offsets offsets) + { + if (offsets instanceof Immutable) + return (Immutable) offsets; + return copy(offsets); + } + + @Override + public Immutable create(CoordinatorLogId logId, int[] bounds, int size) + { + return new Immutable(logId, bounds, size); + } + }; + private static int[] EMPTY = new int[0]; + + public Immutable(CoordinatorLogId logId, int[] bounds) + { + super(logId, bounds); + } + + public Immutable(CoordinatorLogId logId, int[] bounds, int size) + { + super(logId, bounds, size); + } + + public Immutable(CoordinatorLogId logId) + { + this(logId, EMPTY); + } + + public static Immutable copy(Offsets offsets) + { + if (offsets instanceof Immutable) + return (Immutable) offsets; + return new Immutable(offsets.logId, Arrays.copyOf(offsets.bounds, offsets.size)); + } + + public static class Builder extends AbstractMutable + { + public Builder(CoordinatorLogId logId) + { + super(logId); + } + + public Builder(CoordinatorLogId logId, int[] bounds) + { + super(logId, bounds); + } + + public Immutable build() + { + return new Immutable(logId, bounds, size); + } + + public static Builder createOrNull(RangeIterator rangeIterator) + { + return AbstractMutable.createOrNull(rangeIterator, Builder::new); + } + + public static Builder copy(Offsets offsets) + { + return new Builder(offsets.logId, Arrays.copyOf(offsets.bounds, offsets.size)); + } + } + + public static Immutable union(Offsets a, Offsets b) + { + return Offsets.union(a, b, adaptor); + } + + public static Immutable difference(Offsets a, Offsets b) + { + return Offsets.difference(a, b, adaptor); + } + + public static Immutable intersection(Offsets a, Offsets b) + { + return Offsets.intersection(a, b, adaptor); + } + } + + private static int rangeStart(int range) + { + return range * 2; + } + + private static int rangeEnd(int range) + { + return rangeStart(range) + 1; + } + + public RangeIterator rangeIterator() + { + int numRanges = rangeCount(); + + return new RangeIterator() + { + int range = -1; + + @Override + public int start() + { + Preconditions.checkState(range >= 0 && range <= numRanges); + return bounds[rangeStart(range)]; + } + + @Override + public int end() + { + Preconditions.checkState(range >= 0 && range <= numRanges); + return bounds[rangeEnd(range)]; + } + + @Override + public boolean tryAdvance() + { + if (isFinished()) + return false; + range++; + return !isFinished(); + } + + @Override + public CoordinatorLogId logId() + { + return logId; + } + + @Override + public boolean isFinished() + { + return range >= numRanges; + } + }; + } + + private static class SetSupport + { + private static final int NO_SPLIT_SENTINEL = Integer.MIN_VALUE; + + interface Adaptor + { + O empty(CoordinatorLogId logId); + O passThrough(Offsets offsets); + O create(CoordinatorLogId logId, int[] bounds, int size); + } + + private enum RangeOverlap + { + BEFORE, BEFORE_ADJACENT, AFTER, AFTER_ADJACENT, INTERSECTING + } + + private static RangeOverlap calculateRangeOverlap(int aStart, int aEnd, int bStart, int bEnd) + { + Preconditions.checkState(aStart <= aEnd); + Preconditions.checkState(bStart <= bEnd); + + if (aEnd < bStart) + return aEnd == bStart - 1 ? RangeOverlap.BEFORE_ADJACENT : RangeOverlap.BEFORE; + + if (bEnd < aStart) + return bEnd == aStart - 1 ? RangeOverlap.AFTER_ADJACENT : RangeOverlap.AFTER; + + return RangeOverlap.INTERSECTING; + } + + private static RangeOverlap calculateRangeOverlap(int aSplit, int aStart, int aEnd, int bSplit, int bStart, int bEnd) + { + aStart = aSplit != NO_SPLIT_SENTINEL ? aSplit : aStart; + bStart = bSplit != NO_SPLIT_SENTINEL ? bSplit : bStart; + return calculateRangeOverlap(aStart, aEnd, bStart, bEnd); + } + + private static RangeOverlap calculateRangeOverlap(int aSplit, int[] a, int aRange, int bSplit, int[] b, int bRange) + { + return calculateRangeOverlap(aSplit, a[rangeStart(aRange)], a[rangeEnd(aRange)], bSplit, b[rangeStart(bRange)], b[rangeEnd(bRange)]); + } + + private static RangeOverlap calculateRangeOverlap(int[] a, int aRange, int[] b, int bRange) + { + return calculateRangeOverlap(NO_SPLIT_SENTINEL, a, aRange, NO_SPLIT_SENTINEL, b, bRange); + } + + private static int[] ensureCapacity(int[] offsets, int capacity, int expectedMaxCapacity) + { + Preconditions.checkArgument(capacity > 0); + if (capacity <= offsets.length) + return offsets; + + int newCapacity = capacity * 2; + + // if we overflowed, set to max + if (newCapacity < 0) + newCapacity = Integer.MAX_VALUE; + + if (newCapacity > expectedMaxCapacity && offsets.length < expectedMaxCapacity) + newCapacity = expectedMaxCapacity; + + int[] newOffsets = new int[newCapacity]; + System.arraycopy(offsets, 0, newOffsets, 0, offsets.length); + + return newOffsets; + } + + private static O addRemainder(CoordinatorLogId logId, int dstSplit, int[] dst, int dstRange, int[] src, int srcRange, int srcNumRanges, Adaptor adaptor) + { + int capacity = (dstRange + srcNumRanges - srcRange) * 2; + dst = ensureCapacity(dst, capacity, capacity); + while (srcRange < srcNumRanges) + { + int addStart = dstSplit != NO_SPLIT_SENTINEL ? dstSplit : src[rangeStart(srcRange)]; + int addEnd = src[rangeEnd(srcRange)]; + // extend last range if possible + if (dstRange > 0 && addStart <= dst[rangeEnd(dstRange - 1)] + 1) + { + dst[rangeEnd(dstRange - 1)] = addEnd; + } + else + { + dst[rangeStart(dstRange)] = addStart; + dst[rangeEnd(dstRange)] = addEnd; + dstRange++; + } + dstSplit = NO_SPLIT_SENTINEL; + srcRange++; + } + return adaptor.create(logId, dst, dstRange * 2); + } + + private static O addRemainder(CoordinatorLogId logId, int[] dst, int dstRange, int[] src, int srcRange, int srcNumRanges, Adaptor adaptor) + { + return addRemainder(logId, NO_SPLIT_SENTINEL, dst, dstRange, src, srcRange, srcNumRanges, adaptor); + } + + private static int[] ensureCapacity(int[] offsets, int capacity) + { + return ensureCapacity(offsets, capacity, Integer.MAX_VALUE); + } + } + + private static O union(Offsets a, Offsets b, SetSupport.Adaptor adaptor) + { + if (a == null) + return adaptor.passThrough(b); + + if (b == null) + return adaptor.passThrough(a); + + Preconditions.checkArgument(a.logId.equals(b.logId)); + CoordinatorLogId logId = a.logId; + + int aNumRanges = a.rangeCount(); + int bNumRanges = b.rangeCount(); + + if (aNumRanges == 0) + return adaptor.passThrough(b); + + if (bNumRanges == 0) + return adaptor.passThrough(a); + + int aRange = 0; + int bRange = 0; + + int cRange = 0; + int[] c = new int[Math.max(aNumRanges, bNumRanges) * 2]; + + while (aRange < aNumRanges && bRange < bNumRanges) + { + int addStart; + int addEnd; + SetSupport.RangeOverlap rangeOverlap = SetSupport.calculateRangeOverlap(a.bounds, aRange, b.bounds, bRange); + switch (rangeOverlap) + { + case BEFORE: + addStart = a.bounds[rangeStart(aRange)]; + addEnd = a.bounds[rangeEnd(aRange)]; + aRange++; + break; + case AFTER: + addStart = b.bounds[rangeStart(bRange)]; + addEnd = b.bounds[rangeEnd(bRange)]; + bRange++; + break; + case BEFORE_ADJACENT: + case AFTER_ADJACENT: + case INTERSECTING: + addStart = Math.min(a.bounds[rangeStart(aRange)], b.bounds[rangeStart(bRange)]); + addEnd = Math.max(a.bounds[rangeEnd(aRange)], b.bounds[rangeEnd(bRange)]); + aRange++; + bRange++; + break; + default: + throw new IllegalStateException("Unhandled union op: " + rangeOverlap); + } + + // extend the tail if we can + if (cRange > 0 && addStart <= c[rangeEnd(cRange-1)] + 1) + { + c[rangeEnd(cRange - 1)] = addEnd; + } + else + { + c = SetSupport.ensureCapacity(c, (cRange + 1) * 2); + c[rangeStart(cRange)] = addStart; + c[rangeEnd(cRange)] = addEnd; + cRange++; + } + } + + if (aRange < aNumRanges) + { + Preconditions.checkState(bRange == bNumRanges); + return SetSupport.addRemainder(logId, c, cRange, a.bounds, aRange, aNumRanges, adaptor); + } + + if (bRange < bNumRanges) + { + Preconditions.checkState(aRange == aNumRanges); + return SetSupport.addRemainder(logId, c, cRange, b.bounds, bRange, bNumRanges, adaptor); + } + + return adaptor.create(logId, c, cRange * 2); + } + + private static abstract class AbstractSetIterator implements RangeIterator + { + enum State + { + INITIALIZED, VALID, FINISHED; + + boolean isValid() + { + return this == VALID; + } + + boolean isFinished() + { + return this == FINISHED; + } + } + final RangeIterator a; + final RangeIterator b; + int start; + int end; + private State state = State.INITIALIZED; + + public AbstractSetIterator(RangeIterator a, RangeIterator b) + { + Preconditions.checkNotNull(a); + Preconditions.checkNotNull(b); + Preconditions.checkArgument(a.logId().equals(b.logId())); + this.a = a; + this.b = b; + } + + @Override + public int start() + { + Preconditions.checkState(state.isValid()); + return start; + } + + @Override + public int end() + { + Preconditions.checkState(state.isValid()); + return end; + } + + protected abstract State computeNext(); + + @Override + public boolean tryAdvance() + { + switch (state) + { + case INITIALIZED: + a.tryAdvance(); + b.tryAdvance(); + state = State.VALID; + case VALID: + state = computeNext(); + if (!state.isFinished()) + break;; + case FINISHED: + return false; + default: + throw new IllegalStateException("Unhandled state: " + state); + } + return true; + } + + @Override + public boolean isFinished() + { + return state.isFinished(); + } + + @Override + public CoordinatorLogId logId() + { + return a.logId(); + } + } + + private static class UnionIterator extends AbstractSetIterator + { + public UnionIterator(RangeIterator a, RangeIterator b) + { + super(a, b); + } + + void extendEnd(RangeIterator iter) + { + while (iter.tryAdvance()) + { + SetSupport.RangeOverlap rangeOverlap = SetSupport.calculateRangeOverlap(iter.start(), iter.end(), start, end); + switch (rangeOverlap) + { + case BEFORE: + case BEFORE_ADJACENT: + throw new IllegalStateException(); + case AFTER: + return; + case INTERSECTING: + case AFTER_ADJACENT: + end = Math.max(end, iter.end()); + continue; + default: + throw new IllegalStateException("Unhandled union op: " + rangeOverlap); + } + } + } + + @Override + protected State computeNext() + { + + if (a.isFinished()) + { + if (b.isFinished()) + return State.FINISHED; + + start = b.start(); + end = b.end(); + b.tryAdvance(); + return State.VALID; + } + + if (b.isFinished()) + { + start = a.start(); + end = a.end(); + a.tryAdvance(); + return State.VALID; + } + + SetSupport.RangeOverlap rangeOverlap = SetSupport.calculateRangeOverlap(a.start(), a.end(), b.start(), b.end()); + switch (rangeOverlap) + { + case BEFORE: + start = a.start(); + end = a.end(); + a.tryAdvance(); + break; + case AFTER: + start = b.start(); + end = b.end(); + b.tryAdvance(); + break; + case BEFORE_ADJACENT: + case AFTER_ADJACENT: + case INTERSECTING: + start = Math.min(a.start(), b.start()); + end = Math.max(a.end(), b.end()); + extendEnd(a); + extendEnd(b); + break; + default: + throw new IllegalStateException("Unhandled union op: " + rangeOverlap); + } + + return State.VALID; + } + } + + public static RangeIterator union(RangeIterator a, RangeIterator b) + { + return new UnionIterator(a, b); + } + + /** + * Subtract b from a + */ + private static O difference(Offsets a, Offsets b, SetSupport.Adaptor adaptor) + { + Preconditions.checkArgument(a != null); + if (b == null) + return adaptor.passThrough(a); + + Preconditions.checkArgument(b == null || a.logId.equals(b.logId)); + CoordinatorLogId logId = a.logId; + int aNumRanges = a.rangeCount(); + int bNumRanges = b.rangeCount(); + + if (aNumRanges == 0) + return adaptor.empty(logId); + + if (bNumRanges == 0) + return adaptor.passThrough(a); + + int aRange = 0; + int bRange = 0; + + int cRange = 0; + int[] c = new int[Math.max(aNumRanges, bNumRanges) * 2]; + + int aSplit = SetSupport.NO_SPLIT_SENTINEL; + while (aRange < aNumRanges && bRange < bNumRanges) + { + int addStart; + int addEnd; + + SetSupport.RangeOverlap rangeOverlap = SetSupport.calculateRangeOverlap(aSplit, a.bounds, aRange, SetSupport.NO_SPLIT_SENTINEL, b.bounds, bRange); + switch (rangeOverlap) + { + case BEFORE: + case BEFORE_ADJACENT: + addStart = aSplit != SetSupport.NO_SPLIT_SENTINEL ? aSplit : a.bounds[rangeStart(aRange)]; + addEnd = a.bounds[rangeEnd(aRange)]; + aSplit = SetSupport.NO_SPLIT_SENTINEL; + aRange++; + break; + case AFTER: + case AFTER_ADJACENT: + aSplit = SetSupport.NO_SPLIT_SENTINEL; + bRange++; + continue; + case INTERSECTING: + int aStart = aSplit != SetSupport.NO_SPLIT_SENTINEL ? aSplit : a.bounds[rangeStart(aRange)]; + int aEnd = a.bounds[rangeEnd(aRange)]; + int bStart = b.bounds[rangeStart(bRange)]; + int bEnd = b.bounds[rangeEnd(bRange)]; + + if (bStart <= aStart) + { + if (bEnd >= aEnd) + { + // b consumes the entire a range + aSplit = SetSupport.NO_SPLIT_SENTINEL; + aRange++; + } + else + { + // set the split and start over, the next b range may intersect with the current range + aSplit = bEnd + 1; + bRange++; + } + continue; + } + + // does not consume the start of the range + Preconditions.checkState(bStart > aStart); + addStart = aStart; + addEnd = bStart - 1; + + if (bEnd < aEnd) + { + // b splits the range + aSplit = bEnd + 1; + bRange++; + } + else + { + // b consumes the rest of the a range + aSplit = SetSupport.NO_SPLIT_SENTINEL; + aRange++; + } + break; + default: + throw new IllegalStateException("Unhandled union op: " + rangeOverlap); + } + + // extend the tail if we can + if (cRange > 0 && addStart <= c[rangeEnd(cRange-1)] + 1) + { + c[rangeEnd(cRange - 1)] = addEnd; + } + else + { + c = SetSupport.ensureCapacity(c, (cRange + 1) * 2); + c[rangeStart(cRange)] = addStart; + c[rangeEnd(cRange)] = addEnd; + cRange++; + } + } + + if (aRange < aNumRanges) + { + Preconditions.checkState(bRange == bNumRanges); + return SetSupport.addRemainder(logId, aSplit, c, cRange, a.bounds, aRange, aNumRanges, adaptor); + } + + return adaptor.create(logId, c, cRange * 2); + } + + private static class DifferenceIterator extends AbstractSetIterator + { + private int aSplit = SetSupport.NO_SPLIT_SENTINEL; + + public DifferenceIterator(RangeIterator a, RangeIterator b) + { + super(a, b); + } + + @Override + protected State computeNext() + { + while (!a.isFinished() && !b.isFinished()) + { + SetSupport.RangeOverlap rangeOverlap = SetSupport.calculateRangeOverlap(aSplit, a.start(), a.end(), + SetSupport.NO_SPLIT_SENTINEL, b.start(), b.end()); + switch (rangeOverlap) + { + case BEFORE: + case BEFORE_ADJACENT: + start = aSplit != SetSupport.NO_SPLIT_SENTINEL ? aSplit : a.start(); + end = a.end(); + aSplit = SetSupport.NO_SPLIT_SENTINEL; + a.tryAdvance(); + return State.VALID; + case AFTER: + case AFTER_ADJACENT: + aSplit = SetSupport.NO_SPLIT_SENTINEL; + b.tryAdvance(); + continue; + case INTERSECTING: + int aStart = aSplit != SetSupport.NO_SPLIT_SENTINEL ? aSplit : a.start(); + int aEnd = a.end(); + int bStart = b.start(); + int bEnd = b.end(); + + if (bStart <= aStart) + { + if (bEnd >= aEnd) + { + // b consumes the entire a range + aSplit = SetSupport.NO_SPLIT_SENTINEL; + a.tryAdvance(); + } + else + { + // set the split and start over, the next b range may intersect with the current range + aSplit = bEnd + 1; + b.tryAdvance(); + } + continue; + } + + // does not consume the start of the range + Preconditions.checkState(bStart > aStart); + start = aStart; + end = bStart - 1; + + if (bEnd < aEnd) + { + // b splits the range + aSplit = bEnd + 1; + b.tryAdvance(); + } + else + { + // b consumes the rest of the a range + aSplit = SetSupport.NO_SPLIT_SENTINEL; + a.tryAdvance(); + } + return State.VALID; + default: + throw new IllegalStateException("Unhandled union op: " + rangeOverlap); + } + } + + if (a.isFinished()) + return State.FINISHED; + + if (b.isFinished()) + { + start = aSplit != SetSupport.NO_SPLIT_SENTINEL ? aSplit : a.start(); + end = a.end(); + aSplit = SetSupport.NO_SPLIT_SENTINEL; + a.tryAdvance(); + return State.VALID; + } + + // shouldn't be possible to get here + throw new IllegalStateException(); + } + } + + public static RangeIterator difference(RangeIterator a, RangeIterator b) + { + return new DifferenceIterator(a, b); + } + + private static O intersection(Offsets a, Offsets b, SetSupport.Adaptor adaptor) + { + Preconditions.checkArgument(a.logId.equals(b.logId)); + CoordinatorLogId logId = a.logId; + + int aNumRanges = a.rangeCount(); + int bNumRanges = b.rangeCount(); + + if (aNumRanges == 0 || bNumRanges == 0) + return adaptor.empty(logId); + + int aRange = 0; + int bRange = 0; + + int cRange = 0; + int[] c = new int[Math.max(aNumRanges, bNumRanges) * 2]; + + int aSplit = SetSupport.NO_SPLIT_SENTINEL; + int bSplit = SetSupport.NO_SPLIT_SENTINEL; + while (aRange < aNumRanges && bRange < bNumRanges) + { + int addStart; + int addEnd; + + SetSupport.RangeOverlap rangeOverlap = SetSupport.calculateRangeOverlap(aSplit, a.bounds, aRange, bSplit, b.bounds, bRange); + switch (rangeOverlap) + { + case BEFORE: + case BEFORE_ADJACENT: + aSplit = SetSupport.NO_SPLIT_SENTINEL; + bSplit = SetSupport.NO_SPLIT_SENTINEL; + aRange++; + continue; + case AFTER: + case AFTER_ADJACENT: + aSplit = SetSupport.NO_SPLIT_SENTINEL; + bSplit = SetSupport.NO_SPLIT_SENTINEL; + bRange++; + continue; + case INTERSECTING: + int aStart = aSplit != SetSupport.NO_SPLIT_SENTINEL ? aSplit : a.bounds[rangeStart(aRange)]; + int aEnd = a.bounds[rangeEnd(aRange)]; + int bStart = bSplit != SetSupport.NO_SPLIT_SENTINEL ? bSplit : b.bounds[rangeStart(bRange)]; + int bEnd = b.bounds[rangeEnd(bRange)]; + + addStart = Math.max(aStart, bStart); + addEnd = Math.min(aEnd, bEnd); + + if (aEnd < bEnd) + { + aSplit = SetSupport.NO_SPLIT_SENTINEL; + bSplit = aEnd; + aRange++; + } + else if (bEnd < aEnd) + { + aSplit = bEnd; + bSplit = SetSupport.NO_SPLIT_SENTINEL; + bRange++; + } + else + { + aSplit = SetSupport.NO_SPLIT_SENTINEL; + bSplit = SetSupport.NO_SPLIT_SENTINEL; + aRange++; + bRange++; + } + break; + + default: + throw new IllegalStateException("Unhandled union op: " + rangeOverlap); + } + + // extend the tail if we can, though it shouldn't be possible unless one of the input lists were malformed + if (cRange > 0 && addStart <= c[rangeEnd(cRange-1)] + 1) + { + c[rangeEnd(cRange - 1)] = addEnd; + } + else + { + c = SetSupport.ensureCapacity(c, (cRange + 1) * 2); + c[rangeStart(cRange)] = addStart; + c[rangeEnd(cRange)] = addEnd; + cRange++; + } + } + + return adaptor.create(logId, c, cRange * 2); + } + + public interface OffsetReciever + { + boolean add(int offset); + + void addAll(Offsets offsets); + } + + public interface OffsetConsumer + { + void accept(CoordinatorLogId logId, int offset); + } + + public interface RangeConsumer + { + RangeConsumer NONE = (log, start, end) -> {}; + + void consume(CoordinatorLogId logId, int startOffset, int endOffset); + } + + /** + * Intrusive iterator. Adjacent ranges may or may not be merged + * + * iterator is created in an invalid state, and tryAdvance needs to be called before calling start or + * stop. This is to support empty iterators, and makes iterating through the contents less verbose than + * have to check isFinished before reading the values, and makes iterator initialization less error-prone; + */ + public interface RangeIterator + { + int start(); + int end(); + boolean tryAdvance(); + boolean isFinished(); + CoordinatorLogId logId(); + } + + // TODO (consider): delta-encoding + vints + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(Offsets.Immutable offsets, DataOutputPlus out, int version) throws IOException + { + CoordinatorLogId.serializer.serialize(offsets.logId, out, version); + out.writeInt(offsets.size); + for (int i = 0; i < offsets.size; i++) + out.writeInt(offsets.bounds[i]); + } + + @Override + public Offsets.Immutable deserialize(DataInputPlus in, int version) throws IOException + { + CoordinatorLogId logId = CoordinatorLogId.serializer.deserialize(in, version); + int size = in.readInt(); + Preconditions.checkArgument(size >= 0 && size % 2 == 0); + int[] bounds = new int[size]; + for (int i = 0; i < size; i++) + bounds[i] = in.readInt(); + return new Offsets.Immutable(logId, bounds); + } + + @Override + public long serializedSize(Offsets.Immutable offsets, int version) + { + long size = CoordinatorLogId.serializer.serializedSize(offsets.logId, version); + size += TypeSizes.sizeof(offsets.size); + size += (long) TypeSizes.INT_SIZE * offsets.size; + return size; + } + }; +} diff --git a/src/java/org/apache/cassandra/replication/Participants.java b/src/java/org/apache/cassandra/replication/Participants.java new file mode 100644 index 0000000000..c665d63a99 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/Participants.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.util.Arrays; +import java.util.Collection; + +public class Participants +{ + private final int[] hosts; + + Participants(Collection participants) + { + int i = 0; + int[] hosts = new int[participants.size()]; + for (int host : participants) hosts[i++] = host; + Arrays.sort(hosts); + this.hosts = hosts; + } + + int size() + { + return hosts.length; + } + + int indexOf(int hostId) + { + int idx = Arrays.binarySearch(hosts, hostId); + if (idx < 0) + throw new IllegalArgumentException("Unknown host id " + hostId); + return idx; + } + + int get(int idx) + { + if (idx < 0 || idx >= hosts.length) + throw new IllegalArgumentException("Out of bounds host idx " + idx); + return hosts[idx]; + } +} diff --git a/src/java/org/apache/cassandra/replication/ReconciliationPlan.java b/src/java/org/apache/cassandra/replication/ReconciliationPlan.java new file mode 100644 index 0000000000..ac085bf839 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/ReconciliationPlan.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.replication.MutationSummary.CoordinatorSummary; + +public class ReconciliationPlan +{ + private final ImmutableMap txPlan; + + public ReconciliationPlan(ImmutableMap txPlan) + { + this.txPlan = txPlan; + } + + public Set nodes() + { + return txPlan.keySet(); + } + + public Log2OffsetsMap.Immutable peerReconciliation(InetAddressAndPort to) + { + return txPlan.get(to); + } + + public Log2OffsetsMap.Immutable offsetsFor(InetAddressAndPort node) + { + return txPlan.get(node); + } + + public boolean isEmpty() + { + return txPlan.isEmpty(); + } + + private static class PlanBuilder + { + final InetAddressAndPort node; + + final MutationSummary summary; + final Map peerReconciliations = new HashMap<>(); + + public PlanBuilder(InetAddressAndPort node, MutationSummary summary) + { + this.node = node; + this.summary = summary; + } + + public void send(InetAddressAndPort to, Offsets sequenceIds) + { + peerReconciliations.computeIfAbsent(to, ep -> new Log2OffsetsMap.Immutable.Builder()).add(sequenceIds); + } + + ReconciliationPlan build() + { + ImmutableMap.Builder builder = ImmutableMap.builder(); + peerReconciliations.forEach((to, ids) -> builder.put(to, ids.build())); + return new ReconciliationPlan(builder.build()); + } + } + + // TODO (desired): rework Offsets set logic usage to use Offsets#RangeIterator instead of rematerializing sets + private static class CoordinatorLogReconciliation + { + final CoordinatorLogId logId; + Offsets.Immutable reconciled; + Offsets.Immutable unreconciled; + + Map unreconciledNodes = new HashMap<>(); + + CoordinatorLogReconciliation(CoordinatorLogId logId) + { + this.logId = logId; + } + + void addPeerSummary(InetAddressAndPort peer, CoordinatorSummary summary) + { + Preconditions.checkArgument(summary.logId().equals(logId)); + reconciled = Offsets.Immutable.union(reconciled, summary.reconciled); + unreconciled = Offsets.Immutable.union(unreconciled, summary.unreconciled); + unreconciledNodes.put(peer, summary.unreconciled); + } + + void createPlan(Map plan) + { + // remove reconciled ids + Offsets.Immutable allIds = Offsets.Immutable.difference(unreconciled, reconciled); + for (InetAddressAndPort receiver : plan.keySet()) + { + Offsets.Immutable missing = Offsets.Immutable.difference(allIds, unreconciledNodes.get(receiver)); + if (missing.isEmpty()) + continue; + + // TODO: look into more intelligent ways to distribute mutation requests + for (Map.Entry sender : unreconciledNodes.entrySet()) + { + if (sender.getKey().equals(receiver)) + continue; + + Offsets senderIds = sender.getValue(); + PlanBuilder senderPlan = plan.get(sender.getKey()); + + Offsets.Immutable requestedIds = Offsets.Immutable.intersection(missing, senderIds); + senderPlan.send(receiver, requestedIds); + + missing = Offsets.Immutable.difference(missing, requestedIds); + if (missing.rangeCount() == 0) + break; + } + } + } + } + + public static Map calculateReconciliation(Map summaries) + { + Map planBuilders = new HashMap<>(); + Map coordinatorReconciliations = new HashMap<>(); + + // organize data by peer and log id + summaries.forEach((node, summary) -> { + + planBuilders.put(node, new PlanBuilder(node, summary)); + + for (int i=0; i planBuilder.createPlan(planBuilders)); + + Map plans = new HashMap<>(); + planBuilders.forEach((node, planBuilder) -> { + ReconciliationPlan plan = planBuilder.build(); + if (!plan.isEmpty()) + plans.put(node, plan); + }); + return plans; + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(ReconciliationPlan plan, DataOutputPlus out, int version) throws IOException + { + out.writeInt(plan.txPlan.size()); + for (Map.Entry entry : plan.txPlan.entrySet()) + { + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(entry.getKey(), out, version); + Log2OffsetsMap.Immutable.serializer.serialize(entry.getValue(), out, version); + } + } + + @Override + public ReconciliationPlan deserialize(DataInputPlus in, int version) throws IOException + { + int size = in.readInt(); + ImmutableMap.Builder builder = ImmutableMap.builderWithExpectedSize(size); + for (int i = 0; i < size; i++) + { + InetAddressAndPort endpoint = InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version); + Log2OffsetsMap.Immutable offsets = Log2OffsetsMap.Immutable.serializer.deserialize(in, version); + builder.put(endpoint, offsets); + } + return new ReconciliationPlan(builder.build()); + } + + @Override + public long serializedSize(ReconciliationPlan plan, int version) + { + long size = TypeSizes.sizeof(plan.txPlan.size()); + for (Map.Entry entry : plan.txPlan.entrySet()) + { + size += InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(entry.getKey(), version); + size += Log2OffsetsMap.Immutable.serializer.serializedSize(entry.getValue(), version); + } + return size; + } + }; +} diff --git a/src/java/org/apache/cassandra/replication/Shard.java b/src/java/org/apache/cassandra/replication/Shard.java new file mode 100644 index 0000000000..538e9a57c0 --- /dev/null +++ b/src/java/org/apache/cassandra/replication/Shard.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.util.function.IntSupplier; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.replication.CoordinatorLog.CoordinatorLogPrimary; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.tcm.Epoch; +import org.jctools.maps.NonBlockingHashMapLong; + +public class Shard +{ + private final String keyspace; + private final Range tokenRange; + private final int localHostId; + private final Participants participants; + private final Epoch sinceEpoch; + private final NonBlockingHashMapLong logs; + // TODO (expected): add support for log rotation + private final CoordinatorLogPrimary currentLocalLog; + + Shard(String keyspace, Range tokenRange, int localHostId, Participants participants, Epoch sinceEpoch, IntSupplier logIdProvider) + { + this.keyspace = keyspace; + this.tokenRange = tokenRange; + this.localHostId = localHostId; + this.participants = participants; + this.sinceEpoch = sinceEpoch; + this.logs = new NonBlockingHashMapLong<>(); + this.currentLocalLog = startNewLog(localHostId, logIdProvider.getAsInt(), participants); + logs.put(currentLocalLog.logId.asLong(), currentLocalLog); + } + + MutationId nextId() + { + return currentLocalLog.nextId(); + } + + void witnessedRemoteMutation(MutationId mutationId, InetAddressAndPort onHost) + { + int onHostId = ClusterMetadata.current().directory.peerId(onHost).id(); + get(mutationId).witnessedRemoteMutation(mutationId, onHostId); + } + + void startWriting(Mutation mutation) + { + get(mutation.id()).startWriting(mutation); + } + + void finishWriting(Mutation mutation) + { + get(mutation.id()).finishWriting(mutation); + } + + void addSummaryForKey(Token token, boolean includePending, MutationSummary.Builder builder) + { + logs.forEach((id, log) -> { + MutationSummary.CoordinatorSummary.Builder summaryBuilder = builder.builderForLog(log.logId); + log.collectOffsetsFor(token, builder.tableId, includePending, summaryBuilder.unreconciled, summaryBuilder.reconciled); + }); + } + + void addSummaryForRange(AbstractBounds range, boolean includePending, MutationSummary.Builder builder) + { + logs.forEach((id, log) -> { + MutationSummary.CoordinatorSummary.Builder summaryBuilder = builder.builderForLog(log.logId); + log.collectOffsetsFor(range, builder.tableId, includePending, summaryBuilder.unreconciled, summaryBuilder.reconciled); + }); + } + + /** + * Creates a new coordinator log for this host. Primarily on Shard init (node startup or topology change). + * Also on keyspace creation. + */ + private static CoordinatorLog.CoordinatorLogPrimary startNewLog(int localHostId, int hostLogId, Participants participants) + { + CoordinatorLogId logId = new CoordinatorLogId(localHostId, hostLogId); + return new CoordinatorLog.CoordinatorLogPrimary(localHostId, logId, participants); + } + + private CoordinatorLog get(MutationId mutationId) + { + Preconditions.checkArgument(!mutationId.isNone()); + return get(mutationId.logId()); + } + + private CoordinatorLog get(long logId) + { + CoordinatorLog log = logs.get(logId); + return log != null + ? log : logs.computeIfAbsent(logId, ignore -> CoordinatorLog.create(localHostId, new CoordinatorLogId(logId), participants)); + } +} diff --git a/src/java/org/apache/cassandra/replication/ShortMutationId.java b/src/java/org/apache/cassandra/replication/ShortMutationId.java new file mode 100644 index 0000000000..93ea294cbe --- /dev/null +++ b/src/java/org/apache/cassandra/replication/ShortMutationId.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; + +/** + * MutationId without the timestamp component. This is sufficient for uniquely identifying a mutation, + * and for lookup in the journal and most tracking data structures. + */ +public class ShortMutationId extends CoordinatorLogId +{ + /** + * 4 byte offset. Offest is incremented, is alone is sufficient to identify + * the entry within a coordinator log. + * MutationId adds a timestamp for correlation purposes. + */ + protected final int offset; + + public ShortMutationId(long logId, int offset) + { + super(logId); + this.offset = offset; + } + + public ShortMutationId(CoordinatorLogId logId, int offset) + { + super(logId.hostId(), logId.hostLogId()); + this.offset = offset; + } + + public ShortMutationId(MutationId mutationId) + { + this(mutationId.logId(), mutationId.offset()); + } + + public long logId() + { + return super.asLong(); + } + + public int offset() + { + return offset; + } + + @Override + public boolean equals(Object o) + { + if (this == o) return true; + if (!(o instanceof ShortMutationId)) return false; + ShortMutationId that = (ShortMutationId) o; + return this.logId() == that.logId() && this.offset == that.offset; + } + + @Override + public int hashCode() + { + return Integer.hashCode(offset) + 31 * super.hashCode(); + } + + @Override + public String toString() + { + return "ShortMutationId{" + hostId() + ", " + hostLogId() + ", " + offset() + '}'; + } + + public static final Comparator comparator = (l, r) -> { + int cmp = CoordinatorLogId.comparator.compare(l, r); + return cmp != 0 ? cmp : Integer.compare(l.offset, r.offset); + }; + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(ShortMutationId id, DataOutputPlus out, int version) throws IOException + { + out.writeLong(id.logId()); + out.writeInt(id.offset()); + } + + @Override + public ShortMutationId deserialize(DataInputPlus in, int version) throws IOException + { + long logId = in.readLong(); + int offset = in.readInt(); + return new ShortMutationId(logId, offset); + } + + @Override + public long serializedSize(ShortMutationId id, int version) + { + return TypeSizes.sizeof(id.logId()) + TypeSizes.sizeof(id.offset()); + } + }; +} diff --git a/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java new file mode 100644 index 0000000000..5ff3acb34e --- /dev/null +++ b/src/java/org/apache/cassandra/replication/TrackedWriteRequest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.concurrent.DebuggableTask; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.IMutation; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.DynamicEndpointSnitch; +import org.apache.cassandra.locator.EndpointsForToken; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.net.ForwardingInfo; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessageFlag; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.TrackedWriteResponseHandler; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.MonotonicClock; + +import static java.util.Collections.singletonList; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.writeMetrics; +import static org.apache.cassandra.net.Verb.MUTATION_REQ; + +public class TrackedWriteRequest +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedWriteRequest.class); + + /** + * Coordinate write of a tracked mutation. Assumes the replica is a coordinator. + * + * @param mutation the mutation to be applied + * @param consistencyLevel the consistency level for the write operation + * @param requestTime object holding times when request got enqueued and started execution + */ + public static TrackedWriteResponseHandler perform( + Mutation mutation, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + { + Tracing.trace("Determining replicas for mutation"); + + String keyspaceName = mutation.getKeyspaceName(); + Keyspace keyspace = Keyspace.open(keyspaceName); + Token token = mutation.key().getToken(); + + MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, token); + mutation = mutation.withMutationId(id); + + ReplicaPlan.ForWrite plan = ReplicaPlans.forWrite(keyspace, consistencyLevel, token, ReplicaPlans.writeNormal); + + if (plan.lookup(FBUtilities.getBroadcastAddressAndPort()) != null) + writeMetrics.localRequests.mark(); + else + writeMetrics.remoteRequests.mark(); + + AbstractReplicationStrategy rs = plan.replicationStrategy(); + + TrackedWriteResponseHandler handler = + TrackedWriteResponseHandler.wrap(rs.getWriteResponseHandler(plan, null, WriteType.SIMPLE, null, requestTime), + keyspaceName, + mutation.key().getToken(), + id); + applyLocallyAndSendToReplicas(mutation, plan, handler); + + return handler; + } + + public static void applyLocallyAndSendToReplicas(Mutation mutation, ReplicaPlan.ForWrite plan, TrackedWriteResponseHandler handler) + { + String localDataCenter = DatabaseDescriptor.getLocator().local().datacenter; + + boolean applyLocally = false; + + // this DC replicas + List localDCReplicas = null; + + // extra-DC, grouped by DC + Map> remoteDCReplicas = null; + + // only need to create a Message for non-local writes + Message message = null; + + // For performance, Mutation caches serialized buffers that are computed lazily in serializedBuffer(). That + // computation is not synchronized however, and we will potentially call that method concurrently for each + // dispatched message (not that concurrent calls to serializedBuffer() are "unsafe" per se, just that they + // may result in multiple computations, making the caching optimization moot). So forcing the serialization + // here to make sure it's already cached/computed when it's concurrently used later. + // Side note: we have one cached buffers for each used EncodingVersion and this only pre-compute the one for + // the current version, but it's just an optimization, and we're ok not optimizing for mixed-version clusters. + Mutation.serializer.prepareSerializedBuffer(mutation, MessagingService.current_version); + + for (Replica destination : plan.contacts()) + { + if (!plan.isAlive(destination)) + { + handler.expired(); // immediately mark the response as expired since the request will not be sent + continue; + } + + if (destination.isSelf()) + { + applyLocally = true; + continue; + } + + if (message == null) + message = Message.outWithFlags(MUTATION_REQ, mutation, handler.getRequestTime(), singletonList(MessageFlag.CALL_BACK_ON_FAILURE)); + + String dc = DatabaseDescriptor.getLocator().location(destination.endpoint()).datacenter; + + if (localDataCenter.equals(dc)) + { + if (localDCReplicas == null) + localDCReplicas = new ArrayList<>(plan.contacts().size()); + localDCReplicas.add(destination); + } + else + { + if (remoteDCReplicas == null) + remoteDCReplicas = new HashMap<>(); + + List messages = remoteDCReplicas.get(dc); + if (messages == null) + messages = remoteDCReplicas.computeIfAbsent(dc, ignore -> new ArrayList<>(3)); // most DCs will have <= 3 replicas + messages.add(destination); + } + } + + Preconditions.checkState(applyLocally); // the coordinator is always a replica + applyMutationLocally(mutation, handler); + + if (localDCReplicas != null) + for (Replica destination : localDCReplicas) + MessagingService.instance().sendWriteWithCallback(message, destination, handler); + + if (remoteDCReplicas != null) + { + // for each datacenter, send the message to one node to relay the write to other replicas + for (List dcReplicas : remoteDCReplicas.values()) + sendMessagesToRemoteDC(message, EndpointsForToken.copyOf(mutation.key().getToken(), dcReplicas), handler); + } + } + + private static void applyMutationLocally(Mutation mutation, TrackedWriteResponseHandler handler) + { + Stage.MUTATION.maybeExecuteImmediately(new LocalMutationRunnable(mutation, handler)); + } + + private static class LocalMutationRunnable implements DebuggableTask.RunnableDebuggableTask + { + private final Mutation mutation; + private final TrackedWriteResponseHandler handler; + + LocalMutationRunnable(Mutation mutation, TrackedWriteResponseHandler handler) + { + this.mutation = mutation; + this.handler = handler; + } + + @Override + public final void run() + { + long now = MonotonicClock.Global.approxTime.now(); + long deadline = handler.getRequestTime().computeDeadline(MUTATION_REQ.expiresAfterNanos()); + + if (now > deadline) + { + long timeTakenNanos = now - startTimeNanos(); + MessagingService.instance().metrics.recordSelfDroppedMessage(Verb.MUTATION_REQ, timeTakenNanos, NANOSECONDS); + return; + } + + try + { + mutation.apply(); + handler.onResponse(null); + } + catch (Exception ex) + { + if (!(ex instanceof WriteTimeoutException)) + logger.error("Failed to apply mutation locally : ", ex); + handler.onFailure(FBUtilities.getBroadcastAddressAndPort(), RequestFailureReason.forException(ex)); + } + } + + @Override + public long creationTimeNanos() + { + return handler.getRequestTime().enqueuedAtNanos(); + } + + @Override + public long startTimeNanos() + { + return handler.getRequestTime().startedAtNanos(); + } + + @Override + public String description() + { + // description is an Object and toString() called so we do not have to evaluate the Mutation.toString() + // unless expliclitly checked + return mutation.toString(); + } + } + + /* + * Send the message to the first replica of targets, and have it forward the message to others in its DC + */ + private static void sendMessagesToRemoteDC(Message message, + EndpointsForToken targets, + TrackedWriteResponseHandler handler) + { + final Replica target; + + if (targets.size() > 1) + { + target = pickReplica(targets); + EndpointsForToken forwardToReplicas = targets.filter(r -> r != target, targets.size()); + + for (Replica replica : forwardToReplicas) + { + MessagingService.instance().callbacks.addWithExpiration(handler, message, replica); + logger.trace("Adding FWD message to {}@{}", message.id(), replica); + } + + // starting with 4.0, use the same message id for all replicas + long[] messageIds = new long[forwardToReplicas.size()]; + Arrays.fill(messageIds, message.id()); + + message = message.withForwardTo(new ForwardingInfo(forwardToReplicas.endpointList(), messageIds)); + } + else + { + target = targets.get(0); + } + + Tracing.trace("Sending mutation to remote replica {}", target); + MessagingService.instance().sendWriteWithCallback(message, target, handler); + logger.trace("Sending message to {}@{}", message.id(), target); + } + + private static Replica pickReplica(EndpointsForToken targets) + { + EndpointsForToken healthy = targets.filter(r -> DynamicEndpointSnitch.getSeverity(r.endpoint()) == 0); + EndpointsForToken select = healthy.isEmpty() ? targets : healthy; + return select.get(ThreadLocalRandom.current().nextInt(0, select.size())); + } +} diff --git a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java index 82adbb6d80..9bd0378064 100644 --- a/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java +++ b/src/java/org/apache/cassandra/schema/DistributedMetadataLogKeyspace.java @@ -245,7 +245,7 @@ public final class DistributedMetadataLogKeyspace public static KeyspaceMetadata initialMetadata(Set knownDatacenters) { - return KeyspaceMetadata.create(SchemaConstants.METADATA_KEYSPACE_NAME, new KeyspaceParams(true, ReplicationParams.simpleMeta(1, knownDatacenters), FastPathStrategy.simple()), Tables.of(Log)); + return KeyspaceMetadata.create(SchemaConstants.METADATA_KEYSPACE_NAME, new KeyspaceParams(true, ReplicationParams.simpleMeta(1, knownDatacenters), FastPathStrategy.simple(), ReplicationType.untracked), Tables.of(Log)); } public static KeyspaceMetadata initialMetadata(String datacenter) diff --git a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java index 5239790b19..122c265356 100644 --- a/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java +++ b/src/java/org/apache/cassandra/schema/KeyspaceMetadata.java @@ -120,27 +120,27 @@ public final class KeyspaceMetadata implements SchemaElement public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables) { - return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, Views.none(), Types.none(), UserFunctions.none()); + return new KeyspaceMetadata(name, Kind.REGULAR, params, tables.withKeyspaceReplicationType(params.replicationType), Views.none(), Types.none(), UserFunctions.none()); } public static KeyspaceMetadata create(String name, KeyspaceParams params, Tables tables, Views views, Types types, UserFunctions functions) { - return new KeyspaceMetadata(name, Kind.REGULAR, params, tables, views, types, functions); + return new KeyspaceMetadata(name, Kind.REGULAR, params, tables.withKeyspaceReplicationType(params.replicationType), views, types, functions); } public static KeyspaceMetadata virtual(String name, Tables tables) { - return new KeyspaceMetadata(name, Kind.VIRTUAL, KeyspaceParams.local(), tables, Views.none(), Types.none(), UserFunctions.none()); + return new KeyspaceMetadata(name, Kind.VIRTUAL, KeyspaceParams.local(), tables.withKeyspaceReplicationType(ReplicationType.untracked), Views.none(), Types.none(), UserFunctions.none()); } public KeyspaceMetadata withSwapped(KeyspaceParams params) { - return new KeyspaceMetadata(name, kind, params, tables, views, types, userFunctions); + return new KeyspaceMetadata(name, kind, params, tables.withKeyspaceReplicationType(params.replicationType), views, types, userFunctions); } public KeyspaceMetadata withSwapped(Tables regular) { - return new KeyspaceMetadata(name, kind, params, regular, views, types, userFunctions); + return new KeyspaceMetadata(name, kind, params, regular.withKeyspaceReplicationType(params.replicationType), views, types, userFunctions); } public KeyspaceMetadata withSwapped(Views views) @@ -290,6 +290,11 @@ public final class KeyspaceMetadata implements SchemaElement return Optional.empty(); } + public boolean useMutationTracking() + { + return params.replicationType.isTracked(); + } + @Override public int hashCode() { @@ -384,7 +389,9 @@ public final class KeyspaceMetadata implements SchemaElement params.replication.appendCqlTo(builder); builder.append(" AND durable_writes = ") - .append(params.durableWrites); + .append(params.durableWrites) + .append(" AND replication_type = ") + .appendWithSingleQuotes(params.replicationType.name()); if (params.fastPath != null) { diff --git a/src/java/org/apache/cassandra/schema/KeyspaceParams.java b/src/java/org/apache/cassandra/schema/KeyspaceParams.java index 4f41235367..38b203dee3 100644 --- a/src/java/org/apache/cassandra/schema/KeyspaceParams.java +++ b/src/java/org/apache/cassandra/schema/KeyspaceParams.java @@ -55,11 +55,14 @@ public final class KeyspaceParams @VisibleForTesting public static boolean DEFAULT_LOCAL_DURABLE_WRITES = true; + public static final ReplicationType DEFAULT_REPLICATION_TYPE = ReplicationType.untracked; + public enum Option { DURABLE_WRITES, REPLICATION, - FAST_PATH; + FAST_PATH, + REPLICATION_TYPE; @Override public String toString() @@ -73,74 +76,97 @@ public final class KeyspaceParams public final FastPathStrategy fastPath; public final String comment; public final String securityLabel; + public final ReplicationType replicationType; - public KeyspaceParams(boolean durableWrites, ReplicationParams replication, FastPathStrategy fastPath) + public KeyspaceParams(boolean durableWrites, ReplicationParams replication, FastPathStrategy fastPath, ReplicationType replicationType) { - this(durableWrites, replication, fastPath, EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + this(durableWrites, replication, fastPath, EMPTY_COMMENT, EMPTY_SECURITY_LABEL, replicationType); } - public KeyspaceParams(boolean durableWrites, ReplicationParams replication, FastPathStrategy fastPath, String comment, String securityLabel) + public KeyspaceParams(boolean durableWrites, ReplicationParams replication, FastPathStrategy fastPath, String comment, String securityLabel, ReplicationType replicationType) { this.durableWrites = durableWrites; this.replication = replication; this.fastPath = fastPath; this.comment = comment == null ? EMPTY_COMMENT : comment; this.securityLabel = securityLabel == null ? EMPTY_SECURITY_LABEL : securityLabel; + this.replicationType = replicationType; } - public static KeyspaceParams create(boolean durableWrites, Map replication, FastPathStrategy fastPath) + public static KeyspaceParams create(boolean durableWrites, Map replication, FastPathStrategy fastPath, ReplicationType replicationType) { - return new KeyspaceParams(durableWrites, ReplicationParams.fromMap(replication), fastPath, EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + return new KeyspaceParams(durableWrites, ReplicationParams.fromMap(replication), fastPath, EMPTY_COMMENT, EMPTY_SECURITY_LABEL, replicationType); + } + + public static KeyspaceParams create(boolean durableWrites, Map replication, ReplicationType replicationType) + { + return create(durableWrites, replication, FastPathStrategy.simple(), replicationType); } public static KeyspaceParams create(boolean durableWrites, Map replication, Map fastPath) { - return create(durableWrites, replication, FastPathStrategy.fromMap(fastPath)); + return create(durableWrites, replication, FastPathStrategy.fromMap(fastPath), ReplicationType.untracked); + } + + public static KeyspaceParams create(boolean durableWrites, Map replication, Map fastPath, ReplicationType replicationType) + { + return create(durableWrites, replication, FastPathStrategy.fromMap(fastPath), replicationType); } public static KeyspaceParams create(boolean durableWrites, Map replication) { - return create(durableWrites, replication, FastPathStrategy.simple()); + return create(durableWrites, replication, FastPathStrategy.simple(), ReplicationType.untracked); } public static KeyspaceParams local() { - return new KeyspaceParams(DEFAULT_LOCAL_DURABLE_WRITES, ReplicationParams.local(), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + return new KeyspaceParams(DEFAULT_LOCAL_DURABLE_WRITES, ReplicationParams.local(), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL, ReplicationType.untracked); + } + + public static KeyspaceParams simple(int replicationFactor, ReplicationType replicationType) + { + return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), replicationType); } public static KeyspaceParams simple(int replicationFactor) { - return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL, ReplicationType.untracked); } public static KeyspaceParams simple(String replicationFactor) { - return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + return new KeyspaceParams(true, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL, ReplicationType.untracked); } public static KeyspaceParams simpleTransient(int replicationFactor) { - return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + return new KeyspaceParams(false, ReplicationParams.simple(replicationFactor), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL, ReplicationType.untracked); } + public static KeyspaceParams nts(ReplicationType replicationType, Object... args) + { + return new KeyspaceParams(true, ReplicationParams.nts(args), FastPathStrategy.simple(), replicationType); + } + + public static KeyspaceParams nts(Object... args) { - return new KeyspaceParams(true, ReplicationParams.nts(args), FastPathStrategy.simple(), EMPTY_COMMENT, EMPTY_SECURITY_LABEL); + return nts(ReplicationType.untracked, args); } public KeyspaceParams withSwapped(ReplicationParams params) { - return new KeyspaceParams(durableWrites, params, fastPath, comment, securityLabel); + return new KeyspaceParams(durableWrites, params, fastPath, comment, securityLabel, replicationType); } public KeyspaceParams withComment(String comment) { - return new KeyspaceParams(durableWrites, replication, fastPath, comment, securityLabel); + return new KeyspaceParams(durableWrites, replication, fastPath, comment, securityLabel, replicationType); } public KeyspaceParams withSecurityLabel(String securityLabel) { - return new KeyspaceParams(durableWrites, replication, fastPath, comment, securityLabel); + return new KeyspaceParams(durableWrites, replication, fastPath, comment, securityLabel, replicationType); } public void validate(String name, ClientState state, ClusterMetadata metadata) @@ -163,13 +189,14 @@ public final class KeyspaceParams && replication.equals(p.replication) && fastPath.equals(p.fastPath) && comment.equals(p.comment) - && securityLabel.equals(p.securityLabel); + && securityLabel.equals(p.securityLabel) + && replicationType == p.replicationType; } @Override public int hashCode() { - return Objects.hashCode(durableWrites, replication, fastPath, comment, securityLabel); + return Objects.hashCode(durableWrites, replication, fastPath, comment, securityLabel, replicationType); } @Override @@ -181,6 +208,7 @@ public final class KeyspaceParams .add(Option.FAST_PATH.toString(), fastPath.toString()) .add("COMMENT", comment) .add("SECURITY_LABEL", securityLabel) + .add(Option.REPLICATION_TYPE.toString(), replicationType) .toString(); } @@ -188,6 +216,7 @@ public final class KeyspaceParams { public void serialize(KeyspaceParams t, DataOutputPlus out, Version version) throws IOException { + ReplicationType.serializer.serialize(t.replicationType, out, version); ReplicationParams.serializer.serialize(t.replication, out, version); out.writeBoolean(t.durableWrites); if (version.isAtLeast(MIN_ACCORD_VERSION)) @@ -201,6 +230,7 @@ public final class KeyspaceParams public KeyspaceParams deserialize(DataInputPlus in, Version version) throws IOException { + ReplicationType rtype = ReplicationType.serializer.deserialize(in, version); ReplicationParams params = ReplicationParams.serializer.deserialize(in, version); boolean durableWrites = in.readBoolean(); FastPathStrategy fastPath = version.isAtLeast(MIN_ACCORD_VERSION) @@ -213,12 +243,13 @@ public final class KeyspaceParams comment = in.readUTF(); securityLabel = in.readUTF(); } - return new KeyspaceParams(durableWrites, params, fastPath, comment, securityLabel); + return new KeyspaceParams(durableWrites, params, fastPath, comment, securityLabel, rtype); } public long serializedSize(KeyspaceParams t, Version version) { - return ReplicationParams.serializer.serializedSize(t.replication, version) + + return ReplicationType.serializer.serializedSize(t.replicationType, version) + + ReplicationParams.serializer.serializedSize(t.replication, version) + TypeSizes.sizeof(t.durableWrites) + (version.isAtLeast(Version.V8) ? TypeSizes.sizeof(t.comment) : 0) + (version.isAtLeast(Version.V8) ? TypeSizes.sizeof(t.securityLabel) : 0) + diff --git a/src/java/org/apache/cassandra/schema/ReplicationType.java b/src/java/org/apache/cassandra/schema/ReplicationType.java new file mode 100644 index 0000000000..575e9b7f78 --- /dev/null +++ b/src/java/org/apache/cassandra/schema/ReplicationType.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.schema; + +import java.io.IOException; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.tcm.serialization.MetadataSerializer; +import org.apache.cassandra.tcm.serialization.Version; + +public enum ReplicationType +{ + untracked, tracked; + + public static final MetadataSerializer serializer = new MetadataSerializer<>() + { + @Override + public void serialize(ReplicationType t, DataOutputPlus out, Version version) throws IOException + { + if (version.isBefore(Version.V9)) + return; + + switch (t) + { + case untracked: + out.writeByte(0); + break; + case tracked: + out.writeByte(1); + break; + default: + throw new IllegalArgumentException("Unsupported replication type: " + t); + } + } + + @Override + public ReplicationType deserialize(DataInputPlus in, Version version) throws IOException + { + if (version.isBefore(Version.V9)) + return untracked; + + byte t = in.readByte(); + + switch (t) + { + case 0: + return untracked; + case 1: + return tracked; + default: + throw new IllegalArgumentException("Unsupported replication type: " + t); + } + } + + @Override + public long serializedSize(ReplicationType t, Version version) + { + if (version.isBefore(Version.V9)) + return 0; + return TypeSizes.BYTE_SIZE; + } + }; + + public boolean isTracked() + { + return this == tracked; + } + + // FIXME: used in lieu of adding support for tracked reads in parameterized tests, fix usages of this method + public static ReplicationType[] fixmeValues() + { + return new ReplicationType[]{ untracked }; + } +} diff --git a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java index f8057ed8b7..a0934755ee 100644 --- a/src/java/org/apache/cassandra/schema/SchemaKeyspace.java +++ b/src/java/org/apache/cassandra/schema/SchemaKeyspace.java @@ -82,6 +82,7 @@ import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.RowIterators; import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata.ClusteringOrder; import org.apache.cassandra.schema.Keyspaces.KeyspacesDiff; import org.apache.cassandra.service.accord.fastpath.FastPathStrategy; @@ -145,6 +146,7 @@ public final class SchemaKeyspace "CREATE TABLE %s (" + "keyspace_name text," + "durable_writes boolean," + + "replication_type text," + "replication frozen>," + "fast_path frozen>," + "PRIMARY KEY ((keyspace_name)))"); @@ -488,7 +490,7 @@ public final class SchemaKeyspace continue; DecoratedKey key = partition.partitionKey(); - Mutation.PartitionUpdateCollector puCollector = mutationMap.computeIfAbsent(key, k -> new Mutation.PartitionUpdateCollector(SchemaConstants.SCHEMA_KEYSPACE_NAME, key)); + Mutation.PartitionUpdateCollector puCollector = mutationMap.computeIfAbsent(key, k -> new Mutation.PartitionUpdateCollector(MutationId.none(), SchemaConstants.SCHEMA_KEYSPACE_NAME, key)); puCollector.add(makeUpdateForSchema(partition, cmd.columnFilter()).withOnlyPresentColumns()); } } @@ -537,12 +539,13 @@ public final class SchemaKeyspace private static Mutation.SimpleBuilder makeCreateKeyspaceMutation(String name, KeyspaceParams params, long timestamp) { - Mutation.SimpleBuilder builder = Mutation.simpleBuilder(Keyspaces.keyspace, decorate(Keyspaces, name)) + Mutation.SimpleBuilder builder = Mutation.simpleBuilder(MutationId.none(), Keyspaces.keyspace, decorate(Keyspaces, name)) .timestamp(timestamp); builder.update(Keyspaces) .row() .add(KeyspaceParams.Option.DURABLE_WRITES.toString(), params.durableWrites) + .add(KeyspaceParams.Option.REPLICATION_TYPE.toString(), params.replicationType.toString()) .add(KeyspaceParams.Option.REPLICATION.toString(), (params.replication.isMeta() ? params.replication.asNonMeta() : params.replication).asMap()) .add(KeyspaceParams.Option.FAST_PATH.toString(), params.fastPath.asMap()); @@ -566,7 +569,7 @@ public final class SchemaKeyspace private static Mutation.SimpleBuilder makeDropKeyspaceMutation(KeyspaceMetadata keyspace, long timestamp) { - Mutation.SimpleBuilder builder = Mutation.simpleBuilder(SchemaConstants.SCHEMA_KEYSPACE_NAME, decorate(Keyspaces, keyspace.name)) + Mutation.SimpleBuilder builder = Mutation.simpleBuilder(MutationId.none(), SchemaConstants.SCHEMA_KEYSPACE_NAME, decorate(Keyspaces, keyspace.name)) .timestamp(timestamp); for (TableMetadata schemaTable : ALL_TABLE_METADATA) @@ -1030,12 +1033,24 @@ public final class SchemaKeyspace UntypedResultSet.Row row = query(query, keyspaceName).one(); boolean durableWrites = row.getBoolean(KeyspaceParams.Option.DURABLE_WRITES.toString()); + + ReplicationType replicationType; + if (row.has(KeyspaceParams.Option.REPLICATION_TYPE.toString())) + { + String replicationTypeName = row.getString(KeyspaceParams.Option.REPLICATION_TYPE.toString()); + replicationType = replicationTypeName != null ? ReplicationType.valueOf(replicationTypeName) : ReplicationType.untracked; + } + else + { + replicationType = ReplicationType.untracked; + } + Map replication = row.getFrozenTextMap(KeyspaceParams.Option.REPLICATION.toString()); Map fastPath = row.getFrozenTextMap(KeyspaceParams.Option.FAST_PATH.toString()); - KeyspaceParams params = KeyspaceParams.create(durableWrites, replication, fastPath); + KeyspaceParams params = KeyspaceParams.create(durableWrites, replication, fastPath, replicationType); if (keyspaceName.equals(SchemaConstants.METADATA_KEYSPACE_NAME)) - params = new KeyspaceParams(params.durableWrites, params.replication.asMeta(), FastPathStrategy.simple()); + params = new KeyspaceParams(params.durableWrites, params.replication.asMeta(), FastPathStrategy.simple(), ReplicationType.untracked); return params; } diff --git a/src/java/org/apache/cassandra/schema/TableMetadata.java b/src/java/org/apache/cassandra/schema/TableMetadata.java index 9589062c8b..a22ccfebb1 100644 --- a/src/java/org/apache/cassandra/schema/TableMetadata.java +++ b/src/java/org/apache/cassandra/schema/TableMetadata.java @@ -185,6 +185,7 @@ public class TableMetadata implements SchemaElement public final IPartitioner partitioner; public final Kind kind; public final TableParams params; + public final ReplicationType keyspaceReplicationType; public final ImmutableSet flags; @Nullable @@ -233,6 +234,7 @@ public class TableMetadata implements SchemaElement partitioner = builder.partitioner; kind = builder.kind; params = builder.params.build(); + keyspaceReplicationType = builder.keyspaceReplicationType; indexName = kind == Kind.INDEX ? name.substring(name.indexOf('.') + 1) : null; @@ -316,6 +318,7 @@ public class TableMetadata implements SchemaElement .partitioner(partitioner) .kind(kind) .params(params) + .keyspaceReplicationType(keyspaceReplicationType) .flags(flags) .addColumns(columns()) .droppedColumns(droppedColumns) @@ -329,6 +332,11 @@ public class TableMetadata implements SchemaElement return kind == Kind.INDEX; } + public TableMetadata withKeyspaceReplicationType(ReplicationType type) + { + return unbuild().keyspaceReplicationType(type).build(); + } + public TableMetadata withSwapped(TableParams params) { return unbuild().params(params).build(); @@ -364,6 +372,13 @@ public class TableMetadata implements SchemaElement return kind == Kind.VIRTUAL; } + public ReplicationType replicationType() + { + if (kind != Kind.REGULAR) + return ReplicationType.untracked; + return keyspaceReplicationType; + } + public Optional indexName() { return Optional.ofNullable(indexName); @@ -975,6 +990,7 @@ public class TableMetadata implements SchemaElement private IPartitioner partitioner; private Kind kind = Kind.REGULAR; private TableParams.Builder params = TableParams.builder(); + private ReplicationType keyspaceReplicationType = ReplicationType.untracked; // See the comment on Flag.COMPOUND definition for why we (still) inconditionally add this flag. private Set flags = EnumSet.of(Flag.COMPOUND); @@ -1077,6 +1093,12 @@ public class TableMetadata implements SchemaElement return new ColumnMetadata(prev.ksName, prev.cfName, prev.name, prev.type, uniqueId, prev.position(), prev.kind, prev.getMask(), prev.getColumnConstraints(), prev.comment, prev.securityLabel); } + public Builder keyspaceReplicationType(ReplicationType type) + { + keyspaceReplicationType = type; + return this; + } + public Builder id(TableId val) { id = val; @@ -1102,6 +1124,8 @@ public class TableMetadata implements SchemaElement public Builder kind(Kind val) { + if (val != Kind.REGULAR) + keyspaceReplicationType = null; kind = val; return this; } diff --git a/src/java/org/apache/cassandra/schema/Tables.java b/src/java/org/apache/cassandra/schema/Tables.java index f0e7ad03e5..9a576f6c49 100644 --- a/src/java/org/apache/cassandra/schema/Tables.java +++ b/src/java/org/apache/cassandra/schema/Tables.java @@ -192,6 +192,13 @@ public final class Tables implements Iterable : this; } + public Tables withKeyspaceReplicationType(ReplicationType type) + { + return any(this, t -> t.keyspaceReplicationType != type) + ? builder().add(transform(this, t -> t.withKeyspaceReplicationType(type))).build() + : this; + } + MapDifference indexesDiff(Tables other) { Map thisIndexTables = new HashMap<>(); diff --git a/src/java/org/apache/cassandra/service/CassandraDaemon.java b/src/java/org/apache/cassandra/service/CassandraDaemon.java index 4a5c2f1d9d..9a40915a60 100644 --- a/src/java/org/apache/cassandra/service/CassandraDaemon.java +++ b/src/java/org/apache/cassandra/service/CassandraDaemon.java @@ -80,6 +80,7 @@ import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Locator; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.metrics.CassandraMetricsRegistry; import org.apache.cassandra.metrics.DefaultNameFactory; import org.apache.cassandra.net.StartupClusterConnectivityChecker; @@ -271,6 +272,7 @@ public class CassandraDaemon Keyspace.setInitialized(); CommitLog.instance.start(); + MutationJournal.instance.start(); SnapshotManager.instance.start(false); SnapshotManager.instance.clearExpiredSnapshots(); diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index cc97da02aa..78f67ced46 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -134,6 +134,7 @@ import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.RequestCallback; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.TrackedWriteRequest; import org.apache.cassandra.schema.PartitionDenylist; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; @@ -173,6 +174,7 @@ import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.range.RangeCommands; import org.apache.cassandra.service.reads.repair.ReadRepair; +import org.apache.cassandra.service.reads.tracked.TrackedRead; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tcm.membership.NodeState; import org.apache.cassandra.tcm.ownership.VersionedEndpoints; @@ -194,6 +196,7 @@ import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; import static accord.primitives.Txn.Kind.Read; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.Iterables.concat; +import static java.util.Collections.singleton; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.NANOSECONDS; import static org.apache.cassandra.db.ConsistencyLevel.SERIAL; @@ -961,6 +964,71 @@ public class StorageProxy implements StorageProxyMBean }); } + /** + * Performs a coordinated write with mutation tracking. + * Assumes that local coordinator is a replica (forwarding implementation pending). + * + * @param mutation + * @param consistencyLevel + * @param requestTime + */ + public static void mutateWithTracking(Mutation mutation, ConsistencyLevel consistencyLevel, Dispatcher.RequestTime requestTime) + { + try + { + TrackedWriteRequest.perform(mutation, consistencyLevel, requestTime).get(); + } + catch (WriteTimeoutException|WriteFailureException ex) + { + if (consistencyLevel == ConsistencyLevel.ANY) + { + // TODO (expected): what exactly? + } + else + { + if (ex instanceof WriteFailureException) + { + writeMetrics.failures.mark(); + writeMetricsForLevel(consistencyLevel).failures.mark(); + WriteFailureException fe = (WriteFailureException)ex; + Tracing.trace("Write failure; received {} of {} required replies, failed {} requests", + fe.received, fe.blockFor, fe.failureReasonByEndpoint.size()); + } + else + { + writeMetrics.timeouts.mark(); + writeMetricsForLevel(consistencyLevel).timeouts.mark(); + WriteTimeoutException te = (WriteTimeoutException)ex; + Tracing.trace("Write timeout; received {} of {} required replies", te.received, te.blockFor); + } + throw ex; + } + } + catch (UnavailableException e) + { + writeMetrics.unavailables.mark(); + writeMetricsForLevel(consistencyLevel).unavailables.mark(); + Tracing.trace("Unavailable"); + throw e; + } + catch (OverloadedException e) + { + writeMetrics.unavailables.mark(); + writeMetricsForLevel(consistencyLevel).unavailables.mark(); + Tracing.trace("Overloaded"); + throw e; + } + finally + { + // We track latency based on request processing time, since the amount of time that request spends in the queue + // is not a representative metric of replica performance. + long latency = nanoTime() - requestTime.startedAtNanos(); + writeMetrics.addNano(latency); + writeMetricsForLevel(consistencyLevel).addNano(latency); + updateCoordinatorWriteLatencyTableMetric(singleton(mutation), latency); + } + } + /** * Use this method to have these Mutations applied * across all replicas. This method will take care @@ -1220,11 +1288,11 @@ public class StorageProxy implements StorageProxyMBean } @SuppressWarnings("unchecked") - public static void mutateWithTriggers(List mutations, - ConsistencyLevel consistencyLevel, - boolean mutateAtomically, - Dispatcher.RequestTime requestTime, - PreserveTimestamp preserveTimestamps) + public static void mutateWithoutConditions(List mutations, + ConsistencyLevel consistencyLevel, + boolean mutateAtomically, + Dispatcher.RequestTime requestTime, + PreserveTimestamp preserveTimestamps) throws WriteTimeoutException, WriteFailureException, UnavailableException, OverloadedException, InvalidRequestException { if (DatabaseDescriptor.getPartitionDenylistEnabled() && DatabaseDescriptor.getDenylistWritesEnabled()) @@ -1256,7 +1324,24 @@ public class StorageProxy implements StorageProxyMBean long size = IMutation.dataSize(augmented != null ? augmented : mutations); writeMetrics.mutationSize.update(size); writeMetricsForLevel(consistencyLevel).mutationSize.update(size); - if (augmented != null || mutateAtomically || updatesView) + + boolean isTracked = Schema.instance.getKeyspaceMetadata(mutations.get(0).getKeyspaceName()).params.replicationType.isTracked(); + if (isTracked) + { + if (mutations.stream().anyMatch(m -> m instanceof CounterMutation)) + throw new InvalidRequestException("Mutation tracking is currently unsupported with counters"); + if (augmented != null) + throw new InvalidRequestException("Mutation tracking is currently unsupported with triggers"); + if (mutateAtomically) + throw new InvalidRequestException("Mutation tracking is currently unsupported with logged batches"); + if (mutations.size() > 1) + throw new InvalidRequestException("Mutation tracking is currently unsupported with unlogged batches"); + if (updatesView) + throw new InvalidRequestException("Mutation tracking is currently unsupported with materialized views"); + + mutateWithTracking((Mutation) mutations.get(0), consistencyLevel, requestTime); + } + else if (augmented != null || mutateAtomically || updatesView) mutateAtomically(augmented != null ? augmented : (List)mutations, consistencyLevel, updatesView, requestTime); else dispatchMutationsWithRetryOnDifferentSystem(mutations, consistencyLevel, requestTime, preserveTimestamps); @@ -2628,6 +2713,30 @@ public class StorageProxy implements StorageProxyMBean }; } + private static PartitionIterator fetchRowsTracked(List commands, + ConsistencyLevel consistencyLevel, + Dispatcher.RequestTime requestTime) + { + int cmdCount = commands.size(); + TrackedRead.Partition[] reads = new TrackedRead.Partition[cmdCount]; + ClusterMetadata metadata = ClusterMetadata.current(); + + for (int i=0; i iterators = new ArrayList<>(cmdCount); + for (TrackedRead.Partition read : reads) + iterators.add(read.awaitResults()); + + return PartitionIterators.concat(iterators); + } + /** * This function executes local and remote reads, and blocks for the results: * @@ -2642,10 +2751,10 @@ public class StorageProxy implements StorageProxyMBean * This should not be called directly because it bypasses statistics and error handling. It is public * so it can be used by Accord to fetch rows and the statistics will be tracked by Accord. */ - public static PartitionIterator fetchRows(List commands, - ConsistencyLevel consistencyLevel, - ReadCoordinator coordinator, - Dispatcher.RequestTime requestTime) + public static PartitionIterator fetchRowsUntracked(List commands, + ConsistencyLevel consistencyLevel, + ReadCoordinator coordinator, + Dispatcher.RequestTime requestTime) throws UnavailableException, ReadFailureException, ReadTimeoutException { int cmdCount = commands.size(); @@ -2714,6 +2823,22 @@ public class StorageProxy implements StorageProxyMBean return concatAndBlockOnRepair(results, repairs); } + private static PartitionIterator fetchRows(List commands, + ConsistencyLevel consistencyLevel, + ReadCoordinator coordinator, + Dispatcher.RequestTime requestTime) + { + if (commands.get(0).metadata().replicationType().isTracked()) + { + return fetchRowsTracked(commands, consistencyLevel, requestTime); + } + else + { + return fetchRowsUntracked(commands, consistencyLevel, coordinator, requestTime); + } + + } + public static class LocalReadRunnable extends DroppableRunnable implements RunnableDebuggableTask { private final ReadCommand command; diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 424df75b97..1b953d7f96 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -165,6 +165,8 @@ import org.apache.cassandra.repair.RepairParallelism; import org.apache.cassandra.repair.SharedContext; import org.apache.cassandra.repair.autorepair.AutoRepair; import org.apache.cassandra.repair.messages.RepairOption; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.CompactionParams.TombstoneOption; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; @@ -3963,6 +3965,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE SnapshotManager.instance.shutdownAndWait(1L, MINUTES); HintsService.instance.shutdownBlocking(); + MutationTrackingService.instance.shutdownBlocking(); // Interrupt ongoing compactions and shutdown CM to prevent further compactions. CompactionManager.instance.forceShutdown(); @@ -3972,6 +3975,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE CommitLog.instance.forceRecycleAllSegments(); CommitLog.instance.shutdownBlocking(); + MutationJournal.instance.shutdownBlocking(); AutoRepair.instance.shutdownBlocking(); diff --git a/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java new file mode 100644 index 0000000000..b224cbf1c0 --- /dev/null +++ b/src/java/org/apache/cassandra/service/TrackedWriteResponseHandler.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service; + +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.exceptions.WriteFailureException; +import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.NoPayload; +import org.apache.cassandra.replication.MutationId; +import org.apache.cassandra.replication.MutationTrackingService; + +public class TrackedWriteResponseHandler extends AbstractWriteResponseHandler +{ + private final AbstractWriteResponseHandler wrapped; + + private final String keyspace; + private final Token token; + private final MutationId mutationId; + + private TrackedWriteResponseHandler( + AbstractWriteResponseHandler wrapped, String keyspace, Token token, MutationId mutationId) + { + super(wrapped.replicaPlan, wrapped.callback, wrapped.writeType, null, wrapped.getRequestTime()); + this.wrapped = wrapped; + this.keyspace = keyspace; + this.token = token; + this.mutationId = mutationId; + } + + public static TrackedWriteResponseHandler wrap( + AbstractWriteResponseHandler handler, String keyspace, Token token, MutationId mutationId) + { + return new TrackedWriteResponseHandler(handler, keyspace, token, mutationId); + } + + @Override + public void onResponse(Message msg) + { + /* local mutations are witnessed from Keyspace.applyInternalTracked */ + if (msg != null) + MutationTrackingService.instance.witnessedRemoteMutation(keyspace, token, mutationId, msg.from()); + + wrapped.onResponse(msg); + } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + wrapped.onFailure(from, failureReason); + } + + @Override + public boolean trackLatencyForSnitch() + { + return wrapped.trackLatencyForSnitch(); + } + + @Override + protected int ackCount() + { + return wrapped.ackCount(); + } + + @Override + public boolean invokeOnFailure() + { + return wrapped.invokeOnFailure(); + } + + @Override + public void get() throws WriteTimeoutException, WriteFailureException + { + wrapped.get(); + } + + @Override + protected int blockFor() + { + return wrapped.blockFor(); + } + + @Override + protected int candidateReplicaCount() + { + return wrapped.candidateReplicaCount(); + } + + @Override + protected boolean waitingFor(InetAddressAndPort from) + { + return wrapped.waitingFor(from); + } + + @Override + protected void signal() + { + wrapped.signal(); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/Commit.java b/src/java/org/apache/cassandra/service/paxos/Commit.java index fd4102e038..896beabdf2 100644 --- a/src/java/org/apache/cassandra/service/paxos/Commit.java +++ b/src/java/org/apache/cassandra/service/paxos/Commit.java @@ -36,6 +36,7 @@ import org.apache.cassandra.db.rows.DeserializationHelper; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import static org.apache.cassandra.db.SystemKeyspace.legacyPaxosTtlSec; @@ -316,7 +317,15 @@ public class Commit public Mutation makeMutation() { - return new Mutation(update, PotentialTxnConflicts.ALLOW); + // TODO (expected): what's the best thing to do here? Deriving the mutation id from the ballot seems like the best + // thing to do, like we do with the partition update timestamps, but there are caveats related to id collisions + // and the assumption that a mutation id is unique amonth other mutations, which is not the case w/ paxos ballots, + // which only need to be unique to a given partition key to be accepted. It may be best to keep them separate anyway, + // since the reconciliation process as currently planned will not allow writes with ids before some point in time, + // and there might be edge cases where that locks up paxos execution or has other side effects. The downside of + // not making paxos mutation ids deterministic is that the same commit may create multiple mutation ids if a paxos + // operation is not fully committed, then re-committed on repair or the next operation + return new Mutation(MutationId.fixme(), update, PotentialTxnConflicts.ALLOW); } @Override diff --git a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java index 814aa74481..f73c33de5c 100644 --- a/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java +++ b/src/java/org/apache/cassandra/service/reads/AbstractReadExecutor.java @@ -199,6 +199,7 @@ public abstract class AbstractReadExecutor ReadCoordinator coordinator, Dispatcher.RequestTime requestTime) throws UnavailableException { + Preconditions.checkArgument(!command.metadata().replicationType().isTracked()); Keyspace keyspace = Keyspace.open(command.metadata().keyspace); ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id); SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry; diff --git a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java index 62725a1d6d..bf9885e224 100644 --- a/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java +++ b/src/java/org/apache/cassandra/service/reads/range/RangeCommandIterator.java @@ -61,6 +61,7 @@ import org.apache.cassandra.service.reads.DataResolver; import org.apache.cassandra.service.reads.ReadCallback; import org.apache.cassandra.service.reads.ReadCoordinator; import org.apache.cassandra.service.reads.repair.ReadRepair; +import org.apache.cassandra.service.reads.tracked.TrackedRead; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.tracing.Tracing; import org.apache.cassandra.transport.Dispatcher; @@ -208,7 +209,8 @@ public class RangeCommandIterator extends AbstractIterator implemen // If enabled, request repaired data tracking info from full replicas, but // only if there are multiple full replicas to compare results from. boolean trackRepairedStatus = DatabaseDescriptor.getRepairedDataTrackingForRangeReadsEnabled() - && replicaPlan.contacts().filter(Replica::isFull).size() > 1; + && replicaPlan.contacts().filter(Replica::isFull).size() > 1 + && !command.metadata().replicationType().isTracked(); ReplicaPlan.SharedForRangeRead sharedReplicaPlan = ReplicaPlan.shared(replicaPlan); ReadRepair readRepair = @@ -329,7 +331,39 @@ public class RangeCommandIterator extends AbstractIterator implemen command.metadata().enforceStrictLiveness()); } - PartitionIterator sendNextRequests() + private PartitionIterator sendNextRequestsTracked() + { + List concurrentQueries = new ArrayList<>(concurrencyFactor); + + try + { + for (int i = 0; i < concurrencyFactor && replicaPlans.hasNext(); ) + { + ReplicaPlan.ForRangeRead replicaPlan = replicaPlans.next(); + PartitionRangeReadCommand rangeCommand = command.forSubRange(replicaPlan.range(), i == 0); + + TrackedRead.Range read = TrackedRead.Range.create(rangeCommand, replicaPlan); + read.start(requestTime); + concurrentQueries.add(read.iterator()); + + // due to RangeMerger, coordinator may fetch more ranges than required by concurrency factor. + rangesQueried += replicaPlan.vnodeCount(); + i += replicaPlan.vnodeCount(); + } + batchesRequested++; + } + catch (Throwable t) + { + for (PartitionIterator response : concurrentQueries) + response.close(); + throw t; + } + Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size()); + + return PartitionIterators.concat(concurrentQueries); + } + + PartitionIterator sendNextRequestsUntracked() { List concurrentQueries = new ArrayList<>(concurrencyFactor); List> readRepairs = new ArrayList<>(concurrencyFactor); @@ -366,13 +400,29 @@ public class RangeCommandIterator extends AbstractIterator implemen response.close(); throw t; } - Tracing.trace("Submitted {} concurrent range requests", concurrentQueries.size()); + + return StorageProxy.concatAndBlockOnRepair(concurrentQueries, readRepairs); + } + + PartitionIterator sendNextRequests() + { + PartitionIterator result; + if (command.metadata().replicationType().isTracked()) + { + result = sendNextRequestsTracked(); + + } + else + { + result = sendNextRequestsUntracked(); + } + // We want to count the results for the sake of updating the concurrency factor (see updateConcurrencyFactor) // but we don't want to enforce any particular limit at this point (this could break code than rely on // postReconciliationProcessing), hence the DataLimits.NONE. counter = DataLimits.NONE.newCounter(command.nowInSec(), true, command.selectsFullPartition(), enforceStrictLiveness); - return counter.applyTo(StorageProxy.concatAndBlockOnRepair(concurrentQueries, readRepairs)); + return counter.applyTo(result); } // Wrap the iterator to retry if request routing is incorrect diff --git a/src/java/org/apache/cassandra/service/reads/range/SingleRangeResponse.java b/src/java/org/apache/cassandra/service/reads/range/SingleRangeResponse.java index 0274058030..a12b2fc984 100644 --- a/src/java/org/apache/cassandra/service/reads/range/SingleRangeResponse.java +++ b/src/java/org/apache/cassandra/service/reads/range/SingleRangeResponse.java @@ -33,7 +33,7 @@ class SingleRangeResponse extends AbstractIterator implements Parti { private final DataResolver resolver; private final ReadCallback handler; - private final ReadRepair readRepair; + protected final ReadRepair readRepair; private PartitionIterator result; diff --git a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java index 3986a42bf4..2dc64d101c 100644 --- a/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java +++ b/src/java/org/apache/cassandra/service/reads/repair/BlockingReadRepairs.java @@ -31,6 +31,7 @@ import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tracing.Tracing; @@ -53,7 +54,7 @@ public class BlockingReadRepairs return null; DecoratedKey key = update.partitionKey(); - Mutation mutation = new Mutation(update, potentialTxnConflicts); + Mutation mutation = new Mutation(MutationId.fixme(), update, potentialTxnConflicts); int messagingVersion = MessagingService.instance().versions.get(destination); try diff --git a/src/java/org/apache/cassandra/service/reads/tracked/AbstractPartialTrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/AbstractPartialTrackedRead.java new file mode 100644 index 0000000000..0cbb3be38d --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/AbstractPartialTrackedRead.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import java.util.List; + +import com.google.common.base.Preconditions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.db.transform.RTBoundValidator; +import org.apache.cassandra.index.Index; + +import static org.apache.cassandra.db.partitions.UnfilteredPartitionIterators.MergeListener.NOOP; + +public abstract class AbstractPartialTrackedRead implements PartialTrackedRead +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractPartialTrackedRead.class); + + private enum State + { + INITIALIZED, + PREPARED, + READING, + FINISHED + } + + final ReadExecutionController executionController; + final Index.Searcher searcher; + final ColumnFamilyStore cfs; + final long startTimeNanos; + volatile State state = State.INITIALIZED; + + public AbstractPartialTrackedRead(ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos) + { + this.executionController = executionController; + this.searcher = searcher; + this.cfs = cfs; + this.startTimeNanos = startTimeNanos; + } + + @Override + public ReadExecutionController executionController() + { + return executionController; + } + + @Override + public Index.Searcher searcher() + { + return searcher; + } + + @Override + public ColumnFamilyStore cfs() + { + return cfs; + } + + @Override + public long startTimeNanos() + { + return startTimeNanos; + } + + abstract void freezeInitialData(); + + abstract UnfilteredPartitionIterator initialData(); + + abstract UnfilteredPartitionIterator augmentedData(); + + abstract void augmentResponse(PartitionUpdate update); + + /** + * Implementors need to call this before returning this from createInProgressRead + */ + synchronized void prepare() + { + logger.trace("Preparing read {}", this); + Preconditions.checkState(state == State.INITIALIZED); + freezeInitialData(); + state = State.PREPARED; + } + + @Override + public void augment(Mutation mutation) + { + Preconditions.checkState(state == State.PREPARED); + PartitionUpdate update = mutation.getPartitionUpdate(command().metadata()); + if (update != null) + augmentResponse(update); + } + + private UnfilteredPartitionIterator complete(UnfilteredPartitionIterator iterator) + { + return command().completeTrackedRead(iterator, this); + } + + abstract CompletedRead createResult(UnfilteredPartitionIterator iterator); + + @Override + public synchronized CompletedRead complete() + { + Preconditions.checkState(state == State.PREPARED); + state = State.READING; + + UnfilteredPartitionIterator initial = initialData(); + UnfilteredPartitionIterator augmented = augmentedData(); + + UnfilteredPartitionIterator result = augmented != null ? + UnfilteredPartitionIterators.merge(List.of(initial, augmented), NOOP) : + initial; + + result = complete(result); + // validate that the sequence of RT markers is correct: open is followed by close, deletion times for both + // ends equal, and there are no dangling RT bound in any partition. + result = RTBoundValidator.validate(result, RTBoundValidator.Stage.PROCESSED, true); + return createResult(complete(result)); + } + + @Override + public synchronized void close() + { + if (state == State.FINISHED) + return; + + logger.trace("Closing read {}", this); + executionController.close(); + state = State.FINISHED; + } +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java new file mode 100644 index 0000000000..4f50442950 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRangeRead.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import java.util.Iterator; +import java.util.SortedMap; +import java.util.TreeMap; + +import com.google.common.base.Preconditions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.DataRange; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.PartitionRangeReadCommand; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.partitions.AbstractBTreePartition; +import org.apache.cassandra.db.partitions.AbstractUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.SimpleBTreePartition; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.transform.EmptyPartitionsDiscarder; +import org.apache.cassandra.db.transform.Transformation; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.ExcludingBounds; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.index.transactions.UpdateTransaction; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.FBUtilities; + +public class PartialTrackedRangeRead extends AbstractPartialTrackedRead +{ + private static final Logger logger = LoggerFactory.getLogger(PartialTrackedRangeRead.class); + + private final PartitionRangeReadCommand command; + private final SortedMap data = new TreeMap<>(); + private final UnfilteredPartitionIterator initialData; + private final boolean enforceStrictLiveness; + + // short read support + private DecoratedKey lastPartitionKey; // key of the last observed partition + private boolean partitionsFetched; // whether we've seen any new partitions since iteration start or last moreContents() call + private boolean initialIteratorExhausted; + private boolean wasAugmented; + AbstractBounds followUpBounds; + + public PartialTrackedRangeRead(ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos, PartitionRangeReadCommand command, UnfilteredPartitionIterator initialData) + { + super(executionController, searcher, cfs, startTimeNanos); + this.command = command; + this.initialData = initialData; + this.enforceStrictLiveness = command.metadata().enforceStrictLiveness(); + } + + public static PartialTrackedRangeRead create(ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos, PartitionRangeReadCommand command, UnfilteredPartitionIterator initialData) + { + PartialTrackedRangeRead read = new PartialTrackedRangeRead(executionController, searcher, cfs, startTimeNanos, command, initialData); + try + { + read.prepare(); + return read; + } + catch (Throwable e) + { + read.close(); + throw e; + } + } + + @Override + public ReadCommand command() + { + return command; + } + + UnfilteredRowIterator queryPartition(AbstractBTreePartition partition) + { + return partition.unfilteredIterator(command.columnFilter(), + command.requestedSlices(), + command.clusteringIndexFilter(partition.partitionKey()).isReversed()); + } + + private static void consume(UnfilteredPartitionIterator iterator) + { + while (iterator.hasNext()) + { + try (UnfilteredRowIterator partition = iterator.next()) + { + while (partition.hasNext()) + partition.next(); + } + } + } + + @Override + void freezeInitialData() + { + // memtable contents are frozen at read completion time, when the iterator is evaluated, not at the beginning + // of the read, when references to memtables and sstables are collected. Because of this, replica coordinated + // reads can cause read monotonicity to be broken by returning data that hasn't been replicated to at least + // CL other nodes via reconciliation. To prevent this, the contents of the initial iterator are materialized + // onto heap at partition granularity until the limits of the read are reached. + + UnfilteredPartitionIterator materializer = new AbstractUnfilteredPartitionIterator() + { + @Override + public TableMetadata metadata() + { + return initialData.metadata(); + } + + @Override + public boolean hasNext() + { + return initialData.hasNext(); + } + + @Override + public UnfilteredRowIterator next() + { + try (UnfilteredRowIterator rowIterator = initialData.next()) + { + SimpleBTreePartition partition = augmentResponseInternal(PartitionUpdate.fromIterator(rowIterator, command.columnFilter())); + lastPartitionKey = partition.partitionKey(); + partitionsFetched = true; + return queryPartition(partition); + } + } + + @Override + public void close() + { + super.close(); + initialData.close(); + } + }; + + // unmerged per-source counter + final DataLimits.Counter singleResultCounter = command.limits().newCounter(command.nowInSec(), + false, + command.selectsFullPartition(), + enforceStrictLiveness); + try (UnfilteredPartitionIterator iterator = singleResultCounter.applyTo(materializer)) + { + consume(iterator); + } + initialIteratorExhausted = command.limits().isExhausted(singleResultCounter); + if (partitionsFetched) + { + AbstractBounds bounds = command.dataRange().keyRange(); + followUpBounds = bounds.inclusiveRight() + ? new Range<>(lastPartitionKey, bounds.right) + : new ExcludingBounds<>(lastPartitionKey, bounds.right); + Preconditions.checkState(!followUpBounds.contains(lastPartitionKey)); + } + wasAugmented = false; + } + + @Override + UnfilteredPartitionIterator initialData() + { + Iterator iterator = data.values().iterator(); + return new AbstractUnfilteredPartitionIterator() + { + @Override + public TableMetadata metadata() + { + return command.metadata(); + } + + @Override + public boolean hasNext() + { + return iterator.hasNext(); + } + + @Override + public UnfilteredRowIterator next() + { + return queryPartition(iterator.next()); + } + }; + } + + @Override + UnfilteredPartitionIterator augmentedData() + { + return null; + } + + private SimpleBTreePartition augmentResponseInternal(PartitionUpdate update) + { + SimpleBTreePartition partition = data.computeIfAbsent(update.partitionKey(), key -> new SimpleBTreePartition(key, update.metadata(), UpdateTransaction.NO_OP)); + partition.update(update); + return partition; + } + + @Override + void augmentResponse(PartitionUpdate update) + { + // if the input iterator reached the row limit, then we can't apply any augmenting mutations that are past + // the last materialized key. Since we wouldn't have materialized the local data for that key, applying an + // update would cause us to return incomplete data for it. + if (initialIteratorExhausted || !followUpBounds.contains(update.partitionKey())) + augmentResponseInternal(update); + wasAugmented = true; + } + + private class ExtendingCompletedRead implements CompletedRead + { + + final UnfilteredPartitionIterator iterator; + // merged end-result counter + final DataLimits.Counter mergedResultCounter = command.limits().newCounter(command.nowInSec(), + true, + command.selectsFullPartition(), + enforceStrictLiveness); + + public ExtendingCompletedRead(UnfilteredPartitionIterator iterator) + { + this.iterator = iterator; + } + + @Override + public PartitionIterator iterator() + { + PartitionIterator filtered = UnfilteredPartitionIterators.filter(iterator, command.nowInSec()); + PartitionIterator counted = Transformation.apply(filtered, mergedResultCounter); + return Transformation.apply(counted, new EmptyPartitionsDiscarder()); + } + + @Override + public TrackedRead followupRead(ConsistencyLevel consistencyLevel, long expiresAtNanos) + { + // never try to request additional partitions from replicas if our reconciled partitions are already filled to the limit + if (mergedResultCounter.isDone()) + return null; + + // we do not apply short read protection when we have no limits at all + if (command.limits().isUnlimited()) + return null; + + /* + * If this is a single partition read command or an (indexed) partition range read command with + * a partition key specified, then we can't and shouldn't try fetch more partitions. + */ + if (command.isLimitedToOnePartition()) + return null; + + /* + * If the returned result doesn't have enough rows/partitions to satisfy even the original limit, don't ask for more. + * + * Can only take the short cut if there is no per partition limit set. Otherwise it's possible to hit false + * positives due to some rows being uncounted for in certain scenarios (see CASSANDRA-13911). + */ + if (initialIteratorExhausted && command.limits().perPartitionCount() == DataLimits.NO_LIMIT) + return null; + + /* + * Either we had an empty iterator as the initial response, or our moreContents() call got us an empty iterator. + * There is no point to ask the replica for more rows - it has no more in the requested range. + */ + if (!partitionsFetched) + return null; + partitionsFetched = false; + + /* + * We are going to fetch one partition at a time for thrift and potentially more for CQL. + * The row limit will either be set to the per partition limit - if the command has no total row limit set, or + * the total # of rows remaining - if it has some. If we don't grab enough rows in some of the partitions, + * then future ShortReadRowsProtection.moreContents() calls will fetch the missing ones. + */ + int toQuery = command.limits().count() != DataLimits.NO_LIMIT + ? command.limits().count() - mergedResultCounter.rowsCounted() + : command.limits().perPartitionCount(); + + ColumnFamilyStore.metricsFor(command.metadata().id).shortReadProtectionRequests.mark(); + Tracing.trace("Requesting {} extra rows from {} for short read protection", toQuery, FBUtilities.getBroadcastAddressAndPort()); + logger.info("Requesting {} extra rows from {} for short read protection", toQuery, FBUtilities.getBroadcastAddressAndPort()); + + return makeFollowupRead(toQuery, consistencyLevel, expiresAtNanos); + } + + private TrackedRead makeFollowupRead(int toQuery, ConsistencyLevel consistencyLevel, long expiresAtNanos) + { + DataLimits newLimits = command.limits().forShortReadRetry(toQuery); + + DataRange newDataRange = command.dataRange().forSubRange(followUpBounds); + + Keyspace keyspace = Keyspace.open(command.metadata().keyspace); + PartitionRangeReadCommand followUpCmd = command.withUpdatedLimitsAndDataRange(newLimits, newDataRange); + ReplicaPlan.ForRangeRead replicaPlan = ReplicaPlans.forRangeRead(keyspace, + followUpCmd.indexQueryPlan(), + consistencyLevel, + followUpCmd.dataRange().keyRange(), + 1); + + TrackedRead.Range read = TrackedRead.Range.create(followUpCmd, replicaPlan); + logger.trace("Short read detected, starting followup read {}", read); + read.start(expiresAtNanos); + return read; + } + + @Override + public void close() + { + iterator.close(); + } + } + + @Override + CompletedRead createResult(UnfilteredPartitionIterator iterator) + { + if (wasAugmented) + return new ExtendingCompletedRead(iterator); + return CompletedRead.simple(iterator, command().nowInSec()); + } +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java new file mode 100644 index 0000000000..daf222239c --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedRead.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import java.util.Collection; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterators; +import org.apache.cassandra.index.Index; + +public interface PartialTrackedRead +{ + interface CompletedRead extends AutoCloseable + { + PartitionIterator iterator(); + TrackedRead followupRead(ConsistencyLevel consistencyLevel, long expiresAtNanos); + + @Override + void close(); + + static CompletedRead simple(UnfilteredPartitionIterator partition, long nowInSec) + { + return new CompletedRead() + { + @Override + public PartitionIterator iterator() + { + return UnfilteredPartitionIterators.filter(partition, nowInSec); + } + + @Override + public TrackedRead followupRead(ConsistencyLevel consistencyLevel, long expiresAtNanos) + { + return null; + } + + @Override + public void close() + { + partition.close(); + } + }; + } + } + + CompletedRead complete(); + + void augment(Mutation mutation); + + default void augment(Collection mutations) + { + mutations.forEach(this::augment); + } + + ReadExecutionController executionController(); + + Index.Searcher searcher(); + + ColumnFamilyStore cfs(); + + long startTimeNanos(); + + ReadCommand command(); + + void close(); +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java new file mode 100644 index 0000000000..6aa64a2985 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/PartialTrackedSinglePartitionRead.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import com.google.common.base.Preconditions; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.SimpleBTreePartition; +import org.apache.cassandra.db.partitions.SingletonUnfilteredPartitionIterator; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.index.transactions.UpdateTransaction; + +public class PartialTrackedSinglePartitionRead extends AbstractPartialTrackedRead +{ + private final SinglePartitionReadCommand command; + private final UnfilteredPartitionIterator initialData; + private SimpleBTreePartition augmentedData; + + public PartialTrackedSinglePartitionRead(ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos, SinglePartitionReadCommand command, UnfilteredPartitionIterator initialData) + { + super(executionController, searcher, cfs, startTimeNanos); + this.command = command; + this.initialData = initialData; + } + + public static PartialTrackedSinglePartitionRead create(ReadExecutionController executionController, Index.Searcher searcher, ColumnFamilyStore cfs, long startTimeNanos, SinglePartitionReadCommand command, UnfilteredPartitionIterator initialData) + { + PartialTrackedSinglePartitionRead read = new PartialTrackedSinglePartitionRead(executionController, searcher, cfs, startTimeNanos, command, initialData); + try + { + read.prepare(); + return read; + } + catch (Throwable e) + { + read.close(); + throw e; + } + } + + @Override + void freezeInitialData() + { + // the iterators from queryStorage grabs sstable references and a + // snapshot of the memtable partition so we don't need to do anything here + } + + @Override + public ReadCommand command() + { + return command; + } + + @Override + UnfilteredPartitionIterator initialData() + { + return initialData; + } + + @Override + UnfilteredPartitionIterator augmentedData() + { + if (augmentedData == null) + return null; + Slices slices = command.clusteringIndexFilter().getSlices(command.metadata()); + UnfilteredRowIterator augmented = augmentedData.unfilteredIterator(command.columnFilter(), slices, command.clusteringIndexFilter().isReversed()); + return new SingletonUnfilteredPartitionIterator(augmented); + } + + @Override + void augmentResponse(PartitionUpdate update) + { + Preconditions.checkArgument(update.partitionKey().equals(command.partitionKey())); + if (augmentedData == null) + augmentedData = new SimpleBTreePartition(command.partitionKey(), command.metadata(), UpdateTransaction.NO_OP); + + augmentedData.update(update); + } + + @Override + CompletedRead createResult(UnfilteredPartitionIterator iterator) + { + return CompletedRead.simple(iterator, command().nowInSec()); + } +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java new file mode 100644 index 0000000000..3f0e3ca3a5 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileNotify.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.replication.MutationTrackingService; + +/** + * Notifies the read coordinator that this node has received mutations + * from the sending node + */ +public class ReadReconcileNotify +{ + private static final Logger logger = LoggerFactory.getLogger(ReadReconcileNotify.class); + + public final TrackedRead.Id readId; + public final int syncId; + + public ReadReconcileNotify(TrackedRead.Id readId, int syncId) + { + this.readId = readId; + this.syncId = syncId; + } + + @Override + public String toString() + { + return "ReadReconcileNotify{" + + "reconciliationId=" + readId + + ", syncId=" + syncId + + '}'; + } + + public static final IVerbHandler verbHandler = new IVerbHandler() + { + @Override + public void doVerb(Message message) throws IOException + { + ReadReconcileNotify notify = message.payload; + logger.trace("Received read reconcile notify from {}: {}", message.from(), notify); + MutationTrackingService.instance.localReads().acknowledgeSync(notify.readId, notify.syncId); + } + }; + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(ReadReconcileNotify notify, DataOutputPlus out, int version) throws IOException + { + TrackedRead.Id.serializer.serialize(notify.readId, out, version); + out.writeInt(notify.syncId); + } + + @Override + public ReadReconcileNotify deserialize(DataInputPlus in, int version) throws IOException + { + TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version); + int syncId = in.readInt(); + return new ReadReconcileNotify(readId, syncId); + } + + @Override + public long serializedSize(ReadReconcileNotify notify, int version) + { + return TrackedRead.Id.serializer.serializedSize(notify.readId, version) + + TypeSizes.sizeof(notify.syncId); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java new file mode 100644 index 0000000000..f39f347e57 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileReceive.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.utils.CollectionSerializer; + +import static org.apache.cassandra.locator.InetAddressAndPort.Serializer.inetAddressAndPortSerializer; + +public class ReadReconcileReceive +{ + private static final Logger logger = LoggerFactory.getLogger(ReadReconcileReceive.class); + + public final TrackedRead.Id readId; + public final int syncId; + public final InetAddressAndPort coordinator; + public final List mutations; + + public ReadReconcileReceive(TrackedRead.Id readId, int syncId, InetAddressAndPort coordinator, List mutations) + { + this.readId = readId; + this.syncId = syncId; + this.coordinator = coordinator; + this.mutations = mutations; + } + + private static String mutationString(List mutations) + { + StringBuilder builder = new StringBuilder('['); + boolean isFirst = true; + for (Mutation mutation : mutations) + { + if (!isFirst) + { + builder.append(", "); + } + isFirst = false; + builder.append(mutation.id()); + } + builder.append(']'); + return builder.toString(); + } + + @Override + public String toString() + { + return "ReadReconcileReceive{" + + "reconciliationId=" + readId + + ", syncId=" + syncId + + ", coordinator=" + coordinator + + ", mutations=" + mutationString(mutations) + + '}'; + } + + public static final IVerbHandler verbHandler = new IVerbHandler() + { + @Override + public void doVerb(Message message) throws IOException + { + // TODO: check epoch and tokens? + ReadReconcileReceive receive = message.payload; + logger.trace("Received read reconciliation from {}: {}", message.from(), receive); + receive.mutations.forEach(Mutation::apply); + + if (!MutationTrackingService.instance.localReads().receiveMutations(receive.readId, receive.syncId, receive.mutations)) + { + // if this isn't a locally coordinated read, notify the coordinator + ReadReconcileNotify notify = new ReadReconcileNotify(receive.readId, receive.syncId); + MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_NOTIFY, notify), receive.coordinator); + } + } + }; + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(ReadReconcileReceive rcv, DataOutputPlus out, int version) throws IOException + { + TrackedRead.Id.serializer.serialize(rcv.readId, out, version); + out.writeInt(rcv.syncId); + inetAddressAndPortSerializer.serialize(rcv.coordinator, out, version); + CollectionSerializer.serializeCollection(Mutation.serializer, rcv.mutations, out, version); + + } + + @Override + public ReadReconcileReceive deserialize(DataInputPlus in, int version) throws IOException + { + TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version); + int syncId = in.readInt(); + InetAddressAndPort coordinator = inetAddressAndPortSerializer.deserialize(in, version); + List mutations = CollectionSerializer.deserializeCollection(Mutation.serializer, ArrayList::new, in, version); + return new ReadReconcileReceive(readId, syncId, coordinator, mutations); + } + + @Override + public long serializedSize(ReadReconcileReceive t, int version) + { + return TrackedRead.Id.serializer.serializedSize(t.readId, version) + + TypeSizes.sizeof(t.syncId) + + inetAddressAndPortSerializer.serializedSize(t.coordinator, version) + + CollectionSerializer.serializedSizeCollection(Mutation.serializer, t.mutations, version); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java new file mode 100644 index 0000000000..e51ea282b0 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/ReadReconcileSend.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.reads.tracked; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.Log2OffsetsMap; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.utils.CollectionSerializer; + +/** + * Instructs a node to send mutations to other node + */ +public class ReadReconcileSend +{ + private static final Logger logger = LoggerFactory.getLogger(ReadReconcileSend.class); + + public static class PeerSync + { + final int syncId; + final InetAddressAndPort to; + final Log2OffsetsMap.Immutable plan; + + public PeerSync(int syncId, InetAddressAndPort to, Log2OffsetsMap.Immutable plan) + { + this.syncId = syncId; + this.to = to; + this.plan = plan; + } + + @Override + public String toString() + { + return "PeerSync{" + + "syncId=" + syncId + + ", to=" + to + + ", plan=" + plan + + '}'; + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(PeerSync sync, DataOutputPlus out, int version) throws IOException + { + out.writeInt(sync.syncId); + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serialize(sync.to, out, version); + Log2OffsetsMap.Immutable.serializer.serialize(sync.plan, out, version); + } + + @Override + public PeerSync deserialize(DataInputPlus in, int version) throws IOException + { + return new PeerSync(in.readInt(), + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.deserialize(in, version), + Log2OffsetsMap.Immutable.serializer.deserialize(in, version)); + } + + @Override + public long serializedSize(PeerSync sync, int version) + { + return TypeSizes.sizeof(sync.syncId) + + InetAddressAndPort.Serializer.inetAddressAndPortSerializer.serializedSize(sync.to, version) + + Log2OffsetsMap.Immutable.serializer.serializedSize(sync.plan, version); + } + }; + } + + public final TrackedRead.Id reconcileId; + public final ImmutableList syncTasks; + + public ReadReconcileSend(TrackedRead.Id reconcileId, List syncTasks) + { + this.reconcileId = reconcileId; + this.syncTasks = ImmutableList.copyOf(syncTasks); + } + + @Override + public String toString() + { + return "ReadReconcileSend{" + + "reconcileId=" + reconcileId + + ", syncTasks=" + syncTasks + + '}'; + } + + public static final IVerbHandler verbHandler = new IVerbHandler<>() + { + @Override + public void doVerb(Message message) + { + logger.trace("Received ReadReconcileSend from {}: {}", message.from(), message.payload); + // TODO: check epoch and tokens? + ReadReconcileSend payload = message.payload; + for (PeerSync sync : message.payload.syncTasks) + { + // TODO (expected): do not deser just to serialize again, if same messaging versions (common case) + // TODO (expected): don't materialize mutation ids, look up from offset collections + int mutationCount = sync.plan.idCount(); + List mutations = new ArrayList<>(mutationCount); + MutationJournal.instance.readAll(sync.plan, mutations); + Preconditions.checkArgument(mutationCount == mutations.size()); + + ReadReconcileReceive receive = new ReadReconcileReceive(payload.reconcileId, sync.syncId, message.from(), mutations); + logger.trace("Sending {} to replica {}", receive, sync.to); + MessagingService.instance().send(Message.out(Verb.READ_RECONCILE_RCV, receive), sync.to); + } + } + }; + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(ReadReconcileSend send, DataOutputPlus out, int version) throws IOException + { + TrackedRead.Id.serializer.serialize(send.reconcileId, out, version); + CollectionSerializer.serializeList(PeerSync.serializer, send.syncTasks, out, version); + } + + @Override + public ReadReconcileSend deserialize(DataInputPlus in, int version) throws IOException + { + TrackedRead.Id readId = TrackedRead.Id.serializer.deserialize(in, version); + List syncTasks = CollectionSerializer.deserializeCollection(PeerSync.serializer, ArrayList::new, in, version); + return new ReadReconcileSend(readId, syncTasks); + } + + @Override + public long serializedSize(ReadReconcileSend send, int version) + { + return TrackedRead.Id.serializer.serializedSize(send.reconcileId, version) + CollectionSerializer.serializedSizeCollection(PeerSync.serializer, send.syncTasks, version); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedDataResponse.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedDataResponse.java new file mode 100644 index 0000000000..b55095bf02 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedDataResponse.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterators; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.utils.ByteBufferUtil; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class TrackedDataResponse +{ + private final int serializationVersion; + private final ByteBuffer data; + + public TrackedDataResponse(int serializationVersion, ByteBuffer data) + { + this.serializationVersion = serializationVersion; + this.data = data; + } + + public static TrackedDataResponse create(PartitionIterator iter, ColumnFilter selection) + { + try (DataOutputBuffer buffer = new DataOutputBuffer()) + { + PartitionIterators.Serializer.serialize(iter, selection, buffer, MessagingService.current_version); + return new TrackedDataResponse(MessagingService.current_version, buffer.buffer(false)); + } + catch (IOException e) + { + // We're serializing in memory so this shouldn't happen + throw new RuntimeException(e); + } + } + + public PartitionIterator makeIterator(ReadCommand command) + { + try (DataInputBuffer in = new DataInputBuffer(data, true)) + { + return PartitionIterators.Serializer.deserialize(command.metadata(), command.columnFilter(), in, serializationVersion); + } + catch (IOException e) + { + // We're deserializing in memory so this shouldn't happen + throw new RuntimeException(e); + } + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer() + { + @Override + public void serialize(TrackedDataResponse response, DataOutputPlus out, int version) throws IOException + { + out.writeInt(response.serializationVersion); + ByteBufferUtil.writeWithVIntLength(response.data, out); + } + + @Override + public TrackedDataResponse deserialize(DataInputPlus in, int version) throws IOException + { + int serializationVersion = in.readInt(); + ByteBuffer data = ByteBufferUtil.readWithVIntLength(in); + return new TrackedDataResponse(serializationVersion, data); + } + + @Override + public long serializedSize(TrackedDataResponse response, int version) + { + return TypeSizes.sizeof(response.serializationVersion) + + ByteBufferUtil.serializedSizeWithVIntLength(response.data); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java new file mode 100644 index 0000000000..6732adca4b --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReadCoordinator.java @@ -0,0 +1,617 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.reads.tracked; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.ConsistencyLevel; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.ReadCommand; +import org.apache.cassandra.db.ReadExecutionController; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterators; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.metrics.ReadRepairMetrics; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.Log2OffsetsMap; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.ReconciliationPlan; +import org.apache.cassandra.replication.ShortMutationId; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.AbstractFuture; +import org.apache.cassandra.utils.concurrent.Accumulator; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +public class TrackedLocalReadCoordinator +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedLocalReadCoordinator.class); + + private final TrackedRead.Id readId; + private final AsyncPromise promise; + + private volatile State state; + + public TrackedLocalReadCoordinator(TrackedRead.Id readId) + { + this.readId = readId; + this.promise = new AsyncPromise<>(); + this.state = INITIALIZED; + } + + public AbstractFuture addCallback(BiConsumer callback) + { + return promise.addCallback(callback); + } + + enum Status { INITIALIZED, AWAITING_READ, READING, RECONCILING, ABORTED, COMPLETED } + + private static abstract class State + { + abstract Status status(); + + State startLocalRead(TrackedRead.Id readId, AsyncPromise promise, ReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, int[] summaryNodes, long expiresAtNanos) + { + // TODO: validate permitted state instead of just ignoring events? + return this; + } + + State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary) + { + throw new IllegalStateException("Received in-progress read with status " + status()); + } + + State receiveSummary(InetAddressAndPort from, MutationSummary summary) + { + logger.trace("Ignoring summary from {} with status {}", from, status()); + return this; + } + + State acknowledgeSync(int syncId) + { + logger.trace("Ignoring sync ack {} with status {}", syncId, status()); + return this; + } + + State receiveMutations(List mutations) + { + logger.trace("Ignoring {} mutations with status {}", mutations.size(), status()); + return this; + } + + State abort() + { + return ABORTED; + } + + boolean isPurgeable(long nanoTime) + { + return false; + } + + boolean isReading() + { + return is(Status.READING); + } + + boolean isCompleted() + { + return is(Status.COMPLETED); + } + + boolean is(Status status) + { + return status() == status; + } + + @Override + public String toString() + { + return status().toString(); + } + } + + static final State INITIALIZED = new State() + { + @Override + Status status() { return Status.INITIALIZED; } + + @Override + State startLocalRead(TrackedRead.Id readId, AsyncPromise promise, ReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, int[] summaryNodes, long expiresAtNanos) + { + return new Reading(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos); + } + + @Override + State receiveSummary(InetAddressAndPort from, MutationSummary summary) + { + return new AwaitingRead(from, summary); + } + }; + + static final State ABORTED = new State() + { + @Override + Status status() { return Status.ABORTED; } + + @Override + State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary) + { + return this; // the read can't complete without data, but it could have been aborted in the meantime + } + + @Override + boolean isPurgeable(long nanoTime) { return true; } + }; + + static final State COMPLETED = new State() + { + @Override + Status status() { return Status.COMPLETED; } + + @Override + boolean isPurgeable(long nanoTime) { return true; } + }; + + private static class ReceivedSummary + { + final InetAddressAndPort from; + final MutationSummary summary; + + ReceivedSummary(InetAddressAndPort from, MutationSummary summary) + { + this.from = from; + this.summary = summary; + } + } + + // if we start receiving summaries before we receive the read command, they're + // collected here + private static class AwaitingRead extends State + { + private long lastUpdateNanos; + private final List summaries; + + AwaitingRead(InetAddressAndPort summaryFrom, MutationSummary summary) + { + summaries = new ArrayList<>(); + summaries.add(new ReceivedSummary(summaryFrom, summary)); + } + + @Override + Status status() + { + return Status.AWAITING_READ; + } + + @Override + State startLocalRead(TrackedRead.Id readId, AsyncPromise promise, ReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, int[] summaryNodes, long expiresAtNanos) + { + return new Reading(readId, promise, command, replicaPlan, summaryNodes, summaries, expiresAtNanos); + } + + @Override + State receiveSummary(InetAddressAndPort from, MutationSummary summary) + { + summaries.add(new ReceivedSummary(from, summary)); + lastUpdateNanos = Clock.Global.nanoTime(); + return this; + } + + @Override + boolean isPurgeable(long nanoTime) + { + return nanoTime - lastUpdateNanos > DatabaseDescriptor.getReadRpcTimeout(TimeUnit.NANOSECONDS); + } + } + + private static class Reading extends State + { + private final TrackedRead.Id readId; + private final AsyncPromise promise; + private final ReadCommand command; + private volatile PartialTrackedRead read; + private final long expiresAtNanos; + private final ReplicaPlan.AbstractForRead replicaPlan; + private final Accumulator summaries; + private final int[] summaryNodes; // for speculating when we haven't received enough summaries + + Reading( + TrackedRead.Id readId, + AsyncPromise promise, + ReadCommand command, + ReplicaPlan.AbstractForRead replicaPlan, + int[] summaryNodes, + long expiresAtNanos) + { + this.readId = readId; + this.promise = promise; + this.expiresAtNanos = expiresAtNanos; + this.command = command; + this.replicaPlan = replicaPlan; + this.summaries = new Accumulator<>(replicaPlan.readCandidates().size()); + this.summaryNodes = summaryNodes; + } + + Reading( + TrackedRead.Id readId, + AsyncPromise promise, + ReadCommand command, + ReplicaPlan.AbstractForRead replicaPlan, + int[] summaryNodes, + List summaries, + long expiresAtNanos) + { + this(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos); + for (ReceivedSummary summary : summaries) this.summaries.add(summary); + } + + @Override + Status status() + { + return Status.READING; + } + + @Override + boolean isPurgeable(long nanoTime) + { + return nanoTime - expiresAtNanos > 0; + } + + private State maybeComplete() + { + if (read == null || summaries.size() < replicaPlan.readQuorum()) + return this; + + Map summaryMap = new HashMap<>(); + summaries.snapshot().forEach(rc -> summaryMap.put(rc.from, rc.summary)); + + Map reconciliations = ReconciliationPlan.calculateReconciliation(summaryMap); + + if (reconciliations.isEmpty()) + { + logger.trace("Read complete for {}", readId); + complete(promise, read, command.columnFilter(), replicaPlan.consistencyLevel(), expiresAtNanos); + return COMPLETED; + } + else + { + logger.trace("Beginning reconciliation for {}", readId); + Reconciling reconciling = new Reconciling(readId, promise, command, read, replicaPlan.consistencyLevel(), expiresAtNanos, reconciliations); + reconciling.start(); // TODO: don't do this until after the coordinator state is set to reconciling if converting to lock free + return reconciling; + } + } + + @Override + State receiveInProgressRead(PartialTrackedRead read, MutationSummary summary) + { + if (this.read != null) + return this; + + logger.trace("In progress read received for {}", readId); + this.read = read; + summaries.add(new ReceivedSummary(FBUtilities.getBroadcastAddressAndPort(), summary)); + + return maybeComplete(); + } + + @Override + State receiveSummary(InetAddressAndPort from, MutationSummary summary) + { + logger.trace("Summary received from {} for {}", from, readId); + summaries.add(new ReceivedSummary(from, summary)); + return maybeComplete(); + } + + @Override + State abort() + { + logger.trace("Aborting read {}", readId); + if (read != null) + read.close(); + return ABORTED; + } + } + + private static class PendingSync + { + final int syncId; + final InetAddressAndPort from; + final InetAddressAndPort to; + final Log2OffsetsMap.Immutable plan; + + PendingSync(int syncId, InetAddressAndPort from, InetAddressAndPort to, Log2OffsetsMap.Immutable plan) + { + this.syncId = syncId; + this.from = from; + this.to = to; + this.plan = plan; + } + + ReadReconcileSend.PeerSync toPeerSync() + { + return new ReadReconcileSend.PeerSync(syncId, to, plan); + } + } + + private static class Reconciling extends State + { + private final TrackedRead.Id readId; + private final AsyncPromise promise; + private final ReadCommand command; + private final PartialTrackedRead read; + private final ConsistencyLevel consistencyLevel; + private final long expiresAtNanos; + + final Map plans; + final Log2OffsetsMap.Mutable outstandingMutations = new Log2OffsetsMap.Mutable(); + final Map pendingSync = new ConcurrentHashMap<>(); + final int blockFor; + + Reconciling( + TrackedRead.Id readId, + AsyncPromise promise, + ReadCommand command, + PartialTrackedRead read, + ConsistencyLevel consistencyLevel, + long expiresAtNanos, + Map plans) + { + this.readId = readId; + this.promise = promise; + this.command = command; + this.read = Preconditions.checkNotNull(read); + this.consistencyLevel = consistencyLevel; + this.expiresAtNanos = expiresAtNanos; + this.plans = plans; + + int syncs = 0; + int nextSyncId = 0; + for (Map.Entry entry : plans.entrySet()) + { + InetAddressAndPort from = entry.getKey(); + ReconciliationPlan plan = entry.getValue(); + for (InetAddressAndPort to : plan.nodes()) + { + int syncId = nextSyncId++; + PendingSync sync = new PendingSync(syncId, from, to, plan.peerReconciliation(to)); + pendingSync.put(syncId, sync); + syncs++; + if (to.equals(FBUtilities.getBroadcastAddressAndPort())) + { + outstandingMutations.addAll(plan.offsetsFor(to)); + } + } + } + + if (logger.isTraceEnabled()) + logger.trace("Reconciling {} syncs, {} mutations for {}", syncs, outstandingMutations.idCount(), readId); + + this.blockFor = syncs; + } + + @Override + Status status() + { + return Status.RECONCILING; + } + + @Override + boolean isPurgeable(long nanoTime) + { + return nanoTime - expiresAtNanos > 0; + } + + void start() + { + ReadRepairMetrics.trackedReconcile.mark(); + ColumnFamilyStore.metricsFor(command.metadata().id).readRepairRequests.mark(); + + Map> peerSync = new HashMap<>(); + pendingSync.values().forEach(pending -> + peerSync.computeIfAbsent(pending.from, node -> new ArrayList<>()).add(pending.toPeerSync())); + + for (Map.Entry> entry : peerSync.entrySet()) + { + Message message = Message.out(Verb.READ_RECONCILE_SEND, new ReadReconcileSend(readId, entry.getValue())); + logger.trace("Sending read reconciliation for {} {} to {}", readId, message.payload, entry.getKey()); + MessagingService.instance().send(message, entry.getKey()); + } + } + + private State maybeComplete() + { + if (!pendingSync.isEmpty() || !outstandingMutations.isEmpty()) + return this; + + logger.trace("Reconciliation completed for read {}", readId); + complete(promise, read, command.columnFilter(), consistencyLevel, expiresAtNanos); + return COMPLETED; + } + + @Override + State acknowledgeSync(int syncId) + { + logger.trace("Reconciliation sync {} received for {}", syncId, readId); + pendingSync.remove(syncId); + return maybeComplete(); + } + + @Override + State receiveMutations(List mutations) + { + Log2OffsetsMap.Mutable received = new Log2OffsetsMap.Mutable(); + mutations.forEach(mutation -> { + logger.trace("Received mutation {} for read {}", mutation.id(), readId); + received.add(mutation.id()); + }); + outstandingMutations.removeAll(received); + + if (logger.isTraceEnabled()) + logger.trace("Received {} mutations, {} mutations outstanding for {}", mutations.size(), outstandingMutations.idCount(), readId); + read.augment(mutations); + return maybeComplete(); + } + + @Override + State abort() + { + read.close(); + return ABORTED; + } + } + + @Override + public String toString() + { + return "TrackedLocalReadCoordinator{" + readId + ':' + state.status() + '}'; + } + + @VisibleForTesting + public static void processDelta(PartialTrackedRead read, MutationSummary initialSummary, MutationSummary secondarySummary) + { + // Compute any mutations that we could've missed during initial read execution. + ArrayList delta = new ArrayList<>(); + MutationSummary.difference(secondarySummary, initialSummary, delta); + + delta.forEach(mutationId -> { + Mutation mutation = MutationJournal.instance.read(mutationId); + Preconditions.checkNotNull(mutation); + read.augment(mutation); + }); + } + + public void startLocalRead(TrackedRead.Id readId, ReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, int[] summaryNodes, long expiresAtNanos) + { + synchronized (this) + { + if (!(state = state.startLocalRead(readId, promise, command, replicaPlan, summaryNodes, expiresAtNanos)).isReading()) + return; + } + + PartialTrackedRead read; + MutationSummary secondarySummary; + + MutationSummary initialSummary = command.createMutationSummary(false); + ReadExecutionController controller = command.executionController(false); + try + { + read = command.beginTrackedRead(controller); + // Create another summary once initial data has been read fully. We do this to catch + // any mutations that may have arrived during initial read execution. + secondarySummary = command.createMutationSummary(true); + processDelta(read, initialSummary, secondarySummary); + } + catch (Exception e) + { + controller.close(); + abort(); + throw e; + } + + synchronized (this) + { + state = state.receiveInProgressRead(read, secondarySummary); + } + } + + private static void complete(AsyncPromise promise, PartialTrackedRead read, ColumnFilter selection, ConsistencyLevel consistencyLevel, long expiresAtNanos) + { + Stage.READ.submit(() -> completeInternal(promise, read, selection, consistencyLevel, expiresAtNanos)); + } + + private static void completeInternal(AsyncPromise promise, PartialTrackedRead read, ColumnFilter selection, ConsistencyLevel consistencyLevel, long expiresAtNanos) + { + try (PartialTrackedRead.CompletedRead completedRead = read.complete()) + { + TrackedDataResponse response = TrackedDataResponse.create(completedRead.iterator(), selection); + TrackedRead followUp = completedRead.followupRead(consistencyLevel, expiresAtNanos); + + if (followUp != null) + { + ReadCommand command = read.command(); + followUp.future().addCallback((iterator, error) -> { + if (error != null) + { + promise.tryFailure(error); + return; + } + PartitionIterator previous = response.makeIterator(command); + TrackedDataResponse newResponse = TrackedDataResponse.create(PartitionIterators.concat(List.of(previous, iterator)), selection); + promise.trySuccess(newResponse); + }); + } + else + { + promise.trySuccess(response); + } + } + catch (Exception e) + { + promise.tryFailure(e); + throw e; + } + finally + { + read.close(); + } + } + + synchronized void receiveSummary(InetAddressAndPort from, MutationSummary summary) + { + if (logger.isTraceEnabled()) + logger.trace("Received summary {} from {}, for {}", summary, from, state); + state = state.receiveSummary(from, summary); + } + + synchronized boolean acknowledgeSync(int syncId) + { + return (state = state.acknowledgeSync(syncId)).isCompleted(); + } + + synchronized boolean receiveMutations(List mutations) + { + return (state = state.receiveMutations(mutations)).isCompleted(); + } + + synchronized void abort() + { + state = state.abort(); + } + + boolean isTimedOutOrComplete(long nanoTime) + { + return state.isPurgeable(nanoTime); + } +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java new file mode 100644 index 0000000000..5c2417e6b5 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedLocalReads.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import org.apache.cassandra.concurrent.ScheduledExecutorPlus; +import org.apache.cassandra.concurrent.Shutdownable; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.locator.ReplicaPlans; +import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.utils.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.apache.cassandra.concurrent.ExecutorFactory.Global.executorFactory; +import static org.apache.cassandra.concurrent.ExecutorFactory.SimulatorSemantics.NORMAL; + +/** + * Since the read reconciliations don't use 2 way callbacks, a map of active reconciliations + * are maintained and expired here. + * + * Borrowed heavily from RequestCallbacks + */ +public class TrackedLocalReads implements Shutdownable +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedLocalReads.class); + + private final ConcurrentMap reads = new ConcurrentHashMap<>(); + private final ScheduledExecutorPlus executor = executorFactory().scheduled("Reconciliation-Map-Reaper", NORMAL); + + public TrackedLocalReads() + { + long expirationInterval = defaultExpirationInterval(); + executor.scheduleWithFixedDelay(this::expire, expirationInterval, expirationInterval, NANOSECONDS); + } + + private void expire() + { + long start = Clock.Global.nanoTime(); + int n = 0; + for (Map.Entry entry : reads.entrySet()) + { + TrackedLocalReadCoordinator read = entry.getValue(); + if (read.isTimedOutOrComplete(start)) + { + read.abort(); + if (reads.remove(entry.getKey(), entry.getValue())) + n++; + } + } + if (n > 0) + logger.trace("Expired {} entries", n); + } + + private TrackedLocalReadCoordinator getOrCreate(TrackedRead.Id id) + { + return reads.computeIfAbsent(id, TrackedLocalReadCoordinator::new); + } + + public void receiveSummary(InetAddressAndPort from, TrackedSummaryResponse summary) + { + getOrCreate(summary.readId()).receiveSummary(from, summary.summary()); + } + + public TrackedLocalReadCoordinator beginRead(TrackedRead.Id readId, ClusterMetadata metadata, ReadCommand command, ConsistencyLevel consistencyLevel, int[] summaryNodes, long expiresAtNanos) + { + Keyspace keyspace = Keyspace.open(command.metadata().keyspace); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id); + SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry; + ReplicaPlan.AbstractForRead replicaPlan; + if (command instanceof SinglePartitionReadCommand) + { + replicaPlan = ReplicaPlans.forRead(metadata, + keyspace, + ((SinglePartitionReadCommand) command).partitionKey().getToken(), + command.indexQueryPlan(), + consistencyLevel, + retry); + } + else + { + // TODO: confirm range we're reading doesn't span multiple replica sets + replicaPlan = ReplicaPlans.forRangeRead(keyspace, + command.indexQueryPlan(), + consistencyLevel, + command.dataRange().keyRange(), + 1); + } + // TODO: confirm all summaryNodes are present in the replica plan + + TrackedLocalReadCoordinator coordinator = getOrCreate(readId); + coordinator.startLocalRead(readId, command, replicaPlan, summaryNodes, expiresAtNanos); + return coordinator; + } + + public void acknowledgeSync(TrackedRead.Id readId, int syncId) + { + TrackedLocalReadCoordinator read = reads.get(readId); + if (read == null) + return; + + if (read.acknowledgeSync(syncId)) + reads.remove(readId); + } + + public boolean receiveMutations(TrackedRead.Id readId, int syncId, List mutations) + { + TrackedLocalReadCoordinator read = reads.get(readId); + if (read == null) + return false; + + read.receiveMutations(mutations); + if (read.acknowledgeSync(syncId)) + reads.remove(readId); + return true; + } + + public static long defaultExpirationInterval() + { + return DatabaseDescriptor.getMinRpcTimeout(NANOSECONDS) / 2; + } + + @Override + public void shutdown() + { + executor.shutdown(); + } + + @Override + public boolean isTerminated() + { + return executor.isTerminated(); + } + + @Override + public Object shutdownNow() + { + return executor.shutdownNow(); + } + + public void shutdownBlocking() throws InterruptedException + { + if (executor == null || executor.isTerminated()) + return; + + executor.shutdown(); + executor.awaitTermination(1, MINUTES); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit units) throws InterruptedException + { + return executor.awaitTermination(timeout, units); + } +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java new file mode 100644 index 0000000000..9109b69c01 --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedRead.java @@ -0,0 +1,520 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.reads.tracked; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterables; + +import org.apache.cassandra.concurrent.Stage; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.AbstractPartitionIterator; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.exceptions.ReadFailureException; +import org.apache.cassandra.exceptions.ReadTimeoutException; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.locator.*; +import org.apache.cassandra.net.*; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.service.reads.ReadCoordinator; +import org.apache.cassandra.service.reads.SpeculativeRetryPolicy; +import org.apache.cassandra.tcm.ClusterMetadata; +import org.apache.cassandra.transport.Dispatcher; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.concurrent.AsyncPromise; +import org.apache.cassandra.utils.concurrent.Future; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.cassandra.metrics.ClientRequestsMetricsHolder.readMetrics; + +public abstract class TrackedRead, P extends ReplicaPlan.ForRead> implements RequestCallback +{ + private static final Logger logger = LoggerFactory.getLogger(TrackedRead.class); + + public static class Id + { + private static final int nodeId = ClusterMetadata.current().myNodeId().id(); + private static final AtomicLong lastHlc = new AtomicLong(); + + private final int node; + private final long hlc; + + public Id(int node, long hlc) + { + this.node = node; + this.hlc = hlc; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + Id id = (Id) o; + return node == id.node && hlc == id.hlc; + } + + @Override + public int hashCode() + { + return Integer.hashCode(node) * 31 + Long.hashCode(hlc); + } + + @Override + public String toString() + { + return "Id{" + node + ':' + hlc + '}'; + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(Id id, DataOutputPlus out, int version) throws IOException + { + out.writeInt(id.node); + out.writeLong(id.hlc); + } + + @Override + public Id deserialize(DataInputPlus in, int version) throws IOException + { + int node = in.readInt(); + long hlc = in.readLong(); + return new Id(node, hlc); + } + + @Override + public long serializedSize(Id id, int version) + { + return TypeSizes.sizeof(id.node) + TypeSizes.sizeof(id.hlc); + } + }; + + public static Id nextId() + { + while (true) + { + long lastMicros = lastHlc.get(); + long nextMicros = Math.max(lastMicros + 1, TimeUnit.MILLISECONDS.toMicros(Clock.Global.currentTimeMillis())); + if (lastHlc.compareAndSet(lastMicros, nextMicros)) + return new Id(nodeId, nextMicros); + } + } + } + + private final AsyncPromise future = new AsyncPromise<>(); + + private final Id readId = Id.nextId(); + private final ReadCommand command; + private final ReplicaPlan.AbstractForRead replicaPlan; + private final ConsistencyLevel consistencyLevel; + + private static class RequestFailure extends Throwable + { + private final InetAddressAndPort from; + private final RequestFailureReason reason; + + public RequestFailure(InetAddressAndPort from, RequestFailureReason reason) + { + this.from = from; + this.reason = reason; + } + + public Map reasonByEndpoint() + { + return Map.of(from, reason); + } + } + + public TrackedRead(ReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, ConsistencyLevel consistencyLevel) + { + this.command = command; + this.replicaPlan = replicaPlan; + this.consistencyLevel = consistencyLevel; + } + + @Override + public String toString() + { + return "TrackedRead." + getClass().getSimpleName() + '{' + readId + '}'; + } + + protected abstract Verb verb(); + + public static class Partition extends TrackedRead + { + private Partition(SinglePartitionReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, ConsistencyLevel consistencyLevel) + { + super(command, replicaPlan, consistencyLevel); + } + + public static Partition create(ClusterMetadata metadata, SinglePartitionReadCommand command, ConsistencyLevel consistencyLevel) + { + Preconditions.checkArgument(command.metadata().replicationType().isTracked()); + Keyspace keyspace = Keyspace.open(command.metadata().keyspace); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(command.metadata().id); + SpeculativeRetryPolicy retry = cfs.metadata().params.speculativeRetry; + ReplicaPlan.ForTokenRead replicaPlan = ReplicaPlans.forRead(metadata, + keyspace, + cfs.getTableId(), + command.partitionKey().getToken(), + command.indexQueryPlan(), + consistencyLevel, + retry, + ReadCoordinator.DEFAULT); + return new Partition(command, replicaPlan, consistencyLevel); + } + + @Override + protected Verb verb() + { + return Verb.TRACKED_PARTITION_READ_REQ; + } + } + + public static class Range extends TrackedRead + { + private Range(PartitionRangeReadCommand command, ReplicaPlan.AbstractForRead replicaPlan, ConsistencyLevel consistencyLevel) + { + super(command, replicaPlan, consistencyLevel); + } + + public static TrackedRead.Range create(PartitionRangeReadCommand command, ReplicaPlan.ForRangeRead replicaPlan) + { + Preconditions.checkArgument(command.metadata().replicationType().isTracked()); + return new Range(command, replicaPlan, replicaPlan.consistencyLevel()); + } + + @Override + protected Verb verb() + { + return Verb.TRACKED_RANGE_READ_REQ; + } + } + + private static > int[] endpointsToHostIds(E endpoints) + { + int[] hostids = new int[endpoints.size()]; + int idx = 0; + ClusterMetadata metadata = ClusterMetadata.current(); + for (Replica replica : endpoints) + hostids[idx++] = metadata.directory.peerId(replica.endpoint()).id(); + return hostids; + } + + public void start(long expiresAt) + { + // TODO: skip local coordination if this node knows its recovering from an outage + // TODO: read speculation + Replica localReplica = replicaPlan.lookup(FBUtilities.getBroadcastAddressAndPort()); + if (localReplica != null) + readMetrics.localRequests.mark(); + else + readMetrics.remoteRequests.mark(); + + // create an id + // select data node + // select summary nodes + E selected = replicaPlan.contacts(); + Replica dataNode = localReplica != null && localReplica.isFull() + ? localReplica + : Iterables.getOnlyElement(selected.filter(Replica::isFull, 1)); + E summaryNodes = selected.filter(r -> r != dataNode); + int[] summaryHostIds = endpointsToHostIds(summaryNodes); + + + if (dataNode == localReplica) + { + Stage.READ.submit(() -> { + TrackedLocalReadCoordinator coordinator = MutationTrackingService.instance.localReads().beginRead(readId, ClusterMetadata.current(), command, consistencyLevel, summaryHostIds, expiresAt); + coordinator.addCallback((response, error) -> { + if (error != null) + { + // TODO: notify coordinator that read has failed + logger.error("Error while processing read", error); + return; + } + logger.trace("Finished locally coordinating {}", this); + onResponse(response); + }); + }); + } + else + { + DataRequest dataRequest = new DataRequest(readId, command, consistencyLevel, summaryHostIds); + Message dataMessage = Message.outWithFlag(verb(), dataRequest, MessageFlag.CALL_BACK_ON_FAILURE); + MessagingService.instance().sendWithCallback(dataMessage, dataNode.endpoint(), this); + } + + if (summaryNodes.isEmpty()) + return; + + SummaryRequest summaryRequest = new SummaryRequest(readId, command); + Message summaryMessage = Message.outWithParam(Verb.TRACKED_SUMMARY_REQ, + summaryRequest, + ParamType.RESPOND_TO, + dataNode.endpoint()); + for (Replica replica : summaryNodes) + { + if (localReplica == replica) + { + Stage.READ.submit(() -> summaryRequest.executeLocally(summaryMessage, ClusterMetadata.current())); + } + else + { + MessagingService.instance().send(summaryMessage, replica.endpoint()); + } + } + } + + public void start(Dispatcher.RequestTime requestTime) + { + start(requestTime.computeDeadline(verb().expiresAfterNanos())); + } + + private void onResponse(TrackedDataResponse response) + { + future.trySuccess(response.makeIterator(command)); + } + + @Override + public void onResponse(Message msg) + { + onResponse(msg.payload); + } + + @Override + public void onFailure(InetAddressAndPort from, org.apache.cassandra.exceptions.RequestFailure failure) + { + future.tryFailure(new RequestFailure(from, failure.reason)); + } + + @Override + public boolean invokeOnFailure() + { + return true; + } + + public Future future() + { + return future; + } + + public PartitionIterator awaitResults() + { + try + { + return future.get(command.getTimeout(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + catch (ExecutionException e) + { + Throwable ex = e.getCause(); + Map reasons = Collections.emptyMap(); + if (ex instanceof RequestFailure) + { + RequestFailure failure = (RequestFailure) ex; + if (failure.reason == RequestFailureReason.TIMEOUT) + { + throw new ReadTimeoutException(replicaPlan.consistencyLevel(), 0, replicaPlan.readQuorum(), false); + } + + reasons = failure.reasonByEndpoint(); + } + + throw new ReadFailureException(replicaPlan.consistencyLevel(), 0, replicaPlan.readQuorum(), false, reasons); + } + catch (TimeoutException e) + { + throw new ReadTimeoutException(replicaPlan.consistencyLevel(), 0, replicaPlan.readQuorum(), false); + } + } + + public PartitionIterator iterator() + { + return new AbstractPartitionIterator() + { + PartitionIterator result = null; + + @Override + protected RowIterator computeNext() + { + if (result == null) + result = awaitResults(); + + if (!result.hasNext()) + return endOfData(); + + return result.next(); + } + }; + } + + public abstract static class Request + { + protected final Id readId; + protected final ReadCommand command; + + protected Request(Id readId, ReadCommand command) + { + this.readId = readId; + this.command = command; + } + + public abstract void executeLocally(Message message, ClusterMetadata metadata); + } + + public static class DataRequest extends Request + { + private final ConsistencyLevel consistencyLevel; + private final int[] summaryNodes; + + public DataRequest(Id readId, ReadCommand command, ConsistencyLevel consistencyLevel, int[] summaryNodes) + { + super(readId, command); + this.consistencyLevel = consistencyLevel; + this.summaryNodes = summaryNodes; + } + + @Override + public void executeLocally(Message message, ClusterMetadata metadata) + { + TrackedLocalReadCoordinator coordinator = MutationTrackingService.instance.localReads().beginRead(readId, metadata, command, consistencyLevel, summaryNodes, message.expiresAtNanos()); + coordinator.addCallback((response, error) -> { + if (error != null) + { + // TODO: notify coordinator that read has failed + logger.error("Error while processing read", error); + return; + } + MessagingService.instance().send(message.responseWith(response), message.from()); + }); + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(DataRequest request, DataOutputPlus out, int version) throws IOException + { + Id.serializer.serialize(request.readId, out, version); + ReadCommand.serializer.serialize(request.command, out, version); + out.writeInt(request.consistencyLevel.code); + out.writeInt(request.summaryNodes.length); + for (int hostid : request.summaryNodes) + out.writeInt(hostid); + } + + @Override + public DataRequest deserialize(DataInputPlus in, int version) throws IOException + { + Id readId = Id.serializer.deserialize(in, version); + ReadCommand command = ReadCommand.serializer.deserialize(in, version); + ConsistencyLevel consistencyLevel = ConsistencyLevel.fromCode(in.readInt()); + int[] summaryNodes = new int[in.readInt()]; + for (int i = 0; i < summaryNodes.length; i++) + summaryNodes[i] = in.readInt(); + return new DataRequest(readId, command, consistencyLevel, summaryNodes); + } + + @Override + public long serializedSize(DataRequest request, int version) + { + return Id.serializer.serializedSize(request.readId, version) + + ReadCommand.serializer.serializedSize(request.command, version) + + TypeSizes.sizeof(request.consistencyLevel.code) + + TypeSizes.sizeof(request.summaryNodes.length) + + ((long) TypeSizes.INT_SIZE * request.summaryNodes.length); + } + }; + } + + public static class SummaryRequest extends Request + { + public SummaryRequest(Id readId, ReadCommand command) + { + super(readId, command); + } + + @Override + public void executeLocally(Message message, ClusterMetadata metadata) + { + MutationSummary summary = command.createMutationSummary(false); + TrackedSummaryResponse response = new TrackedSummaryResponse(readId, summary); + MessagingService.instance().send(message.responseWith(response), message.respondTo()); + } + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(SummaryRequest request, DataOutputPlus out, int version) throws IOException + { + Id.serializer.serialize(request.readId, out, version); + ReadCommand.serializer.serialize(request.command, out, version); + } + + @Override + public SummaryRequest deserialize(DataInputPlus in, int version) throws IOException + { + Id readId = Id.serializer.deserialize(in, version); + ReadCommand command = ReadCommand.serializer.deserialize(in, version); + return new SummaryRequest(readId, command); + } + + @Override + public long serializedSize(SummaryRequest request, int version) + { + return Id.serializer.serializedSize(request.readId, version) + + ReadCommand.serializer.serializedSize(request.command, version); + } + }; + } + + public static final IVerbHandler verbHandler = new AbstractReadCommandVerbHandler<>() + { + @Override + protected void performRead(Message message, ClusterMetadata metadata) + { + message.payload.executeLocally(message, metadata); + } + + @Override + protected ReadCommand getCommand(Request payload) + { + return payload.command; + } + }; +} diff --git a/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java b/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java new file mode 100644 index 0000000000..97f2254eee --- /dev/null +++ b/src/java/org/apache/cassandra/service/reads/tracked/TrackedSummaryResponse.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.service.reads.tracked; + +import org.apache.cassandra.io.IVersionedSerializer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.MutationTrackingService; + +import java.io.IOException; + +public class TrackedSummaryResponse +{ + private final TrackedRead.Id readId; + private final MutationSummary summary; + + public TrackedSummaryResponse(TrackedRead.Id readId, MutationSummary summary) + { + this.readId = readId; + this.summary = summary; + } + + public TrackedRead.Id readId() + { + return readId; + } + + public MutationSummary summary() + { + return summary; + } + + public static final IVerbHandler verbHandler = + message -> MutationTrackingService.instance.localReads().receiveSummary(message.from(), message.payload); + + public static final IVersionedSerializer serializer = new IVersionedSerializer<>() + { + @Override + public void serialize(TrackedSummaryResponse summary, DataOutputPlus out, int version) throws IOException + { + TrackedRead.Id.serializer.serialize(summary.readId, out, version); + MutationSummary.serializer.serialize(summary.summary, out, version); + } + + @Override + public TrackedSummaryResponse deserialize(DataInputPlus in, int version) throws IOException + { + TrackedRead.Id id = TrackedRead.Id.serializer.deserialize(in, version); + MutationSummary summary = MutationSummary.serializer.deserialize(in, version); + return new TrackedSummaryResponse(id, summary); + } + + @Override + public long serializedSize(TrackedSummaryResponse summary, int version) + { + return TrackedRead.Id.serializer.serializedSize(summary.readId, version) + + MutationSummary.serializer.serializedSize(summary.summary, version); + } + }; +} diff --git a/src/java/org/apache/cassandra/service/tracking/MutationId.java b/src/java/org/apache/cassandra/service/tracking/MutationId.java deleted file mode 100644 index 9132676530..0000000000 --- a/src/java/org/apache/cassandra/service/tracking/MutationId.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.service.tracking; - -public class MutationId -{ - /** - * 4 byte TCM host id + 4 byte host log id packed into a long. - * Host log ID is unique within the host, allocated - * anew on host restart - one per token range replicated by the host, - * persisted on allocation, unique within the host. - */ - public final long logId; - - /** - * 4 byte position + 4 byte timestamp packed into a long. - * Position is incremented, the timestamp is monotonically non-decreasing. - * The position is enough to identify the entry within a coordinator log, - * the timestamp is added for correlation purposes. - */ - public final long sequenceId; - - MutationId(long logId, long sequenceId) - { - this.logId = logId; - this.sequenceId = sequenceId; - } - - public int hostId() - { - return (int) (0xffffffffL & (logId >> 32)); - } - - public int hostLogId() - { - return (int) (0xffffffffL & logId); - } - - public int position() - { - return (int) (0xffffffffL & (sequenceId >> 32)); - } - - public int timestamp() - { - return (int) (0xffffffffL & sequenceId); - } - - @Override - public boolean equals(Object o) - { - if (this == o) return true; - if (!(o instanceof MutationId)) return false; - MutationId that = (MutationId) o; - return this.logId == that.logId && this.sequenceId == that.sequenceId; - } - - @Override - public int hashCode() - { - return Long.hashCode(logId) + 31 * Long.hashCode(sequenceId); - } - - @Override - public String toString() - { - return "MutationId{" + logId + ", " + sequenceId + '}'; - } -} diff --git a/src/java/org/apache/cassandra/tcm/Startup.java b/src/java/org/apache/cassandra/tcm/Startup.java index 970ea190c6..a60a934e97 100644 --- a/src/java/org/apache/cassandra/tcm/Startup.java +++ b/src/java/org/apache/cassandra/tcm/Startup.java @@ -53,6 +53,7 @@ import org.apache.cassandra.gms.NewGossiper; import org.apache.cassandra.gms.VersionedValue; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.DistributedSchema; import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.Keyspaces; @@ -158,7 +159,8 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; LocalLog.LogSpec logSpec = LocalLog.logSpec() .withStorage(LogStorage.SystemKeyspace) .afterReplay(Startup::scrubDataDirectories, - (metadata) -> StorageService.instance.registerMBeans()) + (metadata) -> StorageService.instance.registerMBeans(), + MutationTrackingService.instance::start) .withDefaultListeners(); ClusterMetadataService.setInstance(new ClusterMetadataService(new UniformRangePlacement(), wrapProcessor, @@ -285,7 +287,8 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; LocalLog.LogSpec logSpec = LocalLog.logSpec() .withInitialState(emptyFromSystemTables) .afterReplay(Startup::scrubDataDirectories, - (metadata) -> StorageService.instance.registerMBeans()) + (metadata) -> StorageService.instance.registerMBeans(), + MutationTrackingService.instance::start) .withStorage(LogStorage.SystemKeyspace) .withDefaultListeners(); @@ -392,7 +395,8 @@ import static org.apache.cassandra.utils.FBUtilities.getBroadcastAddressAndPort; ClusterMetadataService.unsetInstance(); LocalLog.LogSpec logSpec = LocalLog.logSpec() .afterReplay(Startup::scrubDataDirectories, - (_metadata) -> StorageService.instance.registerMBeans()) + (_metadata) -> StorageService.instance.registerMBeans(), + MutationTrackingService.instance::start) .withPreviousState(prev) .withInitialState(metadata) .withStorage(LogStorage.SystemKeyspace) diff --git a/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java b/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java index de1bc71250..f3b78a1ee9 100644 --- a/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java +++ b/src/java/org/apache/cassandra/tcm/membership/NodeVersion.java @@ -36,7 +36,7 @@ import static org.apache.cassandra.db.TypeSizes.sizeofUnsignedVInt; public class NodeVersion implements Comparable { public static final Serializer serializer = new Serializer(); - public static final Version CURRENT_METADATA_VERSION = Version.V8; + public static final Version CURRENT_METADATA_VERSION = Version.V9; public static final NodeVersion CURRENT = new NodeVersion(new CassandraVersion(FBUtilities.getReleaseVersionString()), CURRENT_METADATA_VERSION); private static final CassandraVersion SINCE_VERSION = CassandraVersion.CASSANDRA_5_1; diff --git a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java index 5ce36e09a8..cd7679d2af 100644 --- a/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/sequences/CancelCMSReconfiguration.java @@ -95,7 +95,7 @@ public class CancelCMSReconfiguration implements Transformation // Also update schema with the corrected params KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); - KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple())); + KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, fromPlacement, FastPathStrategy.simple(), keyspace.params.replicationType)); transformer = transformer.with(new DistributedSchema(prev.schema.getKeyspaces().withAddedOrUpdated(newKeyspace))); } diff --git a/src/java/org/apache/cassandra/tcm/serialization/Version.java b/src/java/org/apache/cassandra/tcm/serialization/Version.java index 9cbcb8587a..c356881782 100644 --- a/src/java/org/apache/cassandra/tcm/serialization/Version.java +++ b/src/java/org/apache/cassandra/tcm/serialization/Version.java @@ -79,6 +79,11 @@ public enum Version */ V8(8), + /** + * - MutationTracking + */ + V9(9), + UNKNOWN(Integer.MAX_VALUE); /** diff --git a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java index e94292831e..7964169b35 100644 --- a/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java +++ b/src/java/org/apache/cassandra/tcm/transformations/cms/PrepareCMSReconfiguration.java @@ -264,7 +264,7 @@ public abstract class PrepareCMSReconfiguration implements Transformation // In a complex reconfiguration, in addition to initiating the sequence of membership changes, // we're modifying the replication params of the metadata keyspace so we supply a function to do that KeyspaceMetadata keyspace = prev.schema.getKeyspaceMetadata(SchemaConstants.METADATA_KEYSPACE_NAME); - KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple())); + KeyspaceMetadata newKeyspace = keyspace.withSwapped(new KeyspaceParams(keyspace.params.durableWrites, replicationParams, FastPathStrategy.simple(), keyspace.params.replicationType)); return executeInternal(prev, transformer -> transformer.with(prev.placements.replaceParams(prev.nextEpoch(), ReplicationParams.meta(prev), replicationParams)) diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index 6f18007c67..c62e8b9c89 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -126,6 +126,8 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.NoPayload; import org.apache.cassandra.net.Verb; import org.apache.cassandra.repair.autorepair.AutoRepair; +import org.apache.cassandra.replication.MutationJournal; +import org.apache.cassandra.replication.MutationTrackingService; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.SchemaConstants; import org.apache.cassandra.service.ActiveRepairService; @@ -787,6 +789,7 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance CassandraDaemon.getInstanceForTesting().migrateSystemDataIfNeeded(); CommitLog.instance.start(); + MutationJournal.instance.start(); SnapshotManager.instance.start(false); SnapshotManager.instance.clearExpiredSnapshots(); @@ -1055,6 +1058,8 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance // CommitLog must shut down after Stage, or threads from the latter may attempt to use the former. // (ex. A Mutation stage thread may attempt to add a mutation to the CommitLog.) error = parallelRun(error, executor, CommitLog.instance::shutdownBlocking); + error = parallelRun(error, executor, MutationJournal.instance::shutdownBlocking); + error = parallelRun(error, executor, MutationTrackingService.instance::shutdownBlocking); error = parallelRun(error, executor, () -> shutdownAndWait(Collections.singletonList(JMXBroadcastExecutor.executor)) ); diff --git a/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java b/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java index 6dcc5f5e26..9ac80fa078 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/GroupByTest.java @@ -18,56 +18,151 @@ package org.apache.cassandra.distributed.test; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.Iterator; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + import com.datastax.driver.core.Row; import com.datastax.driver.core.Session; import com.datastax.driver.core.SimpleStatement; import com.google.common.collect.Iterators; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.ICoordinator; +import org.apache.cassandra.distributed.api.IInstanceConfig; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.serializers.SimpleDateSerializer; import org.apache.cassandra.serializers.TimeSerializer; import org.apache.cassandra.serializers.TimestampSerializer; +import org.apache.cassandra.utils.concurrent.UncheckedInterruptedException; +import static org.apache.cassandra.distributed.api.ConsistencyLevel.ALL; import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; import static org.apache.cassandra.distributed.api.Feature.NATIVE_PROTOCOL; import static org.apache.cassandra.distributed.api.Feature.NETWORK; import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; import static org.apache.cassandra.distributed.shared.AssertUtils.row; +@RunWith(Parameterized.class) public class GroupByTest extends TestBaseImpl { + @Parameterized.Parameter(0) + public ReplicationType replicationType; + + @Parameterized.Parameters(name = "replication={0}") + public static Collection data() + { + List result = new ArrayList<>(); + for (ReplicationType replication : ReplicationType.values()) + result.add(new Object[]{replication}); + return result; + } + + private static Cluster cluster; + private static int nextKeyspaceId = 0; + + @BeforeClass + public static void setupClass() throws Exception + { + Consumer config = cfg -> cfg.with(Feature.GOSSIP, NETWORK, NATIVE_PROTOCOL).set("user_defined_functions_enabled", "true"); + cluster = Cluster.build().withNodes(3).withConfig(config).start(); + } + + @AfterClass + public static void tearDownClass() throws Exception + { + if (cluster != null) + cluster.close(); + } + + @Before + public void setup() + { + Assert.assertNotNull(cluster); + cluster.filters().reset(); + + KEYSPACE = "ks_" + nextKeyspaceId++; + cluster.schemaChange("CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': " + cluster.size() + "} AND REPLICATION_TYPE = '" + replicationType.name() + "';"); + } + + private static void markNodeDown(IInvokableInstance instance, InetAddressAndPort ep) + { + instance.runOnInstance(() -> { + + }); + } + + private static void awaitNodeDown(IInvokableInstance instance, InetAddressAndPort ep) + { + instance.runOnInstance(() -> { + long started = System.currentTimeMillis(); + FailureDetector.instance.forceConviction(ep); + while (true) + { + long elapsed = System.currentTimeMillis() - started; + if (elapsed > TimeUnit.SECONDS.toMillis(30)) + throw new AssertionError("Timed out waiting for node down - " + ep); + + if (!FailureDetector.instance.isAlive(ep)) + return; + + try + { + Thread.sleep(100); + } + catch (InterruptedException e) + { + throw new UncheckedInterruptedException(e); + } + } + }); + } + + private static void awaitNodeDown(IInvokableInstance instance, InetSocketAddress ep) + { + awaitNodeDown(instance, InetAddressAndPort.getByAddress(ep)); + } + + @Test public void groupByWithDeletesAndSrpOnPartitions() throws Throwable { - try (Cluster cluster = init(builder().withNodes(2).withConfig((cfg) -> cfg.set("user_defined_functions_enabled", "true")).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, PRIMARY KEY (pk, ck))")); + initFunctions(cluster); + cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (1, '1') USING TIMESTAMP 0")); + cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (2, '2') USING TIMESTAMP 0")); + cluster.get(1).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='0'")); + + cluster.get(2).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '0') USING TIMESTAMP 0")); + cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=1 AND ck='1'")); + cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=2 AND ck='2'")); + + for (String limitClause : new String[]{ "", "LIMIT 1", "LIMIT 10", "PER PARTITION LIMIT 1", "PER PARTITION LIMIT 10" }) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, PRIMARY KEY (pk, ck))")); - initFunctions(cluster); - cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (1, '1') USING TIMESTAMP 0")); - cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (2, '2') USING TIMESTAMP 0")); - cluster.get(1).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='0'")); - - cluster.get(2).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '0') USING TIMESTAMP 0")); - cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=1 AND ck='1'")); - cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=2 AND ck='2'")); - - for (String limitClause : new String[]{ "", "LIMIT 1", "LIMIT 10", "PER PARTITION LIMIT 1", "PER PARTITION LIMIT 10" }) + String query = withKeyspace("SELECT concat(ck) FROM %s.tbl GROUP BY pk " + limitClause); + for (int i = 1; i <= 4; i++) { - String query = withKeyspace("SELECT concat(ck) FROM %s.tbl GROUP BY pk " + limitClause); - for (int i = 1; i <= 4; i++) - { - Iterator rows = cluster.coordinator(2).executeWithPaging(query, ConsistencyLevel.ALL, i); - assertRows(Iterators.toArray(rows, Object[].class)); - } + Iterator rows = cluster.coordinator(2).executeWithPaging(query, ALL, i); + assertRows(Iterators.toArray(rows, Object[].class)); } } } @@ -75,26 +170,23 @@ public class GroupByTest extends TestBaseImpl @Test public void groupByWithDeletesAndSrpOnRows() throws Throwable { - try (Cluster cluster = init(builder().withNodes(2).withConfig((cfg) -> cfg.set("user_defined_functions_enabled", "true")).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, PRIMARY KEY (pk, ck))")); + initFunctions(cluster); + cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '1') USING TIMESTAMP 0")); + cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '2') USING TIMESTAMP 0")); + cluster.get(1).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='0'")); + + cluster.get(2).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '0') USING TIMESTAMP 0")); + cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='1'")); + cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='2'")); + + for (String limitClause : new String[]{ "", "LIMIT 1", "LIMIT 10", "PER PARTITION LIMIT 1", "PER PARTITION LIMIT 10" }) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck text, PRIMARY KEY (pk, ck))")); - initFunctions(cluster); - cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '1') USING TIMESTAMP 0")); - cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '2') USING TIMESTAMP 0")); - cluster.get(1).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='0'")); - - cluster.get(2).executeInternal(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, '0') USING TIMESTAMP 0")); - cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='1'")); - cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck='2'")); - - for (String limitClause : new String[]{ "", "LIMIT 1", "LIMIT 10", "PER PARTITION LIMIT 1", "PER PARTITION LIMIT 10" }) + String query = withKeyspace("SELECT concat(ck) FROM %s.tbl GROUP BY pk " + limitClause); + for (int i = 1; i <= 4; i++) { - String query = withKeyspace("SELECT concat(ck) FROM %s.tbl GROUP BY pk " + limitClause); - for (int i = 1; i <= 4; i++) - { - Iterator rows = cluster.coordinator(2).executeWithPaging(query, ConsistencyLevel.ALL, i); - assertRows(Iterators.toArray(rows, Object[].class)); - } + Iterator rows = cluster.coordinator(2).executeWithPaging(query, ALL, i); + assertRows(Iterators.toArray(rows, Object[].class)); } } } @@ -102,116 +194,107 @@ public class GroupByTest extends TestBaseImpl @Test public void testGroupByWithAggregatesAndPaging() throws Throwable { - try (Cluster cluster = init(builder().withNodes(2).withConfig((cfg) -> cfg.set("user_defined_functions_enabled", "true")).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v1 text, v2 text, v3 text, primary key (pk, ck))")); + initFunctions(cluster); + + cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck, v1, v2, v3) values (1,1,'1','1','1')"), ALL); + cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck, v1, v2, v3) values (1,2,'2','2','2')"), ALL); + cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck, v1, v2, v3) values (1,3,'3','3','3')"), ALL); + + for (int i = 1; i <= 4; i++) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, v1 text, v2 text, v3 text, primary key (pk, ck))")); - initFunctions(cluster); + assertRows(cluster.coordinator(1).executeWithPaging(withKeyspace("select concat(v1), concat(v2), concat(v3) from %s.tbl where pk = 1 group by pk"), + ALL, i), + row("_ 1 2 3", "_ 1 2 3", "_ 1 2 3")); - cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck, v1, v2, v3) values (1,1,'1','1','1')"), ConsistencyLevel.ALL); - cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck, v1, v2, v3) values (1,2,'2','2','2')"), ConsistencyLevel.ALL); - cluster.coordinator(1).execute(withKeyspace("insert into %s.tbl (pk, ck, v1, v2, v3) values (1,3,'3','3','3')"), ConsistencyLevel.ALL); + assertRows(cluster.coordinator(1).executeWithPaging(withKeyspace("select concat(v1), concat(v2), concat(v3) from %s.tbl where pk = 1 group by pk limit 1"), + ALL, i), + row("_ 1 2 3", "_ 1 2 3", "_ 1 2 3")); - for (int i = 1; i <= 4; i++) - { - assertRows(cluster.coordinator(1).executeWithPaging(withKeyspace("select concat(v1), concat(v2), concat(v3) from %s.tbl where pk = 1 group by pk"), - ConsistencyLevel.ALL, i), - row("_ 1 2 3", "_ 1 2 3", "_ 1 2 3")); - - assertRows(cluster.coordinator(1).executeWithPaging(withKeyspace("select concat(v1), concat(v2), concat(v3) from %s.tbl where pk = 1 group by pk limit 1"), - ConsistencyLevel.ALL, i), - row("_ 1 2 3", "_ 1 2 3", "_ 1 2 3")); - - assertRows(cluster.coordinator(1).executeWithPaging(withKeyspace("select * from %s.tbl where pk = 1 group by pk"), - ConsistencyLevel.ALL, i), - row(1, 1, "1", "1", "1")); - } + assertRows(cluster.coordinator(1).executeWithPaging(withKeyspace("select * from %s.tbl where pk = 1 group by pk"), + ALL, i), + row(1, 1, "1", "1", "1")); } } @Test public void testGroupWithDeletesAndPaging() throws Throwable { - try (Cluster cluster = init(builder().withNodes(2).withConfig(cfg -> cfg.with(Feature.GOSSIP, NETWORK, NATIVE_PROTOCOL)).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, PRIMARY KEY (pk, ck))")); + ICoordinator coordinator = cluster.coordinator(1); + coordinator.execute(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, 0)"), ALL); + coordinator.execute(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (1, 1)"), ALL); + + cluster.get(1).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck=0")); + cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=1 AND ck=1")); + String query = withKeyspace("SELECT * FROM %s.tbl GROUP BY pk"); + Iterator rows = coordinator.executeWithPaging(query, ALL, 1); + assertRows(Iterators.toArray(rows, Object[].class)); + + try (com.datastax.driver.core.Cluster c = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1").build(); + Session session = c.connect()) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (pk int, ck int, PRIMARY KEY (pk, ck))")); - ICoordinator coordinator = cluster.coordinator(1); - coordinator.execute(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (0, 0)"), ConsistencyLevel.ALL); - coordinator.execute(withKeyspace("INSERT INTO %s.tbl (pk, ck) VALUES (1, 1)"), ConsistencyLevel.ALL); - - cluster.get(1).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=0 AND ck=0")); - cluster.get(2).executeInternal(withKeyspace("DELETE FROM %s.tbl WHERE pk=1 AND ck=1")); - String query = withKeyspace("SELECT * FROM %s.tbl GROUP BY pk"); - Iterator rows = coordinator.executeWithPaging(query, ConsistencyLevel.ALL, 1); - assertRows(Iterators.toArray(rows, Object[].class)); - - try (com.datastax.driver.core.Cluster c = com.datastax.driver.core.Cluster.builder().addContactPoint("127.0.0.1").build(); - Session session = c.connect()) - { - SimpleStatement stmt = new SimpleStatement(withKeyspace("select * from %s.tbl where pk = 1 group by pk")); - stmt.setFetchSize(1); - Iterator rs = session.execute(stmt).iterator(); - Assert.assertFalse(rs.hasNext()); - } + SimpleStatement stmt = new SimpleStatement(withKeyspace("select * from %s.tbl where pk = 1 group by pk")); + stmt.setFetchSize(1); + Iterator rs = session.execute(stmt).iterator(); + Assert.assertFalse(rs.hasNext()); } } @Test public void testGroupByTimeRangesWithTimestampType() throws Throwable { - try (Cluster cluster = init(builder().withNodes(3).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithTimestamp (pk int, time timestamp, v int, primary key (pk, time))")); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:10:00 UTC', 1)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:12:00 UTC', 2)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:14:00 UTC', 3)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:15:00 UTC', 4)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:21:00 UTC', 5)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:22:00 UTC', 6)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:26:00 UTC', 7)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:26:20 UTC', 8)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (2, '2016-09-27 16:26:20 UTC', 10)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (2, '2016-09-27 16:30:00 UTC', 11)"), ConsistencyLevel.QUORUM); + + for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithTimestamp (pk int, time timestamp, v int, primary key (pk, time))")); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:10:00 UTC', 1)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:12:00 UTC', 2)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:14:00 UTC', 3)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:15:00 UTC', 4)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:21:00 UTC', 5)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:22:00 UTC', 6)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:26:00 UTC', 7)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (1, '2016-09-27 16:26:20 UTC', 8)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (2, '2016-09-27 16:26:20 UTC', 10)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTimestamp (pk, time, v) VALUES (2, '2016-09-27 16:30:00 UTC', 11)"), ConsistencyLevel.QUORUM); - - for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) + for (String startingTime : new String[] {"", ", '2016-09-27 UTC'"} ) { - for (String startingTime : new String[] {"", ", '2016-09-27 UTC'"} ) - { - String stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ")"; - Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), - row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), - row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), - row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), - row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), - row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); + String stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ")"; + Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L), + row(2, toTimestamp("2016-09-27 16:30:00 UTC"), 11, 11, 1L)); - stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), - row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ") LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L)); - stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), - row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp GROUP BY pk, floor(time, 5m" + startingTime + ") PER PARTITION LIMIT 1"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L), + row(2, toTimestamp("2016-09-27 16:25:00 UTC"), 10, 10, 1L)); - stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), - row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), - row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), - row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L), + row(1, toTimestamp("2016-09-27 16:15:00 UTC"), 4, 4, 1L), + row(1, toTimestamp("2016-09-27 16:10:00 UTC"), 1, 3, 3L)); - stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), - row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); - } + stmt = "SELECT pk, floor(time, 5m" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithTimestamp WHERE pk = 1 GROUP BY pk, floor(time, 5m" + startingTime + ") ORDER BY time DESC LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toTimestamp("2016-09-27 16:25:00 UTC"), 7, 8, 2L), + row(1, toTimestamp("2016-09-27 16:20:00 UTC"), 5, 6, 2L)); } } } @@ -219,60 +302,56 @@ public class GroupByTest extends TestBaseImpl @Test public void testGroupByTimeRangesWithDateType() throws Throwable { - try (Cluster cluster = init(builder().withNodes(3).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithDate (pk int, time date, v int, primary key (pk, time))")); + + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-27', 1)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-28', 2)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-29', 3)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-30', 4)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-01', 5)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-04', 6)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-20', 7)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-11-27', 8)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (2, '2016-11-01', 10)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (2, '2016-11-02', 11)"), ConsistencyLevel.QUORUM); + + for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) { - - cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithDate (pk int, time date, v int, primary key (pk, time))")); - - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-27', 1)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-28', 2)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-29', 3)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-09-30', 4)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-01', 5)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-04', 6)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-10-20', 7)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (1, '2016-11-27', 8)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (2, '2016-11-01', 10)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithDate (pk, time, v) VALUES (2, '2016-11-02', 11)"), ConsistencyLevel.QUORUM); - - for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) + for (String startingTime : new String[] {"", ", '2016-06-01'"} ) { - for (String startingTime : new String[] {"", ", '2016-06-01'"} ) - { - String stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ")"; - Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-01"), 1, 4, 4L), - row(1, toLocalDate("2016-10-01"), 5, 7, 3L), - row(1, toLocalDate("2016-11-01"), 8, 8, 1L), - row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); + String stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ")"; + Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L), + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); - stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ") LIMIT 2"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-01"), 1, 4, 4L), - row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ") LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); - stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ") PER PARTITION LIMIT 1"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-01"), 1, 4, 4L), - row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate GROUP BY pk, floor(time, 1mo" + startingTime + ") PER PARTITION LIMIT 1"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-01"), 1, 4, 4L), + row(2, toLocalDate("2016-11-01"), 10, 11, 2L)); - stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-11-01"), 8, 8, 1L), - row(1, toLocalDate("2016-10-01"), 5, 7, 3L), - row(1, toLocalDate("2016-09-01"), 1, 4, 4L)); + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L), + row(1, toLocalDate("2016-09-01"), 1, 4, 4L)); - stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC LIMIT 2"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-11-01"), 8, 8, 1L), - row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); - } + stmt = "SELECT pk, floor(time, 1mo" + startingTime + "), min(v), max(v), count(v) FROM %s.testWithDate WHERE pk = 1 GROUP BY pk, floor(time, 1mo" + startingTime + ") ORDER BY time DESC LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-11-01"), 8, 8, 1L), + row(1, toLocalDate("2016-10-01"), 5, 7, 3L)); } } } @@ -280,55 +359,51 @@ public class GroupByTest extends TestBaseImpl @Test public void testGroupByTimeRangesWithTimeType() throws Throwable { - try (Cluster cluster = init(builder().withNodes(3).start())) + cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithTime (pk int, date date, time time, v int, primary key (pk, date, time))")); + + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:10:00', 1)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:12:00', 2)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:14:00', 3)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:15:00', 4)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:21:00', 5)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:22:00', 6)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:00', 7)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:20', 8)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:20', 9)"), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:30', 10)"), ConsistencyLevel.QUORUM); + + for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.testWithTime (pk int, date date, time time, v int, primary key (pk, date, time))")); + String stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime GROUP BY pk, date, floor(time, 5m)"; + Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L)); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:10:00', 1)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:12:00', 2)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:14:00', 3)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:15:00', 4)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:21:00', 5)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:22:00', 6)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:00', 7)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-27', '16:26:20', 8)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:20', 9)"), ConsistencyLevel.QUORUM); - cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.testWithTime (pk, date, time, v) VALUES (1, '2016-09-28', '16:26:30', 10)"), ConsistencyLevel.QUORUM); + stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime GROUP BY pk, date, floor(time, 5m) LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L)); - for (int pageSize : new int[] {2, 3, 4, 5, 7, 10}) - { + stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), + row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L)); - String stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime GROUP BY pk, date, floor(time, 5m)"; - Iterator pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), - row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), - row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), - row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), - row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L)); - - stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime GROUP BY pk, date, floor(time, 5m) LIMIT 2"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L), - row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L)); - - stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), - row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L), - row(1, toLocalDate("2016-09-27"), toTime("16:20:00"), 5, 6, 2L), - row(1, toLocalDate("2016-09-27"), toTime("16:15:00"), 4, 4, 1L), - row(1, toLocalDate("2016-09-27"), toTime("16:10:00"), 1, 3, 3L)); - - stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC LIMIT 2"; - pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); - assertRows(pagingRows, - row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), - row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L)); - } + stmt = "SELECT pk, date, floor(time, 5m), min(v), max(v), count(v) FROM %s.testWithTime WHERE pk = 1 GROUP BY pk, date, floor(time, 5m) ORDER BY date DESC, time DESC LIMIT 2"; + pagingRows = cluster.coordinator(1).executeWithPaging(withKeyspace(stmt), QUORUM, pageSize); + assertRows(pagingRows, + row(1, toLocalDate("2016-09-28"), toTime("16:25:00"), 9, 10, 2L), + row(1, toLocalDate("2016-09-27"), toTime("16:25:00"), 7, 8, 2L)); } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairEmptyRangeTombstonesTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairEmptyRangeTombstonesTestBase.java index fa066e8b37..d3ed8d0d7b 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairEmptyRangeTombstonesTestBase.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairEmptyRangeTombstonesTestBase.java @@ -31,6 +31,7 @@ import org.junit.runners.Parameterized; import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import static org.apache.cassandra.distributed.shared.AssertUtils.row; @@ -70,15 +71,19 @@ public abstract class ReadRepairEmptyRangeTombstonesTestBase extends TestBaseImp @Parameterized.Parameter(3) public boolean reverse; - @Parameterized.Parameters(name = "{index}: strategy={0} coordinator={1} paging={2} reverse={3}") + @Parameterized.Parameter(4) + public ReplicationType replicationType; + + @Parameterized.Parameters(name = "{index}: strategy={0} coordinator={1} paging={2} reverse={3} replication={4}") public static Collection data() { List result = new ArrayList<>(); for (int coordinator = 1; coordinator <= NUM_NODES; coordinator++) for (boolean paging : BOOLEANS) for (boolean reverse : BOOLEANS) - result.add(new Object[]{ ReadRepairStrategy.BLOCKING, coordinator, paging, reverse }); - result.add(new Object[]{ ReadRepairStrategy.NONE, 1, false, false }); + for (ReplicationType replication : ReplicationType.fixmeValues()) + result.add(new Object[]{ ReadRepairStrategy.BLOCKING, coordinator, paging, reverse, replication }); + result.add(new Object[]{ ReadRepairStrategy.NONE, 1, false, false, ReplicationType.untracked }); return result; } @@ -250,14 +255,14 @@ public abstract class ReadRepairEmptyRangeTombstonesTestBase extends TestBaseImp private Tester tester() { - return new Tester(cluster, strategy, coordinator, flush(), paging, reverse); + return new Tester(cluster, strategy, coordinator, flush(), paging, reverse, replicationType); } private static class Tester extends ReadRepairTester { - private Tester(Cluster cluster, ReadRepairStrategy strategy, int coordinator, boolean flush, boolean paging, boolean reverse) + private Tester(Cluster cluster, ReadRepairStrategy strategy, int coordinator, boolean flush, boolean paging, boolean reverse, ReplicationType replicationType) { - super(cluster, strategy, coordinator, flush, paging, reverse); + super(cluster, strategy, coordinator, flush, paging, reverse, replicationType); } @Override diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairInQueriesTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairInQueriesTest.java index 0e34d1821c..8a7adeaf08 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairInQueriesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairInQueriesTest.java @@ -18,8 +18,11 @@ package org.apache.cassandra.distributed.test; +import org.junit.Before; import org.junit.Test; +import org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils; + import static org.apache.cassandra.distributed.shared.AssertUtils.row; /** @@ -27,6 +30,13 @@ import static org.apache.cassandra.distributed.shared.AssertUtils.row; */ public class ReadRepairInQueriesTest extends ReadRepairQueryTester { + + @Before + public void setup() + { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType, "Fix read repair for logged IN queries"); + } + /** * Test queries with an IN restriction on a table without clustering columns. */ diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairQueryTester.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairQueryTester.java index 1524513816..447bdbde88 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairQueryTester.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairQueryTester.java @@ -29,6 +29,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import static org.apache.cassandra.distributed.shared.AssertUtils.assertEquals; @@ -97,15 +98,19 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl @Parameterized.Parameter(3) public boolean paging; - @Parameterized.Parameters(name = "{index}: strategy={0} coordinator={1} flush={2} paging={3}") + @Parameterized.Parameter(4) + public ReplicationType replicationType; + + @Parameterized.Parameters(name = "{index}: strategy={0} coordinator={1} flush={2} paging={3} replication={4}") public static Collection data() { List result = new ArrayList<>(); for (int coordinator = 1; coordinator <= NUM_NODES; coordinator++) for (boolean flush : BOOLEANS) for (boolean paging : BOOLEANS) - result.add(new Object[]{ ReadRepairStrategy.BLOCKING, coordinator, flush, paging }); - result.add(new Object[]{ ReadRepairStrategy.NONE, 1, false, false }); + for (ReplicationType replication : ReplicationType.values()) + result.add(new Object[]{ ReadRepairStrategy.BLOCKING, coordinator, flush, paging, replication }); + result.add(new Object[]{ ReadRepairStrategy.NONE, 1, false, false, ReplicationType.untracked }); return result; } @@ -118,7 +123,6 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl .withConfig(config -> config.set("read_request_timeout", "1m") .set("write_request_timeout", "1m")) .start()); - cluster.schemaChange(withKeyspace("CREATE TYPE %s.udt (x int, y int)")); } @AfterClass @@ -130,7 +134,7 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl protected Tester tester(String restriction) { - return new Tester(restriction, cluster, strategy, coordinator, flush, paging); + return new Tester(restriction, cluster, strategy, coordinator, flush, paging, replicationType); } protected static class Tester extends ReadRepairTester @@ -138,9 +142,9 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl private final String restriction; // the tested CQL query WHERE restriction private final String allColumnsQuery; // a SELECT * query for the table using the tested restriction - Tester(String restriction, Cluster cluster, ReadRepairStrategy strategy, int coordinator, boolean flush, boolean paging) + Tester(String restriction, Cluster cluster, ReadRepairStrategy strategy, int coordinator, boolean flush, boolean paging, ReplicationType replicationType) { - super(cluster, strategy, coordinator, flush, paging, false); + super(cluster, strategy, coordinator, flush, paging, false, replicationType); this.restriction = restriction; allColumnsQuery = String.format("SELECT * FROM %s %s", qualifiedTableName, restriction); @@ -173,6 +177,32 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl { // query only the selected columns with CL=ALL to trigger partial read repair on that column String columnsQuery = String.format("SELECT %s FROM %s %s", columns, qualifiedTableName, restriction); + + if (replicationType.isTracked()) + { + // for tracked replication, entire mutations will be replicated, so unlike untracked read repair we'd + // expect the node that missed writes to be completely up to date with the node that was last written to. + // So here we adjust expected rows for tracked replication + switch (lastMutatedNode) + { + case 1: + node2Rows = node1Rows; + break; + case 2: + node1Rows = node2Rows; + break; + default: + throw new AssertionError("Unhandled lastMutatedNode value: " + lastMutatedNode); + } + + // we also expect all pending mutations to be reconciled in the initial read, and none to be reconciled on the verification step + columnsQueryRepairedRows = Math.max(columnsQueryRepairedRows, rowsQueryRepairedRows); + rowsQueryRepairedRows = 0; + + // and all missing mutations should be reconciled as part of a single reconciliation, not one per-partition + columnsQueryRepairedRows = Math.min(columnsQueryRepairedRows, 1); + } + assertRowsDistributed(columnsQuery, columnsQueryRepairedRows, columnsQueryResults); // query entire rows to repair the rest of the columns, that might trigger new repairs for those columns @@ -223,6 +253,12 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl Tester deleteRows(String rowDeletion, long repairedRows, Object[][] node1Rows, Object[][] node2Rows) { mutate(1, rowDeletion); + + // for range reads we still expect all missing mutations to be reconciled as part + // of a single reconciliation operation + if (replicationType.isTracked()) + repairedRows = Math.min(repairedRows, 1); + return verifyQuery(allColumnsQuery, repairedRows, node1Rows, node2Rows); } @@ -259,9 +295,32 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl * Verifies the final status of the nodes with an unrestricted query, to ensure that the main tested query * hasn't triggered any unexpected repairs. Then, it verifies that the node that hasn't been used as coordinator * hasn't triggered any unexpected repairs. Finally, it drops the table. + * + * The expectUnrepaired flag is meant for range query tests where logged replication table special casing + * doesn't apply since we do expect the final query to find and repair missing mutations */ - void tearDown(long repairedRows, Object[][] node1Rows, Object[][] node2Rows) + void tearDown(long repairedRows, Object[][] node1Rows, Object[][] node2Rows, boolean expectUnrepaired) { + if (replicationType.isTracked() && !expectUnrepaired) + { + // for tracked replication, entire mutations will be replicated, so unlike untracked read repair we'd + // expect the node that missed writes to be completely up to date with the node that was last written to. + // So here we adjust expected rows for tracked replication + switch (lastMutatedNode) + { + case 1: + node2Rows = node1Rows; + break; + case 2: + node1Rows = node2Rows; + break; + default: + throw new AssertionError("Unhandled lastMutatedNode value: " + lastMutatedNode); + } + + // we also expect all pending mutations to be reconciled in the initial read, and none to be reconciled on the verification step + repairedRows = 0; + } verifyQuery("SELECT * FROM " + qualifiedTableName, repairedRows, node1Rows, node2Rows); for (int n = 1; n <= cluster.size(); n++) { @@ -275,5 +334,10 @@ public abstract class ReadRepairQueryTester extends TestBaseImpl } schemaChange("DROP TABLE " + qualifiedTableName); } + + void tearDown(long repairedRows, Object[][] node1Rows, Object[][] node2Rows) + { + tearDown(repairedRows, node1Rows, node2Rows, false); + } } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairRangeQueriesTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairRangeQueriesTest.java index f9abb4505d..b2af1f938e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairRangeQueriesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairRangeQueriesTest.java @@ -20,10 +20,13 @@ package org.apache.cassandra.distributed.test; import org.junit.Test; +import org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils; + import static org.apache.cassandra.distributed.shared.AssertUtils.row; /** * {@link ReadRepairQueryTester} for range queries. + * */ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester { @@ -51,7 +54,8 @@ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester rows(row(2, null, 200), row(3, 30, 300))) .tearDown(1, rows(row(1, 10, 100), row(2, null, 200)), - rows(row(2, null, 200))); + rows(row(2, null, 200)), + true); } /** @@ -84,7 +88,8 @@ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester rows(row(2, 20, 200, 2000), row(3, 30, 300, 3000))) .tearDown(1, rows(row(1, 10, 100, 1000), row(3, 30, 300, 3000)), - rows(row(3, 30, 300, 3000))); + rows(row(3, 30, 300, 3000)), + true); } /** @@ -151,6 +156,7 @@ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester @Test public void testRangeQueryWithFilterOnSelectedColumnOnSkinnyTable() { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType, "uses ALLOW FILTERING (CASSANDRA-20555)"); tester("WHERE a=2 ALLOW FILTERING") .createTable("CREATE TABLE %s (k int PRIMARY KEY, a int, b int)") .mutate("INSERT INTO %s (k, a, b) VALUES (1, 2, 3)", @@ -177,6 +183,7 @@ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester @Test public void testRangeQueryWithFilterOnSelectedColumnOnWideTable() { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType, "uses ALLOW FILTERING (CASSANDRA-20555)"); tester("WHERE a=1 ALLOW FILTERING") .createTable("CREATE TABLE %s (k int, c int, a int, b int, PRIMARY KEY(k, c))") .mutate("INSERT INTO %s (k, c, a, b) VALUES (1, 1, 1, 1)", @@ -208,6 +215,8 @@ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester @Test public void testRangeQueryWithFilterOnUnselectedColumnOnSkinnyTable() { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType, "uses ALLOW FILTERING (CASSANDRA-20555)"); + tester("WHERE b=3 ALLOW FILTERING") .createTable("CREATE TABLE %s (k int PRIMARY KEY, a int, b int)") .mutate("INSERT INTO %s (k, a, b) VALUES (1, 2, 3)", @@ -234,6 +243,8 @@ public class ReadRepairRangeQueriesTest extends ReadRepairQueryTester @Test public void testRangeQueryWithFilterOnUnselectedColumnOnWideTable() { + if (coordinator == 2) + MutationTrackingUtils.fixmeSkipIfTracked(replicationType, "Depends on ALLOW FILTERING"); tester("WHERE b=2 ALLOW FILTERING") .createTable("CREATE TABLE %s (k int, c int, a int, b int, PRIMARY KEY(k, c))") .mutate("INSERT INTO %s (k, c, a, b) VALUES (1, 1, 1, 1)", diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairSliceQueriesTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairSliceQueriesTest.java index 62f3702d50..58e9ecca11 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairSliceQueriesTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairSliceQueriesTest.java @@ -20,6 +20,8 @@ package org.apache.cassandra.distributed.test; import org.junit.Test; +import org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils; + import static org.apache.cassandra.distributed.shared.AssertUtils.row; /** @@ -62,6 +64,7 @@ public class ReadRepairSliceQueriesTest extends ReadRepairQueryTester @Test public void testSliceQueryWithFilter() { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType, "Depends on ALLOW FILTERING"); tester("WHERE k=0 AND a>10 AND a<40 ALLOW FILTERING") .createTable("CREATE TABLE %s (k int, c int, a int, b int, PRIMARY KEY(k, c))") .mutate("INSERT INTO %s (k, c, a, b) VALUES (0, 1, 10, 100)", diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java similarity index 67% rename from test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java rename to test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java index 14b81dc9ac..009399beb5 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTestBase.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -57,14 +58,17 @@ import org.apache.cassandra.distributed.api.IMessageFilters.Filter; import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.NetworkTopology; import org.apache.cassandra.distributed.test.accord.AccordTestBase; +import org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.locator.ReplicaPlan; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.service.reads.repair.BlockingReadRepair; import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; import org.apache.cassandra.utils.concurrent.Condition; import org.apache.cassandra.utils.concurrent.Future; +import static java.lang.String.format; import static net.bytebuddy.matcher.ElementMatchers.named; import static org.apache.cassandra.config.CassandraRelevantProperties.ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT; import static org.apache.cassandra.db.Keyspace.open; @@ -73,6 +77,9 @@ import static org.apache.cassandra.distributed.api.ConsistencyLevel.QUORUM; import static org.apache.cassandra.distributed.shared.AssertUtils.assertEquals; import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; import static org.apache.cassandra.distributed.shared.AssertUtils.row; +import static org.apache.cassandra.net.Verb.READ_RECONCILE_NOTIFY; +import static org.apache.cassandra.net.Verb.READ_RECONCILE_RCV; +import static org.apache.cassandra.net.Verb.READ_RECONCILE_SEND; import static org.apache.cassandra.net.Verb.READ_REPAIR_REQ; import static org.apache.cassandra.net.Verb.READ_REPAIR_RSP; import static org.apache.cassandra.net.Verb.READ_REQ; @@ -81,20 +88,32 @@ import static org.apache.cassandra.utils.concurrent.Condition.newOneTimeConditio import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -public class ReadRepairTest extends TestBaseImpl +public abstract class ReadRepairTestBase extends TestBaseImpl { - private static int tableNum = 0; - private String tableName; + protected abstract ReplicationType replicationType(); - private void incrementTableName() - { - tableName = "tbl" + tableNum++; - } + private static final AtomicInteger keyspaceIdx = new AtomicInteger(); + private String keyspaceName; + private String tableName; + private String qualifiedTableName; @Before public void setup() { - incrementTableName(); + keyspaceName = "ks_" + keyspaceIdx.getAndIncrement(); + tableName = "tbl"; + qualifiedTableName = keyspaceName + '.' + tableName; + } + + private void createKeyspace(Cluster cluster) + { + cluster.schemaChange(format("CREATE KEYSPACE %s WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor':%s} AND REPLICATION_TYPE='%s';", + keyspaceName, cluster.size(), replicationType())); + } + + private String withTable(String fmt) + { + return format(fmt, qualifiedTableName); } /** @@ -104,34 +123,32 @@ public class ReadRepairTest extends TestBaseImpl public void testBlockingReadRepair() throws Throwable { testReadRepair(ReadRepairStrategy.BLOCKING, false); - incrementTableName(); - testReadRepair(ReadRepairStrategy.BLOCKING, true); - } - /** - * - * Tests basic behaviour of read repair with {@code NONE} read repair strategy. - */ - @Test - public void testNoneReadRepair() throws Throwable - { - testReadRepair(ReadRepairStrategy.NONE); } - private void testReadRepair(ReadRepairStrategy strategy) throws Throwable + @Test + public void testBlockingReadRepairAccord() throws Throwable + { + testReadRepair(ReadRepairStrategy.BLOCKING, true); + } + + protected void testReadRepair(ReadRepairStrategy strategy) throws Throwable { testReadRepair(strategy, false); } - private void testReadRepair(ReadRepairStrategy strategy, boolean brrThroughAccord) throws Throwable { - try (Cluster cluster = init(Cluster.create(3, c -> c.with(Feature.GOSSIP, Feature.NETWORK)))) { + private void testReadRepair(ReadRepairStrategy strategy, boolean brrThroughAccord) throws Throwable + { + try (Cluster cluster = init(Cluster.create(3, c -> c.with(Feature.GOSSIP, Feature.NETWORK)))) + { + createKeyspace(cluster); TransactionalMode transactionalMode = brrThroughAccord ? TransactionalMode.test_unsafe_writes : TransactionalMode.off; - cluster.schemaChange(withKeyspace("CREATE TABLE %s." + tableName + " (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode.toString().toLowerCase() + '\'' + - String.format(" AND read_repair='%s'", strategy))); + cluster.schemaChange(withTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY (k, c)) WITH transactional_mode='" + transactionalMode.toString().toLowerCase() + '\'' + + format(" AND read_repair='%s'", strategy))); AccordTestBase.ensureTableIsAccordManaged(cluster, KEYSPACE, "t"); Object[] row = row(1, 1, 1); - String insertQuery = withKeyspace("INSERT INTO %s." + tableName + " (k, c, v) VALUES (?, ?, ?)"); - String selectQuery = withKeyspace("SELECT * FROM %s." + tableName + " WHERE k=1"); + String insertQuery = withTable("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)"); + String selectQuery = withTable("SELECT * FROM %s WHERE k=1"); // insert data in two nodes, simulating a quorum write that has missed one node cluster.get(1).executeInternal(insertQuery, row); @@ -159,16 +176,17 @@ public class ReadRepairTest extends TestBaseImpl { try (Cluster cluster = init(Cluster.create(3, c -> c.with(Feature.GOSSIP, Feature.NETWORK)))) { final long reducedReadTimeout = 3000L; + createKeyspace(cluster); cluster.forEach(i -> i.runOnInstance(() -> DatabaseDescriptor.setReadRpcTimeout(reducedReadTimeout))); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); - cluster.get(1).executeInternal("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1)"); - cluster.get(2).executeInternal("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1)"); - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1")); - cluster.verbs(READ_REPAIR_RSP).to(1).drop(); + cluster.schemaChange(withTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'")); + cluster.get(1).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)")); + cluster.get(2).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)")); + assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1"))); + cluster.verbs(replicationType().isTracked() ? READ_RECONCILE_NOTIFY : READ_REPAIR_RSP).to(1).drop(); final long start = currentTimeMillis(); try { - cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1", ConsistencyLevel.ALL); + cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk = 1"), ConsistencyLevel.ALL); fail("Read timeout expected but it did not occur"); } catch (Exception ex) @@ -181,7 +199,7 @@ public class ReadRepairTest extends TestBaseImpl assertTrue(actualTimeTaken > reducedReadTimeout); // But it should not exceed too much assertTrue(actualTimeTaken < reducedReadTimeout + magicDelayAmount); - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1"), + assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1")), row(1, 1, 1)); // the partition happened when the repaired node sending back ack. The mutation should be in fact applied. } } @@ -192,33 +210,37 @@ public class ReadRepairTest extends TestBaseImpl { try (Cluster cluster = init(builder().withNodes(3).start())) { - cluster.schemaChange("CREATE TABLE " + KEYSPACE + "." + tableName + " (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); + createKeyspace(cluster); + cluster.schemaChange(withTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'")); for (int i = 1 ; i <= 2 ; ++i) - cluster.get(i).executeInternal("INSERT INTO " + KEYSPACE + "." + tableName + " (pk, ck, v) VALUES (1, 1, 1)"); + cluster.get(i).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (1, 1, 1)")); - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1")); + assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1"))); cluster.filters().verbs(READ_REPAIR_REQ.id).to(3).drop(); - assertRows(cluster.coordinator(1).execute("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1", - ConsistencyLevel.QUORUM), + cluster.filters().verbs(READ_RECONCILE_SEND.id).to(3).drop(); + cluster.filters().verbs(READ_RECONCILE_RCV.id).to(3).drop(); + assertRows(cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk = 1"), ConsistencyLevel.QUORUM), row(1, 1, 1)); // Data was not repaired - assertRows(cluster.get(3).executeInternal("SELECT * FROM " + KEYSPACE + "." + tableName + " WHERE pk = 1")); + assertRows(cluster.get(3).executeInternal(withTable("SELECT * FROM %s WHERE pk = 1"))); } } @Test @Ignore public void movingTokenReadRepairTest() throws Throwable { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType(), "Token moves not supported"); // TODO: rewrite using FuzzTestBase to control progress through decommission // TODO: fails with vnode enabled try (Cluster cluster = init(Cluster.build(4).withoutVNodes().start(), 3)) { List tokens = cluster.tokens(); - cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'"); + createKeyspace(cluster); + cluster.schemaChange(withTable("CREATE TABLE %s (pk int, ck int, v int, PRIMARY KEY (pk, ck)) WITH read_repair='blocking'")); int i = 0; while (true) @@ -231,7 +253,7 @@ public class ReadRepairTest extends TestBaseImpl } // write only to #4 - cluster.get(4).executeInternal("INSERT INTO " + KEYSPACE + ".tbl (pk, ck, v) VALUES (?, 1, 1)", i); + cluster.get(4).executeInternal(withTable("INSERT INTO %s (pk, ck, v) VALUES (?, 1, 1)"), i); // mark #2 as leaving in #4 // cluster.get(4).acceptsOnInstance((InetSocketAddress endpoint) -> { //// StorageService.instance.getTokenMetadata().addLeavingEndpoint(InetAddressAndPort.getByAddressOverrideDefaults(endpoint.getAddress(), endpoint.getPort())); @@ -244,13 +266,22 @@ public class ReadRepairTest extends TestBaseImpl // (as a speculative repair in this case, as we prefer to send repair mutations to the initial // set of read replicas, which are 2 and 3 here). cluster.filters().verbs(READ_REQ.id).from(4).to(3).drop(); - cluster.filters().verbs(READ_REPAIR_REQ.id).from(4).to(3).drop(); - assertRows(cluster.coordinator(4).execute("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?", + if (replicationType().isTracked()) + { + cluster.filters().verbs(READ_RECONCILE_SEND.id).from(4).to(3).drop(); + cluster.filters().verbs(READ_RECONCILE_RCV.id).from(4).to(3).drop(); + } + else + { + cluster.filters().verbs(READ_REPAIR_REQ.id).from(4).to(3).drop(); + } + + assertRows(cluster.coordinator(4).execute(withTable("SELECT * FROM %s WHERE pk = ?"), ConsistencyLevel.QUORUM, i), row(i, 1, 1)); // verify that #1 receives the write - assertRows(cluster.get(1).executeInternal("SELECT * FROM " + KEYSPACE + ".tbl WHERE pk = ?", i), + assertRows(cluster.get(1).executeInternal(withTable("SELECT * FROM %s WHERE pk = ?"), i), row(i, 1, 1)); } } @@ -262,22 +293,25 @@ public class ReadRepairTest extends TestBaseImpl @Test public void alterRFAndRunReadRepair() throws Throwable { + MutationTrackingUtils.fixmeSkipIfTracked(replicationType(), "RF changes not supported"); try (Cluster cluster = builder().withNodes(2).start()) { - cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " + - "{'class': 'SimpleStrategy', 'replication_factor': 1}")); - cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int PRIMARY KEY, a int, b int)" + + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 1} " + + "AND replication_type='%s'", + keyspaceName, replicationType().toString())); + cluster.schemaChange(withTable("CREATE TABLE %s (k int PRIMARY KEY, a int, b int)" + " WITH read_repair='blocking'")); // insert a row that will only get to one node due to the RF=1 Object[] row = row(1, 1, 1); - cluster.get(1).executeInternal(withKeyspace("INSERT INTO %s.t (k, a, b) VALUES (?, ?, ?)"), row); + cluster.get(1).executeInternal(withTable("INSERT INTO %s (k, a, b) VALUES (?, ?, ?)"), row); // flush to ensure reads come from sstables - cluster.get(1).flush(KEYSPACE); + cluster.get(1).flush(keyspaceName); // at RF=1 it shouldn't matter which node we query, as the data should always come from the only replica - String query = withKeyspace("SELECT * FROM %s.t WHERE k = 1"); + String query = withTable("SELECT * FROM %s WHERE k = 1"); for (int i = 1; i <= cluster.size(); i++) assertRows(cluster.coordinator(i).execute(query, ALL), row); @@ -287,8 +321,8 @@ public class ReadRepairTest extends TestBaseImpl // alter RF ALLOW_ALTER_RF_DURING_RANGE_MOVEMENT.setBoolean(true); - cluster.schemaChange(withKeyspace("ALTER KEYSPACE %s WITH replication = " + - "{'class': 'SimpleStrategy', 'replication_factor': 2}")); + cluster.schemaChange(format("ALTER KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 2}", keyspaceName)); // altering the RF shouldn't have triggered any read repair assertRows(cluster.get(1).executeInternal(query), row); @@ -324,12 +358,13 @@ public class ReadRepairTest extends TestBaseImpl { try (Cluster cluster = init(Cluster.create(2))) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, v int, PRIMARY KEY(k, c))")); + createKeyspace(cluster); + cluster.schemaChange(withTable("CREATE TABLE %s (k int, c int, v int, PRIMARY KEY(k, c))")); ICoordinator coordinator = cluster.coordinator(1); // insert some rows in all nodes - String insertQuery = withKeyspace("INSERT INTO %s.t (k, c, v) VALUES (?, ?, ?)"); + String insertQuery = withTable("INSERT INTO %s (k, c, v) VALUES (?, ?, ?)"); for (int k = 0; k < 10; k++) { for (int c = 0; c < 10; c++) @@ -337,14 +372,14 @@ public class ReadRepairTest extends TestBaseImpl } // delete a subset of the inserted partitions, plus some others that don't exist - String deletePartitionQuery = withKeyspace("DELETE FROM %s.t WHERE k = ?"); + String deletePartitionQuery = withTable("DELETE FROM %s WHERE k = ?"); for (int k = 5; k < 15; k++) { coordinator.execute(deletePartitionQuery, ALL, k); } // delete some of the rows of some of the partitions, including deleted and not deleted partitions - String deleteRowQuery = withKeyspace("DELETE FROM %s.t WHERE k = ? AND c = ?"); + String deleteRowQuery = withTable("DELETE FROM %s WHERE k = ? AND c = ?"); for (int k = 2; k < 7; k++) { for (int c = 0; c < 5; c++) @@ -362,22 +397,22 @@ public class ReadRepairTest extends TestBaseImpl if (flush) { for (int n = 1; n <= cluster.size(); n++) - cluster.get(n).flush(KEYSPACE); + cluster.get(n).flush(keyspaceName); } // run a bunch of queries verifying that they don't trigger read repair - coordinator.execute(withKeyspace("SELECT * FROM %s.t LIMIT 100"), QUORUM); + coordinator.execute(withTable("SELECT * FROM %s LIMIT 100"), QUORUM); for (int k = 0; k < 15; k++) { - coordinator.execute(withKeyspace("SELECT * FROM %s.t WHERE k=?"), QUORUM, k); + coordinator.execute(withTable("SELECT * FROM %s WHERE k=?"), QUORUM, k); for (int c = 0; c < 10; c++) { - coordinator.execute(withKeyspace("SELECT * FROM %s.t WHERE k=? AND c=?"), QUORUM, k, c); - coordinator.execute(withKeyspace("SELECT * FROM %s.t WHERE k=? AND c>?"), QUORUM, k, c); - coordinator.execute(withKeyspace("SELECT * FROM %s.t WHERE k=? AND c?"), QUORUM, k, c); + coordinator.execute(withTable("SELECT * FROM %s WHERE k=? AND c config.with(Feature.GOSSIP, Feature.NETWORK) - .set("read_request_timeout", String.format("%dms", Integer.MAX_VALUE)) - .set("native_transport_timeout", String.format("%dms", Integer.MAX_VALUE)) + .set("read_request_timeout", format("%dms", Integer.MAX_VALUE)) + .set("native_transport_timeout", format("%dms", Integer.MAX_VALUE)) ) .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(4)) .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(4, "dc0", "rack0")) .withNodes(3) .start())) { - cluster.schemaChange("CREATE TABLE distributed_test_keyspace.tbl (\n" + - " key text,\n" + - " column1 int,\n" + - " PRIMARY KEY (key, column1)\n" + - ") WITH CLUSTERING ORDER BY (column1 ASC)"); + createKeyspace(cluster); + cluster.schemaChange(withTable("CREATE TABLE %s (\n" + + " key text,\n" + + " column1 int,\n" + + " PRIMARY KEY (key, column1)\n" + + ") WITH CLUSTERING ORDER BY (column1 ASC)")); - cluster.forEach(i -> i.runOnInstance(() -> open(KEYSPACE).getColumnFamilyStore("tbl").disableAutoCompaction())); + { + String ks = keyspaceName; + String tbl = tableName; + cluster.forEach(i -> i.runOnInstance(() -> open(ks).getColumnFamilyStore(tbl).disableAutoCompaction())); + } for (int i = 1; i <= 2; i++) { - cluster.get(i).executeInternal("DELETE FROM distributed_test_keyspace.tbl USING TIMESTAMP 50 WHERE key=?;", key); - cluster.get(i).executeInternal("DELETE FROM distributed_test_keyspace.tbl USING TIMESTAMP 80 WHERE key=? and column1 >= ? and column1 < ?;", key, 10, 100); - cluster.get(i).executeInternal("DELETE FROM distributed_test_keyspace.tbl USING TIMESTAMP 70 WHERE key=? and column1 = ?;", key, 30); - cluster.get(i).flush(KEYSPACE); + cluster.get(i).executeInternal(withTable("DELETE FROM %s USING TIMESTAMP 50 WHERE key=?;"), key); + cluster.get(i).executeInternal(withTable("DELETE FROM %s USING TIMESTAMP 80 WHERE key=? and column1 >= ? and column1 < ?;"), key, 10, 100); + cluster.get(i).executeInternal(withTable("DELETE FROM %s USING TIMESTAMP 70 WHERE key=? and column1 = ?;"), key, 30); + cluster.get(i).flush(keyspaceName); } - cluster.get(3).executeInternal("DELETE FROM distributed_test_keyspace.tbl USING TIMESTAMP 100 WHERE key=?;", key); - cluster.get(3).flush(KEYSPACE); + cluster.get(3).executeInternal(withTable("DELETE FROM %s USING TIMESTAMP 100 WHERE key=?;"), key); + cluster.get(3).flush(keyspaceName); // pause the read until we have bootstrapped a new node below Condition continueRead = newOneTimeCondition(); @@ -433,7 +472,7 @@ public class ReadRepairTest extends TestBaseImpl return false; }).drop(); - String query = "SELECT * FROM distributed_test_keyspace.tbl WHERE key=? and column1 >= ? and column1 <= ?"; + String query = withTable("SELECT * FROM %s WHERE key=? and column1 >= ? and column1 <= ?"); Future read = es.submit(() -> cluster.coordinator(3).execute(query, ALL, key, 20, 40)); read.addCallback(new FutureCallback() { @@ -486,34 +525,35 @@ public class ReadRepairTest extends TestBaseImpl { try (Cluster cluster = init(Cluster.create(2))) { - cluster.schemaChange(withKeyspace("CREATE TABLE %s.t (k int, c int, PRIMARY KEY(k, c)) " + + createKeyspace(cluster); + cluster.schemaChange(withTable("CREATE TABLE %s (k int, c int, PRIMARY KEY(k, c)) " + "WITH gc_grace_seconds=0 AND compaction = " + "{'class': 'SizeTieredCompactionStrategy', 'enabled': 'false'}")); ICoordinator coordinator = cluster.coordinator(1); // insert some data - coordinator.execute(withKeyspace("INSERT INTO %s.t(k, c) VALUES (0, 0)"), ALL); - coordinator.execute(withKeyspace("INSERT INTO %s.t(k, c) VALUES (1, 1)"), ALL); + coordinator.execute(withTable("INSERT INTO %s (k, c) VALUES (0, 0)"), ALL); + coordinator.execute(withTable("INSERT INTO %s (k, c) VALUES (1, 1)"), ALL); // create partition tombstones in all nodes for both existent and not existent partitions - coordinator.execute(withKeyspace("DELETE FROM %s.t WHERE k=0"), ALL); // exists - coordinator.execute(withKeyspace("DELETE FROM %s.t WHERE k=2"), ALL); // doesn't exist + coordinator.execute(withTable("DELETE FROM %s WHERE k=0"), ALL); // exists + coordinator.execute(withTable("DELETE FROM %s WHERE k=2"), ALL); // doesn't exist // create row tombstones in all nodes for both existent and not existent rows - coordinator.execute(withKeyspace("DELETE FROM %s.t WHERE k=1 AND c=1"), ALL); // exists - coordinator.execute(withKeyspace("DELETE FROM %s.t WHERE k=3 AND c=1"), ALL); // doesn't exist + coordinator.execute(withTable("DELETE FROM %s WHERE k=1 AND c=1"), ALL); // exists + coordinator.execute(withTable("DELETE FROM %s WHERE k=3 AND c=1"), ALL); // doesn't exist // flush single sstable with tombstones - cluster.get(1).flush(KEYSPACE); - cluster.get(2).flush(KEYSPACE); + cluster.get(1).flush(keyspaceName); + cluster.get(2).flush(keyspaceName); // purge tombstones from node2 with compaction (gc_grace_seconds=0) - cluster.get(2).forceCompact(KEYSPACE, "t"); + cluster.get(2).forceCompact(keyspaceName, tableName); // run an unrestricted range query verifying that it doesn't trigger read repair - coordinator.execute(withKeyspace("SELECT * FROM %s.t"), ALL); - long requests = ReadRepairTester.readRepairRequestsCount(cluster.get(1), "t"); + coordinator.execute(withTable("SELECT * FROM %s"), ALL); + long requests = ReadRepairTester.readRepairRequestsCount(cluster.get(1), keyspaceName, tableName); assertEquals("No read repair requests were expected, found " + requests, 0, requests); } } @@ -526,12 +566,13 @@ public class ReadRepairTest extends TestBaseImpl .withInstanceInitializer(RRHelper::install) .start())) { - cluster.schemaChange(withKeyspace("CREATE TABLE distributed_test_keyspace.tbl0 (pk bigint,ck bigint,value bigint, PRIMARY KEY (pk, ck)) WITH CLUSTERING ORDER BY (ck ASC) AND read_repair='blocking';")); + createKeyspace(cluster); + cluster.schemaChange(withTable("CREATE TABLE %s (pk bigint,ck bigint,value bigint, PRIMARY KEY (pk, ck)) WITH CLUSTERING ORDER BY (ck ASC) AND read_repair='blocking';")); long pk = 0L; - cluster.coordinator(1).execute("INSERT INTO distributed_test_keyspace.tbl0 (pk, ck, value) VALUES (?,?,?) USING TIMESTAMP 1", ConsistencyLevel.ALL, pk, 1L, 1L); - cluster.coordinator(1).execute("DELETE FROM distributed_test_keyspace.tbl0 USING TIMESTAMP 2 WHERE pk=? AND ck>?;", ConsistencyLevel.ALL, pk, 2L); - cluster.get(3).executeInternal("DELETE FROM distributed_test_keyspace.tbl0 USING TIMESTAMP 2 WHERE pk=?;", pk); - assertRows(cluster.coordinator(1).execute("SELECT * FROM distributed_test_keyspace.tbl0 WHERE pk=? AND ck>=? AND ck?;"), ConsistencyLevel.ALL, pk, 2L); + cluster.get(3).executeInternal(withTable("DELETE FROM %s USING TIMESTAMP 2 WHERE pk=?;"), pk); + assertRows(cluster.coordinator(1).execute(withTable("SELECT * FROM %s WHERE pk=? AND ck>=? AND ck> { private static final AtomicInteger seqNumber = new AtomicInteger(); - private final String tableName = "t_" + seqNumber.getAndIncrement(); - final String qualifiedTableName = KEYSPACE + '.' + tableName; + private final String keyspaceName = "ks_" + seqNumber.getAndIncrement(); + private static final String tableName = "tbl"; + final String qualifiedTableName = keyspaceName + '.' + tableName; protected final Cluster cluster; protected final ReadRepairStrategy strategy; @@ -53,8 +55,12 @@ public abstract class ReadRepairTester> protected final boolean paging; protected final boolean reverse; protected final int coordinator; + protected final ReplicationType replicationType; - ReadRepairTester(Cluster cluster, ReadRepairStrategy strategy, int coordinator, boolean flush, boolean paging, boolean reverse) + // logged replication test support + protected int lastMutatedNode = -1; + + ReadRepairTester(Cluster cluster, ReadRepairStrategy strategy, int coordinator, boolean flush, boolean paging, boolean reverse, ReplicationType replicationType) { this.cluster = cluster; this.strategy = strategy; @@ -62,6 +68,7 @@ public abstract class ReadRepairTester> this.paging = paging; this.reverse = reverse; this.coordinator = coordinator; + this.replicationType = replicationType; } abstract T self(); @@ -76,6 +83,9 @@ public abstract class ReadRepairTester> T createTable(String createTable) { + cluster.schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION={'class': 'SimpleStrategy', 'replication_factor': " + cluster.size() + "} AND replication_type='%s'", + keyspaceName, replicationType.name())); + cluster.schemaChange(String.format("CREATE TYPE IF NOT EXISTS %s.udt (x int, y int)", keyspaceName)); String query; switch (StringUtils.countMatches(createTable, "%s")) { @@ -100,6 +110,7 @@ public abstract class ReadRepairTester> */ T mutate(int node, String... queries) { + lastMutatedNode = node; // run the write queries only on one node for (String query : queries) cluster.get(node).executeInternal(String.format(query, qualifiedTableName)); @@ -150,13 +161,13 @@ public abstract class ReadRepairTester> long readRepairRequestsCount(int node) { - return readRepairRequestsCount(cluster.get(node), tableName); + return readRepairRequestsCount(cluster.get(node), keyspaceName, tableName); } - static long readRepairRequestsCount(IInvokableInstance node, String table) + static long readRepairRequestsCount(IInvokableInstance node, String keyspace, String table) { return node.callOnInstance(() -> { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(table); + ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); return cfs.metric.readRepairRequests.getCount(); }); } diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTrackedTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTrackedTest.java new file mode 100644 index 0000000000..054a4e7d6c --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTrackedTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import org.apache.cassandra.schema.ReplicationType; + +public class ReadRepairTrackedTest extends ReadRepairTestBase +{ + @Override + protected ReplicationType replicationType() + { + return ReplicationType.tracked; + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairUntrackedTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairUntrackedTest.java new file mode 100644 index 0000000000..5fc9f209d2 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairUntrackedTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import org.junit.Test; + +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.service.reads.repair.ReadRepairStrategy; + +public class ReadRepairUntrackedTest extends ReadRepairTestBase +{ + @Override + protected ReplicationType replicationType() + { + return ReplicationType.untracked; + } + + /** + * + * Tests basic behaviour of read repair with {@code NONE} read repair strategy. + */ + @Test + public void testNoneReadRepair() throws Throwable + { + testReadRepair(ReadRepairStrategy.NONE); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReplicaFilteringProtectionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReplicaFilteringProtectionTest.java index fd8110cba7..fb7753d06a 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReplicaFilteringProtectionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReplicaFilteringProtectionTest.java @@ -31,6 +31,7 @@ import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.exceptions.OverloadedException; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.service.reads.ReplicaFilteringProtection; import static org.apache.cassandra.config.ReplicaFilteringProtectionOptions.DEFAULT_FAIL_THRESHOLD; import static org.apache.cassandra.config.ReplicaFilteringProtectionOptions.DEFAULT_WARN_THRESHOLD; @@ -40,7 +41,7 @@ import static org.apache.cassandra.distributed.shared.AssertUtils.row; import static org.junit.Assert.assertEquals; /** - * Exercises the functionality of {@link org.apache.cassandra.service.reads.ReplicaFilteringProtection}, the + * Exercises the functionality of {@link ReplicaFilteringProtection}, the * mechanism that ensures distributed index and filtering queries at read consistency levels > ONE/LOCAL_ONE * avoid stale replica results. */ diff --git a/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java b/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java index 40458a64f5..21b4d970d6 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ResourceLeakTest.java @@ -146,7 +146,7 @@ public class ResourceLeakTest extends TestBaseImpl void dumpResources(String description) throws IOException, InterruptedException { - dumpHeap(description, false); + dumpHeap(description, true); if (dumpFileHandles) { dumpOpenFiles(description); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java index 559f897510..c3ec8a68cc 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ShortReadProtectionTest.java @@ -43,6 +43,7 @@ import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.shared.AssertUtils; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.service.consensus.TransactionalMode; import org.apache.cassandra.utils.ByteBufferUtil; @@ -88,7 +89,10 @@ public class ShortReadProtectionTest extends TestBaseImpl @Parameterized.Parameter(3) public TransactionalMode transactionalMode; - @Parameterized.Parameters(name = "{index}: read_cl={0} flush={1} paging={2}, transactionalMode={3}") + @Parameterized.Parameter(4) + public ReplicationType replicationType; + + @Parameterized.Parameters(name = "{index}: read_cl={0} flush={1} paging={2}, transactionalMode={3}, replication={4}") public static Collection data() { List result = new ArrayList<>(); @@ -96,7 +100,8 @@ public class ShortReadProtectionTest extends TestBaseImpl for (ConsistencyLevel readConsistencyLevel : Arrays.asList(ALL, QUORUM, SERIAL)) for (boolean flush : BOOLEANS) for (boolean paging : BOOLEANS) - result.add(new Object[]{ readConsistencyLevel, flush, paging, mode}); + for (ReplicationType replication : ReplicationType.values()) + result.add(new Object[]{ readConsistencyLevel, flush, paging, mode, replication}); return result; } @@ -124,7 +129,7 @@ public class ShortReadProtectionTest extends TestBaseImpl @Before public void setupTester() { - tester = new Tester(readConsistencyLevel, flush, paging, transactionalMode); + tester = new Tester(readConsistencyLevel, flush, paging, transactionalMode, replicationType); } @After @@ -162,7 +167,8 @@ public class ShortReadProtectionTest extends TestBaseImpl { tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)") .allNodes(0, 10, i -> format("INSERT INTO %%s (id) VALUES (%d) USING TIMESTAMP 0", i)) // order is 5,1,8,0,2,4,7,6,9,3 - .toNode1("DELETE FROM %s WHERE id IN (1, 0, 4, 6, 3)") // delete every other row +// .toNode1("DELETE FROM %s WHERE id IN (1, 0, 4, 6, 3)") // delete every other row + .toNode1(IntStream.of(1, 0, 4, 6, 3).mapToObj(k -> "DELETE FROM %s WHERE id=" + k).toArray(String[]::new)) // FIXME: revert once mutation tracking supports mutations with pk IN .assertRows("SELECT DISTINCT token(id), id FROM %s", row(token(5), 5), row(token(8), 8), row(token(2), 2), row(token(7), 7), row(token(9), 9)); } @@ -179,8 +185,10 @@ public class ShortReadProtectionTest extends TestBaseImpl { tester.createTable("CREATE TABLE %s (id int PRIMARY KEY)") .allNodes(0, 10, i -> format("INSERT INTO %%s (id) VALUES (%d) USING TIMESTAMP 0", i)) // order is 5,1,8,0,2,4,7,6,9,3 - .toNode1("DELETE FROM %s WHERE id IN (5, 8, 2, 7, 9)") // delete every other row - .toNode2("DELETE FROM %s WHERE id IN (1, 0, 4, 6)") // delete every other row but the last one +// .toNode1("DELETE FROM %s WHERE id IN (5, 8, 2, 7, 9)") // delete every other row + .toNode1(IntStream.of(5, 8, 2, 7, 9).mapToObj(k -> "DELETE FROM %s WHERE id=" + k).toArray(String[]::new)) // FIXME: revert once mutation tracking supports mutations with pk IN +// .toNode2("DELETE FROM %s WHERE id IN (1, 0, 4, 6)") // delete every other row but the last one + .toNode2(IntStream.of(1, 0, 4, 6).mapToObj(k -> "DELETE FROM %s WHERE id=" + k).toArray(String[]::new)) // FIXME: revert once mutation tracking supports mutations with pk IN .assertRows("SELECT id FROM %s LIMIT 1", row(3)) .assertRows("SELECT DISTINCT id FROM %s LIMIT 1", row(3)); } @@ -432,17 +440,21 @@ public class ShortReadProtectionTest extends TestBaseImpl private final String table; private final String qualifiedTableName; private final TransactionalMode transactionalMode; + private final ReplicationType replicationType; + private final String keyspaceName = "ks_" + seqNumber.getAndIncrement(); private boolean flushed = false; - private Tester(ConsistencyLevel readConsistencyLevel, boolean flush, boolean paging, TransactionalMode transactionalMode) + private Tester(ConsistencyLevel readConsistencyLevel, boolean flush, boolean paging, TransactionalMode transactionalMode, ReplicationType replicationType) { this.readConsistencyLevel = readConsistencyLevel; this.flush = flush; this.paging = paging; + this.transactionalMode = transactionalMode; + this.replicationType = replicationType; + this.table = "t_" + seqNumber.getAndIncrement(); qualifiedTableName = KEYSPACE + '.' + table; - this.transactionalMode = transactionalMode; assert readConsistencyLevel == ALL || readConsistencyLevel == QUORUM || readConsistencyLevel == SERIAL : "Only ALL and QUORUM consistency levels are supported"; @@ -450,6 +462,9 @@ public class ShortReadProtectionTest extends TestBaseImpl private Tester createTable(String query) { + cluster.schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH REPLICATION={'class': 'SimpleStrategy', 'replication_factor': " + NUM_NODES + "} AND replication_type='%s'", + keyspaceName, replicationType.name())); + String formattedQuery = format(query) + " WITH read_repair='NONE'"; if (transactionalMode != TransactionalMode.off) { diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java new file mode 100644 index 0000000000..7e17c9c557 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingPendingReadTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.distributed.test.tracking; + +import java.util.Collections; + +import com.google.common.collect.Iterables; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.partitions.PartitionIterator; +import org.apache.cassandra.db.rows.RowIterator; +import org.apache.cassandra.db.tracked.TrackedKeyspaceWriteHandler; +import org.apache.cassandra.replication.*; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.Util; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.partitions.ImmutableBTreePartition; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.reads.tracked.PartialTrackedRead; +import org.apache.cassandra.service.reads.tracked.TrackedLocalReadCoordinator; +import org.apache.cassandra.utils.FBUtilities; + +import static java.lang.String.format; +import static org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils.*; +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; + +public class MutationTrackingPendingReadTest +{ + private static final Logger logger = LoggerFactory.getLogger(MutationTrackingReadReconciliationTest.class); + + private static void assertKcvRow(ImmutableBTreePartition partition, ColumnFamilyStore cfs, int c, int v) + { + Row row = partition.getRow(Clustering.make(bytes(c))); + Assert.assertNotNull(row); + Cell cell = Util.cell(cfs, row, "v"); + Assert.assertEquals(bytes(v), cell.buffer()); + } + + private static void assertKcvRow(Row row, ColumnFamilyStore cfs, int c, int v) + { + Assert.assertNotNull(row); + Assert.assertEquals(bytes(c), row.clustering().bufferAt(0)); + Cell cell = Util.cell(cfs, row, "v"); + Assert.assertEquals(bytes(v), cell.buffer()); + } + + private static void assertNoKcvRow(ImmutableBTreePartition partition, int c) + { + Row row = partition.getRow(Clustering.make(bytes(c))); + Assert.assertNull(row); + } + + /** + * Tests that pending writes are included in read responses + */ + @Test + public void testPendingWriteInclusion() throws Throwable + { + + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + String keyspaceName = "pending_write_inclusion_test"; + String tableName = "tbl"; + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';", keyspaceName)); + + cluster.schemaChange(format("CREATE TABLE %s.%s (k int, c int, v int, primary key (k, c));", keyspaceName, tableName)); + + + // insert a row at all, confirm it's present on all nodes + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 0, 0)", keyspaceName, tableName), ConsistencyLevel.ALL); + + MutationSummary firstSummary = summaryForKey(cluster.get(1), keyspaceName, "tbl", 1); + CoordinatorLogId logId = firstSummary.get(0).logId(); + Offsets firstIds = summaryIdSpace(firstSummary.get(logId)); + Assert.assertEquals(1, firstIds.offsetCount()); + + cluster.forEach(node -> assertMatchingSummaryIdSpaceForKey(node, keyspaceName, "tbl", 1, firstSummary)); + + cluster.get(1).runOnInstance(() -> { + + TableMetadata metadata = Schema.instance.getTableMetadata(keyspaceName, tableName); + DecoratedKey dk = metadata.partitioner.decorateKey(bytes(1)); + + MutationSummary secondSummary = summaryForKey(keyspaceName, tableName, dk); + Offsets secondIds = summaryIdSpace(secondSummary.get(logId)); + Assert.assertEquals(1, secondIds.offsetCount()); + + // create a mutation + MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, dk.getToken()); + SimpleBuilders.MutationBuilder builder = new SimpleBuilders.MutationBuilder(id, keyspaceName, dk); + PartitionUpdate.SimpleBuilder tableBuilder = builder.update(metadata); + tableBuilder.row(bytes(1)).add("v", 1); + Mutation mutation = builder.build(); + MutationId secondId = mutation.id(); + Assert.assertFalse(secondId.isNone()); + + int nowInSeconds = (int) FBUtilities.nowInSeconds(); + // apply it to the journal and open a pending write + PartialTrackedRead read; + MutationSummary initialSummary; + MutationSummary secondarySummary; + + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(metadata, nowInSeconds, dk); + TrackedKeyspaceWriteHandler trackedWriteHandler = new TrackedKeyspaceWriteHandler(); + try (WriteContext ctx = trackedWriteHandler.beginWrite(mutation, true)) + { + + + initialSummary = command.createMutationSummary(false); + ReadExecutionController controller = command.executionController(false); + + MutationTrackingService.instance.startWriting(mutation); + + read = command.beginTrackedRead(controller); + // Create another summary once initial data has been read fully. We do this to catch + // any mutations that may have arrived during initial read execution. + secondarySummary = command.createMutationSummary(true); + TrackedLocalReadCoordinator.processDelta(read, initialSummary, secondarySummary); + } + + ColumnFamilyStore cfs = Keyspace.open(keyspaceName).getColumnFamilyStore(tableName); + // check that the memtable doesn't somehow contain the unapplied mutation + ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, dk)); + Assert.assertTrue(view.sstables.isEmpty()); + try (UnfilteredRowIterator rowIterator = Iterables.getOnlyElement(view.memtables).rowIterator(dk)) + { + ImmutableBTreePartition partition = ImmutableBTreePartition.create(rowIterator); + assertKcvRow(partition, cfs, 0, 0); + assertNoKcvRow(partition, 1); + } + + // confirm that the initial summary was not aware of the unapplied mutation + Offsets initialIds = summaryIdSpace(initialSummary.get(logId)); + Assert.assertEquals(1, initialIds.offsetCount()); + Assert.assertFalse(initialIds.contains(secondId.offset())); + + // check that the summary is aware of the unapplied mutation + Offsets summaryIds = summaryIdSpace(secondarySummary.get(logId)); + Assert.assertEquals(2, summaryIds.offsetCount()); + Assert.assertTrue(summaryIds.contains(secondId.offset())); + + // check that the returned data contains the unapplied mutation + try (PartialTrackedRead.CompletedRead completedRead = read.complete(); + PartitionIterator partitions = completedRead.iterator()) + { + Assert.assertTrue(partitions.hasNext()); + try (RowIterator rowIterator = partitions.next()) + { + Assert.assertTrue(rowIterator.hasNext()); + assertKcvRow(rowIterator.next(), cfs, 0, 0); + + Assert.assertTrue(rowIterator.hasNext()); + assertKcvRow(rowIterator.next(), cfs, 1, 1); + + Assert.assertFalse(rowIterator.hasNext()); + } + Assert.assertFalse(partitions.hasNext()); + } + }); + } + } + + /** + * Confirms that reads are notified of writes that come in while a read is inflight + */ + @Test + public void testPendingReadInclusion() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + String keyspaceName = "pending_read_inclusion_test"; + String tableName = "tbl"; + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';", keyspaceName)); + + cluster.schemaChange(format("CREATE TABLE %s.%s (k int, c int, v int, primary key (k, c));", keyspaceName, tableName)); + + // check that there aren't any mutations for the given key + cluster.forEach(node -> { + assertIdsForKey(node, keyspaceName, tableName, 1, Collections.emptySet()); + }); + + + cluster.get(1).runOnInstance(() -> { + TableMetadata metadata = Schema.instance.getTableMetadata(keyspaceName, tableName); + DecoratedKey dk = metadata.partitioner.decorateKey(bytes(1)); + + + int nowInSeconds = (int) FBUtilities.nowInSeconds(); + SinglePartitionReadCommand command = SinglePartitionReadCommand.fullPartitionRead(metadata, nowInSeconds, dk); +// try (ListeningPendingRead pendingRead = (ListeningPendingRead) MutationTrackingService.instance.startReading(command)) +// { +// Assert.assertTrue(pendingRead.mutationIds().isEmpty()); +// +// // create and apply a mutation +// MutationId id = MutationTrackingService.instance.nextMutationId(keyspaceName, dk.getToken()); +// SimpleBuilders.MutationBuilder builder = new SimpleBuilders.MutationBuilder(id, keyspaceName, dk); +// PartitionUpdate.SimpleBuilder tableBuilder = builder.update(metadata); +// tableBuilder.row(bytes(1)).add("v", 1); +// Mutation mutation = builder.build(); +// mutation.apply(); +// +// // the in flight read should be aware of the racing write +// Assert.assertEquals(Set.of(mutation.id()), pendingRead.mutationIds()); +// } + }); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingReadReconciliationTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingReadReconciliationTest.java new file mode 100644 index 0000000000..306d6e5092 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingReadReconciliationTest.java @@ -0,0 +1,390 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.gms.FailureDetector; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.replication.CoordinatorLogId; +import org.apache.cassandra.replication.MutationSummary; +import org.junit.Assert; +import org.junit.Test; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.replication.Offsets; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.schema.Schema; + +import static java.lang.String.format; +import static org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils.*; + +public class MutationTrackingReadReconciliationTest extends TestBaseImpl +{ + private static final Logger logger = LoggerFactory.getLogger(MutationTrackingReadReconciliationTest.class); + + public static void awaitNodeLiveness(IInvokableInstance from, InetAddressAndPort node, boolean alive) + { + from.runOnInstance(() -> { + for (int i=0; i<30; i++) + { + if (FailureDetector.instance.isAlive(node) == alive) + return; + + try + { + Thread.sleep(1000); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + throw new AssertionError(node + " wasn't marked " + (alive ? "alive" : "dead")); + + }); + } + + public static void awaitNodeDead(IInvokableInstance from, IInvokableInstance node) + { + awaitNodeLiveness(from, InetAddressAndPort.getByAddress(node.broadcastAddress()), false); + } + + public static void awaitNodeAlive(IInvokableInstance from, IInvokableInstance node) + { + awaitNodeLiveness(from, InetAddressAndPort.getByAddress(node.broadcastAddress()), true); + } + + /** + * Test a read reconciliation where the coordinator doesn't have a read response it needs to apply + * additional mutations to + */ + @Test + public void testBasicReadReconciliation() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + String keyspaceName = "basic_reconciliation_test"; + String tableName = "tbl"; + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';", keyspaceName)); + + cluster.forEach(node -> { + node.runOnInstance(() -> { + KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName); + Assert.assertEquals(ReplicationType.tracked, ksm.params.replicationType); + }); + }); + + cluster.schemaChange(format("CREATE TABLE %s.%s (k int, c int, v int, primary key (k, c));", keyspaceName, tableName)); + + // insert a row at ALL, confirm it's present on all nodes + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 0, 0)", keyspaceName, tableName), ConsistencyLevel.ALL); + + MutationSummary firstSummary = summaryForKey(cluster.get(1), keyspaceName, "tbl", 1); + CoordinatorLogId logId = firstSummary.get(0).logId(); + Offsets firstIds = summaryIdSpace(firstSummary.get(logId)); + Assert.assertEquals(1, firstIds.offsetCount()); + + cluster.get(2, 3).forEach(node -> assertMatchingSummaryIdSpaceForKey(node, keyspaceName, tableName, 1, firstSummary)); + + // block messages to node 3 and perform a write at quorum + cluster.filters().allVerbs().to(3).drop(); + cluster.filters().allVerbs().from(3).drop(); + awaitNodeDead(cluster.get(1), cluster.get(3)); + + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 1, 1)", keyspaceName, tableName), ConsistencyLevel.QUORUM); + + MutationSummary finalSummary = summaryForKey(cluster.get(1), keyspaceName, "tbl", 1); + Assert.assertEquals(1, finalSummary.size()); + Offsets finalIds = summaryIdSpace(finalSummary.get(logId)); + + Assert.assertEquals(2, finalIds.offsetCount()); + assertOffsetsIsSuperSet(finalIds, firstIds);; + + Offsets secondIds = Offsets.Immutable.difference(finalIds, firstIds); + Assert.assertEquals(1, secondIds.offsetCount()); + Assert.assertEquals(0, Offsets.Immutable.intersection(firstIds, secondIds).offsetCount()); + + + // second node should have the new id, third should not + assertMatchingSummaryIdSpaceForKey(cluster.get(2), keyspaceName, tableName, 1, finalSummary); + assertMatchingSummaryIdSpaceForKey(cluster.get(3), keyspaceName, tableName, 1, firstSummary); + + // reverse the partition and do a read + cluster.filters().reset(); + cluster.filters().allVerbs().to(2).drop(); + cluster.filters().allVerbs().from(2).drop(); + awaitNodeAlive(cluster.get(1), cluster.get(3)); + awaitNodeDead(cluster.get(1), cluster.get(2)); + + + Assert.assertEquals(0, numLogReconciliations(cluster.get(1))); + Object[][] result = cluster.coordinator(1).execute(format("SELECT * FROM %s.%s WHERE k=1", keyspaceName, tableName), ConsistencyLevel.QUORUM); + Assert.assertEquals(row(row(1, 0, 0), row(1, 1, 1)), result); + + // check that node3 has the new ids + assertMatchingSummaryIdSpaceForKey(cluster.get(3), keyspaceName, tableName, 1, finalSummary); + } + } + + /** + * Test a read reconciliation where the coordinator needs to receive and apply mutations missing + * from its data response + */ + @Test + public void testReadReconciliationApplyMutations() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + String keyspaceName = "basic_reconciliation_test"; + String tableName = "tbl"; + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';", keyspaceName)); + + cluster.forEach(node -> { + logger.info(">>> {}", node); + node.runOnInstance(() -> { + KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName); + Assert.assertEquals(ReplicationType.tracked, ksm.params.replicationType); + }); + }); + + cluster.schemaChange(format("CREATE TABLE %s.%s (k int, c int, v int, primary key (k, c));", keyspaceName, tableName)); + + // insert a row at all, confirm it's present on all nodes + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 0, 0)", keyspaceName, tableName), ConsistencyLevel.ALL); + MutationSummary firstSummary = summaryForKey(cluster.get(1), keyspaceName, "tbl", 1); + CoordinatorLogId logId = firstSummary.get(0).logId(); + Offsets firstIds = summaryIdSpace(firstSummary.get(logId)); + + cluster.get(2, 3).forEach(node -> { + assertMatchingSummaryIdSpaceForKey(node, keyspaceName, tableName, 1, firstSummary); + }); + + // block messages to node 3 and perform a write at quorum + cluster.filters().allVerbs().to(3).drop(); + cluster.filters().allVerbs().from(3).drop(); + + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 1, 1)", keyspaceName, tableName), ConsistencyLevel.QUORUM); + + MutationSummary finalSummary = summaryForKey(cluster.get(1), keyspaceName, "tbl", 1); + Assert.assertEquals(1, finalSummary.size()); + Offsets finalIds = summaryIdSpace(finalSummary.get(logId)); + + Assert.assertEquals(2, finalIds.offsetCount()); + assertOffsetsIsSuperSet(finalIds, firstIds);; + + Offsets secondIds = Offsets.Immutable.difference(finalIds, firstIds); + Assert.assertEquals(1, secondIds.offsetCount()); + Assert.assertEquals(0, Offsets.Immutable.intersection(firstIds, secondIds).offsetCount()); + + + // second node should have the new id, third should not + assertMatchingSummaryIdSpaceForKey(cluster.get(2), keyspaceName, tableName, 1, finalSummary); + assertMatchingSummaryIdSpaceForKey(cluster.get(3), keyspaceName, tableName, 1, firstSummary); + + // reverse the partition and do a read + cluster.filters().reset(); + cluster.filters().allVerbs().to(2).drop(); + cluster.filters().allVerbs().from(2).drop(); + + + Assert.assertEquals(0, numLogReconciliations(cluster.get(1))); + Object[][] result = cluster.coordinator(3).execute(format("SELECT * FROM %s.%s WHERE k=1", keyspaceName, tableName), ConsistencyLevel.QUORUM); + Assert.assertEquals(row(row(1, 0, 0), row(1, 1, 1)), result); + + // check that node3 has the new ids + assertMatchingSummaryIdSpaceForKey(cluster.get(3), keyspaceName, tableName, 1, finalSummary); + } + } + + /** + * Test a read reconciliation where the coordinator doesn't have a read response it needs to apply + * additional mutations to + */ + @Test + public void testBasicRangeReadReconciliation() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + String keyspaceName = "basic_reconciliation_test"; + String tableName = "tbl"; + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';", keyspaceName)); + + cluster.forEach(node -> { + node.runOnInstance(() -> { + KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName); + Assert.assertEquals(ReplicationType.tracked, ksm.params.replicationType); + }); + }); + + cluster.schemaChange(format("CREATE TABLE %s.%s (k int, c int, v int, primary key (k, c));", keyspaceName, tableName)); + + // insert a row at all, confirm it's present on all nodes + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 0, 0)", keyspaceName, tableName), ConsistencyLevel.ALL); + + MutationSummary firstSummary = summaryForTable(cluster.get(1), keyspaceName, "tbl"); + Assert.assertEquals(1, firstSummary.size()); + CoordinatorLogId logId = firstSummary.get(0).logId(); + Offsets firstIds = summaryIdSpace(firstSummary.get(logId)); + Assert.assertEquals(1, firstIds.offsetCount()); + + cluster.get(2, 3).forEach(node -> { + assertMatchingSummaryIdSpaceForKey(node, keyspaceName, tableName, 1, firstSummary); + }); + + // block messages to node 3 and perform a write at quorum + cluster.filters().allVerbs().to(3).drop(); + cluster.filters().allVerbs().from(3).drop(); + + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 1, 1)", keyspaceName, tableName), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (2, 2, 2)", keyspaceName, tableName), ConsistencyLevel.QUORUM); + + MutationSummary finalSummary = summaryForTable(cluster.get(1), keyspaceName, tableName); + Assert.assertEquals(1, finalSummary.size()); + Offsets finalIds = summaryIdSpace(finalSummary.get(logId)); + + Assert.assertEquals(3, finalIds.offsetCount()); + assertSummaryIdSpaceIsSuperSet(finalSummary, firstSummary); + + // second node should have the new id, third should not + assertMatchingSummaryIdSpaceForTable(cluster.get(2), keyspaceName, tableName, finalSummary); + assertMatchingSummaryIdSpaceForTable(cluster.get(3), keyspaceName, tableName, firstSummary); + + // reverse the partition and do a read, read should include coordinator (1)'s ID and replica (3), even though (3) is missing the ID + cluster.filters().reset(); + // cluster.filters().allVerbs().to(2).drop(); + // cluster.filters().allVerbs().from(2).drop(); + cluster.get(2).shutdown().get(); + // wait for node1 gossip to settle, shouldn't see any failures due to node1 coordinating to node2 replica + + // No reconciliation has happened yet + Assert.assertEquals(0, numLogReconciliations(cluster.get(1))); + Object[][] result = cluster.coordinator(1).execute(format("SELECT * FROM %s.%s", keyspaceName, tableName), ConsistencyLevel.QUORUM); + Assert.assertEquals(row(row(1, 0, 0), row(1, 1, 1), row(2, 2, 2)), result); + + // Coordinator sends its missing mutations to 3 on read + Assert.assertEquals(1, numLogReconciliations(cluster.get(1))); + + // check that node3 has the new ids + assertMatchingSummaryIdSpaceForTable(cluster.get(3), keyspaceName, tableName, finalSummary); + } + } + + @Test + public void testBasicRangeReadReconciliationApplyMutations() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + String keyspaceName = "basic_reconciliation_test"; + String tableName = "tbl"; + cluster.schemaChange(format("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';", keyspaceName)); + + cluster.forEach(node -> { + logger.info(">>> {}", node); + node.runOnInstance(() -> { + KeyspaceMetadata ksm = Schema.instance.getKeyspaceMetadata(keyspaceName); + Assert.assertEquals(ReplicationType.tracked, ksm.params.replicationType); + }); + }); + + cluster.schemaChange(format("CREATE TABLE %s.%s (k int, c int, v int, primary key (k, c));", keyspaceName, tableName)); + + // insert a row at all, confirm it's present on all nodes + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 0, 0)", keyspaceName, tableName), ConsistencyLevel.ALL); + + MutationSummary firstSummary = summaryForTable(cluster.get(1), keyspaceName, "tbl"); + Assert.assertEquals(1, firstSummary.size()); + CoordinatorLogId logId = firstSummary.get(0).logId(); + Offsets firstIds = summaryIdSpace(firstSummary.get(logId)); + Assert.assertEquals(1, firstIds.offsetCount()); + + cluster.get(2, 3).forEach(node -> { + assertMatchingSummaryIdSpaceForKey(node, keyspaceName, tableName, 1, firstSummary); + }); + + // block messages to node 3 and perform a write at quorum + cluster.filters().allVerbs().to(3).drop(); + cluster.filters().allVerbs().from(3).drop(); + + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (1, 1, 1)", keyspaceName, tableName), ConsistencyLevel.QUORUM); + cluster.coordinator(1).execute(format("INSERT INTO %s.%s (k, c, v) VALUES (2, 2, 2)", keyspaceName, tableName), ConsistencyLevel.QUORUM); + + MutationSummary finalSummary = summaryForTable(cluster.get(1), keyspaceName, tableName); + Assert.assertEquals(1, finalSummary.size()); + Offsets finalIds = summaryIdSpace(finalSummary.get(logId)); + + Assert.assertEquals(3, finalIds.offsetCount()); + assertSummaryIdSpaceIsSuperSet(finalSummary, firstSummary); + + // second node should have the new id, third should not + assertMatchingSummaryIdSpaceForTable(cluster.get(2), keyspaceName, tableName, finalSummary); + assertMatchingSummaryIdSpaceForTable(cluster.get(3), keyspaceName, tableName, firstSummary); + + // reverse the partition and do a read + cluster.filters().reset(); + cluster.filters().allVerbs().to(2).drop(); + cluster.filters().allVerbs().from(2).drop(); + + + Assert.assertEquals(0, numLogReconciliations(cluster.get(3))); + Object[][] result = cluster.coordinator(3).execute(format("SELECT * FROM %s.%s", keyspaceName, tableName), ConsistencyLevel.QUORUM); + Assert.assertEquals(row(row(1, 0, 0), row(1, 1, 1), row(2, 2, 2)), result); + + // Coordinator sends its missing mutations to 3 on read + Assert.assertEquals(1, numLogReconciliations(cluster.get(3))); + + // check that node3 has the new ids + assertMatchingSummaryIdSpaceForTable(cluster.get(3), keyspaceName, tableName, finalSummary); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java new file mode 100644 index 0000000000..21d18c5b08 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.util.UUID; + +import org.apache.cassandra.replication.CoordinatorLogId; +import org.apache.cassandra.replication.MutationSummary; +import org.apache.cassandra.replication.Offsets; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.test.TestBaseImpl; +import org.apache.cassandra.hints.HintsService; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.replication.MutationTrackingService; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; + +import static org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils.getOnlyLogId; +import static org.apache.cassandra.distributed.test.tracking.MutationTrackingUtils.summaryIdSpace; + +public class MutationTrackingTest extends TestBaseImpl +{ + private static final Logger logger = LoggerFactory.getLogger(MutationTrackingTest.class); + + @Test + public void testBasicWritePath() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true")) + .start()) + { + + cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';")); + + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);")); + + String keyspaceName = KEYSPACE; + cluster.get(1).runOnInstance(() -> { + + KeyspaceMetadata keyspace = Schema.instance.getKeyspaceMetadata(keyspaceName); + Assert.assertEquals(ReplicationType.tracked, keyspace.params.replicationType); + }); + + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM); + + cluster.get(1).runOnInstance(() -> { + TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, "tbl"); + DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(1)); + MutationSummary summary = MutationTrackingService.instance.createSummaryForKey(dk, table.id, false); + CoordinatorLogId logId = getOnlyLogId(summary); + Offsets summaryIds = summaryIdSpace(summary.get(logId)); + Assert.assertEquals(1, summaryIds.offsetCount()); + }); + } + } + + @Test + public void testHintsNotWrittenOnFailedWrite() throws Throwable + { + try (Cluster cluster = Cluster.build(3) + .withConfig(cfg -> cfg.with(Feature.NETWORK) + .with(Feature.GOSSIP) + .set("mutation_tracking_enabled", "true") + .set("write_request_timeout", "1000ms")) + .start()) + { + + String keyspaceName = KEYSPACE; + cluster.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = " + + "{'class': 'SimpleStrategy', 'replication_factor': 3} " + + "AND replication_type='tracked';")); + + cluster.schemaChange(withKeyspace("CREATE TABLE %s.tbl (k int primary key, v int);")); + + // block messages to node 3 + cluster.filters().allVerbs().to(3).drop(); + cluster.filters().allVerbs().from(3).drop(); + UUID node3HostId = cluster.get(3).callOnInstance(() -> StorageService.instance.getLocalHostUUID()); + long hints = cluster.get(1).callOnInstance(() -> StorageMetrics.totalHints.getCount()); + + // confirm no hints for node 3 + cluster.get(1).runOnInstance(() -> Assert.assertEquals(0, HintsService.instance.getTotalHintsSize(node3HostId))); + cluster.coordinator(1).execute(withKeyspace("INSERT INTO %s.tbl (k, v) VALUES (1, 1)"), ConsistencyLevel.QUORUM); + + // wait for write timeout + Thread.sleep(5000); + + // TODO: confirm hints aren't written + cluster.get(1).runOnInstance(() -> { + Assert.assertEquals(hints, StorageMetrics.totalHints.getCount()); + }); + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java new file mode 100644 index 0000000000..527df544ac --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/tracking/MutationTrackingUtils.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test.tracking; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.google.common.primitives.Ints; + +import org.apache.cassandra.replication.*; +import org.junit.Assert; +import org.junit.Assume; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.distributed.api.IInvokableInstance; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.metrics.ReadRepairMetrics; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.replication.MutationSummary.CoordinatorSummary; +import org.apache.cassandra.schema.ReplicationType; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class MutationTrackingUtils +{ + private static final int VERSION = MessagingService.current_version; + + public static byte[] encodeId(MutationId id) + { + int size = Ints.checkedCast(MutationId.serializer.serializedSize(id, VERSION)); + ByteBuffer buffer = ByteBuffer.allocate(size); + try (DataOutputBuffer dob = new DataOutputBuffer(buffer)) + { + MutationId.serializer.serialize(id, dob, VERSION); + return buffer.array(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + public static MutationId decodeId(byte[] bytes) + { + try (DataInputBuffer dib = new DataInputBuffer(bytes)) + { + MutationId id = MutationId.serializer.deserialize(dib, VERSION); + Assert.assertEquals(MutationId.serializer.serializedSize(id, VERSION), bytes.length); + return id; + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + public static byte[] encodeSummary(MutationSummary summary) + { + int size = Ints.checkedCast(MutationSummary.serializer.serializedSize(summary, VERSION)); + ByteBuffer buffer = ByteBuffer.allocate(size); + try (DataOutputBuffer dob = new DataOutputBuffer(buffer)) + { + MutationSummary.serializer.serialize(summary, dob, VERSION); + Assert.assertEquals(size, buffer.position()); + return buffer.array(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + public static MutationSummary decodeSummary(byte[] bytes) + { + try (DataInputBuffer dib = new DataInputBuffer(bytes)) + { + MutationSummary summary = MutationSummary.serializer.deserialize(dib, VERSION); + Assert.assertEquals(MutationSummary.serializer.serializedSize(summary, VERSION), bytes.length); + return summary; + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + public static MutationSummary summaryForKey(String keyspaceName, String tableName, DecoratedKey dk) + { + TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, tableName); + return MutationTrackingService.instance.createSummaryForKey(dk, table.id, false); + } + + public static MutationSummary summaryForTable(String keyspaceName, String tableName) + { + Range range = new Range<>(Murmur3Partitioner.instance.getMinimumToken(), Murmur3Partitioner.instance.getMinimumToken()); + return summaryForRange(keyspaceName, tableName, range); + } + + public static MutationSummary summaryForKey(IInvokableInstance node, String keyspaceName, String tableName, int key) + { + byte[] encodedSummary = node.callOnInstance(() -> { + DecoratedKey dk = Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(key)); + MutationSummary summary = summaryForKey(keyspaceName, tableName, dk); + return encodeSummary(summary); + }); + + return decodeSummary(encodedSummary); + } + + public static MutationSummary summaryForTable(IInvokableInstance node, String keyspaceName, String tableName) + { + byte[] encodedSummary = node.callOnInstance(() -> { + MutationSummary summary = summaryForTable(keyspaceName, tableName); + return encodeSummary(summary); + }); + return decodeSummary(encodedSummary); + } + + public static MutationSummary summaryForKey(String keyspaceName, String tableName, int key) + { + return summaryForKey(keyspaceName, tableName, Murmur3Partitioner.instance.decorateKey(ByteBufferUtil.bytes(key))); + } + + public static MutationSummary summaryForRange(String keyspaceName, String tableName, Range range) + { + TableMetadata table = Schema.instance.getTableMetadata(keyspaceName, tableName); + return MutationTrackingService.instance.createSummaryForRange(range, table.id, false); + } + + public static Offsets summaryIdSpace(CoordinatorSummary summary) + { + return Offsets.Immutable.union(summary.reconciled, summary.unreconciled); + } + + public static Map summaryIdSpace(MutationSummary summary) + { + Map idSpace = new HashMap<>(); + for (int i=0; i expected) + { + Assert.assertEquals(expected.size(), summary.unreconciledIds()); + for (MutationId id : expected) + { + if (!summary.contains(id)) + throw new AssertionError(String.format("%s doesn't contain %s", summary, id)); + } + } + + public static void assertIdsForKey(IInvokableInstance node, String keyspaceName, String tableName, int key, Set expected) + { + MutationSummary summary = summaryForKey(node, keyspaceName, tableName, key); + assertSummaryContents(summary, expected); + } + + public static void assertMatchingSummaryForKey(IInvokableInstance node, String keyspaceName, String tableName, int key, MutationSummary expected) + { + byte[] encodedExpected = encodeSummary(expected); + node.runOnInstance(() -> { + MutationSummary decodedExpected = decodeSummary(encodedExpected); + MutationSummary actual = summaryForKey(keyspaceName, tableName, key); + Assert.assertEquals(decodedExpected, actual); + }); + } + + /** + * Checks that nodes have seen the same ids, regardless of whether they agree on their reconciliation status + */ + public static void assertMatchingSummaryIdSpaceForKey(IInvokableInstance node, String keyspaceName, String tableName, int key, MutationSummary expected) + { + byte[] encodedExpected = encodeSummary(expected); + node.runOnInstance(() -> { + MutationSummary decodedExpected = decodeSummary(encodedExpected); + MutationSummary actual = summaryForKey(keyspaceName, tableName, key); + Assert.assertEquals(summaryIdSpace(decodedExpected), summaryIdSpace(actual)); + }); + } + + public static void assertMatchingSummaryForTable(IInvokableInstance node, String keyspaceName, String tableName, MutationSummary expected) + { + byte[] encodedExpected = encodeSummary(expected); + node.runOnInstance(() -> { + MutationSummary decodedExpected = decodeSummary(encodedExpected); + MutationSummary actual = summaryForTable(keyspaceName, tableName); + Assert.assertEquals(decodedExpected, actual); + }); + } + + public static void assertMatchingSummaryIdSpaceForTable(IInvokableInstance node, String keyspaceName, String tableName, MutationSummary expected) + { + byte[] encodedExpected = encodeSummary(expected); + node.runOnInstance(() -> { + MutationSummary decodedExpected = decodeSummary(encodedExpected); + MutationSummary actual = summaryForTable(keyspaceName, tableName); + Assert.assertEquals(summaryIdSpace(decodedExpected), summaryIdSpace(actual)); + }); + } + + public static void assertOffsetsIsSuperSet(Offsets expectedSuperset, Offsets expectedSubset) + { + Offsets.Immutable diff = Offsets.Immutable.difference(expectedSubset, expectedSuperset); + if (!diff.isEmpty()) + { + String msg = String.format("%s not a super set of %s\n", expectedSuperset, expectedSubset); + msg = msg + String.format("%s found in expected subset that are not in the expected superset", diff); + throw new AssertionError(msg); + } + } + + public static void assertSummaryIsUnreconciledSuperSet(MutationSummary expectedSuperset, MutationSummary expectedSubset) + { + for (int i = 0; i < expectedSubset.size(); i++) + { + CoordinatorSummary subset = expectedSubset.get(i); + CoordinatorSummary superset = expectedSuperset.get(subset.logId()); + + if (superset == null) + throw new AssertionError(String.format("Coordinator summary for %s found in expected subset and not in expected superset", subset.logId())); + + assertOffsetsIsSuperSet(superset.unreconciled, subset.unreconciled); + } + } + + public static void assertSummaryIdSpaceIsSuperSet(MutationSummary expectedSuperset, MutationSummary expectedSubset) + { + for (int i = 0; i < expectedSubset.size(); i++) + { + CoordinatorSummary subset = expectedSubset.get(i); + CoordinatorSummary superset = expectedSuperset.get(subset.logId()); + + if (superset == null) + throw new AssertionError(String.format("Coordinator summary for %s found in expected subset and not in expected superset", subset.logId())); + + assertOffsetsIsSuperSet(summaryIdSpace(superset), summaryIdSpace(subset)); + } + } + + public static void assertIdsForTable(IInvokableInstance node, String keyspaceName, String tableName, Set expected) + { + MutationSummary summary = summaryForTable(node, keyspaceName, tableName); + assertSummaryContents(summary, expected); + } + + public static long numLogReconciliations(IInvokableInstance node) + { + return node.callOnInstance(() -> ReadRepairMetrics.trackedReconcile.getCount()); + } + + public static Object[] row(Object... objs) + { + return objs; + } + + public static Object[][] rows(Object[][]... objs) + { + return objs; + } + + public static void fixmeSkipIfTracked(ReplicationType replicationType, String reason) + { + Assume.assumeFalse(replicationType.isTracked()); + } + + public static CoordinatorLogId getOnlyLogId(MutationSummary summary) + { + Assert.assertEquals(1, summary.size()); + return summary.get(0).logId(); + } +} diff --git a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java index 5d1c005153..a2975090b7 100644 --- a/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java +++ b/test/long/org/apache/cassandra/db/commitlog/CommitLogStressTest.java @@ -66,6 +66,7 @@ import org.apache.cassandra.io.util.DataInputBuffer; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileInputStreamPlus; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.security.EncryptionContext; import org.apache.cassandra.security.EncryptionContextGenerator; @@ -133,6 +134,7 @@ public abstract class CommitLogStressTest } SchemaLoader.loadSchema(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(""); // leave def. blank to maintain old behaviour CommitLog.instance.stopUnsafe(true); diff --git a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java index 2509f7e83a..4fe11c8802 100644 --- a/test/unit/org/apache/cassandra/AbstractSerializationsTester.java +++ b/test/unit/org/apache/cassandra/AbstractSerializationsTester.java @@ -42,7 +42,8 @@ public class AbstractSerializationsTester put("3.0", MessagingService.VERSION_30); put("4.0", MessagingService.VERSION_40); put("5.0", MessagingService.VERSION_50); - put("5.1", MessagingService.VERSION_60); + put("6.0", MessagingService.VERSION_60); + put("6.1", MessagingService.VERSION_61); }}; protected static final boolean EXECUTE_WRITES = TEST_SERIALIZATION_WRITES.getBoolean(); diff --git a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java index f82c25b3ad..555f9fc84e 100644 --- a/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java +++ b/test/unit/org/apache/cassandra/cql3/statements/DescribeStatementTest.java @@ -107,10 +107,10 @@ public class DescribeStatementTest extends CQLTester createFunctionOverload(fOverloaded, "text, ascii", "CREATE FUNCTION %s (input text, other_in ascii) " + - "RETURNS NULL ON NULL INPUT " + - "RETURNS text " + - "LANGUAGE java " + - "AS 'return \"Hello World\";'"); + "RETURNS NULL ON NULL INPUT " + + "RETURNS text " + + "LANGUAGE java " + + "AS 'return \"Hello World\";'"); for (String describeKeyword : new String[]{ "DESCRIBE", "DESC" }) { @@ -307,11 +307,15 @@ public class DescribeStatementTest extends CQLTester row(KEYSPACE, "keyspace", KEYSPACE, "CREATE KEYSPACE " + KEYSPACE + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" + - " AND durable_writes = true AND fast_path = 'simple';"), + " AND durable_writes = true" + + " AND replication_type = 'untracked'" + + " AND fast_path = 'simple';"), row(KEYSPACE_PER_TEST, "keyspace", KEYSPACE_PER_TEST, "CREATE KEYSPACE " + KEYSPACE_PER_TEST + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" + - " AND durable_writes = true AND fast_path = 'simple';"), + " AND durable_writes = true" + + " AND replication_type = 'untracked'" + + " AND fast_path = 'simple';"), row("test", "keyspace", "test", keyspaceOutput()), row("test", "table", "has_all_types", allTypesTable()), row("test", "table", "\"Test\"", testTableOutput()), @@ -713,8 +717,7 @@ public class DescribeStatementTest extends CQLTester assertRowsNet(executeDescribeNet(KEYSPACE_PER_TEST, "DESCRIBE KEYSPACE " + KEYSPACE_PER_TEST), row(KEYSPACE_PER_TEST, "keyspace", KEYSPACE_PER_TEST, "CREATE KEYSPACE " + KEYSPACE_PER_TEST + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" + - " AND durable_writes = true" + - " AND fast_path = 'simple';"), + " AND durable_writes = true AND replication_type = 'untracked' AND fast_path = 'simple';"), row(KEYSPACE_PER_TEST, "type", type2, "CREATE TYPE " + KEYSPACE_PER_TEST + "." + type2 + " (\n" + " x text,\n" + " y text\n" + @@ -820,6 +823,7 @@ public class DescribeStatementTest extends CQLTester String expectedKeyspaceStmt = "CREATE KEYSPACE " + KEYSPACE_PER_TEST + " WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}" + " AND durable_writes = true" + + " AND replication_type = 'untracked'" + " AND fast_path = 'simple';"; String expectedTableStmt = "CREATE TABLE " + KEYSPACE_PER_TEST + "." + table + " (\n" + @@ -1418,7 +1422,7 @@ public class DescribeStatementTest extends CQLTester private static String keyspaceOutput() { - return "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true AND fast_path = 'simple';"; + return "CREATE KEYSPACE test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true AND replication_type = 'untracked' AND fast_path = 'simple';"; } private void describeError(String cql, String msg) throws Throwable diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 00881fedcf..ab03d6d55f 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -78,6 +78,7 @@ import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.File; import org.apache.cassandra.io.util.FileUtils; import org.apache.cassandra.metrics.ClearableHistogram; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.IndexMetadata; import org.apache.cassandra.schema.Indexes; @@ -246,7 +247,7 @@ public class ColumnFamilyStoreTest ByteBuffer val = ByteBufferUtil.bytes("val1"); // insert - Mutation.SimpleBuilder builder = Mutation.simpleBuilder(keyspaceName, cfs.metadata().partitioner.decorateKey(ByteBufferUtil.bytes("val2"))); + Mutation.SimpleBuilder builder = Mutation.simpleBuilder(MutationId.none(), keyspaceName, cfs.metadata().partitioner.decorateKey(ByteBufferUtil.bytes("val2"))); builder.update(cfName).row("Column1").add("val", "val1").build(); new RowUpdateBuilder(cfs.metadata(), 0, "key1").clustering("Column1").add("val", "val1").build().applyUnsafe(); diff --git a/test/unit/org/apache/cassandra/db/CounterMutationTest.java b/test/unit/org/apache/cassandra/db/CounterMutationTest.java index a84c877898..85ed24e030 100644 --- a/test/unit/org/apache/cassandra/db/CounterMutationTest.java +++ b/test/unit/org/apache/cassandra/db/CounterMutationTest.java @@ -26,6 +26,7 @@ import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; @@ -118,7 +119,7 @@ public class CounterMutationTest cfsTwo.truncateBlocking(); // Do the update (+1, -1), (+2, -2) - Mutation.PartitionUpdateCollector batch = new Mutation.PartitionUpdateCollector(KEYSPACE1, Util.dk("key1")); + Mutation.PartitionUpdateCollector batch = new Mutation.PartitionUpdateCollector(MutationId.none(), KEYSPACE1, Util.dk("key1")); batch.add(new RowUpdateBuilder(cfsOne.metadata(), 5, "key1") .clustering("cc") .add("val", 1L) diff --git a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java index 35bb684d6e..ed2b1cd7fa 100644 --- a/test/unit/org/apache/cassandra/db/DeletePartitionTest.java +++ b/test/unit/org/apache/cassandra/db/DeletePartitionTest.java @@ -27,6 +27,7 @@ import org.apache.cassandra.db.partitions.FilteredPartition; import org.apache.cassandra.db.partitions.ImmutableBTreePartition; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; @@ -80,7 +81,7 @@ public class DeletePartitionTest Util.flush(store); // delete the partition - new Mutation.PartitionUpdateCollector(KEYSPACE1, key) + new Mutation.PartitionUpdateCollector(MutationId.none(), KEYSPACE1, key) .add(PartitionUpdate.fullPartitionDelete(store.metadata(), key, 0, FBUtilities.nowInSeconds())) .build() .applyUnsafe(); diff --git a/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java index c01cc41285..1d1e030e11 100644 --- a/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java +++ b/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java @@ -44,6 +44,7 @@ import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; @@ -76,6 +77,7 @@ public class MutationVerbHandlerOutOfRangeTest public static void init() throws Exception { ServerTestUtils.prepareServerNoRegister(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); ServerTestUtils.markCMS(); StorageService.instance.unsafeSetInitialized(); diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java index 5b42b98b1a..8574c74c2e 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java @@ -43,6 +43,7 @@ import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; @@ -76,6 +77,7 @@ public class ReadCommandVerbHandlerOutOfRangeTest public static void init() throws Throwable { ServerTestUtils.prepareServerNoRegister(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); ServerTestUtils.markCMS(); metadata_nonreplicated = Schema.instance.getTableMetadata(KEYSPACE_NONREPLICATED, TABLE); diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java index c005011051..8489943c78 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java @@ -40,6 +40,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessageFlag; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.ParamType; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.tcm.membership.NodeAddresses; @@ -72,6 +73,7 @@ public class ReadCommandVerbHandlerTest public static void init() throws Throwable { ServerTestUtils.prepareServerNoRegister(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE); metadata_with_transient = Schema.instance.getTableMetadata(KEYSPACE_WITH_TRANSIENT, TABLE); diff --git a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java index 5392339faf..a0ed6f1994 100644 --- a/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java +++ b/test/unit/org/apache/cassandra/db/RowUpdateBuilder.java @@ -25,6 +25,7 @@ import org.apache.cassandra.cql3.ColumnIdentifier; import org.apache.cassandra.db.context.CounterContext; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.FBUtilities; @@ -139,7 +140,7 @@ public class RowUpdateBuilder { PartitionUpdate.Builder update = new PartitionUpdate.Builder(metadata, makeKey(metadata, key), metadata.regularAndStaticColumns(), 0); deleteRow(update, timestamp, localDeletionTime, clusteringValues); - return new Mutation.PartitionUpdateCollector(update.metadata().keyspace, update.partitionKey()).add(update.build()).build(); + return new Mutation.PartitionUpdateCollector(MutationId.none(), update.metadata().keyspace, update.partitionKey()).add(update.build()).build(); } private static DecoratedKey makeKey(TableMetadata metadata, Object... partitionKey) diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java index 3fa099b6fe..c5f5e0ffc7 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsPurgeTest.java @@ -48,6 +48,7 @@ import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.CachingParams; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.ByteBufferUtil; @@ -242,7 +243,7 @@ public class CompactionsPurgeTest } Util.flush(cfs); - new Mutation.PartitionUpdateCollector(KEYSPACE1, dk(key)) + new Mutation.PartitionUpdateCollector(MutationId.none(), KEYSPACE1, dk(key)) .add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), Long.MAX_VALUE, FBUtilities.nowInSeconds())) .build() .applyUnsafe(); @@ -475,7 +476,7 @@ public class CompactionsPurgeTest } // deletes partition - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KEYSPACE_CACHED, dk(key)); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KEYSPACE_CACHED, dk(key)); rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), 1, FBUtilities.nowInSeconds())); rm.build().applyUnsafe(); @@ -515,7 +516,7 @@ public class CompactionsPurgeTest } // deletes partition with timestamp such that not all columns are deleted - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KEYSPACE1, dk(key)); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KEYSPACE1, dk(key)); rm.add(PartitionUpdate.fullPartitionDelete(cfs.metadata(), dk(key), 4, FBUtilities.nowInSeconds())); rm.build().applyUnsafe(); diff --git a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java index 32e040f965..f95b131fe0 100644 --- a/test/unit/org/apache/cassandra/dht/BootStrapperTest.java +++ b/test/unit/org/apache/cassandra/dht/BootStrapperTest.java @@ -51,6 +51,7 @@ import org.apache.cassandra.gms.IFailureDetectionEventListener; import org.apache.cassandra.gms.IFailureDetector; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.StreamOperation; @@ -93,6 +94,7 @@ public class BootStrapperTest oldPartitioner = StorageService.instance.setPartitionerUnsafe(Murmur3Partitioner.instance); ServerTestUtils.prepareServerNoRegister(); SchemaLoader.startGossiper(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition("BootStrapperTest"); RangeStreamer.ALIVE_PREDICATE = Predicates.alwaysTrue(); ServerTestUtils.markCMS(); diff --git a/test/unit/org/apache/cassandra/hints/HintTest.java b/test/unit/org/apache/cassandra/hints/HintTest.java index 43533afda8..ee22272023 100644 --- a/test/unit/org/apache/cassandra/hints/HintTest.java +++ b/test/unit/org/apache/cassandra/hints/HintTest.java @@ -32,6 +32,7 @@ import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.ReadExecutionController; import org.apache.cassandra.db.RegularAndStaticColumns; @@ -330,7 +331,7 @@ public class HintTest private static Mutation createMutation(String key, long now) { - Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key)); + Mutation.SimpleBuilder builder = Mutation.simpleBuilder(MutationId.none(), KEYSPACE, dk(key)); builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE0)) .timestamp(now) diff --git a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java index 3b9cf346fe..5d26612cd6 100644 --- a/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java +++ b/test/unit/org/apache/cassandra/hints/HintsCatalogTest.java @@ -30,6 +30,7 @@ import org.junit.rules.TemporaryFolder; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.io.util.File; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.Schema; @@ -209,7 +210,7 @@ public class HintsCatalogTest private static Mutation createMutation(String key, long now) { - Mutation.SimpleBuilder builder = Mutation.simpleBuilder(KEYSPACE, dk(key)); + Mutation.SimpleBuilder builder = Mutation.simpleBuilder(MutationId.none(), KEYSPACE, dk(key)); builder.update(Schema.instance.getTableMetadata(KEYSPACE, TABLE0)) .timestamp(now) diff --git a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index 1b4076692a..9be0e97ebb 100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@ -74,6 +74,7 @@ import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.db.PartitionRangeReadCommand; import org.apache.cassandra.db.ReadCommand; import org.apache.cassandra.db.ReadExecutionController; @@ -907,7 +908,7 @@ public class SASIIndexTest { ColumnFamilyStore store = Keyspace.open(KS_NAME).getColumnFamilyStore(CF_NAME); - Mutation.PartitionUpdateCollector rm1 = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key1"))); + Mutation.PartitionUpdateCollector rm1 = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey(AsciiType.instance.decompose("key1"))); rm1.add(PartitionUpdate.singleRowUpdate(store.metadata(), rm1.key(), buildRow(buildCell(store.metadata(), @@ -915,7 +916,7 @@ public class SASIIndexTest AsciiType.instance.decompose("jason"), 1000)))); - Mutation.PartitionUpdateCollector rm2 = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key2"))); + Mutation.PartitionUpdateCollector rm2 = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey(AsciiType.instance.decompose("key2"))); rm2.add(PartitionUpdate.singleRowUpdate(store.metadata(), rm2.key(), buildRow(buildCell(store.metadata(), @@ -923,7 +924,7 @@ public class SASIIndexTest AsciiType.instance.decompose("pavel"), 2000)))); - Mutation.PartitionUpdateCollector rm3 = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose("key3"))); + Mutation.PartitionUpdateCollector rm3 = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey(AsciiType.instance.decompose("key3"))); rm3.add(PartitionUpdate.singleRowUpdate(store.metadata(), rm3.key(), buildRow(buildCell(store.metadata(), @@ -1321,23 +1322,23 @@ public class SASIIndexTest final ByteBuffer comment = UTF8Type.instance.decompose("comment"); - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1")); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key1")); update(rm, comment, UTF8Type.instance.decompose("ⓈⓅⒺⒸⒾⒶⓁ ⒞⒣⒜⒭⒮ and normal ones"), 1000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key2")); update(rm, comment, UTF8Type.instance.decompose("龍馭鬱"), 2000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key3")); update(rm, comment, UTF8Type.instance.decompose("インディアナ"), 3000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key4")); update(rm, comment, UTF8Type.instance.decompose("レストラン"), 4000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key5")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key5")); update(rm, comment, UTF8Type.instance.decompose("ベンジャミン ウエスト"), 5000); rm.build().apply(); @@ -1397,19 +1398,19 @@ public class SASIIndexTest final ByteBuffer comment = UTF8Type.instance.decompose("comment_suffix_split"); - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1")); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key1")); update(rm, comment, UTF8Type.instance.decompose("龍馭鬱"), 1000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key2")); update(rm, comment, UTF8Type.instance.decompose("インディアナ"), 2000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key3")); update(rm, comment, UTF8Type.instance.decompose("レストラン"), 3000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key4")); update(rm, comment, UTF8Type.instance.decompose("ベンジャミン ウエスト"), 4000); rm.build().apply(); @@ -1466,7 +1467,7 @@ public class SASIIndexTest final ByteBuffer bigValue = UTF8Type.instance.decompose(new String(randomBytes)); - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1")); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key1")); update(rm, comment, bigValue, 1000 + i); rm.build().apply(); @@ -1546,35 +1547,35 @@ public class SASIIndexTest final ByteBuffer fullName = UTF8Type.instance.decompose("/output/full-name/"); - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1")); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key1")); update(rm, fullName, UTF8Type.instance.decompose("美加 八田"), 1000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key2")); update(rm, fullName, UTF8Type.instance.decompose("仁美 瀧澤"), 2000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key3")); update(rm, fullName, UTF8Type.instance.decompose("晃宏 高須"), 3000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key4")); update(rm, fullName, UTF8Type.instance.decompose("弘孝 大竹"), 4000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key5")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key5")); update(rm, fullName, UTF8Type.instance.decompose("満枝 榎本"), 5000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key6")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key6")); update(rm, fullName, UTF8Type.instance.decompose("飛鳥 上原"), 6000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key7")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key7")); update(rm, fullName, UTF8Type.instance.decompose("大輝 鎌田"), 7000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key8")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key8")); update(rm, fullName, UTF8Type.instance.decompose("利久 寺地"), 8000); rm.build().apply(); @@ -1602,15 +1603,15 @@ public class SASIIndexTest final ByteBuffer comment = UTF8Type.instance.decompose("address"); - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1")); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key1")); update(rm, comment, UTF8Type.instance.decompose("577 Rogahn Valleys Apt. 178"), 1000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key2")); update(rm, comment, UTF8Type.instance.decompose("89809 Beverly Course Suite 089"), 2000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key3")); update(rm, comment, UTF8Type.instance.decompose("165 clydie oval apt. 399"), 3000); rm.build().apply(); @@ -1679,38 +1680,38 @@ public class SASIIndexTest Mutation.PartitionUpdateCollector rm; - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key1")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key1")); update(rm, name, UTF8Type.instance.decompose("Pavel"), 1000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key2")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key2")); update(rm, name, UTF8Type.instance.decompose("Jordan"), 2000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key3")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key3")); update(rm, name, UTF8Type.instance.decompose("Mikhail"), 3000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key4")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key4")); update(rm, name, UTF8Type.instance.decompose("Michael"), 4000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key5")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key5")); update(rm, name, UTF8Type.instance.decompose("Johnny"), 5000); rm.build().apply(); // first flush would make interval for name - 'johnny' -> 'pavel' Util.flush(store); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key6")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key6")); update(rm, name, UTF8Type.instance.decompose("Jason"), 6000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key7")); + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key7")); update(rm, name, UTF8Type.instance.decompose("Vijay"), 7000); rm.build().apply(); - rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey("key8")); // this name is going to be tokenized + rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey("key8")); // this name is going to be tokenized update(rm, name, UTF8Type.instance.decompose("Jean-Claude"), 8000); rm.build().apply(); @@ -2738,7 +2739,7 @@ public class SASIIndexTest private static Mutation newMutation(String key, String firstName, String lastName, int age, long timestamp) { - Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(KS_NAME, decoratedKey(AsciiType.instance.decompose(key))); + Mutation.PartitionUpdateCollector rm = new Mutation.PartitionUpdateCollector(MutationId.none(), KS_NAME, decoratedKey(AsciiType.instance.decompose(key))); List> cells = new ArrayList<>(3); if (age >= 0) diff --git a/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java index 2c17751126..2b29379487 100644 --- a/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java +++ b/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java @@ -50,6 +50,7 @@ import org.apache.cassandra.repair.messages.RepairMessage; import org.apache.cassandra.repair.messages.ValidationRequest; import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.repair.state.ParticipateState; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; @@ -91,6 +92,7 @@ public class RepairMessageVerbHandlerOutOfRangeTest SchemaLoader.loadSchema(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); ServerTestUtils.recreateCMS(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); ClusterMetadataTestHelper.register(broadcastAddress); ServerTestUtils.markCMS(); diff --git a/test/unit/org/apache/cassandra/replication/CoordinatorLogTest.java b/test/unit/org/apache/cassandra/replication/CoordinatorLogTest.java new file mode 100644 index 0000000000..37bc1cd080 --- /dev/null +++ b/test/unit/org/apache/cassandra/replication/CoordinatorLogTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.db.Mutation; +import org.apache.cassandra.db.RowUpdateBuilder; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.replication.CoordinatorLog.CoordinatorLogPrimary; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class CoordinatorLogTest +{ + private static final int LOCAL_HOST_ID = 1; + private static final CoordinatorLogId LOG_ID = new CoordinatorLogId(LOCAL_HOST_ID, 1); + private static final Participants PARTICIPANTS = new Participants(List.of(LOCAL_HOST_ID, 2, 3)); + + private static final String KEYSPACE = "cltks"; + private static final String TABLE = "cltt"; + + @BeforeClass + public static void setUp() throws IOException + { + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(3), + TableMetadata.builder(KEYSPACE, TABLE) + .addPartitionKeyColumn("pk", UTF8Type.instance) + .addClusteringColumn("ck", UTF8Type.instance) + .addRegularColumn("value", UTF8Type.instance) + .build()); + } + + private static Token tk(String key) + { + return new ByteOrderedPartitioner.BytesToken(ByteBufferUtil.bytes(key)); + } + + private static Offsets toOffsets(MutationId... ids) + { + Offsets.Mutable list = new Offsets.Mutable(LOG_ID); + for (MutationId id : ids) + list.append(id.offset()); + return list; + } + + private static void assertUnreconciled(Token token, TableId tableId, CoordinatorLog log, boolean includePending, Offsets expectedReconciled, MutationId... expectedIds) + { + Offsets.Mutable reconciled = new Offsets.Mutable(LOG_ID); + Offsets.Mutable unreconciled = new Offsets.Mutable(LOG_ID); + log.collectOffsetsFor(token, tableId, includePending, unreconciled, reconciled); + + for (MutationId mid : expectedIds) + Assert.assertTrue(unreconciled.contains(mid.offset())); + + Assert.assertEquals(toOffsets(expectedIds), unreconciled); + Assert.assertEquals(expectedReconciled, reconciled); + } + + @Test + public void remoteReconciliationTest() + { + Token tk = tk("key"); + TableMetadata metadata = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + TableId tableId = metadata.id; + CoordinatorLogPrimary log = new CoordinatorLogPrimary(LOCAL_HOST_ID, LOG_ID, PARTICIPANTS); + MutationId[] ids = new MutationId[] { log.nextId(), log.nextId(), log.nextId(), }; + + List mutations = new ArrayList<>(ids.length); + for (MutationId id : ids) + { + Mutation mutation = + new RowUpdateBuilder(metadata, 0, "key") + .clustering("ck") + .add("value", "value") + .build() + .withMutationId(id); + + mutations.add(mutation); + log.startWriting(mutation); + } + + Offsets.Mutable reconciled = new Offsets.Mutable(LOG_ID); + // we've only started writing, so the ids shouldn't appear without includePending being true + assertUnreconciled(tk, tableId, log, false, reconciled); + assertUnreconciled(tk, tableId, log, true, reconciled, ids); + + for (Mutation mutation : mutations) + log.finishWriting(mutation); + + // the call to finishWriting will have made the ids visible without the includePending flag + assertUnreconciled(tk, tableId, log, false, reconciled, ids); + + log.witnessedRemoteMutation(ids[0], PARTICIPANTS.get(1)); + assertUnreconciled(tk, tableId, log, false, reconciled, ids); + + log.witnessedRemoteMutation(ids[0], PARTICIPANTS.get(2)); + reconciled.add(ids[0].offset()); + assertUnreconciled(tk, tableId, log, false, reconciled, ids[1], ids[2]); + } +} diff --git a/test/unit/org/apache/cassandra/service/tracking/MutationJournalTest.java b/test/unit/org/apache/cassandra/replication/MutationJournalTest.java similarity index 94% rename from test/unit/org/apache/cassandra/service/tracking/MutationJournalTest.java rename to test/unit/org/apache/cassandra/replication/MutationJournalTest.java index 55e97f57bd..26fcc9f1af 100644 --- a/test/unit/org/apache/cassandra/service/tracking/MutationJournalTest.java +++ b/test/unit/org/apache/cassandra/replication/MutationJournalTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.cassandra.service.tracking; +package org.apache.cassandra.replication; import java.io.IOException; import java.nio.ByteBuffer; @@ -85,7 +85,7 @@ public class MutationJournalTest .add("value", "value") .build(); - MutationId id = new MutationId(100L, 0); + ShortMutationId id = new ShortMutationId(100L, 0); journal.write(id, expected); // regular read @@ -115,9 +115,9 @@ public class MutationJournalTest .build(); List expected = List.of(expected1, expected2); - MutationId id1 = new MutationId(100L, 1); - MutationId id2 = new MutationId(100L, 2); - List ids = List.of(id1, id2); + ShortMutationId id1 = new ShortMutationId(100L, 1); + ShortMutationId id2 = new ShortMutationId(100L, 2); + List ids = List.of(id1, id2); journal.write(id1, expected1); journal.write(id2, expected2); diff --git a/test/unit/org/apache/cassandra/replication/OffsetsTest.java b/test/unit/org/apache/cassandra/replication/OffsetsTest.java new file mode 100644 index 0000000000..539da7d1b0 --- /dev/null +++ b/test/unit/org/apache/cassandra/replication/OffsetsTest.java @@ -0,0 +1,776 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.replication; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +import org.apache.cassandra.io.IVersionedSerializers; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.net.MessagingService; +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class OffsetsTest +{ + private static final CoordinatorLogId LOG_ID = new CoordinatorLogId(0, 0); + private static class TestConsumer implements Offsets.RangeConsumer + { + static class OffsetRange + { + final int start; + final int end; + + public OffsetRange(int start, int end) + { + this.start = start; + this.end = end; + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + OffsetRange range = (OffsetRange) o; + return start == range.start && end == range.end; + } + + @Override + public int hashCode() + { + return Objects.hash(start, end); + } + + @Override + public String toString() + { + return String.format("<%s,%s>", start, end); + } + } + + final List ranges = new ArrayList<>(); + + @Override + public void consume(CoordinatorLogId logId, int start, int end) + { + consumerOffsets(start, end); + } + + public void consumerOffsets(int start, int end) + { + ranges.add(new OffsetRange(start, end)); + } + + @Override + public boolean equals(Object o) + { + if (o == null || getClass() != o.getClass()) return false; + TestConsumer that = (TestConsumer) o; + return Objects.equals(ranges, that.ranges); + } + + @Override + public int hashCode() + { + return Objects.hashCode(ranges); + } + + @Override + public String toString() + { + return ranges.toString(); + } + + public TestConsumer assertOffsetsConsumed(int... expected) + { + Assert.assertTrue(expected.length % 2 == 0); + TestConsumer expectedConsumer = new TestConsumer(); + for (int i = 0; i < expected.length; i+=2) + expectedConsumer.consumerOffsets(expected[i], expected[i+1]); + + Assert.assertEquals(expectedConsumer, this); + return this; + } + + public TestConsumer assertConsumed(int... expected) + { + int[] offsets = new int[expected.length]; + System.arraycopy(expected, 0, offsets, 0, expected.length); + return assertOffsetsConsumed(offsets); + } + + void clear() + { + ranges.clear(); + } + + boolean isEmpty() + { + return ranges.isEmpty(); + } + } + + private static Offsets.Mutable offsets(int... bounds) + { + Assert.assertTrue(bounds.length % 2 == 0); + Offsets.Mutable ids = new Offsets.Mutable(LOG_ID); + int keys = 0; + int last = 0; + for (int i=0; i 0) + Assert.assertTrue(start > last + 1); + + ids.add(start, end); + last = end; + } + + Assert.assertEquals(bounds.length/2, ids.rangeCount()); + Assert.assertEquals(keys, ids.offsetCount()); + return ids; + } + + private static void assertOffsetsEqual(Offsets expected, Offsets actual) + { + if (!Offsets.contentsEqual(expected, actual)) + throw new AssertionError("expected: " + expected.toString() + " but was: " + actual.toString()); + } + + + @Test + public void testEmptyAndAddExisting() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + assertEquals(0, offsets.rangeCount()); + assertEquals(0, offsets.offsetCount()); + + assertTrue(offsets.add(10)); + assertEquals(1, offsets.rangeCount()); + assertEquals(1, offsets.offsetCount()); + + assertFalse(offsets.add(10)); + assertEquals(1, offsets.rangeCount()); + assertEquals(1, offsets.offsetCount()); + } + + @Test + public void testAppend() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + + assertTrue(offsets.add(10)); + assertEquals(1, offsets.rangeCount()); + assertEquals(1, offsets.offsetCount()); + + // should extend + assertTrue(offsets.add(11)); + assertEquals(1, offsets.rangeCount()); + assertEquals(2, offsets.offsetCount()); + + // should append + assertTrue(offsets.add(13)); + assertEquals(2, offsets.rangeCount()); + assertEquals(3, offsets.offsetCount()); + } + + @Test + public void testPrepend() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + + assertTrue(offsets.add(10)); + assertEquals(1, offsets.rangeCount()); + assertEquals(1, offsets.offsetCount()); + + // should extend + assertTrue(offsets.add(9)); + assertEquals(1, offsets.rangeCount()); + assertEquals(2, offsets.offsetCount()); + + // should prepend + assertTrue(offsets.add(7)); + assertEquals(2, offsets.rangeCount()); + assertEquals(3, offsets.offsetCount()); + } + + @Test + public void testClosesGaps() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + + assertTrue(offsets.add(10)); + assertEquals(1, offsets.rangeCount()); + assertEquals(1, offsets.offsetCount()); + + // should prepend + assertTrue(offsets.add(6)); + assertEquals(2, offsets.rangeCount()); + assertEquals(2, offsets.offsetCount()); + + // should extend left range + assertTrue(offsets.add(7)); + assertEquals(2, offsets.rangeCount()); + assertEquals(3, offsets.offsetCount()); + + // should extend right range + assertTrue(offsets.add(9)); + assertEquals(2, offsets.rangeCount()); + assertEquals(4, offsets.offsetCount()); + + // should close the gap and collapse all into one range + assertTrue(offsets.add(8)); + assertEquals(1, offsets.rangeCount()); + assertEquals(5, offsets.offsetCount()); + } + + @Test + public void testCreatesMoreGaps() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + + assertTrue(offsets.add(10)); + assertEquals(1, offsets.rangeCount()); + assertEquals(1, offsets.offsetCount()); + + // should prepend + assertTrue(offsets.add(6)); + assertEquals(2, offsets.rangeCount()); + assertEquals(2, offsets.offsetCount()); + + // should insert in the middle + assertTrue(offsets.add(8)); + assertEquals(3, offsets.rangeCount()); + assertEquals(3, offsets.offsetCount()); + } + + @Test + public void testRangeAppend() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + offsets.add(5, 7); + TestConsumer consumer = new TestConsumer(); + + // add overlapping range 1 + assertTrue(offsets.add(6, 8, consumer)); + assertEquals(1, offsets.rangeCount()); + assertEquals(4, offsets.offsetCount()); + consumer.assertOffsetsConsumed(8, 8).clear(); + + // add overlapping range 2 + assertTrue(offsets.add(8, 9, consumer)); + assertEquals(1, offsets.rangeCount()); + assertEquals(5, offsets.offsetCount()); + consumer.assertOffsetsConsumed(9, 9).clear(); + + // add adjacent range + assertTrue(offsets.add(10, 12, consumer)); + assertEquals(1, offsets.rangeCount()); + assertEquals(8, offsets.offsetCount()); + consumer.assertOffsetsConsumed(10, 12).clear(); + + // add disjoint range + assertTrue(offsets.add(14, 16, consumer)); + assertEquals(2, offsets.rangeCount()); + assertEquals(11, offsets.offsetCount()); + consumer.assertOffsetsConsumed(14, 16).clear(); + + } + + @Test + public void testRangePrepend() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + offsets.add(10, 12); + TestConsumer consumer = new TestConsumer(); + + assertEquals(1, offsets.rangeCount()); + assertEquals(3, offsets.offsetCount()); + + // add overlapping range 1 + assertTrue(offsets.add(9, 11, consumer)); + assertEquals(1, offsets.rangeCount()); + assertEquals(4, offsets.offsetCount()); + consumer.assertOffsetsConsumed(9, 9).clear(); + + // add overlapping range 2 + assertTrue(offsets.add(8, 9, consumer)); + assertEquals(1, offsets.rangeCount()); + assertEquals(5, offsets.offsetCount()); + consumer.assertOffsetsConsumed(8, 8).clear(); + + // add adjacent range + assertTrue(offsets.add(6, 7, consumer)); + assertEquals(1, offsets.rangeCount()); + assertEquals(7, offsets.offsetCount()); + consumer.assertOffsetsConsumed(6, 7).clear(); + + // add disjoint range + assertTrue(offsets.add(0, 3, consumer)); + assertEquals(2, offsets.rangeCount()); + assertEquals(11, offsets.offsetCount()); + consumer.assertOffsetsConsumed(0, 3).clear(); + } + + @Test + public void testRangeAddition() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + offsets.add(5, 7); + + assertEquals(1, offsets.rangeCount()); + assertEquals(3, offsets.offsetCount()); + } + + /** + * adding ranges fully contained in existing ranges should noop + */ + @Test + public void testRangeInclusion() + { + Offsets.Mutable offsets = new Offsets.Mutable(LOG_ID); + TestConsumer consumer = new TestConsumer(); + offsets.add(0, 3); + offsets.add(7, 10); + offsets.add(15, 17); + + assertEquals(3, offsets.rangeCount()); + assertEquals(11, offsets.offsetCount()); + + // fully contained in first + assertFalse(offsets.add(0, 2, consumer)); + assertFalse(offsets.add(1, 2, consumer)); + assertFalse(offsets.add(1, 3, consumer)); + assertFalse(offsets.add(0, 3, consumer)); + + + // fully contained in second + assertFalse(offsets.add(7, 9, consumer)); + assertFalse(offsets.add(8, 9, consumer)); + assertFalse(offsets.add(8, 10, consumer)); + assertFalse(offsets.add(7, 10, consumer)); + + // fully contained in third + assertFalse(offsets.add(16, 16, consumer)); + assertFalse(offsets.add(16, 17, consumer)); + assertFalse(offsets.add(15, 16, consumer)); + assertFalse(offsets.add(15, 17, consumer)); + + // nothing should have changed + assertEquals(3, offsets.rangeCount()); + assertEquals(11, offsets.offsetCount()); + assertTrue(consumer.isEmpty()); + } + + @Test + public void testRangeInsert() + { + Supplier sequenceIds = () -> { + Offsets.Mutable ids0 = new Offsets.Mutable(LOG_ID); + ids0.add(0, 3); + ids0.add(7, 10); + ids0.add(15, 17); + + assertEquals(3, ids0.rangeCount()); + assertEquals(11, ids0.offsetCount()); + return ids0; + }; + + // disjoint insert + { + Offsets.Mutable ids = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(ids.add(12, 13, consumer)); + assertEquals(4, ids.rangeCount()); + assertEquals(13, ids.offsetCount()); + consumer.assertOffsetsConsumed(12, 13).clear(); + } + + // left adjacent insert + { + Offsets.Mutable offsets = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(offsets.add(5, 6, consumer)); + assertEquals(3, offsets.rangeCount()); + assertEquals(13, offsets.offsetCount()); + consumer.assertOffsetsConsumed(5, 6).clear(); + } + + // right adjacent insert + { + Offsets.Mutable ids = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(ids.add(11, 12, consumer)); + assertEquals(3, ids.rangeCount()); + assertEquals(13, ids.offsetCount()); + consumer.assertOffsetsConsumed(11, 12).clear(); + } + + // both adjacent insert + { + Offsets.Mutable offsets = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(offsets.add(11, 14, consumer)); + assertEquals(2, offsets.rangeCount()); + assertEquals(15, offsets.offsetCount()); + consumer.assertOffsetsConsumed(11, 14).clear(); + } + } + + + @Test + public void testRangeMerging() + { + Supplier sequenceIds = () -> { + Offsets.Mutable ids0 = new Offsets.Mutable(LOG_ID); + ids0.add(0, 3); + ids0.add(7, 10); + ids0.add(15, 17); + + assertEquals(3, ids0.rangeCount()); + assertEquals(11, ids0.offsetCount()); + return ids0; + }; + + // left merge + { + Offsets.Mutable offsets = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(offsets.add(5, 8, consumer)); + assertEquals(3, offsets.rangeCount()); + assertEquals(13, offsets.offsetCount()); + consumer.assertOffsetsConsumed(5, 6).clear(); + } + + // right merge + { + Offsets.Mutable offsets = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(offsets.add(8, 12, consumer)); + assertEquals(3, offsets.rangeCount()); + assertEquals(13, offsets.offsetCount()); + consumer.assertOffsetsConsumed(11, 12).clear(); + } + + // right and left merge + { + Offsets.Mutable offsets = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(offsets.add(6, 11, consumer)); + assertEquals(3, offsets.rangeCount()); + assertEquals(13, offsets.offsetCount()); + consumer.assertOffsetsConsumed(6, 6, 11, 11).clear(); + } + + // 2 range merge + { + Offsets.Mutable ids = sequenceIds.get(); + TestConsumer consumer = new TestConsumer(); + + assertTrue(ids.add(2, 8, consumer)); + assertEquals(2, ids.rangeCount()); + assertEquals(14, ids.offsetCount()); + consumer.assertOffsetsConsumed(4, 6).clear(); + } + } + + @Test + public void appendTest() + { + Offsets.Mutable ids = new Offsets.Mutable(LOG_ID); + ids.append(5); + assertEquals(1, ids.rangeCount()); + assertEquals(1, ids.offsetCount()); + + ids.append(6); + assertEquals(1, ids.rangeCount()); + assertEquals(2, ids.offsetCount()); + + ids.append(8); + assertEquals(2, ids.rangeCount()); + assertEquals(3, ids.offsetCount()); + + // insert before tail + try + { + ids.append(8); + Assert.fail(); + } + catch (IllegalArgumentException e) + { + // expected + assertEquals(2, ids.rangeCount()); + assertEquals(3, ids.offsetCount()); + } + + // insert before tail + try + { + ids.append(7); + Assert.fail(); + } + catch (IllegalArgumentException e) + { + // expected + assertEquals(2, ids.rangeCount()); + assertEquals(3, ids.offsetCount()); + } + } + + private static void testUnion(Offsets expected, Offsets a, Offsets b) + { + assertOffsetsEqual(expected, Offsets.Mutable.union(a, b)); + assertOffsetsEqual(expected, Offsets.Immutable.union(a, b)); + assertOffsetsEqual(expected, new Offsets.Mutable(Offsets.union(a.rangeIterator(), b.rangeIterator()))); + assertOffsetsEqual(expected, Offsets.Mutable.union(b, a)); + assertOffsetsEqual(expected, Offsets.Immutable.union(b, a)); + assertOffsetsEqual(expected, new Offsets.Mutable(Offsets.union(b.rangeIterator(), a.rangeIterator()))); + } + + @Test + public void unionTest() + { + // empty + testUnion(offsets(1, 1, 5, 6), + offsets(1, 1, 5, 6), + offsets()); + + // left union + testUnion(offsets(0, 3, 6, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(6, 9)); + + // left adjacent union + testUnion(offsets(0, 3, 5, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(5, 6)); + + // right union + testUnion(offsets(0, 3, 7, 11, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(9, 11)); + + // right adjacent + testUnion(offsets(0, 3, 7, 12, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(11, 12)); + + // superset union + testUnion(offsets(0, 3, 5, 12, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(5, 12)); + + // join union + testUnion(offsets(0, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(2, 8)); + + // disjoint + testUnion(offsets(0, 10, 12, 13, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(2, 8, 12, 13)); + + } + + private static void testDifference(Offsets expected, Offsets a, Offsets b) + { + Offsets.Mutable bPlus = Offsets.Mutable.copy(b); + bPlus.add(50, 55); + + // check copy-remaining + assertOffsetsEqual(expected, Offsets.Mutable.difference(a, b)); + assertOffsetsEqual(expected, Offsets.Immutable.difference(a, b)); + assertOffsetsEqual(expected, new Offsets.Mutable(Offsets.difference(a.rangeIterator(), b.rangeIterator()))); + + // check discarded tail + assertOffsetsEqual(expected, Offsets.Mutable.difference(a, bPlus)); + assertOffsetsEqual(expected, Offsets.Immutable.difference(a, bPlus)); + assertOffsetsEqual(expected, new Offsets.Mutable(Offsets.difference(a.rangeIterator(), bPlus.rangeIterator()))); + } + + @Test + public void differenceTest() + { + // empty input + testDifference(offsets(1, 1), + offsets(1, 1), + offsets()); + + testDifference(offsets(), + offsets(), + offsets(1, 1)); + + // empty result + testDifference(offsets(), + offsets(1, 1), + offsets(1, 1)); + + + // noop + testDifference(offsets(0, 3, 7, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(5, 5)); + + // noop before adjacent + testDifference(offsets(0, 3, 7, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(5, 6)); + + // noop after adjacent + testDifference(offsets(0, 3, 7, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(4, 5)); + + // before + testDifference(offsets(0, 3, 9, 10, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(6, 8)); + + + // after + testDifference(offsets(0, 3, 7, 8, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(9, 11)); + + // both sides + testDifference(offsets(0, 3, 8, 9, 15, 17), + offsets(0, 3, 7, 10, 15, 17), + offsets(6, 7, 10, 12)); + + // multi-split + testDifference(offsets(0, 3, 7, 8, 11, 11, 14, 15, 20, 22), + offsets(0, 3, 7, 15, 20, 22), + offsets(9, 10, 12, 13)); + + // multi-split w/ edges + testDifference(offsets(0, 3, 8, 9, 11, 13, 20, 22), + offsets(0, 3, 7, 15, 20, 22), + offsets(6, 7, 10, 10, 14, 16)); + + } + + private static void testIntersection(Offsets expected, Offsets a, Offsets b) + { + Offsets.Mutable aPlus = Offsets.Mutable.copy(a); + aPlus.add(50, 55); + Offsets.Mutable bPlus = Offsets.Mutable.copy(b); + bPlus.add(50, 55); + + assertOffsetsEqual(expected, Offsets.Mutable.intersection(a, b)); + assertOffsetsEqual(expected, Offsets.Immutable.intersection(a, b)); + assertOffsetsEqual(expected, Offsets.Mutable.intersection(aPlus, b)); + assertOffsetsEqual(expected, Offsets.Immutable.intersection(aPlus, b)); + assertOffsetsEqual(expected, Offsets.Mutable.intersection(a, bPlus)); + assertOffsetsEqual(expected, Offsets.Immutable.intersection(a, bPlus)); + assertOffsetsEqual(expected, Offsets.Mutable.intersection(b, a)); + assertOffsetsEqual(expected, Offsets.Immutable.intersection(b, a)); + assertOffsetsEqual(expected, Offsets.Mutable.intersection(bPlus, a)); + assertOffsetsEqual(expected, Offsets.Immutable.intersection(bPlus, a)); + assertOffsetsEqual(expected, Offsets.Mutable.intersection(b, aPlus)); + assertOffsetsEqual(expected, Offsets.Immutable.intersection(b, aPlus)); + } + + @Test + public void intersectionTest() + { + // emtpy input + testIntersection(offsets(), + offsets(0, 3, 7, 10, 15, 17), + offsets()); + + // disjoint test + testIntersection(offsets(), + offsets(0, 3, 7, 10, 15, 17), + offsets(4, 6, 11, 14)); + + // left intersect test + testIntersection(offsets(7, 9), + offsets(0, 3, 7, 10, 15, 17), + offsets(6, 9)); + + + // right intersect test + testIntersection(offsets(8, 10), + offsets(0, 3, 7, 10, 15, 17), + offsets(8, 11)); + + // superset test + testIntersection(offsets(7, 10), + offsets(0, 3, 7, 10, 15, 17), + offsets(6, 11)); + + // multi-intersect test + testIntersection(offsets(8, 9, 11, 13, 15, 16), + offsets(0, 3, 7, 17, 25, 30), + offsets(8, 9, 11, 13, 15, 16)); + + // multi-intersect test w/ ends + testIntersection(offsets(7, 9, 11, 13, 16, 17), + offsets(0, 3, 7, 17, 25, 30), + offsets(6, 9, 11, 13, 16, 18)); + } + + @Test + public void serializerTest() throws IOException + { + + DataOutputBuffer buffer = new DataOutputBuffer(); + IVersionedSerializers.testSerde(buffer, Offsets.serializer, Offsets.Immutable.copy(offsets(0, 3, 7, 10, 15, 17)), MessagingService.current_version); + } + + private static List ids(int... offsets) + { + List ids = new ArrayList<>(offsets.length); + for (int offset : offsets) + ids.add(new ShortMutationId(LOG_ID, offset)); + return ids; + } + + private static List ids(Offsets offsets) + { + final List ids = new ArrayList<>(); + for (ShortMutationId id : offsets) + ids.add(id); + return ids; + } + + @Test + public void iteratorTest() + { + Assert.assertEquals(ids(), ids(offsets())); + Assert.assertEquals(ids(1, 2, 3), ids(offsets(1, 3))); + Assert.assertEquals(ids(1, 2, 3, 5, 6, 7), ids(offsets(1, 3, 5, 7))); + } +} diff --git a/test/unit/org/apache/cassandra/schema/SchemaMetadataSerializationTest.java b/test/unit/org/apache/cassandra/schema/SchemaMetadataSerializationTest.java index 297b047e4f..13929600c0 100644 --- a/test/unit/org/apache/cassandra/schema/SchemaMetadataSerializationTest.java +++ b/test/unit/org/apache/cassandra/schema/SchemaMetadataSerializationTest.java @@ -258,19 +258,19 @@ public class SchemaMetadataSerializationTest private KeyspaceMetadata serializeAndDeserializeKeyspace(KeyspaceMetadata original) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); - KeyspaceMetadata.serializer.serialize(original, out, Version.V8); + KeyspaceMetadata.serializer.serialize(original, out, Version.V9); DataInputBuffer in = new DataInputBuffer(out.toByteArray()); - return KeyspaceMetadata.serializer.deserialize(in, Version.V8); + return KeyspaceMetadata.serializer.deserialize(in, Version.V9); } private TableMetadata serializeAndDeserializeTable(TableMetadata original) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); - TableMetadata.serializer.serialize(original, out, Version.V8); + TableMetadata.serializer.serialize(original, out, Version.V9); DataInputBuffer in = new DataInputBuffer(out.toByteArray()); - return TableMetadata.serializer.deserialize(in, Types.none(), UserFunctions.none(), Version.V8); + return TableMetadata.serializer.deserialize(in, Types.none(), UserFunctions.none(), Version.V9); } private UserType serializeAndDeserializeType(UserType original) throws IOException @@ -282,9 +282,9 @@ public class SchemaMetadataSerializationTest private Types serializeAndDeserializeTypes(Types original) throws IOException { DataOutputBuffer out = new DataOutputBuffer(); - Types.serializer.serialize(original, out, Version.V8); + Types.serializer.serialize(original, out, Version.V9); DataInputBuffer in = new DataInputBuffer(out.toByteArray()); - return Types.serializer.deserialize(KEYSPACE, in, Version.V8); + return Types.serializer.deserialize(KEYSPACE, in, Version.V9); } } diff --git a/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java b/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java index ec2a373ccb..55b5e8dce8 100644 --- a/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java +++ b/test/unit/org/apache/cassandra/service/JoinTokenRingTest.java @@ -34,6 +34,7 @@ import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.index.StubIndex; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.tcm.ClusterMetadata; import org.apache.cassandra.utils.FBUtilities; @@ -45,6 +46,7 @@ public class JoinTokenRingTest DatabaseDescriptor.daemonInitialization(); ServerTestUtils.prepareServerNoRegister(); SchemaLoader.startGossiper(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition("JoinTokenRingTest"); } diff --git a/test/unit/org/apache/cassandra/service/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/PaxosStateTest.java index 23f12543c2..4d25c18530 100644 --- a/test/unit/org/apache/cassandra/service/PaxosStateTest.java +++ b/test/unit/org/apache/cassandra/service/PaxosStateTest.java @@ -46,6 +46,7 @@ import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.exceptions.ReadTimeoutException; import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.Ballot; import org.apache.cassandra.service.paxos.Commit; @@ -68,6 +69,7 @@ public class PaxosStateTest public static void setUpClass() throws Throwable { SchemaLoader.loadSchema(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition("PaxosStateTest"); } diff --git a/test/unit/org/apache/cassandra/service/StorageServiceTest.java b/test/unit/org/apache/cassandra/service/StorageServiceTest.java index 8742cdac23..4616df8b5b 100644 --- a/test/unit/org/apache/cassandra/service/StorageServiceTest.java +++ b/test/unit/org/apache/cassandra/service/StorageServiceTest.java @@ -21,6 +21,7 @@ package org.apache.cassandra.service; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.cassandra.replication.MutationJournal; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -74,6 +75,7 @@ public class StorageServiceTest extends TestBaseImpl SimpleLocationProvider.LOCATION, NodeVersion.CURRENT)); CommitLog.instance.start(); + MutationJournal.instance.start(); } @Before diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java index b06263850f..28f0ecc44e 100644 --- a/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosStateTest.java @@ -38,6 +38,7 @@ import org.apache.cassandra.db.SystemKeyspace; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.db.rows.Row; import org.apache.cassandra.gms.Gossiper; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.paxos.PaxosState.Snapshot; import org.apache.cassandra.utils.ByteBufferUtil; @@ -63,6 +64,7 @@ public class PaxosStateTest public static void setUpClass() throws Throwable { SchemaLoader.loadSchema(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition("PaxosStateTest"); metadata = Keyspace.open("PaxosStateTestKeyspace1").getColumnFamilyStore("Standard1").metadata.get(); metadata.withSwapped(metadata.params.unbuild().gcGraceSeconds(3600).build()); diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java index 9c6e75c510..c8cd6b5b06 100644 --- a/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java @@ -39,6 +39,7 @@ import org.apache.cassandra.metrics.StorageMetrics; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.net.Verb; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.service.StorageService; @@ -75,6 +76,7 @@ public class PaxosVerbHandlerOutOfRangeTest // PaxosV1 out of range tests - V2 i public static void init() throws Exception { ServerTestUtils.prepareServerNoRegister(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); ServerTestUtils.markCMS(); StorageService.instance.unsafeSetInitialized(); diff --git a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java index f4c230e790..65515175bb 100644 --- a/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java +++ b/test/unit/org/apache/cassandra/service/reads/repair/RepairedDataVerifierTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.filter.RowFilter; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.metrics.TableMetrics; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.schema.Schema; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; @@ -61,6 +62,7 @@ public class RepairedDataVerifierTest public static void init() { SchemaLoader.loadSchema(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); DatabaseDescriptor.reportUnconfirmedRepairedDataMismatches(true); } diff --git a/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java b/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java index c9de3fa6b4..dc450d47db 100644 --- a/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java +++ b/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java @@ -37,6 +37,7 @@ import org.apache.cassandra.distributed.test.log.ClusterMetadataTestHelper; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.replication.MutationJournal; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.messages.SessionFailedMessage; import org.apache.cassandra.streaming.messages.StreamMessage; @@ -77,6 +78,7 @@ public class StreamSessionOwnedRangesTest SchemaLoader.loadSchema(); DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); ServerTestUtils.recreateCMS(); + MutationJournal.instance.start(); SchemaLoader.schemaDefinition(TEST_NAME); ClusterMetadataTestHelper.register(broadcastAddress); ServerTestUtils.markCMS(); diff --git a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java index 9c950fda80..bc793e8775 100644 --- a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java +++ b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java @@ -80,7 +80,8 @@ public abstract class OfflineToolUtils "JNA Cleaner", // spawned by JNA "ThreadLocalMetrics-Cleaner", // spawned by org.apache.cassandra.metrics.ThreadLocalMetrics "Native reference cleanup thread", - "^ForkJoinPool\\.commonPool-worker-\\d+$" + "^ForkJoinPool\\.commonPool-worker-\\d+$", + "Reconciliation-Map-Reaper:[1-9]", }; static final String[] NON_DEFAULT_MEMTABLE_THREADS = diff --git a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java index be1c617a7b..d94630b570 100644 --- a/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java +++ b/test/unit/org/apache/cassandra/triggers/TriggerExecutorTest.java @@ -44,6 +44,7 @@ import org.apache.cassandra.db.rows.RowIterator; import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.replication.MutationId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TriggerMetadata; import org.apache.cassandra.schema.Triggers; @@ -128,8 +129,8 @@ public class TriggerExecutorTest TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfTrigger.class.getName())); PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); - Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build(); - Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build(); + Mutation rm1 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf1.partitionKey()).add(cf1).build(); + Mutation rm2 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf2.partitionKey()).add(cf2).build(); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(2, tmutations.size()); @@ -154,8 +155,8 @@ public class TriggerExecutorTest TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeySameCfPartialTrigger.class.getName())); PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); - Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build(); - Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build(); + Mutation rm1 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf1.partitionKey()).add(cf1).build(); + Mutation rm2 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf2.partitionKey()).add(cf2).build(); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(2, tmutations.size()); @@ -180,8 +181,8 @@ public class TriggerExecutorTest TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentCfTrigger.class.getName())); PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); - Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build(); - Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build(); + Mutation rm1 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf1.partitionKey()).add(cf1).build(); + Mutation rm2 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf2.partitionKey()).add(cf2).build(); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(2, tmutations.size()); @@ -231,8 +232,8 @@ public class TriggerExecutorTest TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", SameKeyDifferentKsTrigger.class.getName())); PartitionUpdate cf1 = makeCf(metadata, "k1", "k1v1", null); PartitionUpdate cf2 = makeCf(metadata, "k2", "k2v1", null); - Mutation rm1 = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build(); - Mutation rm2 = new Mutation.PartitionUpdateCollector("ks1", cf2.partitionKey()).add(cf2).build(); + Mutation rm1 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf1.partitionKey()).add(cf1).build(); + Mutation rm2 = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf2.partitionKey()).add(cf2).build(); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm1, rm2))); assertEquals(4, tmutations.size()); @@ -269,7 +270,7 @@ public class TriggerExecutorTest TableMetadata metadata = makeTableMetadata("ks1", "cf1", TriggerMetadata.create("test", DifferentKeyTrigger.class.getName())); PartitionUpdate cf1 = makeCf(metadata, "k1", "v1", null); - Mutation rm = new Mutation.PartitionUpdateCollector("ks1", cf1.partitionKey()).add(cf1).build(); + Mutation rm = new Mutation.PartitionUpdateCollector(MutationId.none(), "ks1", cf1.partitionKey()).add(cf1).build(); List tmutations = new ArrayList<>(TriggerExecutor.instance.execute(Arrays.asList(rm))); assertEquals(2, tmutations.size()); diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java index 36ce78e21a..df21931fda 100644 --- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java +++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java @@ -124,6 +124,7 @@ import org.apache.cassandra.schema.KeyspaceMetadata; import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.schema.MemtableParams; import org.apache.cassandra.schema.ReplicationParams; +import org.apache.cassandra.schema.ReplicationType; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.schema.TableParams; @@ -516,7 +517,8 @@ public final class CassandraGenerators AbstractReplicationStrategy replication = replicationGen.generate(rs).withKeyspace(nameGen).build().generate(rs); ReplicationParams replicationParams = ReplicationParams.fromStrategy(replication); boolean durableWrites = durableWritesGen.generate(rs); - KeyspaceParams params = new KeyspaceParams(durableWrites, replicationParams, FastPathStrategy.simple()); + // TODO: Support tracked + KeyspaceParams params = new KeyspaceParams(durableWrites, replicationParams, FastPathStrategy.simple(), ReplicationType.untracked); Tables tables = Tables.none(); Views views = Views.none(); Types types = Types.none(); @@ -962,6 +964,7 @@ public final class CassandraGenerators private Gen numClusteringColumnsGen = SourceDSL.integers().between(1, 2); private Gen numRegularColumnsGen = SourceDSL.integers().between(1, 5); private Gen numStaticColumnsGen = SourceDSL.integers().between(0, 2); + private Gen replicationTypeGen = SourceDSL.arbitrary().enumValues(ReplicationType.class); @Nullable private ColumnNameGen columnNameGen = null; private TableParamsBuilder paramsBuilder = new TableParamsBuilder(); @@ -1002,6 +1005,19 @@ public final class CassandraGenerators return withPartitioner(i -> partitioner); } + public TableMetadataBuilder withReplicationType(Gen replicationTypeGen) + { + this.replicationTypeGen = Objects.requireNonNull(replicationTypeGen); + return this; + } + + public TableMetadataBuilder withReplicationType(ReplicationType replicationType) + { + return withReplicationType(r -> replicationType); + } + + + public TableMetadataBuilder withUseCounter(boolean useCounter) { return withUseCounter(ignore -> useCounter); @@ -1189,10 +1205,12 @@ public final class CassandraGenerators String tableName = tableNameGen.generate(rnd); TableParams params = paramsBuilder.build().generate(rnd); boolean isCounter = useCounter.generate(rnd); + ReplicationType replicationType = replicationTypeGen.generate(rnd); TableMetadata.Builder builder = TableMetadata.builder(ks, tableName, tableIdGen.generate(rnd)) .partitioner(partitionerGen.generate(rnd)) .kind(tableKindGen.generate(rnd)) .isCounter(isCounter) + .keyspaceReplicationType(replicationType) .params(params); int numPartitionColumns = numPartitionColumnsGen.generate(rnd);