From b4f1c6d250991fc27c5f87a2dc0fd81e7ad0d272 Mon Sep 17 00:00:00 2001 From: Caleb Rackliffe Date: Thu, 28 May 2020 14:26:25 -0600 Subject: [PATCH] Safer handling of out-of-range tokens patch by Caleb Rackliffe; reviewed by Mick Semb Wever for CASSANDRA-13704 Co-authored-by: Sam Tunnicliffe Co-authored-by: Caleb Rackliffe Co-authored-by: Mick Semb Wever Co-authored-by: Ariel Weisberg --- CHANGES.txt | 1 + NEWS.txt | 18 + .../org/apache/cassandra/config/Config.java | 7 + .../cassandra/config/DatabaseDescriptor.java | 20 + .../db/AbstractMutationVerbHandler.java | 81 +++ .../db/CounterMutationVerbHandler.java | 9 +- .../cassandra/db/MutationVerbHandler.java | 17 +- .../db/PartitionRangeReadCommand.java | 3 +- .../org/apache/cassandra/db/ReadCommand.java | 10 + .../cassandra/db/ReadCommandVerbHandler.java | 45 ++ .../cassandra/db/ReadRepairVerbHandler.java | 19 +- .../CassandraCompressedStreamReader.java | 2 +- .../db/streaming/CassandraStreamReader.java | 142 +++- .../org/apache/cassandra/dht/OwnedRanges.java | 139 ++++ .../cassandra/hints/HintVerbHandler.java | 21 +- .../io/sstable/SSTableSimpleIterator.java | 26 + .../locator/AbstractReplicationStrategy.java | 16 +- .../cassandra/locator/EndpointsForToken.java | 6 + .../cassandra/locator/PendingRangeMaps.java | 47 +- .../cassandra/locator/TokenMetadata.java | 14 + .../metrics/HintedHandoffMetrics.java | 8 + .../cassandra/metrics/KeyspaceMetrics.java | 10 + .../cassandra/metrics/StorageMetrics.java | 2 + .../repair/RepairMessageVerbHandler.java | 49 +- .../cassandra/service/StorageService.java | 71 ++ .../service/StorageServiceMBean.java | 18 + .../service/paxos/AbstractPaxosCallback.java | 18 + .../paxos/AbstractPaxosVerbHandler.java | 79 +++ .../service/paxos/CommitVerbHandler.java | 6 +- .../service/paxos/PrepareVerbHandler.java | 6 +- .../service/paxos/ProposeVerbHandler.java | 4 +- ...treamReceivedOutOfTokenRangeException.java | 50 ++ .../cassandra/streaming/StreamRequest.java | 14 + ...StreamRequestOutOfTokenRangeException.java | 36 + .../cassandra/streaming/StreamSession.java | 115 +++- .../streaming/StreamingMessageSender.java | 2 +- .../async/NettyStreamingMessageSender.java | 2 +- .../org/apache/cassandra/tools/NodeProbe.java | 5 + .../apache/cassandra/tools/nodetool/Info.java | 17 + .../distributed/test/OutOfTokenRangeTest.java | 76 +++ .../distributed/test/ReadRepairTest.java | 14 +- ...nterMutationVerbHandlerOutOfRangeTest.java | 214 ++++++ .../db/MutationVerbHandlerOutOfRangeTest.java | 246 +++++++ .../ReadCommandVerbHandlerOutOfRangeTest.java | 258 ++++++++ .../apache/cassandra/dht/OwnedRangesTest.java | 217 ++++++ .../locator/PendingRangeMapsTest.java | 71 +- ...epairMessageVerbHandlerOutOfRangeTest.java | 254 +++++++ .../paxos/PaxosVerbHandlerOutOfRangeTest.java | 252 +++++++ .../cassandra/streaming/StreamReaderTest.java | 623 ++++++++++++++++++ .../StreamSessionOwnedRangesTest.java | 187 ++++++ .../cassandra/streaming/StreamTestUtils.java | 102 +++ .../NettyStreamingMessageSenderTest.java | 4 +- .../cassandra/utils/TokenRangeTestUtil.java | 166 +++++ 53 files changed, 3749 insertions(+), 90 deletions(-) create mode 100644 src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java create mode 100644 src/java/org/apache/cassandra/dht/OwnedRanges.java create mode 100644 src/java/org/apache/cassandra/service/paxos/AbstractPaxosVerbHandler.java create mode 100644 src/java/org/apache/cassandra/streaming/StreamReceivedOutOfTokenRangeException.java create mode 100644 src/java/org/apache/cassandra/streaming/StreamRequestOutOfTokenRangeException.java create mode 100644 test/distributed/org/apache/cassandra/distributed/test/OutOfTokenRangeTest.java create mode 100644 test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java create mode 100644 test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java create mode 100644 test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java create mode 100644 test/unit/org/apache/cassandra/dht/OwnedRangesTest.java create mode 100644 test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java create mode 100644 test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java create mode 100644 test/unit/org/apache/cassandra/streaming/StreamReaderTest.java create mode 100644 test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java create mode 100644 test/unit/org/apache/cassandra/streaming/StreamTestUtils.java create mode 100644 test/unit/org/apache/cassandra/utils/TokenRangeTestUtil.java diff --git a/CHANGES.txt b/CHANGES.txt index 8511f34eae..d4f9f914d4 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0.14 + * Safer handling of out-of-range tokens (CASSANDRA-13704) * Fix memory leak in BTree.FastBuilder (CASSANDRA-19785) * Fix millisecond and microsecond precision for commit log replay (CASSANDRA-19448) * Improve accuracy of memtable heap usage tracking (CASSANDRA-17298) diff --git a/NEWS.txt b/NEWS.txt index c74403c260..0b842f3167 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -51,6 +51,24 @@ restore snapshots created with the previous major version using the 'sstableloader' tool. You can upgrade the file format of your snapshots using the provided 'sstableupgrade' tool. +4.0.14 +====== + +Upgrading +--------- + - This release introduces safeguards and observability into possible data loss scenarios when nodes have a + divergent view of the cluster. This happens around edge-cases on unsafe bootstrapping, decommissions, or when a + node has a corrupted topology. Two configuration options have been added: `log_out_of_token_range_requests` and + `reject_out_of_token_range_requests`, both enabled by default. The former will make nodes log requests they + receive that don't belong in their current or pending token ranges. The latter will reject those requests + altogether, which prevents the vast majority of eventual data loss that can occur, but may also incur small + windows of degraded availability during range movements in an unhealthy cluster. Both options apply to streaming, + mutations, hints, single-partition reads, read-repair, and Paxos operations. Logging these requests is recommended + in all situations, and operators are encouraged to be aware when such events occur and to investigate if writes + have only been received on incorrect nodes. Operators that are confident of topology parity across all nodes may + disable rejection to maximize availability. See CASSANDRA-13704 for further details. Transactional Cluster + Metadata in version 5.1 will remove all such edge-cases, providing appropriate guarantees. + 4.0.11 ====== diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index dc17639f98..a9cdca46a1 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -526,6 +526,13 @@ public class Config */ public volatile int validation_preview_purge_head_start_in_sec = 60 * 60; + /* + * Toggles to turn on the logging or rejection of operations for token ranges that the node does not own, + * or is not about to acquire. + */ + public volatile boolean log_out_of_token_range_requests = true; + public volatile boolean reject_out_of_token_range_requests = true; + /** * The intial capacity for creating RangeTombstoneList. */ diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 561fc24116..b7d47ec9a5 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -3473,6 +3473,26 @@ public class DatabaseDescriptor conf.consecutive_message_errors_threshold = value; } + public static boolean getLogOutOfTokenRangeRequests() + { + return conf.log_out_of_token_range_requests; + } + + public static void setLogOutOfTokenRangeRequests(boolean enabled) + { + conf.log_out_of_token_range_requests = enabled; + } + + public static boolean getRejectOutOfTokenRangeRequests() + { + return conf.reject_out_of_token_range_requests; + } + + public static void setRejectOutOfTokenRangeRequests(boolean enabled) + { + conf.reject_out_of_token_range_requests = enabled; + } + public static boolean getForceNewPreparedStatementBehaviour() { return conf.force_new_prepared_statement_behaviour; diff --git a/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java new file mode 100644 index 0000000000..3b94286d17 --- /dev/null +++ b/src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.NoSpamLogger; + +public abstract class AbstractMutationVerbHandler implements IVerbHandler +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractMutationVerbHandler.class); + private static final String logMessageTemplate = "Receiving mutation(s) for token(s) neither owned nor pending. Example: from {} for token {} in keyspace {}"; + + @Override + public void doVerb(Message message) throws IOException + { + processMessage(message, message.from()); + } + + public void processMessage(Message message, InetAddressAndPort respondTo) + { + boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); + + DecoratedKey key = message.payload.key(); + boolean isOutOfRangeMutation = isOutOfRangeMutation(message.payload.getKeyspaceName(), key); + if (isOutOfRangeMutation) + { + StorageService.instance.incOutOfRangeOperationCount(); + Keyspace.open(message.payload.getKeyspaceName()).metric.outOfRangeTokenWrites.inc(); + + if (outOfRangeTokenLogging) + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, + respondTo, key.getToken(), message.payload.getKeyspaceName()); + } + if (outOfRangeTokenRejection && isOutOfRangeMutation) + sendFailureResponse(message, respondTo); + else + applyMutation(message, respondTo); + } + + abstract void applyMutation(Message message, InetAddressAndPort respondToAddress); + + private void sendFailureResponse(Message respondTo, InetAddressAndPort respondToAddress) + { + MessagingService.instance().send(respondTo.failureResponse(RequestFailureReason.UNKNOWN), respondToAddress); + + } + + private static boolean isOutOfRangeMutation(String keyspace, DecoratedKey key) + { + return !StorageService.instance.isEndpointValidForWrite(keyspace, key.getToken()); + } +} diff --git a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java index a30ce665be..f9625e9c6d 100644 --- a/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java @@ -21,18 +21,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageProxy; -public class CounterMutationVerbHandler implements IVerbHandler +public class CounterMutationVerbHandler extends AbstractMutationVerbHandler { public static final CounterMutationVerbHandler instance = new CounterMutationVerbHandler(); private static final Logger logger = LoggerFactory.getLogger(CounterMutationVerbHandler.class); - public void doVerb(final Message message) + @Override + protected void applyMutation(final Message message, InetAddressAndPort respondToAddress) { long queryStartNanoTime = System.nanoTime(); final CounterMutation cm = message.payload; @@ -48,7 +49,7 @@ public class CounterMutationVerbHandler implements IVerbHandler // it's own in that case. StorageProxy.applyCounterMutationOnLeader(cm, localDataCenter, - () -> MessagingService.instance().send(message.emptyResponse(), message.from()), + () -> MessagingService.instance().send(message.emptyResponse(), respondToAddress), queryStartNanoTime); } } diff --git a/src/java/org/apache/cassandra/db/MutationVerbHandler.java b/src/java/org/apache/cassandra/db/MutationVerbHandler.java index 1d4f868b7b..840286a02e 100644 --- a/src/java/org/apache/cassandra/db/MutationVerbHandler.java +++ b/src/java/org/apache/cassandra/db/MutationVerbHandler.java @@ -22,7 +22,7 @@ import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.*; import org.apache.cassandra.tracing.Tracing; -public class MutationVerbHandler implements IVerbHandler +public class MutationVerbHandler extends AbstractMutationVerbHandler { public static final MutationVerbHandler instance = new MutationVerbHandler(); @@ -37,6 +37,7 @@ public class MutationVerbHandler implements IVerbHandler Tracing.trace("Payload application resulted in WriteTimeout, not replying"); } + @Override public void doVerb(Message message) { // Check if there were any forwarding headers in this message @@ -55,10 +56,7 @@ public class MutationVerbHandler implements IVerbHandler try { - message.payload.applyFuture().thenAccept(o -> respond(message, respondToAddress)).exceptionally(wto -> { - failed(); - return null; - }); + processMessage(message, respondToAddress); } catch (WriteTimeoutException wto) { @@ -66,6 +64,15 @@ public class MutationVerbHandler implements IVerbHandler } } + @Override + protected void applyMutation(Message message, InetAddressAndPort respondToAddress) + { + message.payload.applyFuture().thenAccept(o -> respond(message, respondToAddress)).exceptionally(wto -> { + failed(); + return null; + }); + } + private static void forwardToLocalNodes(Message originalMessage, ForwardingInfo forwardTo) { Message.Builder builder = diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index e55ab63333..88dfc00ac1 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -54,7 +54,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR private final DataRange dataRange; - private PartitionRangeReadCommand(boolean isDigest, + @VisibleForTesting + protected PartitionRangeReadCommand(boolean isDigest, int digestVersion, boolean acceptsTransient, TableMetadata metadata, diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index d3aef4cd8a..1df23cc7fd 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -149,6 +149,16 @@ public abstract class ReadCommand extends AbstractReadQuery public abstract boolean isLimitedToOnePartition(); + /** + * Whether this command is a single partition read + * + * @return true if the command is a single partition read, false otherwise + */ + public boolean isSinglePartitionRead() + { + return kind == Kind.SINGLE_PARTITION; + } + public abstract boolean isRangeRequest(); /** diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index a86852fabd..7bad9e6d11 100644 --- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -17,6 +17,8 @@ */ package org.apache.cassandra.db; +import java.util.concurrent.TimeUnit; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +32,7 @@ import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.tracing.Tracing; +import org.apache.cassandra.utils.NoSpamLogger; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -38,6 +41,8 @@ public class ReadCommandVerbHandler implements IVerbHandler public static final ReadCommandVerbHandler instance = new ReadCommandVerbHandler(); private static final Logger logger = LoggerFactory.getLogger(ReadCommandVerbHandler.class); + private static final String logMessageTemplate = "Receiving read(s) for token(s) neither owned nor pending. Example: from {} for token {} in {}.{}"; + private static final String exceptionMessageTemplate = "Exception thrown checking if token {} outside valid range for keyspace {} - permitting"; public void doVerb(Message message) { @@ -47,6 +52,30 @@ public class ReadCommandVerbHandler implements IVerbHandler } ReadCommand command = message.payload; + + // no out of token range checking for partition range reads yet + if (command.isSinglePartitionRead()) + { + boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); + + DecoratedKey key = ((SinglePartitionReadCommand)command).partitionKey(); + if (!command.metadata().isVirtual() && isOutOfRangeRead(command.metadata().keyspace, key)) + { + StorageService.instance.incOutOfRangeOperationCount(); + Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc(); + + // Log at most 1 message per second + if (outOfRangeTokenLogging) + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, + message.from(), key.getToken(), command.metadata().keyspace, command.metadata().name); + + if (outOfRangeTokenRejection) + // no need to respond, just drop the request + return; + } + } + validateTransientStatus(message); long timeout = message.expiresAtNanos() - message.createdAtNanos(); @@ -102,4 +131,20 @@ public class ReadCommandVerbHandler implements IVerbHandler this)); } } + + private boolean isOutOfRangeRead(String keyspace, DecoratedKey key) + { + try + { + return Keyspace.open(keyspace) + .getReplicationStrategy() + .getNaturalReplicas(key) + .selfIfPresent() == null; + } + catch (Throwable tr) + { + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, exceptionMessageTemplate, keyspace, key.getKey(), tr); + return false; + } + } } diff --git a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java index 903b3d43bd..a34be9d9c1 100644 --- a/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java @@ -17,17 +17,28 @@ */ package org.apache.cassandra.db; -import org.apache.cassandra.net.IVerbHandler; +import java.io.IOException; + +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; -public class ReadRepairVerbHandler implements IVerbHandler +public class ReadRepairVerbHandler extends AbstractMutationVerbHandler { public static final ReadRepairVerbHandler instance = new ReadRepairVerbHandler(); - public void doVerb(Message message) + @Override + public void doVerb(Message message) throws IOException + { + // This method exists as python dtest relies on byte-code rewriting via Byteman, so requires this + // class defines a "doVerb" method for some tests. + super.doVerb(message); + } + + @Override + void applyMutation(Message message, InetAddressAndPort respondToAddress) { message.payload.apply(); - MessagingService.instance().send(message.emptyResponse(), message.from()); + MessagingService.instance().send(message.emptyResponse(), respondToAddress); } } diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java index ff9e6f7ea1..35d0bf7830 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java @@ -79,8 +79,8 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance)) { TrackedDataInputPlus in = new TrackedDataInputPlus(cis); - deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata())); writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()), session, writer); String filename = writer.getFilename(); int sectionIdx = 0; for (SSTableReader.PartitionPositionBounds section : sections) diff --git a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java index 6835fadb5e..38476bafe4 100644 --- a/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java +++ b/src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java @@ -17,9 +17,13 @@ */ package org.apache.cassandra.db.streaming; -import java.io.*; +import java.io.IOError; +import java.io.IOException; import java.util.Collection; +import java.util.List; +import java.util.ListIterator; import java.util.UUID; +import java.util.concurrent.TimeUnit; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; @@ -28,27 +32,44 @@ import com.google.common.collect.UnmodifiableIterator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.exceptions.UnknownColumnException; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.util.TrackedDataInputPlus; -import org.apache.cassandra.schema.TableId; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.lifecycle.LifecycleNewTracker; -import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.db.rows.DeserializationHelper; +import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.UnknownColumnException; import org.apache.cassandra.io.sstable.SSTableMultiWriter; import org.apache.cassandra.io.sstable.SSTableSimpleIterator; import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter; import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.RewindableDataInputStreamPlus; +import org.apache.cassandra.io.util.TrackedDataInputPlus; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.ProgressInfo; +import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException; import org.apache.cassandra.streaming.StreamReceiver; import org.apache.cassandra.streaming.StreamSession; import org.apache.cassandra.streaming.compress.StreamCompressionInputStream; import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.NoSpamLogger; import static org.apache.cassandra.net.MessagingService.current_version; @@ -58,6 +79,7 @@ import static org.apache.cassandra.net.MessagingService.current_version; public class CassandraStreamReader implements IStreamReader { private static final Logger logger = LoggerFactory.getLogger(CassandraStreamReader.class); + private static final String logMessageTemplate = "[Stream #{}] Received streamed SSTable {} from {} containing key(s) outside valid ranges {}. Example: {}"; protected final TableId tableId; protected final long estimatedKeys; protected final Collection sections; @@ -118,8 +140,8 @@ public class CassandraStreamReader implements IStreamReader try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version)) { TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream); - deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata())); writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + deserializer = getDeserializer(cfs.metadata(), in, inputVersion, session, writer); while (in.getBytesRead() < totalSize) { writePartition(deserializer, writer); @@ -143,6 +165,15 @@ public class CassandraStreamReader implements IStreamReader } } + protected StreamDeserializer getDeserializer(TableMetadata metadata, + TrackedDataInputPlus in, + Version inputVersion, + StreamSession session, + SSTableMultiWriter writer) throws IOException + { + return new StreamDeserializer(metadata, in, inputVersion, getHeader(metadata), session, writer); + } + protected SerializationHeader getHeader(TableMetadata metadata) throws UnknownColumnException { return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader @@ -183,27 +214,57 @@ public class CassandraStreamReader implements IStreamReader private final SerializationHeader header; private final DeserializationHelper helper; - private DecoratedKey key; - private DeletionTime partitionLevelDeletion; - private SSTableSimpleIterator iterator; - private Row staticRow; + private final List> ownedRanges; + private final boolean outOfRangeTokenLogging; + private final boolean outOfRangeTokenRejection; + private final StreamSession session; + private final SSTableMultiWriter writer; + + private int lastCheckedRangeIndex; + protected DecoratedKey key; + protected DeletionTime partitionLevelDeletion; + protected SSTableSimpleIterator iterator; + protected Row staticRow; private IOException exception; - public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header) throws IOException + public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header, StreamSession session, SSTableMultiWriter writer) throws IOException { this.metadata = metadata; this.in = in; this.helper = new DeserializationHelper(metadata, version.correspondingMessagingVersion(), DeserializationHelper.Flag.PRESERVE_SIZE); this.header = header; + this.session = session; + this.writer = writer; + + ownedRanges = Range.normalize(StorageService.instance.getLocalAndPendingRanges(metadata.keyspace)); + lastCheckedRangeIndex = 0; + outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); } - public StreamDeserializer newPartition() throws IOException + public UnfilteredRowIterator newPartition() throws IOException + { + readKey(); + readPartition(); + return this; + } + + protected void readKey() throws IOException { key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in)); + + lastCheckedRangeIndex = verifyKeyInOwnedRanges(key, + ownedRanges, + lastCheckedRangeIndex, + outOfRangeTokenLogging, + outOfRangeTokenRejection); + } + + protected void readPartition() throws IOException + { partitionLevelDeletion = DeletionTime.serializer.deserialize(in); iterator = SSTableSimpleIterator.create(metadata, in, header, helper, partitionLevelDeletion); staticRow = iterator.readStaticRow(); - return this; } public TableMetadata metadata() @@ -284,5 +345,54 @@ public class CassandraStreamReader implements IStreamReader public void close() { } + + /* We have a separate cleanup method because sometimes close is called before exhausting the + StreamDeserializer (for instance, when enclosed in an try-with-resources wrapper, such as in + BigTableWriter.append()). + */ + public void cleanup() + { + if (in instanceof RewindableDataInputStreamPlus) + { + try + { + ((RewindableDataInputStreamPlus) in).close(false); + } + catch (IOException e) + { + logger.warn("Error while closing RewindableDataInputStreamPlus.", e); + } + } + } + + private int verifyKeyInOwnedRanges(final DecoratedKey key, + List> ownedRanges, + int lastCheckedRangeIndex, + boolean outOfRangeTokenLogging, + boolean outOfRangeTokenRejection) + { + if (lastCheckedRangeIndex < ownedRanges.size()) + { + ListIterator> rangesToCheck = ownedRanges.listIterator(lastCheckedRangeIndex); + while (rangesToCheck.hasNext()) + { + Range range = rangesToCheck.next(); + if (range.contains(key.getToken())) + return lastCheckedRangeIndex; + + lastCheckedRangeIndex++; + } + } + + StorageMetrics.totalOpsForInvalidToken.inc(); + + if (outOfRangeTokenLogging) + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, session.planId(), writer.getFilename(), session.peer, ownedRanges, key); + + if (outOfRangeTokenRejection) + throw new StreamReceivedOutOfTokenRangeException(ownedRanges, key, writer.getFilename()); + + return lastCheckedRangeIndex; + } } } diff --git a/src/java/org/apache/cassandra/dht/OwnedRanges.java b/src/java/org/apache/cassandra/dht/OwnedRanges.java new file mode 100644 index 0000000000..cc5eec4b81 --- /dev/null +++ b/src/java/org/apache/cassandra/dht/OwnedRanges.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.dht; + +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.metrics.StorageMetrics; + +public final class OwnedRanges +{ + private static final Logger logger = LoggerFactory.getLogger(OwnedRanges.class); + + private static final Comparator> rangeComparator = Comparator.comparing((Range r) -> r.left).thenComparing(r -> r.right); + + // the set of token ranges that this node is a replica for + private final List> ownedRanges; + + public OwnedRanges(Collection> ownedRanges) + { + this.ownedRanges = Range.normalize(ownedRanges); + } + + /** + * Check that all ranges in a requested set are contained by those in the owned set. Used in several contexts, such + * as validating StreamRequests in StreamSession & PrepareMessage and ValidationRequest in RepairMessageVerbHandler. + * In those callers, we want to verify that the token ranges specified in some request from a peer are not outside + * the ranges owned by the local node. There are 2 levels of response if invalid ranges are detected, controlled + * by options in Config; logging the event and rejecting the request and either/neither/both of these options may be + * enabled. If neither are enabled, we short ciruit and immediately return success without any further processing. + * If either option is enabled, and we do detect unowned ranges in the request, we increment a metric then take further + * action depending on the config. + * + * @param requestedRanges the set of token ranges contained in a request from a peer + * @param requestId an identifier for the peer request, to be used in logging (e.g. Stream or Repair Session #) + * @param requestType description of the request type, to be used in logging (e.g. "prepare request" or "validation") + * @param from the originator of the request + * + * @return true if the request should be accepted (either because no checking was performed, invalid ranges were + * identified but only the logging action is enabled, or because all request ranges were valid). + * Otherwise, returns false to indicate the request should be rejected. + */ + public boolean validateRangeRequest(Collection> requestedRanges, String requestId, String requestType, InetAddressAndPort from) + { + boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); + + Collection> unownedRanges = testRanges(requestedRanges); + + if (!unownedRanges.isEmpty()) + { + StorageMetrics.totalOpsForInvalidToken.inc(); + + if (outOfRangeTokenLogging) + { + logger.warn("[{}] Received {} from {} containing ranges {} outside valid ranges {}", + requestId, requestType, from, unownedRanges, ownedRanges); + } + } + + return !outOfRangeTokenRejection || unownedRanges.isEmpty(); + } + + /** + * Takes a collection of ranges and returns ranges from that collection that are not covered by this node's owned ranges. + *

+ * This normalizes the range collections internally, so: + * a) be cautious about using this in any hot path + * b) any returned ranges may not be identical to those present. That is, the returned values are post-normalization. + *

+ * e.g. Given two collections: + * { (0, 100], (100, 200] } + * { (90, 100], (100, 110], (110, 300] } + * the normalized forms are: + * { (0, 200] } + * { (90, 300] } + * and so the return value would be: + * { (90, 300] } + * which is equivalent, but not strictly equal to any member of the original supplied collection. + * + * @param testedRanges collection of candidate ranges to be checked + * @return the ranges in testedRanges which are not covered by the owned ranges + */ + @VisibleForTesting + Collection> testRanges(final Collection> testedRanges) + { + if (ownedRanges.isEmpty()) + return testedRanges; + + // now normalize the second and check coverage of its members in the normalized first collection + return Range.normalize(testedRanges).stream().filter(requested -> + { + // Find the point at which the target range would insert into the superset + int index = Collections.binarySearch(ownedRanges, requested, rangeComparator); + + // an index >= 0 means an exact match was found, so we can definitely accept this range + if (index >= 0) + return false; + + // convert to an insertion point in the superset + index = Math.abs(index) - 1; + + // target sorts before the last list item, so we only need to check that one + if (index >= ownedRanges.size()) + return !ownedRanges.get(index - 1).contains(requested); + + // target sorts before the first list item, so we only need to check that one + if (index == 0) + return !ownedRanges.get(index).contains(requested); + + // otherwise, check if the range on either side of the insertion point wholly contains the target + return !(ownedRanges.get(index - 1).contains(requested) || ownedRanges.get(index).contains(requested)); + }).collect(Collectors.toSet()); + } +} diff --git a/src/java/org/apache/cassandra/hints/HintVerbHandler.java b/src/java/org/apache/cassandra/hints/HintVerbHandler.java index 2fbe4754b6..af4b6ce217 100644 --- a/src/java/org/apache/cassandra/hints/HintVerbHandler.java +++ b/src/java/org/apache/cassandra/hints/HintVerbHandler.java @@ -19,10 +19,12 @@ package org.apache.cassandra.hints; import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.partitions.PartitionUpdate; import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; @@ -31,10 +33,11 @@ import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.serializers.MarshalException; import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.NoSpamLogger; /** * Verb handler used both for hint dispatch and streaming. - * + *

* With the non-sstable format, we cannot just stream hint sstables on node decommission. So sometimes, at decommission * time, we might have to stream hints to a non-owning host (say, if the owning host B is down during decommission of host A). * In that case the handler just stores the received hint in its local hint store. @@ -89,6 +92,22 @@ public final class HintVerbHandler implements IVerbHandler // the topology has changed, and we are no longer a replica of the mutation - since we don't know which node(s) // it has been handed over to, re-address the hint to all replicas; see CASSANDRA-5902. HintsService.instance.writeForAllReplicas(hint); + + HintsService.instance.metrics.incrHintsReceivedForUnownedRanges(); + if (DatabaseDescriptor.getLogOutOfTokenRangeRequests()) + { + // Log at most 1 message per second + NoSpamLogger.log(logger, + NoSpamLogger.Level.WARN, + 1, + TimeUnit.SECONDS, + "Receiving hint(s) for token(s) neither owned nor pending. Example: from {} for token {} in {}.{}", + message.from(), + hint.mutation.key().getToken(), + hint.mutation.getKeyspaceName(), + hint.mutation.getPartitionUpdates().iterator().next().metadata().name); + } + respond(message); } else diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java index fd1b6a0a81..92643b21b8 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.io.IOError; import java.util.Iterator; +import com.google.common.annotations.VisibleForTesting; + import org.apache.cassandra.db.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.io.util.DataInputPlus; @@ -125,4 +127,28 @@ public abstract class SSTableSimpleIterator extends AbstractIterator } } } + + @VisibleForTesting + public static class EmptySSTableSimpleIterator extends SSTableSimpleIterator + { + public EmptySSTableSimpleIterator(TableMetadata metadata) + { + super(metadata, null, null); + } + + public Row readStaticRow() throws IOException + { + return Rows.EMPTY_STATIC_ROW; + } + + protected Unfiltered computeNext() + { + return null; + } + + public boolean hasNext() + { + return false; + } + } } diff --git a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java index 909a7f6d2c..f14c730101 100644 --- a/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java +++ b/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java @@ -103,11 +103,19 @@ public abstract class AbstractReplicationStrategy return endpoints; } - public Replica getLocalReplicaFor(RingPosition searchPosition) + /** + * Check if the token is in a naturally replicated range or pending range + * @param token the position to check + * @return true if the token is in a natural or pending locally replicationed range, false otherwise + */ + public boolean isTokenInLocalNaturalOrPendingRange(Token token) { - return getNaturalReplicas(searchPosition) - .byEndpoint() - .get(FBUtilities.getBroadcastAddressAndPort()); + return getLocalReplicaFor(token) != null || tokenMetadata.isTokenInLocalPendingRange(keyspaceName, token); + } + + public Replica getLocalReplicaFor(Token searchPosition) + { + return getNaturalReplicas(searchPosition).byEndpoint().get(FBUtilities.getBroadcastAddressAndPort()); } /** diff --git a/src/java/org/apache/cassandra/locator/EndpointsForToken.java b/src/java/org/apache/cassandra/locator/EndpointsForToken.java index c709988762..5fce0fde06 100644 --- a/src/java/org/apache/cassandra/locator/EndpointsForToken.java +++ b/src/java/org/apache/cassandra/locator/EndpointsForToken.java @@ -146,4 +146,10 @@ public class EndpointsForToken extends Endpoints if (replicas.isEmpty()) return empty(token); return builder(token, replicas.size()).addAll(replicas).build(); } + + public static EndpointsForToken copyOf(Token token, Iterable replicas) + { + if (!replicas.iterator().hasNext()) return empty(token); + return builder(token).addAll(replicas).build(); + } } diff --git a/src/java/org/apache/cassandra/locator/PendingRangeMaps.java b/src/java/org/apache/cassandra/locator/PendingRangeMaps.java index 14870c3b10..f768e3e69b 100644 --- a/src/java/org/apache/cassandra/locator/PendingRangeMaps.java +++ b/src/java/org/apache/cassandra/locator/PendingRangeMaps.java @@ -20,13 +20,18 @@ */ package org.apache.cassandra.locator; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; + import com.google.common.collect.Iterators; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict; - -import java.util.*; +import org.apache.cassandra.utils.FBUtilities; import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR; @@ -149,6 +154,44 @@ public class PendingRangeMaps implements Iterable, Endpoi } } + public boolean isTokenInLocalPendingRange(Token token) + { + Range searchRange = new Range<>(token, token); + + // search for non-wrap-around maps + NavigableMap, EndpointsForRange.Builder> ascendingTailMap = ascendingMap.tailMap(searchRange, true); + NavigableMap, EndpointsForRange.Builder> descendingTailMap = descendingMap.tailMap(searchRange, false); + + boolean ascMapSizeLTDescMapSize = ascendingTailMap.size() < descendingTailMap.size(); + NavigableMap, EndpointsForRange.Builder> smallerMap = ascMapSizeLTDescMapSize ? ascendingTailMap : descendingTailMap; + NavigableMap, EndpointsForRange.Builder> biggerMap = ascMapSizeLTDescMapSize ? descendingTailMap : ascendingTailMap; + + // find the intersection of two sets + for (Range range : smallerMap.keySet()) + { + EndpointsForRange.Builder replicas = biggerMap.get(range); + if (replicas != null && replicas.selfIfPresent() != null) + return true; + } + + // search for wrap-around sets + ascendingTailMap = ascendingMapForWrapAround.tailMap(searchRange, true); + descendingTailMap = descendingMapForWrapAround.tailMap(searchRange, false); + + for (EndpointsForRange.Builder endpointsForRange : ascendingTailMap.values()) + { + if (endpointsForRange.selfIfPresent() != null) + return true; + } + for (EndpointsForRange.Builder endpointsForRange : descendingTailMap.values()) + { + if (endpointsForRange.selfIfPresent() != null) + return true; + } + + return false; + } + public EndpointsForToken pendingEndpointsFor(Token token) { EndpointsForToken.Builder replicas = EndpointsForToken.builder(token); diff --git a/src/java/org/apache/cassandra/locator/TokenMetadata.java b/src/java/org/apache/cassandra/locator/TokenMetadata.java index 39f43289da..4897856cac 100644 --- a/src/java/org/apache/cassandra/locator/TokenMetadata.java +++ b/src/java/org/apache/cassandra/locator/TokenMetadata.java @@ -812,6 +812,14 @@ public class TokenMetadata return builder.build(); } + @VisibleForTesting + public void setPendingRangesUnsafe(String keyspaceName, Multimap, Replica> rangeMap) + { + PendingRangeMaps prm = new PendingRangeMaps(); + rangeMap.entries().forEach(entry -> prm.addPendingRange(entry.getKey(), entry.getValue())); + pendingRanges.put(keyspaceName, prm); + } + /** * Calculate pending ranges according to bootsrapping and leaving nodes. Reasoning is: * @@ -1400,6 +1408,12 @@ public class TokenMetadata return partitioner.decorateKey(key); } + public boolean isTokenInLocalPendingRange(String keyspaceName, Token token) + { + PendingRangeMaps pending = pendingRanges.get(keyspaceName); + return pending == null ? false : pending.isTokenInLocalPendingRange(token); + } + /** * Tracks the assignment of racks and endpoints in each datacenter for all the "normal" endpoints * in this TokenMetadata. This allows faster calculation of endpoints in NetworkTopologyStrategy. diff --git a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java index 0261e8ed52..06a5ff86c4 100644 --- a/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java +++ b/src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java @@ -52,6 +52,9 @@ public class HintedHandoffMetrics .executor(MoreExecutors.directExecutor()) .build(address -> Metrics.counter(factory.createMetricName("Hints_created-" + address.toString().replace(':', '.')))); + /** Counter of hints received whose mutations are for keys outside the owned and pending ranges for this node */ + private final Counter hintsReceivedForUnownedRanges = Metrics.counter(factory.createMetricName("Hints_for_unowned_ranges")); + public void incrCreatedHints(InetAddressAndPort address) { createdHintCounts.get(address).inc(); @@ -62,6 +65,11 @@ public class HintedHandoffMetrics notStored.get(address).mark(); } + public void incrHintsReceivedForUnownedRanges() + { + hintsReceivedForUnownedRanges.inc(); + } + public void log() { for (Entry entry : notStored.asMap().entrySet()) diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index dadbe47608..e876b9e8a6 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -125,6 +125,12 @@ public class KeyspaceMetrics public final Histogram bytesValidated; /** histogram over the number of partitions we have validated */ public final Histogram partitionsValidated; + /** Lifetime count of reads for keys outside the node's owned token ranges for this keyspace **/ + public final Counter outOfRangeTokenReads; + /** Lifetime count of writes for keys outside the node's owned token ranges for this keyspace **/ + public final Counter outOfRangeTokenWrites; + /** Lifetime count of paxos requests for keys outside the node's owned token ranges for this keyspace **/ + public final Counter outOfRangeTokenPaxosRequests; /* * Metrics for inconsistencies detected between repaired data sets across replicas. These @@ -235,6 +241,10 @@ public class KeyspaceMetrics repairedDataTrackingOverreadRows = createKeyspaceHistogram("RepairedDataTrackingOverreadRows", false); repairedDataTrackingOverreadTime = createKeyspaceTimer("RepairedDataTrackingOverreadTime"); + + outOfRangeTokenReads = createKeyspaceCounter("ReadOutOfRangeToken"); + outOfRangeTokenWrites = createKeyspaceCounter("WriteOutOfRangeToken"); + outOfRangeTokenPaxosRequests = createKeyspaceCounter("PaxosOutOfRangeToken"); } /** diff --git a/src/java/org/apache/cassandra/metrics/StorageMetrics.java b/src/java/org/apache/cassandra/metrics/StorageMetrics.java index 9399ba6789..8d24537b49 100644 --- a/src/java/org/apache/cassandra/metrics/StorageMetrics.java +++ b/src/java/org/apache/cassandra/metrics/StorageMetrics.java @@ -33,4 +33,6 @@ public class StorageMetrics public static final Counter totalHintsInProgress = Metrics.counter(factory.createMetricName("TotalHintsInProgress")); public static final Counter totalHints = Metrics.counter(factory.createMetricName("TotalHints")); public static final Counter repairExceptions = Metrics.counter(factory.createMetricName("RepairExceptions")); + public static final Counter totalOpsForInvalidToken = Metrics.counter(factory.createMetricName("TotalOpsForInvalidToken")); + public static final Counter startupOpsForInvalidToken = Metrics.counter(factory.createMetricName("StartupOpsForInvalidToken")); } diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index 7488f2ebc1..82856a009e 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -17,20 +17,38 @@ */ package org.apache.cassandra.repair; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; -import org.apache.cassandra.repair.messages.*; +import org.apache.cassandra.repair.messages.CleanupMessage; +import org.apache.cassandra.repair.messages.FailSession; +import org.apache.cassandra.repair.messages.FinalizeCommit; +import org.apache.cassandra.repair.messages.FinalizePromise; +import org.apache.cassandra.repair.messages.FinalizePropose; +import org.apache.cassandra.repair.messages.PrepareConsistentRequest; +import org.apache.cassandra.repair.messages.PrepareConsistentResponse; +import org.apache.cassandra.repair.messages.PrepareMessage; +import org.apache.cassandra.repair.messages.RepairMessage; +import org.apache.cassandra.repair.messages.StatusRequest; +import org.apache.cassandra.repair.messages.StatusResponse; +import org.apache.cassandra.repair.messages.SyncRequest; +import org.apache.cassandra.repair.messages.ValidationRequest; +import org.apache.cassandra.repair.messages.ValidationResponse; import org.apache.cassandra.schema.TableId; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.streaming.PreviewKind; +import org.apache.cassandra.service.StorageService; import static org.apache.cassandra.net.Verb.VALIDATION_RSP; @@ -56,6 +74,7 @@ public class RepairMessageVerbHandler implements IVerbHandler return prs != null ? prs.previewKind : PreviewKind.NONE; } + @Override public void doVerb(final Message message) { // TODO add cancel/interrupt message @@ -82,7 +101,7 @@ public class RepairMessageVerbHandler implements IVerbHandler if (columnFamilyStore == null) { logErrorAndSendFailureResponse(String.format("Table with id %s was dropped during prepare phase of repair", - tableId), message); + tableId.toString()), message); return; } columnFamilyStores.add(columnFamilyStore); @@ -141,7 +160,14 @@ public class RepairMessageVerbHandler implements IVerbHandler ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId); Validator validator = new Validator(desc, message.from(), validationRequest.nowInSec, isIncremental(desc.parentSessionId), previewKind(desc.parentSessionId)); - ValidationManager.instance.submitValidation(store, validator); + if (acceptMessage(validationRequest, message.from())) + { + ValidationManager.instance.submitValidation(store, validator); + } + else + { + validator.fail(); + } break; case SYNC_REQ: @@ -227,4 +253,19 @@ public class RepairMessageVerbHandler implements IVerbHandler Message reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN); MessagingService.instance().send(reply, respondTo.from()); } + + private static boolean acceptMessage(final ValidationRequest validationRequest, final InetAddressAndPort from) + { + boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); + + if (!outOfRangeTokenLogging && !outOfRangeTokenRejection) + return true; + + return StorageService.instance.getNormalizedLocalRanges(validationRequest.desc.keyspace) + .validateRangeRequest(validationRequest.desc.ranges, + "RepairSession #" + validationRequest.desc.parentSessionId, + "validation request", + from); + } } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 081e133db8..221b9c62ae 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -118,6 +118,7 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.dht.BootStrapper; import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.OwnedRanges; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.RangeStreamer; import org.apache.cassandra.dht.RangeStreamer.FetchReplica; @@ -279,6 +280,35 @@ public class StorageService extends NotificationBroadcasterSupport implements IE public static final StorageService instance = new StorageService(); + private final java.util.function.Predicate anyOutOfRangeOpsRecorded + = keyspace -> keyspace.metric.outOfRangeTokenReads.getCount() > 0 + || keyspace.metric.outOfRangeTokenWrites.getCount() > 0 + || keyspace.metric.outOfRangeTokenPaxosRequests.getCount() > 0; + + private long[] getOutOfRangeOperationCounts(Keyspace keyspace) + { + return new long[] + { + keyspace.metric.outOfRangeTokenReads.getCount(), + keyspace.metric.outOfRangeTokenWrites.getCount(), + keyspace.metric.outOfRangeTokenPaxosRequests.getCount() + }; + } + + public Map getOutOfRangeOperationCounts() + { + return Schema.instance.getKeyspaces() + .stream() + .map(Keyspace::open) + .filter(anyOutOfRangeOpsRecorded) + .collect(Collectors.toMap(Keyspace::getName, this::getOutOfRangeOperationCounts)); + } + + public void incOutOfRangeOperationCount() + { + (isStarting() ? StorageMetrics.startupOpsForInvalidToken : StorageMetrics.totalOpsForInvalidToken).inc(); + } + @Deprecated public boolean isInShutdownHook() { @@ -317,6 +347,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return ranges; } + public OwnedRanges getNormalizedLocalRanges(String keyspaceName) + { + return new OwnedRanges(getLocalReplicas(keyspaceName).ranges()); + } + public Collection> getPrimaryRanges(String keyspace) { return getPrimaryRangesForEndpoint(keyspace, FBUtilities.getBroadcastAddressAndPort()); @@ -4503,6 +4538,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalReplicasForToken(token); } + public boolean isEndpointValidForWrite(String keyspace, Token token) + { + AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspace).getReplicationStrategy(); + return replicationStrategy.isTokenInLocalNaturalOrPendingRange(token); + } + public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception { LoggingSupportFactory.getLoggingSupport().setLoggingLevel(classQualifier, rawLevel); @@ -6077,6 +6118,36 @@ public class StorageService extends NotificationBroadcasterSupport implements IE ClientResourceLimits.setEndpointLimit(newLimit); } + public boolean isOutOfTokenRangeRequestLoggingEnabled() + { + return DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + } + + public void setOutOfTokenRangeRequestLoggingEnabled(boolean enabled) + { + if (enabled) + logger.info("Enabling logging of requests on tokens outside owned ranges"); + else + logger.info("Disabling logging of requests on tokens outside owned ranges"); + + DatabaseDescriptor.setLogOutOfTokenRangeRequests(enabled); + } + + public boolean isOutOfTokenRangeRequestRejectionEnabled() + { + return DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); + } + + public void setOutOfTokenRangeRequestRejectionEnabled(boolean enabled) + { + if (enabled) + logger.info("Enabling rejection of requests on tokens outside owned ranges"); + else + logger.info("Disabling rejection of requests on tokens outside owned ranges"); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(enabled); + } + @VisibleForTesting public void shutdownServer() { diff --git a/src/java/org/apache/cassandra/service/StorageServiceMBean.java b/src/java/org/apache/cassandra/service/StorageServiceMBean.java index 31dc6243a4..34b7fb7ff3 100644 --- a/src/java/org/apache/cassandra/service/StorageServiceMBean.java +++ b/src/java/org/apache/cassandra/service/StorageServiceMBean.java @@ -868,4 +868,22 @@ public interface StorageServiceMBean extends NotificationEmitter public Long getRepairRpcTimeout(); public void setRepairRpcTimeout(Long timeoutInMillis); + + /** + * Toggles to turn on the logging or rejection of operations for token ranges that the node does not own, + * or is not about to acquire. + */ + boolean isOutOfTokenRangeRequestLoggingEnabled(); + void setOutOfTokenRangeRequestLoggingEnabled(boolean enabled); + + boolean isOutOfTokenRangeRequestRejectionEnabled(); + void setOutOfTokenRangeRequestRejectionEnabled(boolean enabled); + + /** + * Get the per-keyspace counts of operations that the node has received for tokens outside + * its owned ranges. Represented as a {@code Map}, keys are keyspace names and the + * values are the counts for read, write and paxos ops respectivly. + * e.g. keyspace_name -> [reads, writes, paxos]. + */ + Map getOutOfRangeOperationCounts(); } diff --git a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java index ab24f50efb..04011074df 100644 --- a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java +++ b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java @@ -22,16 +22,22 @@ package org.apache.cassandra.service.paxos; import java.util.concurrent.CountDownLatch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ConsistencyLevel; import org.apache.cassandra.db.WriteType; +import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.exceptions.WriteTimeoutException; +import org.apache.cassandra.locator.InetAddressAndPort; import org.apache.cassandra.net.RequestCallback; import static java.util.concurrent.TimeUnit.NANOSECONDS; public abstract class AbstractPaxosCallback implements RequestCallback { + private static final Logger logger = LoggerFactory.getLogger(AbstractPaxosCallback.class); protected final CountDownLatch latch; protected final int targets; private final ConsistencyLevel consistency; @@ -63,4 +69,16 @@ public abstract class AbstractPaxosCallback implements RequestCallback throw new AssertionError("This latch shouldn't have been interrupted."); } } + + @Override + public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason) + { + logger.debug("Received paxos propose/prepare failure response from {} reason {}", from, failureReason); + } + + @Override + public boolean invokeOnFailure() + { + return true; + } } diff --git a/src/java/org/apache/cassandra/service/paxos/AbstractPaxosVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosVerbHandler.java new file mode 100644 index 0000000000..4b3dc7bc93 --- /dev/null +++ b/src/java/org/apache/cassandra/service/paxos/AbstractPaxosVerbHandler.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.NoSpamLogger; + +public abstract class AbstractPaxosVerbHandler implements IVerbHandler +{ + private static final Logger logger = LoggerFactory.getLogger(AbstractPaxosVerbHandler.class); + private static final String logMessageTemplate = "Receiving Paxos operation(s) for token(s) neither owned nor pending. Example: from {} for token {} in {}.{}"; + + @Override + public void doVerb(Message message) + { + boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests(); + boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests(); + + Commit commit = message.payload; + DecoratedKey key = commit.update.partitionKey(); + boolean isOutOfRangeCommit = isOutOfRangeCommit(commit.update.metadata().keyspace, key); + if (isOutOfRangeCommit) + { + StorageService.instance.incOutOfRangeOperationCount(); + Keyspace.open(commit.update.metadata().keyspace).metric.outOfRangeTokenPaxosRequests.inc(); + + // Log at most 1 message per second + if (outOfRangeTokenLogging) + NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, + message.from(), key.getToken(), commit.update.metadata().keyspace, commit.update.metadata().name); + + } + if (outOfRangeTokenRejection && isOutOfRangeCommit) + sendFailureResponse(message); + else + processMessage(message); + } + + abstract void processMessage(Message message); + + private static void sendFailureResponse(Message respondTo) + { + Message reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN); + MessagingService.instance().send(reply, respondTo.from()); + } + + private static boolean isOutOfRangeCommit(String keyspace, DecoratedKey key) + { + return ! StorageService.instance.isEndpointValidForWrite(keyspace, key.getToken()); + } +} diff --git a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java index 12466ddc69..94ebd9ce16 100644 --- a/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java +++ b/src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java @@ -20,16 +20,16 @@ */ package org.apache.cassandra.service.paxos; -import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.tracing.Tracing; -public class CommitVerbHandler implements IVerbHandler +public class CommitVerbHandler extends AbstractPaxosVerbHandler { public static final CommitVerbHandler instance = new CommitVerbHandler(); - public void doVerb(Message message) + @Override + void processMessage(Message message) { PaxosState.commit(message.payload); diff --git a/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java index 157630f277..86558dc5bc 100644 --- a/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java +++ b/src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java @@ -19,11 +19,10 @@ package org.apache.cassandra.service.paxos; * under the License. * */ -import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; -public class PrepareVerbHandler implements IVerbHandler +public class PrepareVerbHandler extends AbstractPaxosVerbHandler { public static PrepareVerbHandler instance = new PrepareVerbHandler(); @@ -32,7 +31,8 @@ public class PrepareVerbHandler implements IVerbHandler return PaxosState.prepare(toPrepare); } - public void doVerb(Message message) + @Override + public void processMessage(Message message) { Message reply = message.responseWith(doPrepare(message.payload)); MessagingService.instance().send(reply, message.from()); diff --git a/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java b/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java index 5a20b67458..98e188c715 100644 --- a/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java +++ b/src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java @@ -23,7 +23,7 @@ import org.apache.cassandra.net.IVerbHandler; import org.apache.cassandra.net.Message; import org.apache.cassandra.net.MessagingService; -public class ProposeVerbHandler implements IVerbHandler +public class ProposeVerbHandler extends AbstractPaxosVerbHandler implements IVerbHandler { public static final ProposeVerbHandler instance = new ProposeVerbHandler(); @@ -32,7 +32,7 @@ public class ProposeVerbHandler implements IVerbHandler return PaxosState.propose(proposal); } - public void doVerb(Message message) + void processMessage(Message message) { Message reply = message.responseWith(doPropose(message.payload)); MessagingService.instance().send(reply, message.from()); diff --git a/src/java/org/apache/cassandra/streaming/StreamReceivedOutOfTokenRangeException.java b/src/java/org/apache/cassandra/streaming/StreamReceivedOutOfTokenRangeException.java new file mode 100644 index 0000000000..c5a9339ed7 --- /dev/null +++ b/src/java/org/apache/cassandra/streaming/StreamReceivedOutOfTokenRangeException.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.streaming; + +import java.util.Collection; + +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.io.sstable.Descriptor; + +public class StreamReceivedOutOfTokenRangeException extends RuntimeException +{ + private final Collection> ownedRanges; + private final DecoratedKey key; + private final String filename; + + public StreamReceivedOutOfTokenRangeException(Collection> ownedRanges, + DecoratedKey key, + String filename) + { + this.ownedRanges = ownedRanges; + this.key = key; + this.filename = filename; + } + + public String getMessage() + { + return String.format("Received stream for sstable %s containing key %s outside of owned ranges %s ", + filename, + key, + ownedRanges); + } +} diff --git a/src/java/org/apache/cassandra/streaming/StreamRequest.java b/src/java/org/apache/cassandra/streaming/StreamRequest.java index 0c8542fcfa..6589e30379 100644 --- a/src/java/org/apache/cassandra/streaming/StreamRequest.java +++ b/src/java/org/apache/cassandra/streaming/StreamRequest.java @@ -23,6 +23,9 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; + import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; @@ -61,6 +64,17 @@ public class StreamRequest this.columnFamilies.addAll(columnFamilies); } + public String toString() + { + return MoreObjects.toStringHelper(this) + .add("keyspace", keyspace) + .add("tables", columnFamilies) + .add("full", full) + .add("transientReplicas", transientReplicas) + .omitNullValues() + .toString(); + } + public static class StreamRequestSerializer implements IVersionedSerializer { public void serialize(StreamRequest request, DataOutputPlus out, int version) throws IOException diff --git a/src/java/org/apache/cassandra/streaming/StreamRequestOutOfTokenRangeException.java b/src/java/org/apache/cassandra/streaming/StreamRequestOutOfTokenRangeException.java new file mode 100644 index 0000000000..77dbee1524 --- /dev/null +++ b/src/java/org/apache/cassandra/streaming/StreamRequestOutOfTokenRangeException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.streaming; + +import java.util.Collection; + +public class StreamRequestOutOfTokenRangeException extends RuntimeException +{ + private final Collection requests; + + public StreamRequestOutOfTokenRangeException(Collection requests) + { + this.requests = requests; + } + + public String getMessage() + { + return String.format("Received stream requests containing ranges outside of owned ranges: %s", requests); + } +} diff --git a/src/java/org/apache/cassandra/streaming/StreamSession.java b/src/java/org/apache/cassandra/streaming/StreamSession.java index 21d5afe480..786311b1d8 100644 --- a/src/java/org/apache/cassandra/streaming/StreamSession.java +++ b/src/java/org/apache/cassandra/streaming/StreamSession.java @@ -19,34 +19,61 @@ package org.apache.cassandra.streaming; import java.io.EOFException; import java.net.SocketTimeoutException; -import java.util.*; -import java.util.concurrent.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.*; -import org.apache.cassandra.concurrent.ScheduledExecutors; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.locator.RangesAtEndpoint; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.netty.channel.Channel; import io.netty.channel.ChannelId; +import org.apache.cassandra.concurrent.ScheduledExecutors; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.lifecycle.TransactionAlreadyCompletedException; +import org.apache.cassandra.dht.OwnedRanges; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.apache.cassandra.gms.*; +import org.apache.cassandra.gms.EndpointState; +import org.apache.cassandra.gms.IEndpointStateChangeSubscriber; import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; import org.apache.cassandra.locator.Replica; import org.apache.cassandra.metrics.StreamingMetrics; import org.apache.cassandra.net.OutboundConnectionSettings; import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.streaming.async.NettyStreamingMessageSender; -import org.apache.cassandra.streaming.messages.*; +import org.apache.cassandra.streaming.messages.CompleteMessage; +import org.apache.cassandra.streaming.messages.IncomingStreamMessage; +import org.apache.cassandra.streaming.messages.OutgoingStreamMessage; +import org.apache.cassandra.streaming.messages.PrepareAckMessage; +import org.apache.cassandra.streaming.messages.PrepareSynAckMessage; +import org.apache.cassandra.streaming.messages.PrepareSynMessage; +import org.apache.cassandra.streaming.messages.ReceivedMessage; +import org.apache.cassandra.streaming.messages.SessionFailedMessage; +import org.apache.cassandra.streaming.messages.StreamInitMessage; +import org.apache.cassandra.streaming.messages.StreamMessage; +import org.apache.cassandra.streaming.messages.StreamMessageHeader; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.JVMStabilityInspector; import org.apache.cassandra.utils.NoSpamLogger; @@ -163,7 +190,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber final Map>> transferredRangesPerKeyspace = new HashMap<>(); private final boolean isFollower; - private final NettyStreamingMessageSender messageSender; + private final StreamingMessageSender messageSender; // contains both inbound and outbound channels private final ConcurrentMap channels = new ConcurrentHashMap<>(); @@ -313,11 +340,19 @@ public class StreamSession implements IEndpointStateChangeSubscriber { failIfFinished(); - if (!messageSender.hasControlChannel() && isControlChannel) - messageSender.injectControlMessageChannel(channel); + if (messageSender instanceof NettyStreamingMessageSender) + { + NettyStreamingMessageSender nettyStreamingMessageSender = (NettyStreamingMessageSender) messageSender; + if (!nettyStreamingMessageSender.hasControlChannel() && isControlChannel) + nettyStreamingMessageSender.injectControlMessageChannel(channel); - channel.closeFuture().addListener(ignored -> onChannelClose(channel)); - return channels.putIfAbsent(channel.id(), channel) == null; + channel.closeFuture().addListener(ignored -> onChannelClose(channel)); + return channels.putIfAbsent(channel.id(), channel) == null; + } + else + { + throw new IllegalStateException("Tried to attach non-Netty message sender"); + } } /** @@ -361,7 +396,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber logger.info("[Stream #{}] Starting streaming to {}{}", planId(), peer, template.connectTo == null ? "" : " through " + template.connectTo); - messageSender.initialize(); + getMessageSender().initialize(); onInitializationComplete(); } catch (Exception e) @@ -546,7 +581,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber return state; } - public NettyStreamingMessageSender getMessageSender() + public StreamingMessageSender getMessageSender() { return messageSender; } @@ -628,11 +663,9 @@ public class StreamSession implements IEndpointStateChangeSubscriber PrepareSynMessage prepare = new PrepareSynMessage(); prepare.requests.addAll(requests); for (StreamTransferTask task : transfers.values()) - { prepare.summaries.add(task.getSummary()); - } - messageSender.sendMessage(prepare); + getMessageSender().sendMessage(prepare); } /** @@ -673,10 +706,10 @@ public class StreamSession implements IEndpointStateChangeSubscriber logError(e); - if (messageSender.connected()) + if (getMessageSender().connected()) { state(State.FAILED); // make sure subsequent error handling sees the session in a final state - messageSender.sendMessage(new SessionFailedMessage()); + getMessageSender().sendMessage(new SessionFailedMessage()); } return closeSession(State.FAILED); @@ -725,10 +758,10 @@ public class StreamSession implements IEndpointStateChangeSubscriber * Finish preparing the session. This method is blocking (memtables are flushed in {@link #addTransferRanges}), * so the logic should not execute on the main IO thread (read: netty event loop). */ - private void prepareAsync(Collection requests, Collection summaries) + @VisibleForTesting + public void prepareAsync(Collection requests, Collection summaries) { - for (StreamRequest request : requests) - addTransferRanges(request.keyspace, RangesAtEndpoint.concat(request.full, request.transientReplicas), request.columnFamilies, true); // always flush on stream request + processStreamRequests(requests); for (StreamSummary summary : summaries) prepareReceiving(summary); @@ -736,7 +769,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber if (!peer.equals(FBUtilities.getBroadcastAddressAndPort())) for (StreamTransferTask task : transfers.values()) prepareSynAck.summaries.add(task.getSummary()); - messageSender.sendMessage(prepareSynAck); + getMessageSender().sendMessage(prepareSynAck); streamResult.handleSessionPrepared(this); @@ -755,7 +788,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber // only send the (final) ACK if we are expecting the peer to send this node (the initiator) some files if (!isPreview()) - messageSender.sendMessage(new PrepareAckMessage()); + getMessageSender().sendMessage(new PrepareAckMessage()); } if (isPreview()) @@ -771,6 +804,32 @@ public class StreamSession implements IEndpointStateChangeSubscriber startStreamingFiles(true); } + private void processStreamRequests(Collection requests) + { + List rejectedRequests = new ArrayList<>(); + + // group requests by keyspace + Multimap requestsByKeyspace = ArrayListMultimap.create(); + requests.forEach(r -> requestsByKeyspace.put(r.keyspace, r)); + + requestsByKeyspace.asMap().forEach((ks, reqs) -> + { + OwnedRanges ownedRanges = StorageService.instance.getNormalizedLocalRanges(ks); + + reqs.forEach(req -> + { + RangesAtEndpoint allRangesAtEndpoint = RangesAtEndpoint.concat(req.full, req.transientReplicas); + if (ownedRanges.validateRangeRequest(allRangesAtEndpoint.ranges(), "Stream #" + planId(), "stream request", peer)) + addTransferRanges(req.keyspace, allRangesAtEndpoint, req.columnFamilies, true); // always flush on stream request + else + rejectedRequests.add(req); + }); + }); + + if (!rejectedRequests.isEmpty()) + throw new StreamRequestOutOfTokenRangeException(rejectedRequests); + } + /** * Call back after sending StreamMessageHeader. * @@ -812,7 +871,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber StreamingMetrics.totalIncomingBytes.inc(headerSize); metrics.incomingBytes.inc(headerSize); // send back file received message - messageSender.sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber)); + getMessageSender().sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber)); StreamHook.instance.reportIncomingStream(message.header.tableId, message.stream, this, message.header.sequenceNumber); long receivedStartNanos = System.nanoTime(); try @@ -891,7 +950,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber } else { - messageSender.sendMessage(new CompleteMessage()); + getMessageSender().sendMessage(new CompleteMessage()); closeSession(State.COMPLETE); } @@ -998,7 +1057,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber { // pass the session planId/index to the OFM (which is only set at init(), after the transfers have already been created) ofm.header.addSessionInfo(this); - messageSender.sendMessage(ofm); + getMessageSender().sendMessage(ofm); } } else diff --git a/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java b/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java index 96e76267ec..de4b486f7a 100644 --- a/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java +++ b/src/java/org/apache/cassandra/streaming/StreamingMessageSender.java @@ -29,7 +29,7 @@ public interface StreamingMessageSender { void initialize() throws IOException; - void sendMessage(StreamMessage message) throws IOException; + void sendMessage(StreamMessage message); boolean connected(); diff --git a/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java b/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java index 9f8f47608d..4334382be4 100644 --- a/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java +++ b/src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java @@ -288,7 +288,7 @@ public class NettyStreamingMessageSender implements StreamingMessageSender * @return null if the message was processed sucessfully; else, a {@link java.util.concurrent.Future} to indicate * the status of aborting any remaining tasks in the session. */ - java.util.concurrent.Future onControlMessageComplete(Future future, StreamMessage msg) + java.util.concurrent.Future onControlMessageComplete(Future future, StreamMessage msg) { ChannelFuture channelFuture = (ChannelFuture)future; Throwable cause = future.cause(); diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index b06e0f31ee..025371bf4a 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -1370,6 +1370,11 @@ public class NodeProbe implements AutoCloseable return spProxy.getReadRepairRepairedBackground(); } + public Map getOutOfRangeOpCounts() + { + return ssProxy.getOutOfRangeOperationCounts(); + } + // JMX getters for the o.a.c.metrics API below. /** * Retrieve cache metrics based on the cache type (KeyCache, RowCache, or CounterCache) diff --git a/src/java/org/apache/cassandra/tools/nodetool/Info.java b/src/java/org/apache/cassandra/tools/nodetool/Info.java index cf1f894ae8..24ccf36ebd 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/Info.java +++ b/src/java/org/apache/cassandra/tools/nodetool/Info.java @@ -41,6 +41,9 @@ public class Info extends NodeToolCmd @Option(name = {"-T", "--tokens"}, description = "Display all tokens") private boolean tokens = false; + @Option(name = {"-O", "--out-of-range-ops"}, description = "Display per-keyspace counts of operations for invalid tokens") + private boolean outOfRangeOps = false; + @Override public void execute(NodeProbe probe) { @@ -175,6 +178,20 @@ public class Info extends NodeToolCmd { out.printf("%-23s: (node is not joined to the cluster)%n", "Token"); } + + // Operations for out of range tokens + if (this.outOfRangeOps) + { + System.out.printf("%-23s: %-48s %10s %10s %10s%n", "Invalid Token Ops", "Keyspace", "Read", "Write", "Paxos"); + + Map outOfRangeOpCounts = probe.getOutOfRangeOpCounts(); + outOfRangeOpCounts.forEach((ks, counts) -> System.out.printf("%-24s %-48s %10s %10s %10s%n", + "", + ks, + counts[0], + counts[1], + counts[2])); + } } /** diff --git a/test/distributed/org/apache/cassandra/distributed/test/OutOfTokenRangeTest.java b/test/distributed/org/apache/cassandra/distributed/test/OutOfTokenRangeTest.java new file mode 100644 index 0000000000..539d8e4164 --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/OutOfTokenRangeTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.io.IOException; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.distributed.Cluster; +import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.ICluster; + +import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows; +import static org.apache.cassandra.distributed.shared.AssertUtils.row; + +public class OutOfTokenRangeTest extends TestBaseImpl +{ + private static ICluster cluster = null; + + @BeforeClass + public static void setupPartitions() throws IOException + { + cluster = init(Cluster.build().withNodes(2).start(), 2); + cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck text, v text, PRIMARY KEY (pk, ck))"); + + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk,ck,v) VALUES (1, 'a', '1a')", ConsistencyLevel.ALL); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk,ck,v) VALUES (1, 'b', '1b')", ConsistencyLevel.ALL); + cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk,ck,v) VALUES (1, 'c', '1c')", ConsistencyLevel.ALL); + + // Prove the row is there before adding the index + assertRows(cluster.coordinator(1).execute("SELECT pk, ck, v FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck = 'a' ALLOW FILTERING", ConsistencyLevel.ALL), row(1, "a", "1a")); + + cluster.schemaChange("CREATE INDEX ckindex ON " + KEYSPACE + ".tbl (ck)"); + } + + @AfterClass + public static void closeCluster() throws Exception + { + if (cluster != null) + { + cluster.close(); + } + } + + @Test + public void checkSinglePartitionReadCommandTest() + { + Object[][] results = cluster.coordinator(1).execute("SELECT pk, ck, v FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.ALL); + assertRows(results, row(1, "a", "1a"), row(1, "b", "1b"), row(1, "c", "1c")); + } + + @Test + public void checkPartitionRangeReadCommandTest() + { + Object[][] results = cluster.coordinator(1).execute("SELECT pk, ck, v FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck = 'a'", ConsistencyLevel.ALL); + assertRows(results, row(1, "a", "1a")); + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java index c68320e094..bf66369f65 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java @@ -178,7 +178,7 @@ public class ReadRepairTest extends TestBaseImpl @Test public void movingTokenReadRepairTest() throws Throwable { - try (Cluster cluster = init(Cluster.create(4), 3)) + try (Cluster cluster = init(builder().withNodes(4).start(), 3)) { List tokens = cluster.tokens(); @@ -203,6 +203,13 @@ public class ReadRepairTest extends TestBaseImpl PendingRangeCalculatorService.instance.blockUntilFinished(); }).accept(cluster.get(2).broadcastAddress()); + // mark #2 as leaving in #1 + cluster.get(1).acceptsOnInstance((InetSocketAddress endpoint) -> { + StorageService.instance.getTokenMetadata().addLeavingEndpoint(InetAddressAndPort.getByAddressOverrideDefaults(endpoint.getAddress(), endpoint.getPort())); + PendingRangeCalculatorService.instance.update(); + PendingRangeCalculatorService.instance.blockUntilFinished(); + }).accept(cluster.get(2).broadcastAddress()); + // prevent #4 from reading or writing to #3, so our QUORUM must contain #2 and #4 // since #1 is taking over the range, this means any read-repair must make it to #1 as well // (as a speculative repair in this case, as we prefer to send repair mutations to the initial @@ -353,7 +360,9 @@ public class ReadRepairTest extends TestBaseImpl String key = "test1"; try (Cluster cluster = init(Cluster.build() .withConfig(config -> config.with(Feature.GOSSIP, Feature.NETWORK) - .set("read_request_timeout_in_ms", Integer.MAX_VALUE)) + .set("read_request_timeout_in_ms", Integer.MAX_VALUE) + .set("reject_out_of_token_range_requests", false) + ) .withTokenSupplier(TokenSupplier.evenlyDistributedTokens(4)) .withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(4, "dc0", "rack0")) .withNodes(3) @@ -491,6 +500,7 @@ public class ReadRepairTest extends TestBaseImpl // on timestamp tie of RT and partition deletion: we should not generate RT bounds in such case, // since monotonicity is already ensured by the partition deletion, and RT is unnecessary there. // For details, see CASSANDRA-16453. + @SuppressWarnings("unused") public static Object repairPartition(DecoratedKey partitionKey, Map mutations, ReplicaPlan.ForTokenWrite writePlan, @SuperCall Callable r) throws Exception { Assert.assertEquals(2, mutations.size()); diff --git a/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java new file mode 100644 index 0000000000..54dc675a66 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.context.CounterContext; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery; +import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress; +import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt; +import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink; + +public class CounterMutationVerbHandlerOutOfRangeTest +{ + private static final String KEYSPACE = "CounterCacheTest"; + private static final String TABLE = "Counter1"; + + private CounterMutationVerbHandler handler; + private ColumnFamilyStore cfs; + private long startingTotalMetricCount; + private long startingKeyspaceMetricCount; + + @BeforeClass + public static void init() throws Exception + { + SchemaLoader.loadSchema(); + SchemaLoader.createKeyspace(KEYSPACE, + KeyspaceParams.simple(1), + SchemaLoader.counterCFMD(KEYSPACE, TABLE)); + StorageService.instance.initServer(0); + } + + @Before + public void setup() throws Exception + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress); + + MessagingService.instance().inboundSink.clear(); + MessagingService.instance().outboundSink.clear(); + + handler = new CounterMutationVerbHandler(); + cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); + cfs.truncateBlocking(); + startingKeyspaceMetricCount = keyspaceMetricValue(); + startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + + } + + @Test + public void acceptMutationForNaturalEndpoint() throws Exception + { + int messageId = randomInt(); + int value = randomInt(); + int key = 30; + CounterMutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + + // unlike non-counter mutations, we can't verify the response message for a successful write. + // acting as the leader for the mutation, we'll try to forward the writes to the other replicas + // but this will fail as the other node isn't really there. The response message is written + // by the callback to these messages, so it will never get sent. So the best we can do is to check + // that the mutation was actually applied locally. When a counter mutation is rejected by the verb + // handler we *can* verify the failure response message is sent. + verifyWrite(key, value); + assertEquals(startingTotalMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount, keyspaceMetricValue()); + } + + @Test + public void acceptMutationForPendingEndpoint() throws Exception + { + // remove localhost from TM and add it back as pending + StorageService.instance.getTokenMetadata().removeEndpoint(broadcastAddress); + Multimap, Replica> pending = HashMultimap.create(); + Range range = new Range<>(bytesToken(0), bytesToken(100)); + pending.put(range, new Replica(broadcastAddress, range, true)); + StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(KEYSPACE, pending); + + int messageId = randomInt(); + int value = randomInt(); + int key = 50; + CounterMutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + verifyWrite(key, value); + assertEquals(startingTotalMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount, keyspaceMetricValue()); + } + + @Test + public void rejectMutationForTokenOutOfRange() throws Exception + { + // reject a mutation for a token the node neither owns nor is pending + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int value = randomInt(); + int key = 200; + CounterMutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + verifyFailureResponse(messageSink, messageId); + assertEquals(startingTotalMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount + 1, keyspaceMetricValue()); + } + + @Test + public void acceptMutationIfRejectionNotEnabled() throws Exception + { + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + int messageId = randomInt(); + int value = randomInt(); + int key = 500; + CounterMutation mutation = mutation(key, value); + // the node which is the actual natural endpoint for this mutation is not a real + // node, but if we write at CL.ANY we'll generate a hint for it and StorageProxy's + // counterWriterPerformer will blindly apply the mutation, so we can verify it locally + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + + verifyWrite(key, value); + assertEquals(startingTotalMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount + 1, keyspaceMetricValue()); + } + + private void verifyWrite(int key, int value) + { + ReadCommand read = Util.cmd(cfs, bytes(key)).build(); + ColumnMetadata col = cfs.metadata().getColumn(bytes("val")); + assertEquals(value, CounterContext.instance().total(Util.getOnlyRow(read).getCell(col))); + } + + + private static void verifyFailureResponse(ListenableFuture messageSink, int messageId ) throws Exception + { + MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS); + assertEquals(Verb.FAILURE_RSP, response.message.verb()); + assertEquals(broadcastAddress, response.message.from()); + assertTrue(response.message.payload instanceof RequestFailureReason); + assertEquals(messageId, response.message.id()); + assertEquals(node1, response.to); + } + + private CounterMutation mutation(int key, int columnValue) + { + TableMetadata cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey dk = cfs.decorateKey(bytes(key)); + ColumnMetadata col = cfs.metadata().getColumn(bytes("val")); + ByteBuffer val = CounterContext.instance().createLocal(columnValue); + Cell counterCell = BufferCell.live(col, FBUtilities.timestampMicros(), val); + Row row = BTreeRow.singleCellRow(cfs.metadata().comparator.make("clustering_1"), counterCell); + PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, dk, row); + return new CounterMutation(new Mutation(update), ConsistencyLevel.ANY); + } + + private long keyspaceMetricValue() + { + return cfs.keyspace.metric.outOfRangeTokenWrites.getCount(); + } +} diff --git a/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java new file mode 100644 index 0000000000..12f2f8a9a1 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.rows.BTreeRow; +import org.apache.cassandra.db.rows.BufferCell; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.net.IVerbHandler; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.ColumnMetadata; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.assertEquals; + +import static org.apache.cassandra.utils.ByteBufferUtil.bytes; +import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery; +import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress; +import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt; +import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink; + +public class MutationVerbHandlerOutOfRangeTest +{ + private static final String TEST_NAME = "mutation_vh_test_"; + private static final String KEYSPACE = TEST_NAME + "cql_keyspace"; + private static final String TABLE = "table1"; + + private ColumnFamilyStore cfs; + private long startingTotalMetricCount; + private long startingKeyspaceMetricCount; + + @BeforeClass + public static void init() throws Exception + { + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition(TEST_NAME); + StorageService.instance.initServer(0); + + } + + @Before + public void setup() throws Exception + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress); + + MessagingService.instance().inboundSink.clear(); + MessagingService.instance().outboundSink.clear(); + + cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE); + startingKeyspaceMetricCount = keyspaceMetricValue(cfs); + startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + } + + @Test + public void acceptMutationForNaturalEndpoint() throws Exception + { + acceptMutationForNaturalEndpoint(new MutationVerbHandler()); + } + + @Test + public void acceptReadRepairForNaturalEndpoint() throws Exception + { + acceptMutationForNaturalEndpoint(new ReadRepairVerbHandler()); + } + + private void acceptMutationForNaturalEndpoint(IVerbHandler handler) throws Exception + { + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int value = randomInt(); + int key = 50; + Mutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, key, value, false, false); + } + + @Test + public void acceptMutationForPendingEndpoint() throws Exception + { + acceptMutationForPendingEndpoint(new MutationVerbHandler()); + } + + @Test + public void acceptReadRepairForPendingEndpoint() throws Exception + { + acceptMutationForPendingEndpoint(new ReadRepairVerbHandler()); + } + + private void acceptMutationForPendingEndpoint(IVerbHandler handler ) throws Exception + { + // remove localhost from TM and add it back as pending + StorageService.instance.getTokenMetadata().removeEndpoint(broadcastAddress); + Multimap, Replica> pending = HashMultimap.create(); + Range range = new Range<>(bytesToken(0), bytesToken(100)); + pending.put(range, new Replica(broadcastAddress, range, true)); + StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(KEYSPACE, pending); + + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int value = randomInt(); + int key = 50; + Mutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, key, value, false, false); + } + + @Test + public void rejectMutationForTokenOutOfRange() throws Exception + { + rejectMutationForTokenOutOfRange(new MutationVerbHandler()); + } + + @Test + public void rejectReadRepairForTokenOutOfRange() throws Exception + { + rejectMutationForTokenOutOfRange(new ReadRepairVerbHandler()); + } + + private void rejectMutationForTokenOutOfRange(IVerbHandler handler) throws Exception + { + // reject a mutation for a token the node neither owns nor is pending + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int value = randomInt(); + int key = 200; + Mutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, key, value, true, true); + } + + @Test + public void acceptMutationIfRejectionNotEnabled() throws Exception + { + acceptMutationIfRejectionNotEnabled(new MutationVerbHandler()); + } + + @Test + public void acceptReadRepairIfRejectionNotEnabled() throws Exception + { + acceptMutationIfRejectionNotEnabled(new ReadRepairVerbHandler()); + } + + private void acceptMutationIfRejectionNotEnabled(IVerbHandler handler) throws Exception + { + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int value = randomInt(); + int key = 200; + Mutation mutation = mutation(key, value); + handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, key, value, true, false); + } + + static int toInt(Cell cell) + { + return cell.accessor().toInt(cell.value()); + } + + private void getAndVerifyResponse(ListenableFuture messageSink, + int messageId, + int key, + int value, + boolean isOutOfRange, + boolean expectFailure) throws InterruptedException, ExecutionException, TimeoutException + { + MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS); + assertEquals(expectFailure ? Verb.FAILURE_RSP : Verb.MUTATION_RSP, response.message.verb()); + assertEquals(broadcastAddress, response.message.from()); + assertEquals(expectFailure, response.message.payload instanceof RequestFailureReason); + assertEquals(messageId, response.message.id()); + assertEquals(node1, response.to); + assertEquals(startingTotalMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount + (isOutOfRange ? 1 : 0), keyspaceMetricValue(cfs)); + if (!expectFailure) + { + ReadCommand read = Util.cmd(cfs, bytes(key)).build(); + ColumnMetadata col = cfs.metadata().getColumn(bytes("v1")); + assertEquals(value, toInt(Util.getOnlyRow(read).getCell(col))); + } + } + + private static long keyspaceMetricValue(ColumnFamilyStore cfs) + { + return cfs.keyspace.metric.outOfRangeTokenWrites.getCount(); + } + + private Mutation mutation(int key, int columnValue) + { + TableMetadata cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + DecoratedKey dk = cfs.decorateKey(bytes(key)); + ColumnMetadata col = cfs.metadata().getColumn(bytes("v1")); + Cell cell = BufferCell.live(col, FBUtilities.timestampMicros(), bytes(columnValue)); + Row row = BTreeRow.singleCellRow(Clustering.EMPTY, cell); + PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, dk, row); + return new Mutation(update); + } +} diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java new file mode 100644 index 0000000000..2940438904 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.TokenRangeTestUtil; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import static org.apache.cassandra.net.Verb.READ_REQ; +import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress; +import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt; +import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink; + +public class ReadCommandVerbHandlerOutOfRangeTest +{ + private static ReadCommandVerbHandler handler; + private static TableMetadata metadata_nonreplicated; + private ColumnFamilyStore cfs; + private long startingTotalMetricCount; + private long startingKeyspaceMetricCount; + + private static final String TEST_NAME = "read_command_vh_test_"; + private static final String KEYSPACE_NONREPLICATED = TEST_NAME + "cql_keyspace"; + private static final String TABLE = "table1"; + + @BeforeClass + public static void init() throws Throwable + { + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition(TEST_NAME); + StorageService.instance.initServer(0); + metadata_nonreplicated = Schema.instance.getTableMetadata(KEYSPACE_NONREPLICATED, TABLE); + } + + @Before + public void setup() + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress); + + MessagingService.instance().inboundSink.clear(); + MessagingService.instance().outboundSink.clear(); + MessagingService.instance().outboundSink.add((message, to) -> false); + MessagingService.instance().inboundSink.add((message) -> false); + + cfs = Keyspace.open(KEYSPACE_NONREPLICATED).getColumnFamilyStore(TABLE); + startingKeyspaceMetricCount = keyspaceMetricValue(cfs); + startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + handler = new ReadCommandVerbHandler(); + } + + private static DecoratedKey key(TableMetadata metadata, int key) + { + return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)); + } + + @Test + public void acceptReadForNaturalEndpoint() throws Exception + { + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 50; + ReadCommand command = command(key); + handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, false, false); + } + + @Test + public void rejectReadForTokenOutOfRange() throws Exception + { + // reject a read for a key who's token the node doesn't own the range for + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 200; + ReadCommand command = command(key); + handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, true, true); + } + + @Test + public void acceptReadIfRejectionNotEnabled() throws Exception + { + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 200; + ReadCommand command = command(key); + handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, true, false); + } + + @Test + public void rangeReadCommandBoundsAreNotChecked() throws Exception + { + // checking is only currently done for single partition reads, range reads will continue to + // accept any range they are given. So for a range wholly outside the node's ownership we + // expect the metric to remain unchanged and read command to be executed. + // This test is added for 3.0 because the single partition & range commands are now processed + // by the same verb handler. + // rdar://problem/33535104 is to extend checking to range reads + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + Range range = new Range<>(key(metadata_nonreplicated, 150).getToken(), + key(metadata_nonreplicated, 160).getToken()); + ReadCommand command = new StubRangeReadCommand(range, metadata_nonreplicated); + handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, messageId, false, false); + } + + private void getAndVerifyResponse(ListenableFuture messageSink, + int messageId, + boolean isOutOfRange, + boolean expectFailure) throws InterruptedException, ExecutionException, TimeoutException + { + assertEquals(startingTotalMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount + (isOutOfRange ? 1 : 0), keyspaceMetricValue(cfs)); + if (expectFailure) + { + try + { + TokenRangeTestUtil.MessageDelivery response = messageSink.get(10, TimeUnit.MILLISECONDS); + fail(String.format("Didn't expect any message to be sent, but sent %s to %s in response to %s", + response.message.toString(), + response.to, + response.message.id())); + } + catch (TimeoutException e) + { + // expected + } + } + else + { + TokenRangeTestUtil.MessageDelivery response = messageSink.get(10, TimeUnit.MILLISECONDS); + assertEquals(Verb.READ_RSP, response.message.verb()); + assertEquals(broadcastAddress, response.message.from()); + assertEquals(messageId, response.message.id()); + assertEquals(node1, response.to); + } + } + + private ReadCommand command(int key) + { + return new StubReadCommand(key, metadata_nonreplicated); + } + + private static class StubReadCommand extends SinglePartitionReadCommand + { + private final TableMetadata tmd; + + StubReadCommand(int key, TableMetadata tmd) + { + super(false, + 0, + false, + tmd, + FBUtilities.nowInSeconds(), + ColumnFilter.all(tmd), + RowFilter.NONE, + DataLimits.NONE, + key(tmd, key), + null, + null); + + this.tmd = tmd; + } + + public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController) + { + return EmptyIterators.unfilteredPartition(tmd); + } + + public String toString() + { + return "<>"; + } + } + + private static class StubRangeReadCommand extends PartitionRangeReadCommand + { + private final TableMetadata cfm; + + StubRangeReadCommand(Range range, TableMetadata tmd) + { + super(false, + 0, + false, + tmd, + FBUtilities.nowInSeconds(), + ColumnFilter.all(tmd), + RowFilter.NONE, + DataLimits.NONE, + DataRange.forTokenRange(range), + null); + + this.cfm = tmd; + } + + public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController) + { + return EmptyIterators.unfilteredPartition(cfm); + } + } + + private static long keyspaceMetricValue(ColumnFamilyStore cfs) + { + return cfs.keyspace.metric.outOfRangeTokenReads.getCount(); + } +} diff --git a/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java b/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java new file mode 100644 index 0000000000..abeed02dd4 --- /dev/null +++ b/test/unit/org/apache/cassandra/dht/OwnedRangesTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.dht; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRanges; + +public class OwnedRangesTest +{ + @Test + public void testFilterRangesWithEmptySuperset() + { + List> toFilter = generateRanges(0, 50); + Collection> rejected = new OwnedRanges(Collections.emptyList()).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithEmptySupersetEmptySubset() + { + assertTrue(new OwnedRanges(Collections.emptyList()).testRanges(Collections.emptyList()).isEmpty()); + } + + @Test + public void testFilterRangesWithSingleSupersetEmptySubset() + { + assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(Collections.emptyList()).isEmpty()); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetContained() + { + assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(generateRanges(10 , 20)).isEmpty()); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetExactMatch() + { + assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(generateRanges(0, 100)).isEmpty()); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetStrictlyLessThan() + { + Collection> toFilter = generateRanges(-100, 0); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetStrictlyGreaterThan() + { + Collection> toFilter = generateRanges(101, 200); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetOverlapping() + { + Collection> toFilter = generateRanges(80, 120); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetOverlappingOnRight() + { + Collection> toFilter = generateRanges(0, 120); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithSingleSupersetSingleSubsetOverlappingOnLeft() + { + Collection> toFilter = generateRanges(-10, 100); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithSingleSupersetMultipleSubsetAllContained() + { + Collection> toFilter = generateRanges(10, 20, 30, 40, 50, 60); + assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter).isEmpty()); + } + + @Test + public void testFilterRangesWithSingleSupersetMultipleSubsetSomeContained() + { + Collection> toFilter = generateRanges(-20, -10, 0, 10, 20, 30, 110, 120); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(generateRanges(-20, -10, 110, 120), rejected); + } + + @Test + public void testFilterRangesWithSingleSupersetMultipleSubsetNoneContained() + { + Collection> toFilter = generateRanges(-40, -20, -20, -10, 110, 120, 230, 140); + Collection> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterWithMultipleSupersetSingleSubsetContainedInFirst() + { + Collection> toFilter = generateRanges(10, 20); + assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty()); + } + + @Test + public void testFilterWithMultipleSupersetSingleSubsetContainedInLast() + { + Collection> toFilter = generateRanges(450, 460); + assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty()); + } + + @Test + public void testFilterRangesWithMultipleSupersetSingleSubsetStrictlyGreaterThan() + { + Collection> toFilter = generateRanges(510, 520); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithMultipleSupersetSingleSubsetStrictlyLessThan() + { + Collection> toFilter = generateRanges(-20, -10); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithMultipleSupersetSingleSubsetOverlapping() + { + Collection> toFilter = generateRanges(80, 120); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithMultipleSupersetSingleSubsetOverlappingOnRight() + { + Collection> toFilter = generateRanges(0, 120); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithMultipleSupersetSingleSubsetOverlappingOnLeft() + { + Collection> toFilter = generateRanges(-10, 100); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithMultipleSupersetMultipleSubsetAllDisjoint() + { + Collection> toFilter = generateRanges(-20, -10, 110, 120, 310, 320); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(toFilter, rejected); + } + + @Test + public void testFilterRangesWithMultipleSupersetMultipleSubsetAllContained() + { + Collection> toFilter = generateRanges(10, 20, 210, 220, 410, 420); + assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty()); + } + + @Test + public void testFilterRangesWithMultipleSupersetMultipleSubsetSomeContained() + { + Collection> toFilter = generateRanges(10, 20, 210, 220, 310, 320, 410, 420); + Collection> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter); + assertRangesEqual(rejected, generateRanges(310, 320)); + } + + @Test + public void testFilterRangesWithMultipleSupersetMultipleSubsetSomeExactMatchSomeContained() + { + Collection> toFilter = generateRanges(10, 20, 200, 300, 410, 420); + assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty()); + } + + private static void assertRangesEqual(Collection> first, Collection> second) + { + assertTrue(CollectionUtils.isEqualCollection(Range.normalize(first), Range.normalize(second))); + } +} diff --git a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java index 8e0bc00833..f013ade3bd 100644 --- a/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java +++ b/test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java @@ -20,21 +20,24 @@ */ package org.apache.cassandra.locator; +import java.net.UnknownHostException; + +import org.junit.Test; + import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; -import org.junit.Test; - -import java.net.UnknownHostException; +import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class PendingRangeMapsTest { - +public class PendingRangeMapsTest +{ private Range genRange(String left, String right) { - return new Range(new BigIntegerToken(left), new BigIntegerToken(right)); + return new Range<>(new BigIntegerToken(left), new BigIntegerToken(right)); } private static void addPendingRange(PendingRangeMaps pendingRangeMaps, Range range, String endpoint) @@ -105,4 +108,60 @@ public class PendingRangeMapsTest { assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.1"))); assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.7"))); } + + @Test + public void testIsTokenInLocalPendingRanges() + { + PendingRangeMaps pendingRangeMaps = new PendingRangeMaps(); + InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); + String selfHostAddress = self.getHostAddress(true); + + addPendingRange(pendingRangeMaps, genRange("5", "15"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("15", "25"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("25", "35"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("35", "45"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("45", "55"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("45", "65"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange( "66", "67"), "127.0.0.7"); + + assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("0"))); + assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("5"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("10"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("15"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("20"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("25"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("35"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("45"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("55"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("65"))); + assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("66"))); + } + + @Test + public void testIsTokenInLocalPendingRangesWrapAroundRanges() + { + PendingRangeMaps pendingRangeMaps = new PendingRangeMaps(); + InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort(); + String selfHostAddress = self.getHostAddress(true); + + addPendingRange(pendingRangeMaps, genRange("5", "15"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("15", "25"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("25", "35"), "127.0.0.1"); + addPendingRange(pendingRangeMaps, genRange("35", "45"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("45", "55"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("45", "65"), selfHostAddress); + addPendingRange(pendingRangeMaps, genRange("65", "7"), selfHostAddress); + + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("0"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("5"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("7"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("10"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("15"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("20"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("25"))); + assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("35"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("45"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("55"))); + assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("65"))); + } } diff --git a/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java new file mode 100644 index 0000000000..9ce535f54b --- /dev/null +++ b/test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.repair; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.repair.messages.PrepareMessage; +import org.apache.cassandra.repair.messages.RepairMessage; +import org.apache.cassandra.repair.messages.ValidationResponse; +import org.apache.cassandra.repair.messages.ValidationRequest; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.service.ActiveRepairService; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.PreviewKind; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery; +import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress; +import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRange; +import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRanges; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt; +import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink; +import static org.apache.cassandra.utils.TokenRangeTestUtil.setLocalTokens; +import static org.apache.cassandra.utils.TokenRangeTestUtil.token; +import static org.apache.cassandra.utils.TokenRangeTestUtil.uuid; + +public class RepairMessageVerbHandlerOutOfRangeTest +{ + private static final String TEST_NAME = "repair_message_vh_test_"; + private static final String KEYSPACE = TEST_NAME + "cql_keyspace"; + private static final String TABLE = "table1"; + private static List tableIds; + + @BeforeClass + public static void init() throws Exception + { + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition(TEST_NAME); + StorageService.instance.initServer(0); + tableIds = Collections.singletonList(Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata().id); + } + + @Before + public void setup() throws Exception + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + // All tests suppose a 2 node ring, with the other peer having the tokens 0, 200, 300 + // Initially, the local node has no tokens so when indivividual test set owned tokens or + // pending ranges for the local node, they're always in relation to this. + // e.g. test calls setLocalTokens(100, 300) the ring now looks like + // peer -> (min, 0], (100, 200], (300, 400] + // local -> (0, 100], (200, 300], (400, max] + // + // Pending ranges are set in test using start/end pairs. + // Ring is initialised: + // peer -> (min, max] + // local -> (,] + // e.g. test calls setPendingRanges(0, 100, 200, 300) + // the pending ranges for local would be calculated as: + // local -> (0, 100], (200, 300] + StorageService.instance.getTokenMetadata().updateNormalTokens(Lists.newArrayList(token(0), + token(200), + token(400)), + node1); + } + + @Test + public void testPrepareWithAllRequestedRangesWithinOwned() throws Exception + { + setLocalTokens(100); + PrepareMessage prepare = prepareMsg(generateRanges(10, 20)); + tryPrepareExpectingSuccess(prepare); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryPrepareExpectingSuccess(prepare); + } + + @Test + public void testPrepareWithAllRequestedRangesOutsideOwned() throws Exception + { + setLocalTokens(100); + PrepareMessage prepare = prepareMsg(generateRanges(110, 120)); + tryPrepareExpectingSuccess(prepare); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryPrepareExpectingSuccess(prepare); + } + + @Test + public void testPrepareWithSomeRequestedRangesOutsideOwned() throws Exception + { + setLocalTokens(100); + PrepareMessage prepare = prepareMsg(generateRanges(10, 20, 110, 120)); + tryPrepareExpectingSuccess(prepare); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryPrepareExpectingSuccess(prepare); + } + + @Test + public void testValidationRequestWithRequestedRangeWithinOwned() throws Exception + { + setLocalTokens(100); + ValidationRequest request = validationMsg(generateRange(10, 20)); + tryValidationExpectingSuccess(request, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryValidationExpectingSuccess(request, false); + } + + @Test + public void testValidationRequestWithRequestedRangeOutsideOwned() throws Exception + { + setLocalTokens(100); + ValidationRequest request = validationMsg(generateRange(110, 120)); + tryValidationExpectingFailure(request); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryValidationExpectingSuccess(request, true); + } + + @Test + public void testValidationRequestWithRequestedRangeOverlappingOwned() throws Exception + { + setLocalTokens(100); + ValidationRequest request = validationMsg(generateRange(10, 120)); + tryValidationExpectingFailure(request); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryValidationExpectingSuccess(request, true); + } + + private static void tryValidationExpectingFailure(ValidationRequest request) throws Exception + { + tryValidation(request, true, false); + } + + private static void tryValidationExpectingSuccess(ValidationRequest request, boolean isOutOfRange) throws Exception + { + tryValidation(request, isOutOfRange, true); + } + + private static void tryValidation(ValidationRequest request, boolean isOutOfRange, boolean expectSuccess) throws Exception + { + long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + MessagingService.instance().outboundSink.clear(); + MessagingService.instance().inboundSink.clear(); + ListenableFuture messageSink = registerOutgoingMessageSink(Verb.REPAIR_RSP); + RepairMessageVerbHandler handler = new RepairMessageVerbHandler(); + int messageId = randomInt(); + Message message = Message.builder(Verb.VALIDATION_REQ, (RepairMessage)request).from(node1).withId(messageId).build(); + handler.doVerb(message); + MessageDelivery response = messageSink.get(500, TimeUnit.MILLISECONDS); + assertEquals(Verb.VALIDATION_RSP, response.message.verb()); + assertEquals(broadcastAddress, response.message.from()); + assertEquals(node1, response.to); + assertTrue(response.message.payload instanceof ValidationResponse); + ValidationResponse completion = (ValidationResponse) response.message.payload; + assertEquals(expectSuccess, completion.success()); + assertEquals(startMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount()); + } + + private static void tryPrepareExpectingSuccess(PrepareMessage prepare) throws Exception + { + long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + MessagingService.instance().outboundSink.clear(); + MessagingService.instance().inboundSink.clear(); + ListenableFuture messageSink = registerOutgoingMessageSink(); + RepairMessageVerbHandler handler = new RepairMessageVerbHandler(); + int messageId = randomInt(); + Message message = Message.builder(Verb.PREPARE_MSG, (RepairMessage)prepare).from(node1).withId(messageId).build(); + handler.doVerb(message); + + MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS); + assertEquals(Verb.REPAIR_RSP, response.message.verb()); + assertEquals(broadcastAddress, response.message.from()); + assertEquals(messageId, response.message.id()); + assertEquals(node1, response.to); + assertFalse(response.message.payload instanceof RequestFailureReason); + assertEquals(startMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount()); + } + + private static PrepareMessage prepareMsg(Collection> ranges) + { + return new PrepareMessage(uuid(), tableIds, ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, true, PreviewKind.NONE); + } + + private static ValidationRequest validationMsg(Range range) + { + UUID parentId = uuid(); + List stores = tableIds.stream() + .map(Schema.instance::getColumnFamilyStoreInstance) + .collect(Collectors.toList()); + ActiveRepairService.instance.registerParentRepairSession(parentId, + node1, + stores, + Collections.singleton(range), + false, + ActiveRepairService.UNREPAIRED_SSTABLE, + true, + PreviewKind.NONE); + return new ValidationRequest(new RepairJobDesc(parentId, uuid(), KEYSPACE, TABLE, Collections.singleton(range)), + randomInt()); + } +} + diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java new file mode 100644 index 0000000000..2ea10cfc26 --- /dev/null +++ b/test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.service.paxos; + +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.exceptions.RequestFailureReason; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.schema.Schema; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.UUIDGen; + +import static org.junit.Assert.assertEquals; + +import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery; +import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress; +import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt; +import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink; + +public class PaxosVerbHandlerOutOfRangeTest +{ + // For the purposes of this testing, the details of the Commit don't really matter + // as we're just testing the rejection (or lack of) and not the result of doing + // whatever the specific verb handlers are supposed to do when they don't reject + // a given Commit + + private static final String TEST_NAME = "paxos_vh_test_"; + private static final String KEYSPACE = TEST_NAME + "cql_keyspace"; + private static final String TABLE = "table1"; + + private long startingTotalMetricCount; + private long startingKeyspaceMetricCount; + + @BeforeClass + public static void init() throws Exception + { + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition(TEST_NAME); + StorageService.instance.initServer(0); + } + + @Before + public void setup() throws Exception + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1); + StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress); + + MessagingService.instance().inboundSink.clear(); + MessagingService.instance().outboundSink.clear(); + startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + startingKeyspaceMetricCount = keyspaceMetricValue(); + } + private static DecoratedKey key(TableMetadata metadata, int key) + { + return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)); + } + + @Test + public void acceptPrepareForNaturalEndpoint() throws Exception + { + acceptRequestForNaturalEndpoint(Verb.PAXOS_PREPARE_REQ, Verb.PAXOS_PREPARE_RSP, new PrepareVerbHandler()); + } + + @Test + public void acceptProposeForNaturalEndpoint() throws Exception + { + acceptRequestForNaturalEndpoint(Verb.PAXOS_PROPOSE_REQ, Verb.PAXOS_PROPOSE_RSP, new ProposeVerbHandler()); + } + + @Test + public void acceptCommitForNaturalEndpoint() throws Exception + { + acceptRequestForNaturalEndpoint(Verb.PAXOS_COMMIT_REQ, Verb.PAXOS_COMMIT_RSP, new CommitVerbHandler()); + } + + private void acceptRequestForNaturalEndpoint(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception + { + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 50; + Commit commit = commit(key); + handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, responseVerb, messageId, false, false); + } + + @Test + public void acceptPrepareForPendingEndpoint() throws Exception + { + acceptRequestForPendingEndpoint(Verb.PAXOS_PREPARE_REQ, Verb.PAXOS_PREPARE_RSP, new PrepareVerbHandler()); + } + + @Test + public void acceptProposeForPendingEndpoint() throws Exception + { + acceptRequestForPendingEndpoint(Verb.PAXOS_PROPOSE_REQ, Verb.PAXOS_PROPOSE_RSP, new ProposeVerbHandler()); + } + + @Test + public void acceptCommitForPendingEndpoint() throws Exception + { + acceptRequestForPendingEndpoint(Verb.PAXOS_COMMIT_REQ, Verb.PAXOS_COMMIT_RSP, new CommitVerbHandler()); + } + + private void acceptRequestForPendingEndpoint(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception + { + // remove localhost from TM and add it back as pending + StorageService.instance.getTokenMetadata().removeEndpoint(broadcastAddress); + Multimap, Replica> pending = HashMultimap.create(); + Range range = new Range<>(bytesToken(0), bytesToken(100)); + pending.put(range, new Replica(broadcastAddress, range, true)); + StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(KEYSPACE, pending); + + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 50; + Commit commit = commit(key); + handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, responseVerb, messageId, false, false); + } + + @Test + public void rejectPrepareForTokenOutOfRange() throws Exception + { + rejectRequestForTokenOutOfRange(Verb.PAXOS_PREPARE_REQ, Verb.FAILURE_RSP, new PrepareVerbHandler()); + } + + @Test + public void rejectProposeForTokenOutOfRange() throws Exception + { + rejectRequestForTokenOutOfRange(Verb.PAXOS_PROPOSE_REQ, Verb.FAILURE_RSP, new ProposeVerbHandler()); + } + + @Test + public void rejectCommitForTokenOutOfRange() throws Exception + { + rejectRequestForTokenOutOfRange(Verb.PAXOS_COMMIT_REQ, Verb.FAILURE_RSP, new CommitVerbHandler()); + } + + private void rejectRequestForTokenOutOfRange(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception + { + // reject a commit for a token the node neither owns nor is pending + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 200; + Commit commit = commit(key); + handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, responseVerb, messageId, true, true); + } + + @Test + public void acceptPrepareIfRejectionNotEnabled() throws Exception + { + acceptRequestIfRejectionNotEnabled(Verb.PAXOS_PREPARE_REQ, Verb.PAXOS_PREPARE_RSP, new PrepareVerbHandler()); + } + + @Test + public void acceptProposeIfRejectionNotEnabled() throws Exception + { + acceptRequestIfRejectionNotEnabled(Verb.PAXOS_PROPOSE_REQ, Verb.PAXOS_PROPOSE_RSP, new ProposeVerbHandler()); + } + + @Test + public void acceptCommitIfRejectionNotEnabled() throws Exception + { + acceptRequestIfRejectionNotEnabled(Verb.PAXOS_COMMIT_REQ, Verb.PAXOS_COMMIT_RSP, new CommitVerbHandler()); + } + + private void acceptRequestIfRejectionNotEnabled(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception + { + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + ListenableFuture messageSink = registerOutgoingMessageSink(); + int messageId = randomInt(); + int key = 200; + Commit commit = commit(key); + handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build()); + getAndVerifyResponse(messageSink, responseVerb, messageId, true, false); + } + + private void getAndVerifyResponse(ListenableFuture messageSink, + Verb verb, + int messageId, + boolean isOutOfRange, + boolean expectFailure) throws InterruptedException, ExecutionException, TimeoutException + { + MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS); + assertEquals(verb, response.message.verb()); + Assert.assertEquals(broadcastAddress, response.message.from()); + assertEquals(expectFailure, response.message.payload instanceof RequestFailureReason); + assertEquals(messageId, response.message.id()); + Assert.assertEquals(node1, response.to); + assertEquals(startingTotalMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount()); + assertEquals(startingKeyspaceMetricCount + (isOutOfRange ? 1 : 0), keyspaceMetricValue()); + } + + private static Commit commit(int key) + { + TableMetadata tmd = Schema.instance.getTableMetadata(KEYSPACE, TABLE); + UUID ballot = UUIDGen.getRandomTimeUUIDFromMicros(FBUtilities.timestampMicros()); + return Commit.newPrepare(key(tmd, key), tmd, ballot); + } + + private static long keyspaceMetricValue() + { + return Keyspace.open(KEYSPACE).metric.outOfRangeTokenPaxosRequests.getCount(); + } +} diff --git a/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java new file mode 100644 index 0000000000..88d2744291 --- /dev/null +++ b/test/unit/org/apache/cassandra/streaming/StreamReaderTest.java @@ -0,0 +1,623 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.streaming; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import com.google.common.collect.Lists; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import net.jpountz.lz4.LZ4Factory; +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.BufferDecoratedKey; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.streaming.CassandraStreamHeader; +import org.apache.cassandra.db.streaming.CassandraStreamReader; +import org.apache.cassandra.db.streaming.IStreamReader; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.io.sstable.SSTableMultiWriter; +import org.apache.cassandra.io.sstable.SSTableSimpleIterator; +import org.apache.cassandra.io.sstable.format.SSTableFormat; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.Version; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.util.DataInputBuffer; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputBuffer; +import org.apache.cassandra.io.util.TrackedDataInputPlus; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.net.AsyncStreamingOutputPlus; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.async.StreamCompressionSerializer; +import org.apache.cassandra.streaming.messages.StreamMessageHeader; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; + +import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt; +import static org.apache.cassandra.utils.TokenRangeTestUtil.setLocalTokens; +import static org.apache.cassandra.utils.TokenRangeTestUtil.setPendingRanges; +import static org.apache.cassandra.utils.TokenRangeTestUtil.token; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class StreamReaderTest +{ + private static final String TEST_NAME = "streamreader_test_"; + private static final String KEYSPACE = TEST_NAME + "cql_keyspace"; + private static final String TABLE = "table1"; + + @BeforeClass + public static void setupClass() throws Exception + { + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition(TEST_NAME); + StorageService.instance.initServer(0); + } + + @Before + public void setup() throws Exception + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + // All tests suppose a 2 node ring, with the other peer having the tokens 0, 200, 400 + // Initially, the local node has no tokens so when indivividual test set owned tokens or + // pending ranges for the local node, they're always in relation to this. + // e.g. test calls setLocalTokens(100, 300) the ring now looks like + // peer -> (min, 0], (100, 200], (300, 400] + // local -> (0, 100], (200, 300], (400, max] + // + // Pending ranges are set in test using start/end pairs. + // Ring is initialised: + // peer -> (min, max] + // local -> (,] + // e.g. test calls setPendingRanges(0, 100, 200, 300) + // the pending ranges for local would be calculated as: + // local -> (0, 100], (200, 300] + StorageService.instance.getTokenMetadata().updateNormalTokens(Lists.newArrayList(token(0), + token(200), + token(400)), + node1); + } + @Test + public void testReceiveWithNoOwnedRanges() throws Exception + { + int[] tokens = {10, 20}; + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSingleOwnedRangeReceivingTableWithRangeContained() throws Exception + { + int[] tokens = {10, 20}; + setLocalTokens(100); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithSingleOwnedRangeReceivedTableLowestKeyBoundsExclusive() throws Exception + { + // verify that ranges are left exclusive + int[] tokens = {0, 10}; + setLocalTokens(100); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSingleOwnedRangeReceivingTableRangeWithExactMatch() throws Exception + { + // Because ranges are left exlusive, for the range (0, 100] the lowest permissable key is 1 + int[] tokens = {1, 100}; + setLocalTokens(100); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithSingleOwnedRangeReceivingTableRangeLessThanOwned() throws Exception + { + int[] tokens = {-100, 0}; + setLocalTokens(100); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSingleOwnedRangeReceivingTableRangeGreaterThanOwned() throws Exception + { + int[] tokens = {101, 200}; + setLocalTokens(100); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSingleOwnedRangeReceivingTableRangeOverlappingOwned() throws Exception + { + int[] tokens = {80, 120}; + setLocalTokens(100); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSingleOwnedWrappingRangeReceivingTableContainedBeforeMax() throws Exception + { + int[] tokens = {110, 120}; + + // local node owns (min, 0] & (100, max], peer owns (0, 100] + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress); + StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithSingleOwnedWrappingRangeReceivingTableContainedAfterMin() throws Exception + { + int[] tokens = {-150, -140}; + + // local node owns (min, 0] & (100, max], peer owns (0, 100] + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress); + StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithSingleOwnedWrappingRangeReceivingTableOverlappingUpward() throws Exception + { + int[] tokens = {-10, 10}; + + // local node owns (min, 0] & (100, max], peer owns (0, 100] + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress); + StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSingleOwnedWrappingRangeReceivingTableOverlappingDownward() throws Exception + { + int[] tokens = {90, 110}; + + // local node owns (min, 0] & (100, max], peer owns (0, 100] + StorageService.instance.getTokenMetadata().clearUnsafe(); + StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress); + StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithMultipleOwnedRangesReceivingTableRangeContainedInFirstOwned() throws Exception + { + int[] tokens = {10, 20}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithMultipleOwnedRangesReceivingTableRangeContainedInLastOwned() throws Exception + { + int[] tokens = {450, 460}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithMultipleOwnedRangesReceivingTableRangeLessThanOwned() throws Exception + { + int[] tokens = {510, 520}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithMultipleOwnedRangesReceivingTableRangeGreaterThanOwned() throws Exception + { + int[] tokens = {-20, -10}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithMultipleOwnedRangesReceivingTableRangeOverlappingOwned() throws Exception + { + int[] tokens = {80, 120}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithMultipleOwnedRangesAllDisjointFromReceivingTableRange() throws Exception + { + int[] tokens = {310, 320}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithMultipleOwnedRangesDisjointSpannedByReceivingTableRange() throws Exception + { + int[] tokens = {80, 320}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithMultipleOwnedRangesReceivedTableRangeExactMatch() throws Exception + { + // bacause ranges are left exclusive, for the range (200, 300] the lowest permissable key is 201 + int[] tokens = {201, 300}; + setLocalTokens(100, 300, 500); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithOwnedRangeWrappingAndReceivedFileWhollyContained() throws Exception + { + // peer -> (-100, 0], (100, 200], (300, 400] + // local -> (min, -100], (400, max] + setLocalTokens(-100); + int[] tokens = {-200, 500}; + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithOwnedRangeWrappingAndReceivedFilePartiallyContained() throws Exception + { + // peer -> (-100, 0], (100, 200], (300, 400] + // local -> (min, -100], (400, max] + setLocalTokens(-100); + int[] tokens = {-200, 300}; + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithOwnedRangeWrappingAndReceivedFileNotContained() throws Exception + { + // peer -> (-100, 0], (100, 200], (300, 400] + // local -> (min, -100], (400, max] + setLocalTokens(-100); + int[] tokens = {0, 300}; + tryReceiveExpectingFailure(tokens); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, true); + } + + @Test + public void testReceiveWithSinglePendingRangeReceivingTableWithRangeContained() throws Exception + { + int[] tokens = {10, 20}; + setPendingRanges(KEYSPACE, 0, 100); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveWithMultiplePendingRangesReceivingTableRangeContainedInFirstOwned() throws Exception + { + int[] tokens = {10, 20}; + setPendingRanges(KEYSPACE, 0, 100, 200, 300, 400, 500); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + @Test + public void testReceiveNormalizesOwnedAndPendingRanges() throws Exception + { + // Incoming file is not covered by either a single owned or pending range, + // but it is covered by the normalized set of both + int[] tokens = {90, 110}; + setLocalTokens(100); + setPendingRanges(KEYSPACE, 100, 120); + + tryReceiveExpectingSuccess(tokens, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryReceiveExpectingSuccess(tokens, false); + } + + public static StreamSession setupStreamingSessionForTest() + { + StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.REPAIR, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE); + StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.REPAIR, Collections.emptyList(), streamCoordinator); + + InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort(); + streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED)); + + StreamSession session = streamCoordinator.getOrCreateNextSession(peer); + session.init(future); + return session; + } + + private static void tryReceiveExpectingSuccess(int[] tokens, + boolean isOutOfRange) throws IOException + { + StreamSession session = setupStreamingSessionForTest(); + StreamMessageHeader header = streamHeader(); + CassandraStreamHeader streamHeader = streamMessageHeader(tokens); + long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + IStreamReader reader = streamReader(header, streamHeader, session); + StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, 1, 0); + session.prepareReceiving(streamSummary); + reader.read(incomingStream(tokens)); + assertEquals(isOutOfRange, StorageMetrics.totalOpsForInvalidToken.getCount() > startMetricCount); + } + + private static void tryReceiveExpectingFailure(int[] tokens) throws IOException + { + StreamSession session = setupStreamingSessionForTest(); + StreamMessageHeader header = streamHeader(); + CassandraStreamHeader streamHeader = streamMessageHeader(tokens); + long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, 1, 0); + session.prepareReceiving(streamSummary); + try + { + IStreamReader reader = streamReader(header, streamHeader, session); + try (SSTableMultiWriter ignore = reader.read(incomingStream(tokens))) + { + fail("Expected StreamReceivedOfTokenRangeException"); + } + } + catch (StreamReceivedOutOfTokenRangeException e) + { + // expected + } + assertTrue(StorageMetrics.totalOpsForInvalidToken.getCount() > startMetricCount); + } + + + private static class BufferSupplier implements AsyncStreamingOutputPlus.BufferSupplier + { + ByteBuffer supplied; + + @Override + public ByteBuffer get(int capacity) + { + supplied = ByteBuffer.allocateDirect(capacity); + return supplied; + } + + public ByteBuffer getSupplied() + { + return supplied; + } + + } + + private static DataInputPlus incomingStream(int...tokens) throws IOException + { + final net.jpountz.lz4.LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor(); + + DataOutputBuffer out = new DataOutputBuffer(); + for (int token : tokens) + ByteBufferUtil.writeWithShortLength(ByteBufferUtil.bytes((long)token), out); + + int current_version = MessagingService.current_version; + + BufferSupplier bufferSupplier = new BufferSupplier(); + StreamCompressionSerializer.serialize(compressor, out.buffer(), current_version).write(bufferSupplier); + + return new DataInputBuffer(bufferSupplier.getSupplied(), false); + } + + private static IStreamReader streamReader(StreamMessageHeader header, CassandraStreamHeader streamHeader, StreamSession session) + { + return new KeysOnlyStreamReader(header, streamHeader, session); + } + + private static StreamMessageHeader streamHeader() + { + TableMetadata tmd = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata(); + int fakeSession = randomInt(9); + int fakeSeq = randomInt(9); + return new StreamMessageHeader(tmd.id, + null, + null, + true, + fakeSession, + fakeSeq, + System.currentTimeMillis(), + null); + } + + private static CassandraStreamHeader streamMessageHeader(int...tokens) + { + TableMetadata tmd = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata(); + Version version = BigFormat.latestVersion; + List fakeSections = new ArrayList<>(); + // each decorated key takes up (2 + 8) bytes, so this enables the + // StreamReader to calculate the expected number of bytes to read + fakeSections.add(new SSTableReader.PartitionPositionBounds(0L, (tokens.length * 10L) - 1)); + + return CassandraStreamHeader.builder() + .withTableId(tmd.id) + .withSerializationHeader(SerializationHeader.makeWithoutStats(tmd).toComponent()) + .withSSTableFormat(SSTableFormat.Type.BIG) + .withSSTableVersion(version) + .withSections(fakeSections) + .build(); + } + + // Simplifies generating test data as token == key (expects key to be an encoded long) + private static class FakeMurmur3Partitioner extends Murmur3Partitioner + { + public DecoratedKey decorateKey(ByteBuffer key) + { + return new BufferDecoratedKey(new LongToken(ByteBufferUtil.toLong(key)), key); + } + } + + // Stream reader which no-ops the reading/writing of the actual partition data. + // As we only care about keys/tokens here, we don't need to generate the rest + // of the sstable data to simulate a stream + private static class KeysOnlyStreamReader extends CassandraStreamReader + { + public KeysOnlyStreamReader(StreamMessageHeader header, CassandraStreamHeader streamHeader, StreamSession session) + { + super(header, streamHeader, session); + } + + protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, UUID pendingRepair, SSTableFormat.Type format) throws IOException + { + return super.createWriter(cfs, totalSize, repairedAt, pendingRepair, format); + } + + @Override + protected StreamDeserializer getDeserializer(TableMetadata metadata, TrackedDataInputPlus in, Version inputVersion, StreamSession session, SSTableMultiWriter writer) throws IOException + { + return new TestStreamDeserializer(metadata, in, inputVersion, getHeader(metadata), session, writer); + } + + private static class TestStreamDeserializer extends CassandraStreamReader.StreamDeserializer + { + TestStreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header, StreamSession session, SSTableMultiWriter writer) throws IOException + { + super(metadata.unbuild().partitioner(new FakeMurmur3Partitioner()).build(), in, version, header, session, writer); + } + + @Override + protected void readPartition() + { + // no-op, our dummy stream contains only decorated keys + partitionLevelDeletion = DeletionTime.LIVE; + iterator = new SSTableSimpleIterator.EmptySSTableSimpleIterator(metadata()); + } + + @Override + public Row staticRow() + { + return Rows.EMPTY_STATIC_ROW; + } + + @Override + public DeletionTime partitionLevelDeletion() + { + return DeletionTime.LIVE; + } + + + } + } +} diff --git a/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java b/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java new file mode 100644 index 0000000000..3795849073 --- /dev/null +++ b/test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.streaming; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import com.google.common.collect.Lists; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.metrics.StorageMetrics; +import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.streaming.messages.StreamMessage; +import org.apache.cassandra.utils.FBUtilities; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import static org.apache.cassandra.streaming.StreamTestUtils.session; +import static org.apache.cassandra.streaming.messages.StreamMessage.Type.COMPLETE; +import static org.apache.cassandra.streaming.messages.StreamMessage.Type.PREPARE_SYNACK; +import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRangesAtEndpoint; +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; +import static org.apache.cassandra.utils.TokenRangeTestUtil.setLocalTokens; +import static org.apache.cassandra.utils.TokenRangeTestUtil.token; + +public class StreamSessionOwnedRangesTest +{ + private static final String TEST_NAME = "strmsn_owned_ranges_test_"; + private static final String KEYSPACE = TEST_NAME + "cql_keyspace"; + private static final String TABLE = "table1"; + + @BeforeClass + public static void setupClass() throws Exception + { + DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); + SchemaLoader.loadSchema(); + SchemaLoader.schemaDefinition(TEST_NAME); + StorageService.instance.initServer(0); + } + + @Before + public void setup() throws Exception + { + DatabaseDescriptor.setLogOutOfTokenRangeRequests(true); + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true); + + StorageService.instance.getTokenMetadata().clearUnsafe(); + // All tests suppose a 2 node ring, with the other peer having the tokens 0, 200, 300 + // Initially, the local node has no tokens so when indivividual test set owned tokens or + // pending ranges for the local node, they're always in relation to this. + // e.g. test calls setLocalTokens(100, 300) the ring now looks like + // peer -> (min, 0], (100, 200], (300, 400] + // local -> (0, 100], (200, 300], (400, max] + // + // Pending ranges are set in test using start/end pairs. + // Ring is initialised: + // peer -> (min, max] + // local -> (,] + // e.g. test calls setPendingRanges(0, 100, 200, 300) + // the pending ranges for local would be calculated as: + // local -> (0, 100], (200, 300] + StorageService.instance.getTokenMetadata().updateNormalTokens(Lists.newArrayList(token(0), + token(200), + token(400)), + node1); + } + + @Test + public void testPrepareWithAllRequestedRangesWithinOwned() + { + setLocalTokens(100); + InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort(); + Collection requests = streamRequests(generateRangesAtEndpoint(endpoint, 0, 10, 70, 80), + RangesAtEndpoint.empty(endpoint)); + + // prepare request should succeed with or without rejection enabled + tryPrepareExpectingSuccess(requests, false); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryPrepareExpectingSuccess(requests, false); + } + + @Test + public void testPrepareWithAllRequestedRangesOutsideOwned() + { + setLocalTokens(100); + InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort(); + + Collection requests = streamRequests(generateRangesAtEndpoint(endpoint, -20, -10, 110, 120, 310, 320), + RangesAtEndpoint.empty(endpoint)); + + tryPrepareExpectingFailure(requests); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryPrepareExpectingSuccess(requests, true); + } + + @Test + public void testPrepareWithSomeRequestedRangesOutsideOwned() + { + setLocalTokens(100); + InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort(); + + Collection requests = streamRequests(generateRangesAtEndpoint(endpoint, -20, -10, 30, 40, 310, 320), + RangesAtEndpoint.empty(endpoint)); + + tryPrepareExpectingFailure(requests); + + DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false); + tryPrepareExpectingSuccess(requests, true); + } + + private static void tryPrepareExpectingSuccess(Collection requests, boolean isOutOfRange) + { + StreamSession session = session(); + + StreamTestUtils.StubMessageSender handler = (StreamTestUtils.StubMessageSender) session.getMessageSender(); + handler.reset(); + long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + + session.state(StreamSession.State.PREPARING); + session.prepareAsync(requests, Collections.emptySet()); + + List sent = handler.sentMessages; + assertEquals(2, sent.size()); + assertEquals(PREPARE_SYNACK, sent.get(0).type); + assertEquals(COMPLETE, sent.get(1).type); + + assertEquals(startMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount()); + } + + private static void tryPrepareExpectingFailure(Collection requests) + { + StreamSession session = session(); + StreamTestUtils.StubMessageSender handler = (StreamTestUtils.StubMessageSender) session.getMessageSender(); + handler.reset(); + long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount(); + try + { + session.state(StreamSession.State.PREPARING); + session.prepareAsync(requests, Collections.emptySet()); + fail("Expected StreamRequestOfTokenRangeException"); + } + catch (StreamRequestOutOfTokenRangeException e) + { + // expected + } + assertTrue(handler.sentMessages.isEmpty()); + assertEquals(startMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount()); + } + + private static Collection streamRequests(RangesAtEndpoint fullRanges, + RangesAtEndpoint transientRanges) + { + return Collections.singleton(new StreamRequest(KEYSPACE, + fullRanges, + transientRanges, + Collections.singleton(TABLE))); + + } +} diff --git a/test/unit/org/apache/cassandra/streaming/StreamTestUtils.java b/test/unit/org/apache/cassandra/streaming/StreamTestUtils.java new file mode 100644 index 0000000000..282d768e94 --- /dev/null +++ b/test/unit/org/apache/cassandra/streaming/StreamTestUtils.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.streaming; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.apache.cassandra.net.OutboundConnectionSettings; +import org.apache.cassandra.streaming.messages.StreamMessage; + +import static org.apache.cassandra.utils.TokenRangeTestUtil.node1; + +public class StreamTestUtils +{ + static StreamSession session() + { + StubMessageSender messageSender = new StubMessageSender(); + StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.REPAIR, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE); + StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.REPAIR, Collections.emptyList(), streamCoordinator); + + StreamSession session = new StreamSession(StreamOperation.REPAIR, + node1, + connectionFactory, + true, + 0, + null, + PreviewKind.NONE) + { + + @Override + public void progress(String filename, ProgressInfo.Direction direction, long bytes, long total) + { + //no-op + } + + @Override + public StreamingMessageSender getMessageSender() + { + return messageSender; + } + }; + session.init(future); + return session; + } + + static class StubMessageSender implements StreamingMessageSender + { + final List sentMessages = new ArrayList<>(); + + StubMessageSender() + { + } + + public void sendMessage(StreamMessage message) + { + sentMessages.add(message); + } + + void reset() + { + sentMessages.clear(); + } + + public void initialize() throws IOException + { + + } + + public boolean connected() + { + return false; + } + + public void close() + { + + } + } + + static StreamConnectionFactory connectionFactory = (OutboundConnectionSettings template, int messagingVersion) -> { + throw new UnsupportedOperationException(); + }; +} diff --git a/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java b/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java index 76bfa76bdf..6706f771dc 100644 --- a/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java +++ b/test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java @@ -68,7 +68,7 @@ public class NettyStreamingMessageSenderTest session.init(future); session.attachOutbound(channel); - sender = session.getMessageSender(); + sender = (NettyStreamingMessageSender) session.getMessageSender(); sender.setControlMessageChannel(channel); } @@ -194,7 +194,7 @@ public class NettyStreamingMessageSenderTest Assert.assertTrue(sender.connected()); ChannelPromise promise = channel.newPromise(); promise.setFailure(new RuntimeException("this is just a testing exception")); - Future f = sender.onControlMessageComplete(promise, new CompleteMessage()); + Future f = sender.onControlMessageComplete(promise, new CompleteMessage()); f.get(5, TimeUnit.SECONDS); diff --git a/test/unit/org/apache/cassandra/utils/TokenRangeTestUtil.java b/test/unit/org/apache/cassandra/utils/TokenRangeTestUtil.java new file mode 100644 index 0000000000..ae1f380485 --- /dev/null +++ b/test/unit/org/apache/cassandra/utils/TokenRangeTestUtil.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.net.UnknownHostException; +import java.util.*; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; + +import org.apache.cassandra.dht.ByteOrderedPartitioner; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.InetAddressAndPort; +import org.apache.cassandra.locator.RangesAtEndpoint; +import org.apache.cassandra.locator.Replica; +import org.apache.cassandra.net.Message; +import org.apache.cassandra.net.MessagingService; +import org.apache.cassandra.net.Verb; +import org.apache.cassandra.service.StorageService; + +public class TokenRangeTestUtil +{ + public static final InetAddressAndPort node1; + public static final InetAddressAndPort broadcastAddress; + + static + { + try + { + node1 = InetAddressAndPort.getByName("127.0.1.99"); + } + catch (UnknownHostException e) + { + throw new RuntimeException("Error initializing InetAddressAndPort"); + } + broadcastAddress = FBUtilities.getBroadcastAddressAndPort(); + } + + public static UUID uuid() + { + return UUIDGen.getTimeUUID(); + } + + public static void setPendingRanges(String keyspace, int... tokens) + { + Multimap, Replica> pending = HashMultimap.create(); + for (Range range : generateRanges(tokens)) + pending.put(range, new Replica(broadcastAddress, range, true)); + + StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(keyspace, pending); + } + + public static void setLocalTokens(int... tokens) + { + List tokenList = new ArrayList<>(); + for (int token : tokens) + tokenList.add(token(token)); + StorageService.instance.getTokenMetadata().updateNormalTokens(tokenList, broadcastAddress); + } + + private static final Random random = new Random(); + public static int randomInt() + { + return randomInt(Integer.MAX_VALUE); + } + + public static int randomInt(int max) + { + return random.nextInt(max); + } + + public static RangesAtEndpoint generateRangesAtEndpoint(InetAddressAndPort endpoint, int... rangePairs) + { + if (rangePairs.length % 2 == 1) + throw new RuntimeException("generateRangesAtEndpoint argument count should be even"); + + RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint); + + for (int i = 0; i < rangePairs.length; i += 2) + { + builder.add(Replica.fullReplica(endpoint, generateRange(rangePairs[i], rangePairs[i + 1]))); + } + return builder.build(); + } + + public static List> generateRanges(int... rangePairs) + { + if (rangePairs.length % 2 == 1) + throw new RuntimeException("generateRanges argument count should be even"); + + List> ranges = new ArrayList<>(); + + for (int i = 0; i < rangePairs.length; i += 2) + { + ranges.add(generateRange(rangePairs[i], rangePairs[i + 1])); + } + + return ranges; + } + + public static Range generateRange(int left, int right) + { + return new Range<>(token(left), token(right)); + } + + public static Token token(int token) + { + return new Murmur3Partitioner.LongToken(token); + } + + public static Token bytesToken(int token) + { + return new ByteOrderedPartitioner.BytesToken(ByteBufferUtil.bytes(token)); + } + + public static ListenableFuture registerOutgoingMessageSink(Verb... ignored) + { + SettableFuture future = SettableFuture.create(); + Set ignore = Sets.newHashSet(ignored); + + MessagingService.instance().outboundSink.clear(); + MessagingService.instance().outboundSink.add((Message message, InetAddressAndPort to) -> { + if (!ignore.contains(message.verb())) + future.set(new MessageDelivery(message, to)); + return true; + }); + + MessagingService.instance().inboundSink.clear(); + MessagingService.instance().inboundSink.add((Message message) -> false); + + return future; + } + + public static class MessageDelivery + { + public final Message message; + public final InetAddressAndPort to; + + MessageDelivery(Message message, InetAddressAndPort to) + { + this.message = message; + this.to = to; + } + } +}