natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Safer handling of out-of-range tokens 

patch by Caleb Rackliffe; reviewed by Mick Semb Wever for CASSANDRA-13704

Co-authored-by: Sam Tunnicliffe <sam@beobal.com>
Co-authored-by: Caleb Rackliffe <calebrackliffe@gmail.com>
Co-authored-by: Mick Semb Wever <mck@apache.org>
Co-authored-by: Ariel Weisberg <aweisberg@apple.com>
This commit is contained in:
Caleb Rackliffe 2020-05-28 14:26:25 -06:00
parent 525245bdbd
commit b4f1c6d250
53 changed files with 3749 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.0.14
* Safer handling of out-of-range tokens (CASSANDRA-13704)
* Fix memory leak in BTree.FastBuilder (CASSANDRA-19785)
* Fix millisecond and microsecond precision for commit log replay (CASSANDRA-19448)
* Improve accuracy of memtable heap usage tracking (CASSANDRA-17298)

18
NEWS.txt
View File

@ -51,6 +51,24 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
4.0.14
======
Upgrading
---------
- This release introduces safeguards and observability into possible data loss scenarios when nodes have a
divergent view of the cluster. This happens around edge-cases on unsafe bootstrapping, decommissions, or when a
node has a corrupted topology. Two configuration options have been added: `log_out_of_token_range_requests` and
`reject_out_of_token_range_requests`, both enabled by default. The former will make nodes log requests they
receive that don't belong in their current or pending token ranges. The latter will reject those requests
altogether, which prevents the vast majority of eventual data loss that can occur, but may also incur small
windows of degraded availability during range movements in an unhealthy cluster. Both options apply to streaming,
mutations, hints, single-partition reads, read-repair, and Paxos operations. Logging these requests is recommended
in all situations, and operators are encouraged to be aware when such events occur and to investigate if writes
have only been received on incorrect nodes. Operators that are confident of topology parity across all nodes may
disable rejection to maximize availability. See CASSANDRA-13704 for further details. Transactional Cluster
Metadata in version 5.1 will remove all such edge-cases, providing appropriate guarantees.
4.0.11
======

7
src/java/org/apache/cassandra/config/Config.java
View File

@ -526,6 +526,13 @@ public class Config
*/
public volatile int validation_preview_purge_head_start_in_sec = 60 * 60;
/*
* Toggles to turn on the logging or rejection of operations for token ranges that the node does not own,
* or is not about to acquire.
*/
public volatile boolean log_out_of_token_range_requests = true;
public volatile boolean reject_out_of_token_range_requests = true;
/**
* The intial capacity for creating RangeTombstoneList.
*/

20
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -3473,6 +3473,26 @@ public class DatabaseDescriptor
conf.consecutive_message_errors_threshold = value;
}
public static boolean getLogOutOfTokenRangeRequests()
{
return conf.log_out_of_token_range_requests;
}
public static void setLogOutOfTokenRangeRequests(boolean enabled)
{
conf.log_out_of_token_range_requests = enabled;
}
public static boolean getRejectOutOfTokenRangeRequests()
{
return conf.reject_out_of_token_range_requests;
}
public static void setRejectOutOfTokenRangeRequests(boolean enabled)
{
conf.reject_out_of_token_range_requests = enabled;
}
public static boolean getForceNewPreparedStatementBehaviour()
{
return conf.force_new_prepared_statement_behaviour;

81
src/java/org/apache/cassandra/db/AbstractMutationVerbHandler.java Normal file
View File

@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.NoSpamLogger;
public abstract class AbstractMutationVerbHandler<T extends IMutation> implements IVerbHandler<T>
{
private static final Logger logger = LoggerFactory.getLogger(AbstractMutationVerbHandler.class);
private static final String logMessageTemplate = "Receiving mutation(s) for token(s) neither owned nor pending. Example: from {} for token {} in keyspace {}";
@Override
public void doVerb(Message<T> message) throws IOException
{
processMessage(message, message.from());
}
public void processMessage(Message<T> message, InetAddressAndPort respondTo)
{
boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests();
boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
DecoratedKey key = message.payload.key();
boolean isOutOfRangeMutation = isOutOfRangeMutation(message.payload.getKeyspaceName(), key);
if (isOutOfRangeMutation)
{
StorageService.instance.incOutOfRangeOperationCount();
Keyspace.open(message.payload.getKeyspaceName()).metric.outOfRangeTokenWrites.inc();
if (outOfRangeTokenLogging)
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate,
respondTo, key.getToken(), message.payload.getKeyspaceName());
}
if (outOfRangeTokenRejection && isOutOfRangeMutation)
sendFailureResponse(message, respondTo);
else
applyMutation(message, respondTo);
}
abstract void applyMutation(Message<T> message, InetAddressAndPort respondToAddress);
private void sendFailureResponse(Message<T> respondTo, InetAddressAndPort respondToAddress)
{
MessagingService.instance().send(respondTo.failureResponse(RequestFailureReason.UNKNOWN), respondToAddress);
}
private static boolean isOutOfRangeMutation(String keyspace, DecoratedKey key)
{
return !StorageService.instance.isEndpointValidForWrite(keyspace, key.getToken());
}
}

9
src/java/org/apache/cassandra/db/CounterMutationVerbHandler.java
View File

@ -21,18 +21,19 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageProxy;
public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
public class CounterMutationVerbHandler extends AbstractMutationVerbHandler<CounterMutation>
{
public static final CounterMutationVerbHandler instance = new CounterMutationVerbHandler();
private static final Logger logger = LoggerFactory.getLogger(CounterMutationVerbHandler.class);
public void doVerb(final Message<CounterMutation> message)
@Override
protected void applyMutation(final Message<CounterMutation> message, InetAddressAndPort respondToAddress)
{
long queryStartNanoTime = System.nanoTime();
final CounterMutation cm = message.payload;
@ -48,7 +49,7 @@ public class CounterMutationVerbHandler implements IVerbHandler<CounterMutation>
// it's own in that case.
StorageProxy.applyCounterMutationOnLeader(cm,
localDataCenter,
() -> MessagingService.instance().send(message.emptyResponse(), message.from()),
() -> MessagingService.instance().send(message.emptyResponse(), respondToAddress),
queryStartNanoTime);
}
}

17
src/java/org/apache/cassandra/db/MutationVerbHandler.java
View File

@ -22,7 +22,7 @@ import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.*;
import org.apache.cassandra.tracing.Tracing;
public class MutationVerbHandler implements IVerbHandler<Mutation>
public class MutationVerbHandler extends AbstractMutationVerbHandler<Mutation>
{
public static final MutationVerbHandler instance = new MutationVerbHandler();
@ -37,6 +37,7 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
Tracing.trace("Payload application resulted in WriteTimeout, not replying");
}
@Override
public void doVerb(Message<Mutation> message)
{
// Check if there were any forwarding headers in this message
@ -55,10 +56,7 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
try
{
message.payload.applyFuture().thenAccept(o -> respond(message, respondToAddress)).exceptionally(wto -> {
failed();
return null;
});
processMessage(message, respondToAddress);
}
catch (WriteTimeoutException wto)
{
@ -66,6 +64,15 @@ public class MutationVerbHandler implements IVerbHandler<Mutation>
}
}
@Override
protected void applyMutation(Message<Mutation> message, InetAddressAndPort respondToAddress)
{
message.payload.applyFuture().thenAccept(o -> respond(message, respondToAddress)).exceptionally(wto -> {
failed();
return null;
});
}
private static void forwardToLocalNodes(Message<Mutation> originalMessage, ForwardingInfo forwardTo)
{
Message.Builder<Mutation> builder =

3
src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java
View File

@ -54,7 +54,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR
private final DataRange dataRange;
private PartitionRangeReadCommand(boolean isDigest,
@VisibleForTesting
protected PartitionRangeReadCommand(boolean isDigest,
int digestVersion,
boolean acceptsTransient,
TableMetadata metadata,

10
src/java/org/apache/cassandra/db/ReadCommand.java
View File

@ -149,6 +149,16 @@ public abstract class ReadCommand extends AbstractReadQuery
public abstract boolean isLimitedToOnePartition();
/**
* Whether this command is a single partition read
*
* @return true if the command is a single partition read, false otherwise
*/
public boolean isSinglePartitionRead()
{
return kind == Kind.SINGLE_PARTITION;
}
public abstract boolean isRangeRequest();
/**

45
src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.cassandra.db;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -30,6 +32,7 @@ import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.tracing.Tracing;
import org.apache.cassandra.utils.NoSpamLogger;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@ -38,6 +41,8 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
public static final ReadCommandVerbHandler instance = new ReadCommandVerbHandler();
private static final Logger logger = LoggerFactory.getLogger(ReadCommandVerbHandler.class);
private static final String logMessageTemplate = "Receiving read(s) for token(s) neither owned nor pending. Example: from {} for token {} in {}.{}";
private static final String exceptionMessageTemplate = "Exception thrown checking if token {} outside valid range for keyspace {} - permitting";
public void doVerb(Message<ReadCommand> message)
{
@ -47,6 +52,30 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
}
ReadCommand command = message.payload;
// no out of token range checking for partition range reads yet
if (command.isSinglePartitionRead())
{
boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests();
boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
DecoratedKey key = ((SinglePartitionReadCommand)command).partitionKey();
if (!command.metadata().isVirtual() && isOutOfRangeRead(command.metadata().keyspace, key))
{
StorageService.instance.incOutOfRangeOperationCount();
Keyspace.open(command.metadata().keyspace).metric.outOfRangeTokenReads.inc();
// Log at most 1 message per second
if (outOfRangeTokenLogging)
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate,
message.from(), key.getToken(), command.metadata().keyspace, command.metadata().name);
if (outOfRangeTokenRejection)
// no need to respond, just drop the request
return;
}
}
validateTransientStatus(message);
long timeout = message.expiresAtNanos() - message.createdAtNanos();
@ -102,4 +131,20 @@ public class ReadCommandVerbHandler implements IVerbHandler<ReadCommand>
this));
}
}
private boolean isOutOfRangeRead(String keyspace, DecoratedKey key)
{
try
{
return Keyspace.open(keyspace)
.getReplicationStrategy()
.getNaturalReplicas(key)
.selfIfPresent() == null;
}
catch (Throwable tr)
{
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, exceptionMessageTemplate, keyspace, key.getKey(), tr);
return false;
}
}
}

19
src/java/org/apache/cassandra/db/ReadRepairVerbHandler.java
View File

@ -17,17 +17,28 @@
*/
package org.apache.cassandra.db;
import org.apache.cassandra.net.IVerbHandler;
import java.io.IOException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
public class ReadRepairVerbHandler implements IVerbHandler<Mutation>
public class ReadRepairVerbHandler extends AbstractMutationVerbHandler<Mutation>
{
public static final ReadRepairVerbHandler instance = new ReadRepairVerbHandler();
public void doVerb(Message<Mutation> message)
@Override
public void doVerb(Message<Mutation> message) throws IOException
{
// This method exists as python dtest relies on byte-code rewriting via Byteman, so requires this
// class defines a "doVerb" method for some tests.
super.doVerb(message);
}
@Override
void applyMutation(Message<Mutation> message, InetAddressAndPort respondToAddress)
{
message.payload.apply();
MessagingService.instance().send(message.emptyResponse(), message.from());
MessagingService.instance().send(message.emptyResponse(), respondToAddress);
}
}

2
src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java
View File

@ -79,8 +79,8 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader
try (CompressedInputStream cis = new CompressedInputStream(inputPlus, compressionInfo, ChecksumType.CRC32, cfs::getCrcCheckChance))
{
TrackedDataInputPlus in = new TrackedDataInputPlus(cis);
deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()), session, writer);
String filename = writer.getFilename();
int sectionIdx = 0;
for (SSTableReader.PartitionPositionBounds section : sections)

142
src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java
View File

@ -17,9 +17,13 @@
*/
package org.apache.cassandra.db.streaming;
import java.io.*;
import java.io.IOError;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.ListIterator;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
@ -28,27 +32,44 @@ import com.google.common.collect.UnmodifiableIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.TrackedDataInputPlus;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.db.*;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.RegularAndStaticColumns;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.db.rows.DeserializationHelper;
import org.apache.cassandra.db.rows.EncodingStats;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Unfiltered;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.UnknownColumnException;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.SSTableSimpleIterator;
import org.apache.cassandra.io.sstable.format.RangeAwareSSTableWriter;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.RewindableDataInputStreamPlus;
import org.apache.cassandra.io.util.TrackedDataInputPlus;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.ProgressInfo;
import org.apache.cassandra.streaming.StreamReceivedOutOfTokenRangeException;
import org.apache.cassandra.streaming.StreamReceiver;
import org.apache.cassandra.streaming.StreamSession;
import org.apache.cassandra.streaming.compress.StreamCompressionInputStream;
import org.apache.cassandra.streaming.messages.StreamMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.NoSpamLogger;
import static org.apache.cassandra.net.MessagingService.current_version;
@ -58,6 +79,7 @@ import static org.apache.cassandra.net.MessagingService.current_version;
public class CassandraStreamReader implements IStreamReader
{
private static final Logger logger = LoggerFactory.getLogger(CassandraStreamReader.class);
private static final String logMessageTemplate = "[Stream #{}] Received streamed SSTable {} from {} containing key(s) outside valid ranges {}. Example: {}";
protected final TableId tableId;
protected final long estimatedKeys;
protected final Collection<SSTableReader.PartitionPositionBounds> sections;
@ -118,8 +140,8 @@ public class CassandraStreamReader implements IStreamReader
try (StreamCompressionInputStream streamCompressionInputStream = new StreamCompressionInputStream(inputPlus, current_version))
{
TrackedDataInputPlus in = new TrackedDataInputPlus(streamCompressionInputStream);
deserializer = new StreamDeserializer(cfs.metadata(), in, inputVersion, getHeader(cfs.metadata()));
writer = createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
deserializer = getDeserializer(cfs.metadata(), in, inputVersion, session, writer);
while (in.getBytesRead() < totalSize)
{
writePartition(deserializer, writer);
@ -143,6 +165,15 @@ public class CassandraStreamReader implements IStreamReader
}
}
protected StreamDeserializer getDeserializer(TableMetadata metadata,
TrackedDataInputPlus in,
Version inputVersion,
StreamSession session,
SSTableMultiWriter writer) throws IOException
{
return new StreamDeserializer(metadata, in, inputVersion, getHeader(metadata), session, writer);
}
protected SerializationHeader getHeader(TableMetadata metadata) throws UnknownColumnException
{
return header != null? header.toHeader(metadata) : null; //pre-3.0 sstable have no SerializationHeader
@ -183,27 +214,57 @@ public class CassandraStreamReader implements IStreamReader
private final SerializationHeader header;
private final DeserializationHelper helper;
private DecoratedKey key;
private DeletionTime partitionLevelDeletion;
private SSTableSimpleIterator iterator;
private Row staticRow;
private final List<Range<Token>> ownedRanges;
private final boolean outOfRangeTokenLogging;
private final boolean outOfRangeTokenRejection;
private final StreamSession session;
private final SSTableMultiWriter writer;
private int lastCheckedRangeIndex;
protected DecoratedKey key;
protected DeletionTime partitionLevelDeletion;
protected SSTableSimpleIterator iterator;
protected Row staticRow;
private IOException exception;
public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header) throws IOException
public StreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header, StreamSession session, SSTableMultiWriter writer) throws IOException
{
this.metadata = metadata;
this.in = in;
this.helper = new DeserializationHelper(metadata, version.correspondingMessagingVersion(), DeserializationHelper.Flag.PRESERVE_SIZE);
this.header = header;
this.session = session;
this.writer = writer;
ownedRanges = Range.normalize(StorageService.instance.getLocalAndPendingRanges(metadata.keyspace));
lastCheckedRangeIndex = 0;
outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests();
outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
}
public StreamDeserializer newPartition() throws IOException
public UnfilteredRowIterator newPartition() throws IOException
{
readKey();
readPartition();
return this;
}
protected void readKey() throws IOException
{
key = metadata.partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in));
lastCheckedRangeIndex = verifyKeyInOwnedRanges(key,
ownedRanges,
lastCheckedRangeIndex,
outOfRangeTokenLogging,
outOfRangeTokenRejection);
}
protected void readPartition() throws IOException
{
partitionLevelDeletion = DeletionTime.serializer.deserialize(in);
iterator = SSTableSimpleIterator.create(metadata, in, header, helper, partitionLevelDeletion);
staticRow = iterator.readStaticRow();
return this;
}
public TableMetadata metadata()
@ -284,5 +345,54 @@ public class CassandraStreamReader implements IStreamReader
public void close()
{
}
/* We have a separate cleanup method because sometimes close is called before exhausting the
StreamDeserializer (for instance, when enclosed in an try-with-resources wrapper, such as in
BigTableWriter.append()).
*/
public void cleanup()
{
if (in instanceof RewindableDataInputStreamPlus)
{
try
{
((RewindableDataInputStreamPlus) in).close(false);
}
catch (IOException e)
{
logger.warn("Error while closing RewindableDataInputStreamPlus.", e);
}
}
}
private int verifyKeyInOwnedRanges(final DecoratedKey key,
List<Range<Token>> ownedRanges,
int lastCheckedRangeIndex,
boolean outOfRangeTokenLogging,
boolean outOfRangeTokenRejection)
{
if (lastCheckedRangeIndex < ownedRanges.size())
{
ListIterator<Range<Token>> rangesToCheck = ownedRanges.listIterator(lastCheckedRangeIndex);
while (rangesToCheck.hasNext())
{
Range<Token> range = rangesToCheck.next();
if (range.contains(key.getToken()))
return lastCheckedRangeIndex;
lastCheckedRangeIndex++;
}
}
StorageMetrics.totalOpsForInvalidToken.inc();
if (outOfRangeTokenLogging)
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate, session.planId(), writer.getFilename(), session.peer, ownedRanges, key);
if (outOfRangeTokenRejection)
throw new StreamReceivedOutOfTokenRangeException(ownedRanges, key, writer.getFilename());
return lastCheckedRangeIndex;
}
}
}

139
src/java/org/apache/cassandra/dht/OwnedRanges.java Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.dht;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.StorageMetrics;
public final class OwnedRanges
{
private static final Logger logger = LoggerFactory.getLogger(OwnedRanges.class);
private static final Comparator<Range<Token>> rangeComparator = Comparator.comparing((Range<Token> r) -> r.left).thenComparing(r -> r.right);
// the set of token ranges that this node is a replica for
private final List<Range<Token>> ownedRanges;
public OwnedRanges(Collection<Range<Token>> ownedRanges)
{
this.ownedRanges = Range.normalize(ownedRanges);
}
/**
* Check that all ranges in a requested set are contained by those in the owned set. Used in several contexts, such
* as validating StreamRequests in StreamSession & PrepareMessage and ValidationRequest in RepairMessageVerbHandler.
* In those callers, we want to verify that the token ranges specified in some request from a peer are not outside
* the ranges owned by the local node. There are 2 levels of response if invalid ranges are detected, controlled
* by options in Config; logging the event and rejecting the request and either/neither/both of these options may be
* enabled. If neither are enabled, we short ciruit and immediately return success without any further processing.
* If either option is enabled, and we do detect unowned ranges in the request, we increment a metric then take further
* action depending on the config.
*
* @param requestedRanges the set of token ranges contained in a request from a peer
* @param requestId an identifier for the peer request, to be used in logging (e.g. Stream or Repair Session #)
* @param requestType description of the request type, to be used in logging (e.g. "prepare request" or "validation")
* @param from the originator of the request
*
* @return true if the request should be accepted (either because no checking was performed, invalid ranges were
* identified but only the logging action is enabled, or because all request ranges were valid).
* Otherwise, returns false to indicate the request should be rejected.
*/
public boolean validateRangeRequest(Collection<Range<Token>> requestedRanges, String requestId, String requestType, InetAddressAndPort from)
{
boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests();
boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
Collection<Range<Token>> unownedRanges = testRanges(requestedRanges);
if (!unownedRanges.isEmpty())
{
StorageMetrics.totalOpsForInvalidToken.inc();
if (outOfRangeTokenLogging)
{
logger.warn("[{}] Received {} from {} containing ranges {} outside valid ranges {}",
requestId, requestType, from, unownedRanges, ownedRanges);
}
}
return !outOfRangeTokenRejection || unownedRanges.isEmpty();
}
/**
* Takes a collection of ranges and returns ranges from that collection that are not covered by this node's owned ranges.
* <p>
* This normalizes the range collections internally, so:
* a) be cautious about using this in any hot path
* b) any returned ranges may not be identical to those present. That is, the returned values are post-normalization.
* <p>
* e.g. Given two collections:
* { (0, 100], (100, 200] }
* { (90, 100], (100, 110], (110, 300] }
* the normalized forms are:
* { (0, 200] }
* { (90, 300] }
* and so the return value would be:
* { (90, 300] }
* which is equivalent, but not strictly equal to any member of the original supplied collection.
*
* @param testedRanges collection of candidate ranges to be checked
* @return the ranges in testedRanges which are not covered by the owned ranges
*/
@VisibleForTesting
Collection<Range<Token>> testRanges(final Collection<Range<Token>> testedRanges)
{
if (ownedRanges.isEmpty())
return testedRanges;
// now normalize the second and check coverage of its members in the normalized first collection
return Range.normalize(testedRanges).stream().filter(requested ->
{
// Find the point at which the target range would insert into the superset
int index = Collections.binarySearch(ownedRanges, requested, rangeComparator);
// an index >= 0 means an exact match was found, so we can definitely accept this range
if (index >= 0)
return false;
// convert to an insertion point in the superset
index = Math.abs(index) - 1;
// target sorts before the last list item, so we only need to check that one
if (index >= ownedRanges.size())
return !ownedRanges.get(index - 1).contains(requested);
// target sorts before the first list item, so we only need to check that one
if (index == 0)
return !ownedRanges.get(index).contains(requested);
// otherwise, check if the range on either side of the insertion point wholly contains the target
return !(ownedRanges.get(index - 1).contains(requested) || ownedRanges.get(index).contains(requested));
}).collect(Collectors.toSet());
}
}

21
src/java/org/apache/cassandra/hints/HintVerbHandler.java
View File

@ -19,10 +19,12 @@
package org.apache.cassandra.hints;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
@ -31,10 +33,11 @@ import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.serializers.MarshalException;
import org.apache.cassandra.service.StorageProxy;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.NoSpamLogger;
/**
* Verb handler used both for hint dispatch and streaming.
*
* <p>
* With the non-sstable format, we cannot just stream hint sstables on node decommission. So sometimes, at decommission
* time, we might have to stream hints to a non-owning host (say, if the owning host B is down during decommission of host A).
* In that case the handler just stores the received hint in its local hint store.
@ -89,6 +92,22 @@ public final class HintVerbHandler implements IVerbHandler<HintMessage>
// the topology has changed, and we are no longer a replica of the mutation - since we don't know which node(s)
// it has been handed over to, re-address the hint to all replicas; see CASSANDRA-5902.
HintsService.instance.writeForAllReplicas(hint);
HintsService.instance.metrics.incrHintsReceivedForUnownedRanges();
if (DatabaseDescriptor.getLogOutOfTokenRangeRequests())
{
// Log at most 1 message per second
NoSpamLogger.log(logger,
NoSpamLogger.Level.WARN,
1,
TimeUnit.SECONDS,
"Receiving hint(s) for token(s) neither owned nor pending. Example: from {} for token {} in {}.{}",
message.from(),
hint.mutation.key().getToken(),
hint.mutation.getKeyspaceName(),
hint.mutation.getPartitionUpdates().iterator().next().metadata().name);
}
respond(message);
}
else

26
src/java/org/apache/cassandra/io/sstable/SSTableSimpleIterator.java
View File

@ -21,6 +21,8 @@ import java.io.IOException;
import java.io.IOError;
import java.util.Iterator;
import com.google.common.annotations.VisibleForTesting;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.rows.*;
import org.apache.cassandra.io.util.DataInputPlus;
@ -125,4 +127,28 @@ public abstract class SSTableSimpleIterator extends AbstractIterator<Unfiltered>
}
}
}
@VisibleForTesting
public static class EmptySSTableSimpleIterator extends SSTableSimpleIterator
{
public EmptySSTableSimpleIterator(TableMetadata metadata)
{
super(metadata, null, null);
}
public Row readStaticRow() throws IOException
{
return Rows.EMPTY_STATIC_ROW;
}
protected Unfiltered computeNext()
{
return null;
}
public boolean hasNext()
{
return false;
}
}
}

16
src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java
View File

@ -103,11 +103,19 @@ public abstract class AbstractReplicationStrategy
return endpoints;
}
public Replica getLocalReplicaFor(RingPosition<?> searchPosition)
/**
* Check if the token is in a naturally replicated range or pending range
* @param token the position to check
* @return true if the token is in a natural or pending locally replicationed range, false otherwise
*/
public boolean isTokenInLocalNaturalOrPendingRange(Token token)
{
return getNaturalReplicas(searchPosition)
.byEndpoint()
.get(FBUtilities.getBroadcastAddressAndPort());
return getLocalReplicaFor(token) != null || tokenMetadata.isTokenInLocalPendingRange(keyspaceName, token);
}
public Replica getLocalReplicaFor(Token searchPosition)
{
return getNaturalReplicas(searchPosition).byEndpoint().get(FBUtilities.getBroadcastAddressAndPort());
}
/**

6
src/java/org/apache/cassandra/locator/EndpointsForToken.java
View File

@ -146,4 +146,10 @@ public class EndpointsForToken extends Endpoints<EndpointsForToken>
if (replicas.isEmpty()) return empty(token);
return builder(token, replicas.size()).addAll(replicas).build();
}
public static EndpointsForToken copyOf(Token token, Iterable<Replica> replicas)
{
if (!replicas.iterator().hasNext()) return empty(token);
return builder(token).addAll(replicas).build();
}
}

47
src/java/org/apache/cassandra/locator/PendingRangeMaps.java
View File

@ -20,13 +20,18 @@
*/
package org.apache.cassandra.locator;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
import com.google.common.collect.Iterators;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import java.util.*;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.config.CassandraRelevantProperties.LINE_SEPARATOR;
@ -149,6 +154,44 @@ public class PendingRangeMaps implements Iterable<Map.Entry<Range<Token>, Endpoi
}
}
public boolean isTokenInLocalPendingRange(Token token)
{
Range<Token> searchRange = new Range<>(token, token);
// search for non-wrap-around maps
NavigableMap<Range<Token>, EndpointsForRange.Builder> ascendingTailMap = ascendingMap.tailMap(searchRange, true);
NavigableMap<Range<Token>, EndpointsForRange.Builder> descendingTailMap = descendingMap.tailMap(searchRange, false);
boolean ascMapSizeLTDescMapSize = ascendingTailMap.size() < descendingTailMap.size();
NavigableMap<Range<Token>, EndpointsForRange.Builder> smallerMap = ascMapSizeLTDescMapSize ? ascendingTailMap : descendingTailMap;
NavigableMap<Range<Token>, EndpointsForRange.Builder> biggerMap = ascMapSizeLTDescMapSize ? descendingTailMap : ascendingTailMap;
// find the intersection of two sets
for (Range<Token> range : smallerMap.keySet())
{
EndpointsForRange.Builder replicas = biggerMap.get(range);
if (replicas != null && replicas.selfIfPresent() != null)
return true;
}
// search for wrap-around sets
ascendingTailMap = ascendingMapForWrapAround.tailMap(searchRange, true);
descendingTailMap = descendingMapForWrapAround.tailMap(searchRange, false);
for (EndpointsForRange.Builder endpointsForRange : ascendingTailMap.values())
{
if (endpointsForRange.selfIfPresent() != null)
return true;
}
for (EndpointsForRange.Builder endpointsForRange : descendingTailMap.values())
{
if (endpointsForRange.selfIfPresent() != null)
return true;
}
return false;
}
public EndpointsForToken pendingEndpointsFor(Token token)
{
EndpointsForToken.Builder replicas = EndpointsForToken.builder(token);

14
src/java/org/apache/cassandra/locator/TokenMetadata.java
View File

@ -812,6 +812,14 @@ public class TokenMetadata
return builder.build();
}
@VisibleForTesting
public void setPendingRangesUnsafe(String keyspaceName, Multimap<Range<Token>, Replica> rangeMap)
{
PendingRangeMaps prm = new PendingRangeMaps();
rangeMap.entries().forEach(entry -> prm.addPendingRange(entry.getKey(), entry.getValue()));
pendingRanges.put(keyspaceName, prm);
}
/**
* Calculate pending ranges according to bootsrapping and leaving nodes. Reasoning is:
*
@ -1400,6 +1408,12 @@ public class TokenMetadata
return partitioner.decorateKey(key);
}
public boolean isTokenInLocalPendingRange(String keyspaceName, Token token)
{
PendingRangeMaps pending = pendingRanges.get(keyspaceName);
return pending == null ? false : pending.isTokenInLocalPendingRange(token);
}
/**
* Tracks the assignment of racks and endpoints in each datacenter for all the "normal" endpoints
* in this TokenMetadata. This allows faster calculation of endpoints in NetworkTopologyStrategy.

8
src/java/org/apache/cassandra/metrics/HintedHandoffMetrics.java
View File

@ -52,6 +52,9 @@ public class HintedHandoffMetrics
.executor(MoreExecutors.directExecutor())
.build(address -> Metrics.counter(factory.createMetricName("Hints_created-" + address.toString().replace(':', '.'))));
/** Counter of hints received whose mutations are for keys outside the owned and pending ranges for this node */
private final Counter hintsReceivedForUnownedRanges = Metrics.counter(factory.createMetricName("Hints_for_unowned_ranges"));
public void incrCreatedHints(InetAddressAndPort address)
{
createdHintCounts.get(address).inc();
@ -62,6 +65,11 @@ public class HintedHandoffMetrics
notStored.get(address).mark();
}
public void incrHintsReceivedForUnownedRanges()
{
hintsReceivedForUnownedRanges.inc();
}
public void log()
{
for (Entry<InetAddressAndPort, DifferencingCounter> entry : notStored.asMap().entrySet())

10
src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java
View File

@ -125,6 +125,12 @@ public class KeyspaceMetrics
public final Histogram bytesValidated;
/** histogram over the number of partitions we have validated */
public final Histogram partitionsValidated;
/** Lifetime count of reads for keys outside the node's owned token ranges for this keyspace **/
public final Counter outOfRangeTokenReads;
/** Lifetime count of writes for keys outside the node's owned token ranges for this keyspace **/
public final Counter outOfRangeTokenWrites;
/** Lifetime count of paxos requests for keys outside the node's owned token ranges for this keyspace **/
public final Counter outOfRangeTokenPaxosRequests;
/*
* Metrics for inconsistencies detected between repaired data sets across replicas. These
@ -235,6 +241,10 @@ public class KeyspaceMetrics
repairedDataTrackingOverreadRows = createKeyspaceHistogram("RepairedDataTrackingOverreadRows", false);
repairedDataTrackingOverreadTime = createKeyspaceTimer("RepairedDataTrackingOverreadTime");
outOfRangeTokenReads = createKeyspaceCounter("ReadOutOfRangeToken");
outOfRangeTokenWrites = createKeyspaceCounter("WriteOutOfRangeToken");
outOfRangeTokenPaxosRequests = createKeyspaceCounter("PaxosOutOfRangeToken");
}
/**

2
src/java/org/apache/cassandra/metrics/StorageMetrics.java
View File

@ -33,4 +33,6 @@ public class StorageMetrics
public static final Counter totalHintsInProgress = Metrics.counter(factory.createMetricName("TotalHintsInProgress"));
public static final Counter totalHints = Metrics.counter(factory.createMetricName("TotalHints"));
public static final Counter repairExceptions = Metrics.counter(factory.createMetricName("RepairExceptions"));
public static final Counter totalOpsForInvalidToken = Metrics.counter(factory.createMetricName("TotalOpsForInvalidToken"));
public static final Counter startupOpsForInvalidToken = Metrics.counter(factory.createMetricName("StartupOpsForInvalidToken"));
}

49
src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
View File

@ -17,20 +17,38 @@
*/
package org.apache.cassandra.repair;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.repair.messages.*;
import org.apache.cassandra.repair.messages.CleanupMessage;
import org.apache.cassandra.repair.messages.FailSession;
import org.apache.cassandra.repair.messages.FinalizeCommit;
import org.apache.cassandra.repair.messages.FinalizePromise;
import org.apache.cassandra.repair.messages.FinalizePropose;
import org.apache.cassandra.repair.messages.PrepareConsistentRequest;
import org.apache.cassandra.repair.messages.PrepareConsistentResponse;
import org.apache.cassandra.repair.messages.PrepareMessage;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.StatusRequest;
import org.apache.cassandra.repair.messages.StatusResponse;
import org.apache.cassandra.repair.messages.SyncRequest;
import org.apache.cassandra.repair.messages.ValidationRequest;
import org.apache.cassandra.repair.messages.ValidationResponse;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.streaming.PreviewKind;
import org.apache.cassandra.service.StorageService;
import static org.apache.cassandra.net.Verb.VALIDATION_RSP;
@ -56,6 +74,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
return prs != null ? prs.previewKind : PreviewKind.NONE;
}
@Override
public void doVerb(final Message<RepairMessage> message)
{
// TODO add cancel/interrupt message
@ -82,7 +101,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
if (columnFamilyStore == null)
{
logErrorAndSendFailureResponse(String.format("Table with id %s was dropped during prepare phase of repair",
tableId), message);
tableId.toString()), message);
return;
}
columnFamilyStores.add(columnFamilyStore);
@ -141,7 +160,14 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
ActiveRepairService.instance.consistent.local.maybeSetRepairing(desc.parentSessionId);
Validator validator = new Validator(desc, message.from(), validationRequest.nowInSec,
isIncremental(desc.parentSessionId), previewKind(desc.parentSessionId));
ValidationManager.instance.submitValidation(store, validator);
if (acceptMessage(validationRequest, message.from()))
{
ValidationManager.instance.submitValidation(store, validator);
}
else
{
validator.fail();
}
break;
case SYNC_REQ:
@ -227,4 +253,19 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
Message<?> reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN);
MessagingService.instance().send(reply, respondTo.from());
}
private static boolean acceptMessage(final ValidationRequest validationRequest, final InetAddressAndPort from)
{
boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests();
boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
if (!outOfRangeTokenLogging && !outOfRangeTokenRejection)
return true;
return StorageService.instance.getNormalizedLocalRanges(validationRequest.desc.keyspace)
.validateRangeRequest(validationRequest.desc.ranges,
"RepairSession #" + validationRequest.desc.parentSessionId,
"validation request",
from);
}
}

71
src/java/org/apache/cassandra/service/StorageService.java
View File

@ -118,6 +118,7 @@ import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry;
import org.apache.cassandra.dht.BootStrapper;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.OwnedRanges;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.RangeStreamer;
import org.apache.cassandra.dht.RangeStreamer.FetchReplica;
@ -279,6 +280,35 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
public static final StorageService instance = new StorageService();
private final java.util.function.Predicate<Keyspace> anyOutOfRangeOpsRecorded
= keyspace -> keyspace.metric.outOfRangeTokenReads.getCount() > 0
|| keyspace.metric.outOfRangeTokenWrites.getCount() > 0
|| keyspace.metric.outOfRangeTokenPaxosRequests.getCount() > 0;
private long[] getOutOfRangeOperationCounts(Keyspace keyspace)
{
return new long[]
{
keyspace.metric.outOfRangeTokenReads.getCount(),
keyspace.metric.outOfRangeTokenWrites.getCount(),
keyspace.metric.outOfRangeTokenPaxosRequests.getCount()
};
}
public Map<String, long[]> getOutOfRangeOperationCounts()
{
return Schema.instance.getKeyspaces()
.stream()
.map(Keyspace::open)
.filter(anyOutOfRangeOpsRecorded)
.collect(Collectors.toMap(Keyspace::getName, this::getOutOfRangeOperationCounts));
}
public void incOutOfRangeOperationCount()
{
(isStarting() ? StorageMetrics.startupOpsForInvalidToken : StorageMetrics.totalOpsForInvalidToken).inc();
}
@Deprecated
public boolean isInShutdownHook()
{
@ -317,6 +347,11 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return ranges;
}
public OwnedRanges getNormalizedLocalRanges(String keyspaceName)
{
return new OwnedRanges(getLocalReplicas(keyspaceName).ranges());
}
public Collection<Range<Token>> getPrimaryRanges(String keyspace)
{
return getPrimaryRangesForEndpoint(keyspace, FBUtilities.getBroadcastAddressAndPort());
@ -4503,6 +4538,12 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
return Keyspace.open(keyspaceName).getReplicationStrategy().getNaturalReplicasForToken(token);
}
public boolean isEndpointValidForWrite(String keyspace, Token token)
{
AbstractReplicationStrategy replicationStrategy = Keyspace.open(keyspace).getReplicationStrategy();
return replicationStrategy.isTokenInLocalNaturalOrPendingRange(token);
}
public void setLoggingLevel(String classQualifier, String rawLevel) throws Exception
{
LoggingSupportFactory.getLoggingSupport().setLoggingLevel(classQualifier, rawLevel);
@ -6077,6 +6118,36 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
ClientResourceLimits.setEndpointLimit(newLimit);
}
public boolean isOutOfTokenRangeRequestLoggingEnabled()
{
return DatabaseDescriptor.getLogOutOfTokenRangeRequests();
}
public void setOutOfTokenRangeRequestLoggingEnabled(boolean enabled)
{
if (enabled)
logger.info("Enabling logging of requests on tokens outside owned ranges");
else
logger.info("Disabling logging of requests on tokens outside owned ranges");
DatabaseDescriptor.setLogOutOfTokenRangeRequests(enabled);
}
public boolean isOutOfTokenRangeRequestRejectionEnabled()
{
return DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
}
public void setOutOfTokenRangeRequestRejectionEnabled(boolean enabled)
{
if (enabled)
logger.info("Enabling rejection of requests on tokens outside owned ranges");
else
logger.info("Disabling rejection of requests on tokens outside owned ranges");
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(enabled);
}
@VisibleForTesting
public void shutdownServer()
{

18
src/java/org/apache/cassandra/service/StorageServiceMBean.java
View File

@ -868,4 +868,22 @@ public interface StorageServiceMBean extends NotificationEmitter
public Long getRepairRpcTimeout();
public void setRepairRpcTimeout(Long timeoutInMillis);
/**
* Toggles to turn on the logging or rejection of operations for token ranges that the node does not own,
* or is not about to acquire.
*/
boolean isOutOfTokenRangeRequestLoggingEnabled();
void setOutOfTokenRangeRequestLoggingEnabled(boolean enabled);
boolean isOutOfTokenRangeRequestRejectionEnabled();
void setOutOfTokenRangeRequestRejectionEnabled(boolean enabled);
/**
* Get the per-keyspace counts of operations that the node has received for tokens outside
* its owned ranges. Represented as a {@code Map<String, long[]>}, keys are keyspace names and the
* values are the counts for read, write and paxos ops respectivly.
* e.g. keyspace_name -> [reads, writes, paxos].
*/
Map<String, long[]> getOutOfRangeOperationCounts();
}

18
src/java/org/apache/cassandra/service/paxos/AbstractPaxosCallback.java
View File

@ -22,16 +22,22 @@ package org.apache.cassandra.service.paxos;
import java.util.concurrent.CountDownLatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ConsistencyLevel;
import org.apache.cassandra.db.WriteType;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.exceptions.WriteTimeoutException;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.net.RequestCallback;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
{
private static final Logger logger = LoggerFactory.getLogger(AbstractPaxosCallback.class);
protected final CountDownLatch latch;
protected final int targets;
private final ConsistencyLevel consistency;
@ -63,4 +69,16 @@ public abstract class AbstractPaxosCallback<T> implements RequestCallback<T>
throw new AssertionError("This latch shouldn't have been interrupted.");
}
}
@Override
public void onFailure(InetAddressAndPort from, RequestFailureReason failureReason)
{
logger.debug("Received paxos propose/prepare failure response from {} reason {}", from, failureReason);
}
@Override
public boolean invokeOnFailure()
{
return true;
}
}

79
src/java/org/apache/cassandra/service/paxos/AbstractPaxosVerbHandler.java Normal file
View File

@ -0,0 +1,79 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.paxos;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.NoSpamLogger;
public abstract class AbstractPaxosVerbHandler implements IVerbHandler<Commit>
{
private static final Logger logger = LoggerFactory.getLogger(AbstractPaxosVerbHandler.class);
private static final String logMessageTemplate = "Receiving Paxos operation(s) for token(s) neither owned nor pending. Example: from {} for token {} in {}.{}";
@Override
public void doVerb(Message<Commit> message)
{
boolean outOfRangeTokenLogging = DatabaseDescriptor.getLogOutOfTokenRangeRequests();
boolean outOfRangeTokenRejection = DatabaseDescriptor.getRejectOutOfTokenRangeRequests();
Commit commit = message.payload;
DecoratedKey key = commit.update.partitionKey();
boolean isOutOfRangeCommit = isOutOfRangeCommit(commit.update.metadata().keyspace, key);
if (isOutOfRangeCommit)
{
StorageService.instance.incOutOfRangeOperationCount();
Keyspace.open(commit.update.metadata().keyspace).metric.outOfRangeTokenPaxosRequests.inc();
// Log at most 1 message per second
if (outOfRangeTokenLogging)
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.SECONDS, logMessageTemplate,
message.from(), key.getToken(), commit.update.metadata().keyspace, commit.update.metadata().name);
}
if (outOfRangeTokenRejection && isOutOfRangeCommit)
sendFailureResponse(message);
else
processMessage(message);
}
abstract void processMessage(Message<Commit> message);
private static void sendFailureResponse(Message<?> respondTo)
{
Message<?> reply = respondTo.failureResponse(RequestFailureReason.UNKNOWN);
MessagingService.instance().send(reply, respondTo.from());
}
private static boolean isOutOfRangeCommit(String keyspace, DecoratedKey key)
{
return ! StorageService.instance.isEndpointValidForWrite(keyspace, key.getToken());
}
}

6
src/java/org/apache/cassandra/service/paxos/CommitVerbHandler.java
View File

@ -20,16 +20,16 @@
*/
package org.apache.cassandra.service.paxos;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.tracing.Tracing;
public class CommitVerbHandler implements IVerbHandler<Commit>
public class CommitVerbHandler extends AbstractPaxosVerbHandler
{
public static final CommitVerbHandler instance = new CommitVerbHandler();
public void doVerb(Message<Commit> message)
@Override
void processMessage(Message<Commit> message)
{
PaxosState.commit(message.payload);

6
src/java/org/apache/cassandra/service/paxos/PrepareVerbHandler.java
View File

@ -19,11 +19,10 @@ package org.apache.cassandra.service.paxos;
* under the License.
*
*/
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
public class PrepareVerbHandler implements IVerbHandler<Commit>
public class PrepareVerbHandler extends AbstractPaxosVerbHandler
{
public static PrepareVerbHandler instance = new PrepareVerbHandler();
@ -32,7 +31,8 @@ public class PrepareVerbHandler implements IVerbHandler<Commit>
return PaxosState.prepare(toPrepare);
}
public void doVerb(Message<Commit> message)
@Override
public void processMessage(Message<Commit> message)
{
Message<PrepareResponse> reply = message.responseWith(doPrepare(message.payload));
MessagingService.instance().send(reply, message.from());

4
src/java/org/apache/cassandra/service/paxos/ProposeVerbHandler.java
View File

@ -23,7 +23,7 @@ import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
public class ProposeVerbHandler implements IVerbHandler<Commit>
public class ProposeVerbHandler extends AbstractPaxosVerbHandler implements IVerbHandler<Commit>
{
public static final ProposeVerbHandler instance = new ProposeVerbHandler();
@ -32,7 +32,7 @@ public class ProposeVerbHandler implements IVerbHandler<Commit>
return PaxosState.propose(proposal);
}
public void doVerb(Message<Commit> message)
void processMessage(Message<Commit> message)
{
Message<Boolean> reply = message.responseWith(doPropose(message.payload));
MessagingService.instance().send(reply, message.from());

50
src/java/org/apache/cassandra/streaming/StreamReceivedOutOfTokenRangeException.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.streaming;
import java.util.Collection;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.io.sstable.Descriptor;
public class StreamReceivedOutOfTokenRangeException extends RuntimeException
{
private final Collection<Range<Token>> ownedRanges;
private final DecoratedKey key;
private final String filename;
public StreamReceivedOutOfTokenRangeException(Collection<Range<Token>> ownedRanges,
DecoratedKey key,
String filename)
{
this.ownedRanges = ownedRanges;
this.key = key;
this.filename = filename;
}
public String getMessage()
{
return String.format("Received stream for sstable %s containing key %s outside of owned ranges %s ",
filename,
key,
ownedRanges);
}
}

14
src/java/org/apache/cassandra/streaming/StreamRequest.java
View File

@ -23,6 +23,9 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.dht.IPartitioner;
import org.apache.cassandra.dht.Range;
@ -61,6 +64,17 @@ public class StreamRequest
this.columnFamilies.addAll(columnFamilies);
}
public String toString()
{
return MoreObjects.toStringHelper(this)
.add("keyspace", keyspace)
.add("tables", columnFamilies)
.add("full", full)
.add("transientReplicas", transientReplicas)
.omitNullValues()
.toString();
}
public static class StreamRequestSerializer implements IVersionedSerializer<StreamRequest>
{
public void serialize(StreamRequest request, DataOutputPlus out, int version) throws IOException

36
src/java/org/apache/cassandra/streaming/StreamRequestOutOfTokenRangeException.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.streaming;
import java.util.Collection;
public class StreamRequestOutOfTokenRangeException extends RuntimeException
{
private final Collection<StreamRequest> requests;
public StreamRequestOutOfTokenRangeException(Collection<StreamRequest> requests)
{
this.requests = requests;
}
public String getMessage()
{
return String.format("Received stream requests containing ranges outside of owned ranges: %s", requests);
}
}

115
src/java/org/apache/cassandra/streaming/StreamSession.java
View File

@ -19,34 +19,61 @@ package org.apache.cassandra.streaming;
import java.io.EOFException;
import java.net.SocketTimeoutException;
import java.util.*;
import java.util.concurrent.*;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.*;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.locator.RangesAtEndpoint;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.netty.channel.Channel;
import io.netty.channel.ChannelId;
import org.apache.cassandra.concurrent.ScheduledExecutors;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.lifecycle.TransactionAlreadyCompletedException;
import org.apache.cassandra.dht.OwnedRanges;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.gms.*;
import org.apache.cassandra.gms.EndpointState;
import org.apache.cassandra.gms.IEndpointStateChangeSubscriber;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.StreamingMetrics;
import org.apache.cassandra.net.OutboundConnectionSettings;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.async.NettyStreamingMessageSender;
import org.apache.cassandra.streaming.messages.*;
import org.apache.cassandra.streaming.messages.CompleteMessage;
import org.apache.cassandra.streaming.messages.IncomingStreamMessage;
import org.apache.cassandra.streaming.messages.OutgoingStreamMessage;
import org.apache.cassandra.streaming.messages.PrepareAckMessage;
import org.apache.cassandra.streaming.messages.PrepareSynAckMessage;
import org.apache.cassandra.streaming.messages.PrepareSynMessage;
import org.apache.cassandra.streaming.messages.ReceivedMessage;
import org.apache.cassandra.streaming.messages.SessionFailedMessage;
import org.apache.cassandra.streaming.messages.StreamInitMessage;
import org.apache.cassandra.streaming.messages.StreamMessage;
import org.apache.cassandra.streaming.messages.StreamMessageHeader;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.NoSpamLogger;
@ -163,7 +190,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
final Map<String, Set<Range<Token>>> transferredRangesPerKeyspace = new HashMap<>();
private final boolean isFollower;
private final NettyStreamingMessageSender messageSender;
private final StreamingMessageSender messageSender;
// contains both inbound and outbound channels
private final ConcurrentMap<ChannelId, Channel> channels = new ConcurrentHashMap<>();
@ -313,11 +340,19 @@ public class StreamSession implements IEndpointStateChangeSubscriber
{
failIfFinished();
if (!messageSender.hasControlChannel() && isControlChannel)
messageSender.injectControlMessageChannel(channel);
if (messageSender instanceof NettyStreamingMessageSender)
{
NettyStreamingMessageSender nettyStreamingMessageSender = (NettyStreamingMessageSender) messageSender;
if (!nettyStreamingMessageSender.hasControlChannel() && isControlChannel)
nettyStreamingMessageSender.injectControlMessageChannel(channel);
channel.closeFuture().addListener(ignored -> onChannelClose(channel));
return channels.putIfAbsent(channel.id(), channel) == null;
channel.closeFuture().addListener(ignored -> onChannelClose(channel));
return channels.putIfAbsent(channel.id(), channel) == null;
}
else
{
throw new IllegalStateException("Tried to attach non-Netty message sender");
}
}
/**
@ -361,7 +396,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
logger.info("[Stream #{}] Starting streaming to {}{}", planId(),
peer,
template.connectTo == null ? "" : " through " + template.connectTo);
messageSender.initialize();
getMessageSender().initialize();
onInitializationComplete();
}
catch (Exception e)
@ -546,7 +581,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
return state;
}
public NettyStreamingMessageSender getMessageSender()
public StreamingMessageSender getMessageSender()
{
return messageSender;
}
@ -628,11 +663,9 @@ public class StreamSession implements IEndpointStateChangeSubscriber
PrepareSynMessage prepare = new PrepareSynMessage();
prepare.requests.addAll(requests);
for (StreamTransferTask task : transfers.values())
{
prepare.summaries.add(task.getSummary());
}
messageSender.sendMessage(prepare);
getMessageSender().sendMessage(prepare);
}
/**
@ -673,10 +706,10 @@ public class StreamSession implements IEndpointStateChangeSubscriber
logError(e);
if (messageSender.connected())
if (getMessageSender().connected())
{
state(State.FAILED); // make sure subsequent error handling sees the session in a final state
messageSender.sendMessage(new SessionFailedMessage());
getMessageSender().sendMessage(new SessionFailedMessage());
}
return closeSession(State.FAILED);
@ -725,10 +758,10 @@ public class StreamSession implements IEndpointStateChangeSubscriber
* Finish preparing the session. This method is blocking (memtables are flushed in {@link #addTransferRanges}),
* so the logic should not execute on the main IO thread (read: netty event loop).
*/
private void prepareAsync(Collection<StreamRequest> requests, Collection<StreamSummary> summaries)
@VisibleForTesting
public void prepareAsync(Collection<StreamRequest> requests, Collection<StreamSummary> summaries)
{
for (StreamRequest request : requests)
addTransferRanges(request.keyspace, RangesAtEndpoint.concat(request.full, request.transientReplicas), request.columnFamilies, true); // always flush on stream request
processStreamRequests(requests);
for (StreamSummary summary : summaries)
prepareReceiving(summary);
@ -736,7 +769,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
if (!peer.equals(FBUtilities.getBroadcastAddressAndPort()))
for (StreamTransferTask task : transfers.values())
prepareSynAck.summaries.add(task.getSummary());
messageSender.sendMessage(prepareSynAck);
getMessageSender().sendMessage(prepareSynAck);
streamResult.handleSessionPrepared(this);
@ -755,7 +788,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
// only send the (final) ACK if we are expecting the peer to send this node (the initiator) some files
if (!isPreview())
messageSender.sendMessage(new PrepareAckMessage());
getMessageSender().sendMessage(new PrepareAckMessage());
}
if (isPreview())
@ -771,6 +804,32 @@ public class StreamSession implements IEndpointStateChangeSubscriber
startStreamingFiles(true);
}
private void processStreamRequests(Collection<StreamRequest> requests)
{
List<StreamRequest> rejectedRequests = new ArrayList<>();
// group requests by keyspace
Multimap<String, StreamRequest> requestsByKeyspace = ArrayListMultimap.create();
requests.forEach(r -> requestsByKeyspace.put(r.keyspace, r));
requestsByKeyspace.asMap().forEach((ks, reqs) ->
{
OwnedRanges ownedRanges = StorageService.instance.getNormalizedLocalRanges(ks);
reqs.forEach(req ->
{
RangesAtEndpoint allRangesAtEndpoint = RangesAtEndpoint.concat(req.full, req.transientReplicas);
if (ownedRanges.validateRangeRequest(allRangesAtEndpoint.ranges(), "Stream #" + planId(), "stream request", peer))
addTransferRanges(req.keyspace, allRangesAtEndpoint, req.columnFamilies, true); // always flush on stream request
else
rejectedRequests.add(req);
});
});
if (!rejectedRequests.isEmpty())
throw new StreamRequestOutOfTokenRangeException(rejectedRequests);
}
/**
* Call back after sending StreamMessageHeader.
*
@ -812,7 +871,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
StreamingMetrics.totalIncomingBytes.inc(headerSize);
metrics.incomingBytes.inc(headerSize);
// send back file received message
messageSender.sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber));
getMessageSender().sendMessage(new ReceivedMessage(message.header.tableId, message.header.sequenceNumber));
StreamHook.instance.reportIncomingStream(message.header.tableId, message.stream, this, message.header.sequenceNumber);
long receivedStartNanos = System.nanoTime();
try
@ -891,7 +950,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
}
else
{
messageSender.sendMessage(new CompleteMessage());
getMessageSender().sendMessage(new CompleteMessage());
closeSession(State.COMPLETE);
}
@ -998,7 +1057,7 @@ public class StreamSession implements IEndpointStateChangeSubscriber
{
// pass the session planId/index to the OFM (which is only set at init(), after the transfers have already been created)
ofm.header.addSessionInfo(this);
messageSender.sendMessage(ofm);
getMessageSender().sendMessage(ofm);
}
}
else

2
src/java/org/apache/cassandra/streaming/StreamingMessageSender.java
View File

@ -29,7 +29,7 @@ public interface StreamingMessageSender
{
void initialize() throws IOException;
void sendMessage(StreamMessage message) throws IOException;
void sendMessage(StreamMessage message);
boolean connected();

2
src/java/org/apache/cassandra/streaming/async/NettyStreamingMessageSender.java
View File

@ -288,7 +288,7 @@ public class NettyStreamingMessageSender implements StreamingMessageSender
* @return null if the message was processed sucessfully; else, a {@link java.util.concurrent.Future} to indicate
* the status of aborting any remaining tasks in the session.
*/
java.util.concurrent.Future onControlMessageComplete(Future<?> future, StreamMessage msg)
java.util.concurrent.Future<?> onControlMessageComplete(Future<?> future, StreamMessage msg)
{
ChannelFuture channelFuture = (ChannelFuture)future;
Throwable cause = future.cause();

5
src/java/org/apache/cassandra/tools/NodeProbe.java
View File

@ -1370,6 +1370,11 @@ public class NodeProbe implements AutoCloseable
return spProxy.getReadRepairRepairedBackground();
}
public Map<String,long[]> getOutOfRangeOpCounts()
{
return ssProxy.getOutOfRangeOperationCounts();
}
// JMX getters for the o.a.c.metrics API below.
/**
* Retrieve cache metrics based on the cache type (KeyCache, RowCache, or CounterCache)

17
src/java/org/apache/cassandra/tools/nodetool/Info.java
View File

@ -41,6 +41,9 @@ public class Info extends NodeToolCmd
@Option(name = {"-T", "--tokens"}, description = "Display all tokens")
private boolean tokens = false;
@Option(name = {"-O", "--out-of-range-ops"}, description = "Display per-keyspace counts of operations for invalid tokens")
private boolean outOfRangeOps = false;
@Override
public void execute(NodeProbe probe)
{
@ -175,6 +178,20 @@ public class Info extends NodeToolCmd
{
out.printf("%-23s: (node is not joined to the cluster)%n", "Token");
}
// Operations for out of range tokens
if (this.outOfRangeOps)
{
System.out.printf("%-23s: %-48s %10s %10s %10s%n", "Invalid Token Ops", "Keyspace", "Read", "Write", "Paxos");
Map<String, long[]> outOfRangeOpCounts = probe.getOutOfRangeOpCounts();
outOfRangeOpCounts.forEach((ks, counts) -> System.out.printf("%-24s %-48s %10s %10s %10s%n",
"",
ks,
counts[0],
counts[1],
counts[2]));
}
}
/**

76
test/distributed/org/apache/cassandra/distributed/test/OutOfTokenRangeTest.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.io.IOException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.distributed.Cluster;
import org.apache.cassandra.distributed.api.ConsistencyLevel;
import org.apache.cassandra.distributed.api.ICluster;
import static org.apache.cassandra.distributed.shared.AssertUtils.assertRows;
import static org.apache.cassandra.distributed.shared.AssertUtils.row;
public class OutOfTokenRangeTest extends TestBaseImpl
{
private static ICluster<?> cluster = null;
@BeforeClass
public static void setupPartitions() throws IOException
{
cluster = init(Cluster.build().withNodes(2).start(), 2);
cluster.schemaChange("CREATE TABLE " + KEYSPACE + ".tbl (pk int, ck text, v text, PRIMARY KEY (pk, ck))");
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk,ck,v) VALUES (1, 'a', '1a')", ConsistencyLevel.ALL);
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk,ck,v) VALUES (1, 'b', '1b')", ConsistencyLevel.ALL);
cluster.coordinator(1).execute("INSERT INTO " + KEYSPACE + ".tbl (pk,ck,v) VALUES (1, 'c', '1c')", ConsistencyLevel.ALL);
// Prove the row is there before adding the index
assertRows(cluster.coordinator(1).execute("SELECT pk, ck, v FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck = 'a' ALLOW FILTERING", ConsistencyLevel.ALL), row(1, "a", "1a"));
cluster.schemaChange("CREATE INDEX ckindex ON " + KEYSPACE + ".tbl (ck)");
}
@AfterClass
public static void closeCluster() throws Exception
{
if (cluster != null)
{
cluster.close();
}
}
@Test
public void checkSinglePartitionReadCommandTest()
{
Object[][] results = cluster.coordinator(1).execute("SELECT pk, ck, v FROM " + KEYSPACE + ".tbl WHERE pk = 1", ConsistencyLevel.ALL);
assertRows(results, row(1, "a", "1a"), row(1, "b", "1b"), row(1, "c", "1c"));
}
@Test
public void checkPartitionRangeReadCommandTest()
{
Object[][] results = cluster.coordinator(1).execute("SELECT pk, ck, v FROM " + KEYSPACE + ".tbl WHERE pk = 1 AND ck = 'a'", ConsistencyLevel.ALL);
assertRows(results, row(1, "a", "1a"));
}
}

14
test/distributed/org/apache/cassandra/distributed/test/ReadRepairTest.java
View File

@ -178,7 +178,7 @@ public class ReadRepairTest extends TestBaseImpl
@Test
public void movingTokenReadRepairTest() throws Throwable
{
try (Cluster cluster = init(Cluster.create(4), 3))
try (Cluster cluster = init(builder().withNodes(4).start(), 3))
{
List<Token> tokens = cluster.tokens();
@ -203,6 +203,13 @@ public class ReadRepairTest extends TestBaseImpl
PendingRangeCalculatorService.instance.blockUntilFinished();
}).accept(cluster.get(2).broadcastAddress());
// mark #2 as leaving in #1
cluster.get(1).acceptsOnInstance((InetSocketAddress endpoint) -> {
StorageService.instance.getTokenMetadata().addLeavingEndpoint(InetAddressAndPort.getByAddressOverrideDefaults(endpoint.getAddress(), endpoint.getPort()));
PendingRangeCalculatorService.instance.update();
PendingRangeCalculatorService.instance.blockUntilFinished();
}).accept(cluster.get(2).broadcastAddress());
// prevent #4 from reading or writing to #3, so our QUORUM must contain #2 and #4
// since #1 is taking over the range, this means any read-repair must make it to #1 as well
// (as a speculative repair in this case, as we prefer to send repair mutations to the initial
@ -353,7 +360,9 @@ public class ReadRepairTest extends TestBaseImpl
String key = "test1";
try (Cluster cluster = init(Cluster.build()
.withConfig(config -> config.with(Feature.GOSSIP, Feature.NETWORK)
.set("read_request_timeout_in_ms", Integer.MAX_VALUE))
.set("read_request_timeout_in_ms", Integer.MAX_VALUE)
.set("reject_out_of_token_range_requests", false)
)
.withTokenSupplier(TokenSupplier.evenlyDistributedTokens(4))
.withNodeIdTopology(NetworkTopology.singleDcNetworkTopology(4, "dc0", "rack0"))
.withNodes(3)
@ -491,6 +500,7 @@ public class ReadRepairTest extends TestBaseImpl
// on timestamp tie of RT and partition deletion: we should not generate RT bounds in such case,
// since monotonicity is already ensured by the partition deletion, and RT is unnecessary there.
// For details, see CASSANDRA-16453.
@SuppressWarnings("unused")
public static Object repairPartition(DecoratedKey partitionKey, Map<Replica, Mutation> mutations, ReplicaPlan.ForTokenWrite writePlan, @SuperCall Callable<Void> r) throws Exception
{
Assert.assertEquals(2, mutations.size());

214
test/unit/org/apache/cassandra/db/CounterMutationVerbHandlerOutOfRangeTest.java Normal file
View File

@ -0,0 +1,214 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.nio.ByteBuffer;
import java.util.concurrent.TimeUnit;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.context.CounterContext;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery;
import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress;
import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt;
import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink;
public class CounterMutationVerbHandlerOutOfRangeTest
{
private static final String KEYSPACE = "CounterCacheTest";
private static final String TABLE = "Counter1";
private CounterMutationVerbHandler handler;
private ColumnFamilyStore cfs;
private long startingTotalMetricCount;
private long startingKeyspaceMetricCount;
@BeforeClass
public static void init() throws Exception
{
SchemaLoader.loadSchema();
SchemaLoader.createKeyspace(KEYSPACE,
KeyspaceParams.simple(1),
SchemaLoader.counterCFMD(KEYSPACE, TABLE));
StorageService.instance.initServer(0);
}
@Before
public void setup() throws Exception
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1);
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress);
MessagingService.instance().inboundSink.clear();
MessagingService.instance().outboundSink.clear();
handler = new CounterMutationVerbHandler();
cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
cfs.truncateBlocking();
startingKeyspaceMetricCount = keyspaceMetricValue();
startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
}
@Test
public void acceptMutationForNaturalEndpoint() throws Exception
{
int messageId = randomInt();
int value = randomInt();
int key = 30;
CounterMutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
// unlike non-counter mutations, we can't verify the response message for a successful write.
// acting as the leader for the mutation, we'll try to forward the writes to the other replicas
// but this will fail as the other node isn't really there. The response message is written
// by the callback to these messages, so it will never get sent. So the best we can do is to check
// that the mutation was actually applied locally. When a counter mutation is rejected by the verb
// handler we *can* verify the failure response message is sent.
verifyWrite(key, value);
assertEquals(startingTotalMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount, keyspaceMetricValue());
}
@Test
public void acceptMutationForPendingEndpoint() throws Exception
{
// remove localhost from TM and add it back as pending
StorageService.instance.getTokenMetadata().removeEndpoint(broadcastAddress);
Multimap<Range<Token>, Replica> pending = HashMultimap.create();
Range<Token> range = new Range<>(bytesToken(0), bytesToken(100));
pending.put(range, new Replica(broadcastAddress, range, true));
StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(KEYSPACE, pending);
int messageId = randomInt();
int value = randomInt();
int key = 50;
CounterMutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
verifyWrite(key, value);
assertEquals(startingTotalMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount, keyspaceMetricValue());
}
@Test
public void rejectMutationForTokenOutOfRange() throws Exception
{
// reject a mutation for a token the node neither owns nor is pending
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int value = randomInt();
int key = 200;
CounterMutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
verifyFailureResponse(messageSink, messageId);
assertEquals(startingTotalMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount + 1, keyspaceMetricValue());
}
@Test
public void acceptMutationIfRejectionNotEnabled() throws Exception
{
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
int messageId = randomInt();
int value = randomInt();
int key = 500;
CounterMutation mutation = mutation(key, value);
// the node which is the actual natural endpoint for this mutation is not a real
// node, but if we write at CL.ANY we'll generate a hint for it and StorageProxy's
// counterWriterPerformer will blindly apply the mutation, so we can verify it locally
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
verifyWrite(key, value);
assertEquals(startingTotalMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount + 1, keyspaceMetricValue());
}
private void verifyWrite(int key, int value)
{
ReadCommand read = Util.cmd(cfs, bytes(key)).build();
ColumnMetadata col = cfs.metadata().getColumn(bytes("val"));
assertEquals(value, CounterContext.instance().total(Util.getOnlyRow(read).getCell(col)));
}
private static void verifyFailureResponse(ListenableFuture<MessageDelivery> messageSink, int messageId ) throws Exception
{
MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS);
assertEquals(Verb.FAILURE_RSP, response.message.verb());
assertEquals(broadcastAddress, response.message.from());
assertTrue(response.message.payload instanceof RequestFailureReason);
assertEquals(messageId, response.message.id());
assertEquals(node1, response.to);
}
private CounterMutation mutation(int key, int columnValue)
{
TableMetadata cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
DecoratedKey dk = cfs.decorateKey(bytes(key));
ColumnMetadata col = cfs.metadata().getColumn(bytes("val"));
ByteBuffer val = CounterContext.instance().createLocal(columnValue);
Cell<?> counterCell = BufferCell.live(col, FBUtilities.timestampMicros(), val);
Row row = BTreeRow.singleCellRow(cfs.metadata().comparator.make("clustering_1"), counterCell);
PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, dk, row);
return new CounterMutation(new Mutation(update), ConsistencyLevel.ANY);
}
private long keyspaceMetricValue()
{
return cfs.keyspace.metric.outOfRangeTokenWrites.getCount();
}
}

246
test/unit/org/apache/cassandra/db/MutationVerbHandlerOutOfRangeTest.java Normal file
View File

@ -0,0 +1,246 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.Util;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.BTreeRow;
import org.apache.cassandra.db.rows.BufferCell;
import org.apache.cassandra.db.rows.Cell;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.IVerbHandler;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.ColumnMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
import static org.apache.cassandra.utils.ByteBufferUtil.bytes;
import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery;
import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress;
import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt;
import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink;
public class MutationVerbHandlerOutOfRangeTest
{
private static final String TEST_NAME = "mutation_vh_test_";
private static final String KEYSPACE = TEST_NAME + "cql_keyspace";
private static final String TABLE = "table1";
private ColumnFamilyStore cfs;
private long startingTotalMetricCount;
private long startingKeyspaceMetricCount;
@BeforeClass
public static void init() throws Exception
{
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition(TEST_NAME);
StorageService.instance.initServer(0);
}
@Before
public void setup() throws Exception
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1);
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress);
MessagingService.instance().inboundSink.clear();
MessagingService.instance().outboundSink.clear();
cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE);
startingKeyspaceMetricCount = keyspaceMetricValue(cfs);
startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
}
@Test
public void acceptMutationForNaturalEndpoint() throws Exception
{
acceptMutationForNaturalEndpoint(new MutationVerbHandler());
}
@Test
public void acceptReadRepairForNaturalEndpoint() throws Exception
{
acceptMutationForNaturalEndpoint(new ReadRepairVerbHandler());
}
private void acceptMutationForNaturalEndpoint(IVerbHandler<Mutation> handler) throws Exception
{
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int value = randomInt();
int key = 50;
Mutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, key, value, false, false);
}
@Test
public void acceptMutationForPendingEndpoint() throws Exception
{
acceptMutationForPendingEndpoint(new MutationVerbHandler());
}
@Test
public void acceptReadRepairForPendingEndpoint() throws Exception
{
acceptMutationForPendingEndpoint(new ReadRepairVerbHandler());
}
private void acceptMutationForPendingEndpoint(IVerbHandler<Mutation> handler ) throws Exception
{
// remove localhost from TM and add it back as pending
StorageService.instance.getTokenMetadata().removeEndpoint(broadcastAddress);
Multimap<Range<Token>, Replica> pending = HashMultimap.create();
Range<Token> range = new Range<>(bytesToken(0), bytesToken(100));
pending.put(range, new Replica(broadcastAddress, range, true));
StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(KEYSPACE, pending);
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int value = randomInt();
int key = 50;
Mutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, key, value, false, false);
}
@Test
public void rejectMutationForTokenOutOfRange() throws Exception
{
rejectMutationForTokenOutOfRange(new MutationVerbHandler());
}
@Test
public void rejectReadRepairForTokenOutOfRange() throws Exception
{
rejectMutationForTokenOutOfRange(new ReadRepairVerbHandler());
}
private void rejectMutationForTokenOutOfRange(IVerbHandler<Mutation> handler) throws Exception
{
// reject a mutation for a token the node neither owns nor is pending
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int value = randomInt();
int key = 200;
Mutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, key, value, true, true);
}
@Test
public void acceptMutationIfRejectionNotEnabled() throws Exception
{
acceptMutationIfRejectionNotEnabled(new MutationVerbHandler());
}
@Test
public void acceptReadRepairIfRejectionNotEnabled() throws Exception
{
acceptMutationIfRejectionNotEnabled(new ReadRepairVerbHandler());
}
private void acceptMutationIfRejectionNotEnabled(IVerbHandler<Mutation> handler) throws Exception
{
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int value = randomInt();
int key = 200;
Mutation mutation = mutation(key, value);
handler.doVerb(Message.builder(Verb.MUTATION_REQ, mutation).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, key, value, true, false);
}
static <V> int toInt(Cell<V> cell)
{
return cell.accessor().toInt(cell.value());
}
private void getAndVerifyResponse(ListenableFuture<MessageDelivery> messageSink,
int messageId,
int key,
int value,
boolean isOutOfRange,
boolean expectFailure) throws InterruptedException, ExecutionException, TimeoutException
{
MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS);
assertEquals(expectFailure ? Verb.FAILURE_RSP : Verb.MUTATION_RSP, response.message.verb());
assertEquals(broadcastAddress, response.message.from());
assertEquals(expectFailure, response.message.payload instanceof RequestFailureReason);
assertEquals(messageId, response.message.id());
assertEquals(node1, response.to);
assertEquals(startingTotalMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount + (isOutOfRange ? 1 : 0), keyspaceMetricValue(cfs));
if (!expectFailure)
{
ReadCommand read = Util.cmd(cfs, bytes(key)).build();
ColumnMetadata col = cfs.metadata().getColumn(bytes("v1"));
assertEquals(value, toInt(Util.getOnlyRow(read).getCell(col)));
}
}
private static long keyspaceMetricValue(ColumnFamilyStore cfs)
{
return cfs.keyspace.metric.outOfRangeTokenWrites.getCount();
}
private Mutation mutation(int key, int columnValue)
{
TableMetadata cfm = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
DecoratedKey dk = cfs.decorateKey(bytes(key));
ColumnMetadata col = cfs.metadata().getColumn(bytes("v1"));
Cell<?> cell = BufferCell.live(col, FBUtilities.timestampMicros(), bytes(columnValue));
Row row = BTreeRow.singleCellRow(Clustering.EMPTY, cell);
PartitionUpdate update = PartitionUpdate.singleRowUpdate(cfm, dk, row);
return new Mutation(update);
}
}

258
test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerOutOfRangeTest.java Normal file
View File

@ -0,0 +1,258 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.db;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.filter.ColumnFilter;
import org.apache.cassandra.db.filter.DataLimits;
import org.apache.cassandra.db.filter.RowFilter;
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TokenRangeTestUtil;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.apache.cassandra.net.Verb.READ_REQ;
import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress;
import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt;
import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink;
public class ReadCommandVerbHandlerOutOfRangeTest
{
private static ReadCommandVerbHandler handler;
private static TableMetadata metadata_nonreplicated;
private ColumnFamilyStore cfs;
private long startingTotalMetricCount;
private long startingKeyspaceMetricCount;
private static final String TEST_NAME = "read_command_vh_test_";
private static final String KEYSPACE_NONREPLICATED = TEST_NAME + "cql_keyspace";
private static final String TABLE = "table1";
@BeforeClass
public static void init() throws Throwable
{
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition(TEST_NAME);
StorageService.instance.initServer(0);
metadata_nonreplicated = Schema.instance.getTableMetadata(KEYSPACE_NONREPLICATED, TABLE);
}
@Before
public void setup()
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1);
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress);
MessagingService.instance().inboundSink.clear();
MessagingService.instance().outboundSink.clear();
MessagingService.instance().outboundSink.add((message, to) -> false);
MessagingService.instance().inboundSink.add((message) -> false);
cfs = Keyspace.open(KEYSPACE_NONREPLICATED).getColumnFamilyStore(TABLE);
startingKeyspaceMetricCount = keyspaceMetricValue(cfs);
startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
handler = new ReadCommandVerbHandler();
}
private static DecoratedKey key(TableMetadata metadata, int key)
{
return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key));
}
@Test
public void acceptReadForNaturalEndpoint() throws Exception
{
ListenableFuture<TokenRangeTestUtil.MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 50;
ReadCommand command = command(key);
handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, false, false);
}
@Test
public void rejectReadForTokenOutOfRange() throws Exception
{
// reject a read for a key who's token the node doesn't own the range for
ListenableFuture<TokenRangeTestUtil.MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 200;
ReadCommand command = command(key);
handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, true, true);
}
@Test
public void acceptReadIfRejectionNotEnabled() throws Exception
{
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
ListenableFuture<TokenRangeTestUtil.MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 200;
ReadCommand command = command(key);
handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, true, false);
}
@Test
public void rangeReadCommandBoundsAreNotChecked() throws Exception
{
// checking is only currently done for single partition reads, range reads will continue to
// accept any range they are given. So for a range wholly outside the node's ownership we
// expect the metric to remain unchanged and read command to be executed.
// This test is added for 3.0 because the single partition & range commands are now processed
// by the same verb handler.
// rdar://problem/33535104 is to extend checking to range reads
ListenableFuture<TokenRangeTestUtil.MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
Range<Token> range = new Range<>(key(metadata_nonreplicated, 150).getToken(),
key(metadata_nonreplicated, 160).getToken());
ReadCommand command = new StubRangeReadCommand(range, metadata_nonreplicated);
handler.doVerb(Message.builder(READ_REQ, command).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, messageId, false, false);
}
private void getAndVerifyResponse(ListenableFuture<TokenRangeTestUtil.MessageDelivery> messageSink,
int messageId,
boolean isOutOfRange,
boolean expectFailure) throws InterruptedException, ExecutionException, TimeoutException
{
assertEquals(startingTotalMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount + (isOutOfRange ? 1 : 0), keyspaceMetricValue(cfs));
if (expectFailure)
{
try
{
TokenRangeTestUtil.MessageDelivery response = messageSink.get(10, TimeUnit.MILLISECONDS);
fail(String.format("Didn't expect any message to be sent, but sent %s to %s in response to %s",
response.message.toString(),
response.to,
response.message.id()));
}
catch (TimeoutException e)
{
// expected
}
}
else
{
TokenRangeTestUtil.MessageDelivery response = messageSink.get(10, TimeUnit.MILLISECONDS);
assertEquals(Verb.READ_RSP, response.message.verb());
assertEquals(broadcastAddress, response.message.from());
assertEquals(messageId, response.message.id());
assertEquals(node1, response.to);
}
}
private ReadCommand command(int key)
{
return new StubReadCommand(key, metadata_nonreplicated);
}
private static class StubReadCommand extends SinglePartitionReadCommand
{
private final TableMetadata tmd;
StubReadCommand(int key, TableMetadata tmd)
{
super(false,
0,
false,
tmd,
FBUtilities.nowInSeconds(),
ColumnFilter.all(tmd),
RowFilter.NONE,
DataLimits.NONE,
key(tmd, key),
null,
null);
this.tmd = tmd;
}
public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
{
return EmptyIterators.unfilteredPartition(tmd);
}
public String toString()
{
return "<<StubReadCommand>>";
}
}
private static class StubRangeReadCommand extends PartitionRangeReadCommand
{
private final TableMetadata cfm;
StubRangeReadCommand(Range<Token> range, TableMetadata tmd)
{
super(false,
0,
false,
tmd,
FBUtilities.nowInSeconds(),
ColumnFilter.all(tmd),
RowFilter.NONE,
DataLimits.NONE,
DataRange.forTokenRange(range),
null);
this.cfm = tmd;
}
public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController)
{
return EmptyIterators.unfilteredPartition(cfm);
}
}
private static long keyspaceMetricValue(ColumnFamilyStore cfs)
{
return cfs.keyspace.metric.outOfRangeTokenReads.getCount();
}
}

217
test/unit/org/apache/cassandra/dht/OwnedRangesTest.java Normal file
View File

@ -0,0 +1,217 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.dht;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRanges;
public class OwnedRangesTest
{
@Test
public void testFilterRangesWithEmptySuperset()
{
List<Range<Token>> toFilter = generateRanges(0, 50);
Collection<Range<Token>> rejected = new OwnedRanges(Collections.emptyList()).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithEmptySupersetEmptySubset()
{
assertTrue(new OwnedRanges(Collections.emptyList()).testRanges(Collections.emptyList()).isEmpty());
}
@Test
public void testFilterRangesWithSingleSupersetEmptySubset()
{
assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(Collections.emptyList()).isEmpty());
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetContained()
{
assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(generateRanges(10 , 20)).isEmpty());
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetExactMatch()
{
assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(generateRanges(0, 100)).isEmpty());
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetStrictlyLessThan()
{
Collection<Range<Token>> toFilter = generateRanges(-100, 0);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetStrictlyGreaterThan()
{
Collection<Range<Token>> toFilter = generateRanges(101, 200);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetOverlapping()
{
Collection<Range<Token>> toFilter = generateRanges(80, 120);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetOverlappingOnRight()
{
Collection<Range<Token>> toFilter = generateRanges(0, 120);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithSingleSupersetSingleSubsetOverlappingOnLeft()
{
Collection<Range<Token>> toFilter = generateRanges(-10, 100);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithSingleSupersetMultipleSubsetAllContained()
{
Collection<Range<Token>> toFilter = generateRanges(10, 20, 30, 40, 50, 60);
assertTrue(new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter).isEmpty());
}
@Test
public void testFilterRangesWithSingleSupersetMultipleSubsetSomeContained()
{
Collection<Range<Token>> toFilter = generateRanges(-20, -10, 0, 10, 20, 30, 110, 120);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(generateRanges(-20, -10, 110, 120), rejected);
}
@Test
public void testFilterRangesWithSingleSupersetMultipleSubsetNoneContained()
{
Collection<Range<Token>> toFilter = generateRanges(-40, -20, -20, -10, 110, 120, 230, 140);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterWithMultipleSupersetSingleSubsetContainedInFirst()
{
Collection<Range<Token>> toFilter = generateRanges(10, 20);
assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty());
}
@Test
public void testFilterWithMultipleSupersetSingleSubsetContainedInLast()
{
Collection<Range<Token>> toFilter = generateRanges(450, 460);
assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty());
}
@Test
public void testFilterRangesWithMultipleSupersetSingleSubsetStrictlyGreaterThan()
{
Collection<Range<Token>> toFilter = generateRanges(510, 520);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithMultipleSupersetSingleSubsetStrictlyLessThan()
{
Collection<Range<Token>> toFilter = generateRanges(-20, -10);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithMultipleSupersetSingleSubsetOverlapping()
{
Collection<Range<Token>> toFilter = generateRanges(80, 120);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithMultipleSupersetSingleSubsetOverlappingOnRight()
{
Collection<Range<Token>> toFilter = generateRanges(0, 120);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithMultipleSupersetSingleSubsetOverlappingOnLeft()
{
Collection<Range<Token>> toFilter = generateRanges(-10, 100);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithMultipleSupersetMultipleSubsetAllDisjoint()
{
Collection<Range<Token>> toFilter = generateRanges(-20, -10, 110, 120, 310, 320);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(toFilter, rejected);
}
@Test
public void testFilterRangesWithMultipleSupersetMultipleSubsetAllContained()
{
Collection<Range<Token>> toFilter = generateRanges(10, 20, 210, 220, 410, 420);
assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty());
}
@Test
public void testFilterRangesWithMultipleSupersetMultipleSubsetSomeContained()
{
Collection<Range<Token>> toFilter = generateRanges(10, 20, 210, 220, 310, 320, 410, 420);
Collection<Range<Token>> rejected = new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter);
assertRangesEqual(rejected, generateRanges(310, 320));
}
@Test
public void testFilterRangesWithMultipleSupersetMultipleSubsetSomeExactMatchSomeContained()
{
Collection<Range<Token>> toFilter = generateRanges(10, 20, 200, 300, 410, 420);
assertTrue(new OwnedRanges(generateRanges(0, 100, 200, 300, 400, 500)).testRanges(toFilter).isEmpty());
}
private static void assertRangesEqual(Collection<Range<Token>> first, Collection<Range<Token>> second)
{
assertTrue(CollectionUtils.isEqualCollection(Range.normalize(first), Range.normalize(second)));
}
}

71
test/unit/org/apache/cassandra/locator/PendingRangeMapsTest.java
View File

@ -20,21 +20,24 @@
*/
package org.apache.cassandra.locator;
import java.net.UnknownHostException;
import org.junit.Test;
import org.apache.cassandra.dht.RandomPartitioner.BigIntegerToken;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.junit.Test;
import java.net.UnknownHostException;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class PendingRangeMapsTest {
public class PendingRangeMapsTest
{
private Range<Token> genRange(String left, String right)
{
return new Range<Token>(new BigIntegerToken(left), new BigIntegerToken(right));
return new Range<>(new BigIntegerToken(left), new BigIntegerToken(right));
}
private static void addPendingRange(PendingRangeMaps pendingRangeMaps, Range<Token> range, String endpoint)
@ -105,4 +108,60 @@ public class PendingRangeMapsTest {
assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.1")));
assertTrue(replicas.endpoints().contains(InetAddressAndPort.getByName("127.0.0.7")));
}
@Test
public void testIsTokenInLocalPendingRanges()
{
PendingRangeMaps pendingRangeMaps = new PendingRangeMaps();
InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
String selfHostAddress = self.getHostAddress(true);
addPendingRange(pendingRangeMaps, genRange("5", "15"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("15", "25"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("25", "35"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("35", "45"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("45", "55"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("45", "65"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange( "66", "67"), "127.0.0.7");
assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("0")));
assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("5")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("10")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("15")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("20")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("25")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("35")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("45")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("55")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("65")));
assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("66")));
}
@Test
public void testIsTokenInLocalPendingRangesWrapAroundRanges()
{
PendingRangeMaps pendingRangeMaps = new PendingRangeMaps();
InetAddressAndPort self = FBUtilities.getBroadcastAddressAndPort();
String selfHostAddress = self.getHostAddress(true);
addPendingRange(pendingRangeMaps, genRange("5", "15"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("15", "25"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("25", "35"), "127.0.0.1");
addPendingRange(pendingRangeMaps, genRange("35", "45"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("45", "55"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("45", "65"), selfHostAddress);
addPendingRange(pendingRangeMaps, genRange("65", "7"), selfHostAddress);
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("0")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("5")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("7")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("10")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("15")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("20")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("25")));
assertFalse(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("35")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("45")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("55")));
assertTrue(pendingRangeMaps.isTokenInLocalPendingRange(new BigIntegerToken("65")));
}
}

254
test/unit/org/apache/cassandra/repair/RepairMessageVerbHandlerOutOfRangeTest.java Normal file
View File

@ -0,0 +1,254 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.repair;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.repair.messages.PrepareMessage;
import org.apache.cassandra.repair.messages.RepairMessage;
import org.apache.cassandra.repair.messages.ValidationResponse;
import org.apache.cassandra.repair.messages.ValidationRequest;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.PreviewKind;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery;
import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress;
import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRange;
import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRanges;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt;
import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink;
import static org.apache.cassandra.utils.TokenRangeTestUtil.setLocalTokens;
import static org.apache.cassandra.utils.TokenRangeTestUtil.token;
import static org.apache.cassandra.utils.TokenRangeTestUtil.uuid;
public class RepairMessageVerbHandlerOutOfRangeTest
{
private static final String TEST_NAME = "repair_message_vh_test_";
private static final String KEYSPACE = TEST_NAME + "cql_keyspace";
private static final String TABLE = "table1";
private static List<TableId> tableIds;
@BeforeClass
public static void init() throws Exception
{
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition(TEST_NAME);
StorageService.instance.initServer(0);
tableIds = Collections.singletonList(Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata().id);
}
@Before
public void setup() throws Exception
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
// All tests suppose a 2 node ring, with the other peer having the tokens 0, 200, 300
// Initially, the local node has no tokens so when indivividual test set owned tokens or
// pending ranges for the local node, they're always in relation to this.
// e.g. test calls setLocalTokens(100, 300) the ring now looks like
// peer -> (min, 0], (100, 200], (300, 400]
// local -> (0, 100], (200, 300], (400, max]
//
// Pending ranges are set in test using start/end pairs.
// Ring is initialised:
// peer -> (min, max]
// local -> (,]
// e.g. test calls setPendingRanges(0, 100, 200, 300)
// the pending ranges for local would be calculated as:
// local -> (0, 100], (200, 300]
StorageService.instance.getTokenMetadata().updateNormalTokens(Lists.newArrayList(token(0),
token(200),
token(400)),
node1);
}
@Test
public void testPrepareWithAllRequestedRangesWithinOwned() throws Exception
{
setLocalTokens(100);
PrepareMessage prepare = prepareMsg(generateRanges(10, 20));
tryPrepareExpectingSuccess(prepare);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryPrepareExpectingSuccess(prepare);
}
@Test
public void testPrepareWithAllRequestedRangesOutsideOwned() throws Exception
{
setLocalTokens(100);
PrepareMessage prepare = prepareMsg(generateRanges(110, 120));
tryPrepareExpectingSuccess(prepare);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryPrepareExpectingSuccess(prepare);
}
@Test
public void testPrepareWithSomeRequestedRangesOutsideOwned() throws Exception
{
setLocalTokens(100);
PrepareMessage prepare = prepareMsg(generateRanges(10, 20, 110, 120));
tryPrepareExpectingSuccess(prepare);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryPrepareExpectingSuccess(prepare);
}
@Test
public void testValidationRequestWithRequestedRangeWithinOwned() throws Exception
{
setLocalTokens(100);
ValidationRequest request = validationMsg(generateRange(10, 20));
tryValidationExpectingSuccess(request, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryValidationExpectingSuccess(request, false);
}
@Test
public void testValidationRequestWithRequestedRangeOutsideOwned() throws Exception
{
setLocalTokens(100);
ValidationRequest request = validationMsg(generateRange(110, 120));
tryValidationExpectingFailure(request);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryValidationExpectingSuccess(request, true);
}
@Test
public void testValidationRequestWithRequestedRangeOverlappingOwned() throws Exception
{
setLocalTokens(100);
ValidationRequest request = validationMsg(generateRange(10, 120));
tryValidationExpectingFailure(request);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryValidationExpectingSuccess(request, true);
}
private static void tryValidationExpectingFailure(ValidationRequest request) throws Exception
{
tryValidation(request, true, false);
}
private static void tryValidationExpectingSuccess(ValidationRequest request, boolean isOutOfRange) throws Exception
{
tryValidation(request, isOutOfRange, true);
}
private static void tryValidation(ValidationRequest request, boolean isOutOfRange, boolean expectSuccess) throws Exception
{
long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
MessagingService.instance().outboundSink.clear();
MessagingService.instance().inboundSink.clear();
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink(Verb.REPAIR_RSP);
RepairMessageVerbHandler handler = new RepairMessageVerbHandler();
int messageId = randomInt();
Message<RepairMessage> message = Message.builder(Verb.VALIDATION_REQ, (RepairMessage)request).from(node1).withId(messageId).build();
handler.doVerb(message);
MessageDelivery response = messageSink.get(500, TimeUnit.MILLISECONDS);
assertEquals(Verb.VALIDATION_RSP, response.message.verb());
assertEquals(broadcastAddress, response.message.from());
assertEquals(node1, response.to);
assertTrue(response.message.payload instanceof ValidationResponse);
ValidationResponse completion = (ValidationResponse) response.message.payload;
assertEquals(expectSuccess, completion.success());
assertEquals(startMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount());
}
private static void tryPrepareExpectingSuccess(PrepareMessage prepare) throws Exception
{
long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
MessagingService.instance().outboundSink.clear();
MessagingService.instance().inboundSink.clear();
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
RepairMessageVerbHandler handler = new RepairMessageVerbHandler();
int messageId = randomInt();
Message<RepairMessage> message = Message.builder(Verb.PREPARE_MSG, (RepairMessage)prepare).from(node1).withId(messageId).build();
handler.doVerb(message);
MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS);
assertEquals(Verb.REPAIR_RSP, response.message.verb());
assertEquals(broadcastAddress, response.message.from());
assertEquals(messageId, response.message.id());
assertEquals(node1, response.to);
assertFalse(response.message.payload instanceof RequestFailureReason);
assertEquals(startMetricCount, StorageMetrics.totalOpsForInvalidToken.getCount());
}
private static PrepareMessage prepareMsg(Collection<Range<Token>> ranges)
{
return new PrepareMessage(uuid(), tableIds, ranges, false, ActiveRepairService.UNREPAIRED_SSTABLE, true, PreviewKind.NONE);
}
private static ValidationRequest validationMsg(Range<Token> range)
{
UUID parentId = uuid();
List<ColumnFamilyStore> stores = tableIds.stream()
.map(Schema.instance::getColumnFamilyStoreInstance)
.collect(Collectors.toList());
ActiveRepairService.instance.registerParentRepairSession(parentId,
node1,
stores,
Collections.singleton(range),
false,
ActiveRepairService.UNREPAIRED_SSTABLE,
true,
PreviewKind.NONE);
return new ValidationRequest(new RepairJobDesc(parentId, uuid(), KEYSPACE, TABLE, Collections.singleton(range)),
randomInt());
}
}

252
test/unit/org/apache/cassandra/service/paxos/PaxosVerbHandlerOutOfRangeTest.java Normal file
View File

@ -0,0 +1,252 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.service.paxos;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.RequestFailureReason;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
import static org.junit.Assert.assertEquals;
import static org.apache.cassandra.utils.TokenRangeTestUtil.MessageDelivery;
import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress;
import static org.apache.cassandra.utils.TokenRangeTestUtil.bytesToken;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt;
import static org.apache.cassandra.utils.TokenRangeTestUtil.registerOutgoingMessageSink;
public class PaxosVerbHandlerOutOfRangeTest
{
// For the purposes of this testing, the details of the Commit don't really matter
// as we're just testing the rejection (or lack of) and not the result of doing
// whatever the specific verb handlers are supposed to do when they don't reject
// a given Commit
private static final String TEST_NAME = "paxos_vh_test_";
private static final String KEYSPACE = TEST_NAME + "cql_keyspace";
private static final String TABLE = "table1";
private long startingTotalMetricCount;
private long startingKeyspaceMetricCount;
@BeforeClass
public static void init() throws Exception
{
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition(TEST_NAME);
StorageService.instance.initServer(0);
}
@Before
public void setup() throws Exception
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(0), node1);
StorageService.instance.getTokenMetadata().updateNormalToken(bytesToken(100), broadcastAddress);
MessagingService.instance().inboundSink.clear();
MessagingService.instance().outboundSink.clear();
startingTotalMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
startingKeyspaceMetricCount = keyspaceMetricValue();
}
private static DecoratedKey key(TableMetadata metadata, int key)
{
return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key));
}
@Test
public void acceptPrepareForNaturalEndpoint() throws Exception
{
acceptRequestForNaturalEndpoint(Verb.PAXOS_PREPARE_REQ, Verb.PAXOS_PREPARE_RSP, new PrepareVerbHandler());
}
@Test
public void acceptProposeForNaturalEndpoint() throws Exception
{
acceptRequestForNaturalEndpoint(Verb.PAXOS_PROPOSE_REQ, Verb.PAXOS_PROPOSE_RSP, new ProposeVerbHandler());
}
@Test
public void acceptCommitForNaturalEndpoint() throws Exception
{
acceptRequestForNaturalEndpoint(Verb.PAXOS_COMMIT_REQ, Verb.PAXOS_COMMIT_RSP, new CommitVerbHandler());
}
private void acceptRequestForNaturalEndpoint(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception
{
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 50;
Commit commit = commit(key);
handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, responseVerb, messageId, false, false);
}
@Test
public void acceptPrepareForPendingEndpoint() throws Exception
{
acceptRequestForPendingEndpoint(Verb.PAXOS_PREPARE_REQ, Verb.PAXOS_PREPARE_RSP, new PrepareVerbHandler());
}
@Test
public void acceptProposeForPendingEndpoint() throws Exception
{
acceptRequestForPendingEndpoint(Verb.PAXOS_PROPOSE_REQ, Verb.PAXOS_PROPOSE_RSP, new ProposeVerbHandler());
}
@Test
public void acceptCommitForPendingEndpoint() throws Exception
{
acceptRequestForPendingEndpoint(Verb.PAXOS_COMMIT_REQ, Verb.PAXOS_COMMIT_RSP, new CommitVerbHandler());
}
private void acceptRequestForPendingEndpoint(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception
{
// remove localhost from TM and add it back as pending
StorageService.instance.getTokenMetadata().removeEndpoint(broadcastAddress);
Multimap<Range<Token>, Replica> pending = HashMultimap.create();
Range<Token> range = new Range<>(bytesToken(0), bytesToken(100));
pending.put(range, new Replica(broadcastAddress, range, true));
StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(KEYSPACE, pending);
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 50;
Commit commit = commit(key);
handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, responseVerb, messageId, false, false);
}
@Test
public void rejectPrepareForTokenOutOfRange() throws Exception
{
rejectRequestForTokenOutOfRange(Verb.PAXOS_PREPARE_REQ, Verb.FAILURE_RSP, new PrepareVerbHandler());
}
@Test
public void rejectProposeForTokenOutOfRange() throws Exception
{
rejectRequestForTokenOutOfRange(Verb.PAXOS_PROPOSE_REQ, Verb.FAILURE_RSP, new ProposeVerbHandler());
}
@Test
public void rejectCommitForTokenOutOfRange() throws Exception
{
rejectRequestForTokenOutOfRange(Verb.PAXOS_COMMIT_REQ, Verb.FAILURE_RSP, new CommitVerbHandler());
}
private void rejectRequestForTokenOutOfRange(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception
{
// reject a commit for a token the node neither owns nor is pending
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 200;
Commit commit = commit(key);
handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, responseVerb, messageId, true, true);
}
@Test
public void acceptPrepareIfRejectionNotEnabled() throws Exception
{
acceptRequestIfRejectionNotEnabled(Verb.PAXOS_PREPARE_REQ, Verb.PAXOS_PREPARE_RSP, new PrepareVerbHandler());
}
@Test
public void acceptProposeIfRejectionNotEnabled() throws Exception
{
acceptRequestIfRejectionNotEnabled(Verb.PAXOS_PROPOSE_REQ, Verb.PAXOS_PROPOSE_RSP, new ProposeVerbHandler());
}
@Test
public void acceptCommitIfRejectionNotEnabled() throws Exception
{
acceptRequestIfRejectionNotEnabled(Verb.PAXOS_COMMIT_REQ, Verb.PAXOS_COMMIT_RSP, new CommitVerbHandler());
}
private void acceptRequestIfRejectionNotEnabled(Verb requestVerb, Verb responseVerb, AbstractPaxosVerbHandler handler) throws Exception
{
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
ListenableFuture<MessageDelivery> messageSink = registerOutgoingMessageSink();
int messageId = randomInt();
int key = 200;
Commit commit = commit(key);
handler.doVerb(Message.builder(requestVerb, commit).from(node1).withId(messageId).build());
getAndVerifyResponse(messageSink, responseVerb, messageId, true, false);
}
private void getAndVerifyResponse(ListenableFuture<MessageDelivery> messageSink,
Verb verb,
int messageId,
boolean isOutOfRange,
boolean expectFailure) throws InterruptedException, ExecutionException, TimeoutException
{
MessageDelivery response = messageSink.get(100, TimeUnit.MILLISECONDS);
assertEquals(verb, response.message.verb());
Assert.assertEquals(broadcastAddress, response.message.from());
assertEquals(expectFailure, response.message.payload instanceof RequestFailureReason);
assertEquals(messageId, response.message.id());
Assert.assertEquals(node1, response.to);
assertEquals(startingTotalMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount());
assertEquals(startingKeyspaceMetricCount + (isOutOfRange ? 1 : 0), keyspaceMetricValue());
}
private static Commit commit(int key)
{
TableMetadata tmd = Schema.instance.getTableMetadata(KEYSPACE, TABLE);
UUID ballot = UUIDGen.getRandomTimeUUIDFromMicros(FBUtilities.timestampMicros());
return Commit.newPrepare(key(tmd, key), tmd, ballot);
}
private static long keyspaceMetricValue()
{
return Keyspace.open(KEYSPACE).metric.outOfRangeTokenPaxosRequests.getCount();
}
}

623
test/unit/org/apache/cassandra/streaming/StreamReaderTest.java Normal file
View File

@ -0,0 +1,623 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.streaming;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import com.google.common.collect.Lists;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import net.jpountz.lz4.LZ4Factory;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.BufferDecoratedKey;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.DeletionTime;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.db.SerializationHeader;
import org.apache.cassandra.db.rows.Row;
import org.apache.cassandra.db.rows.Rows;
import org.apache.cassandra.db.streaming.CassandraStreamHeader;
import org.apache.cassandra.db.streaming.CassandraStreamReader;
import org.apache.cassandra.db.streaming.IStreamReader;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.io.sstable.SSTableMultiWriter;
import org.apache.cassandra.io.sstable.SSTableSimpleIterator;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.BigFormat;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.io.util.TrackedDataInputPlus;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.net.AsyncStreamingOutputPlus;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.async.StreamCompressionSerializer;
import org.apache.cassandra.streaming.messages.StreamMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.utils.TokenRangeTestUtil.broadcastAddress;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.randomInt;
import static org.apache.cassandra.utils.TokenRangeTestUtil.setLocalTokens;
import static org.apache.cassandra.utils.TokenRangeTestUtil.setPendingRanges;
import static org.apache.cassandra.utils.TokenRangeTestUtil.token;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class StreamReaderTest
{
private static final String TEST_NAME = "streamreader_test_";
private static final String KEYSPACE = TEST_NAME + "cql_keyspace";
private static final String TABLE = "table1";
@BeforeClass
public static void setupClass() throws Exception
{
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition(TEST_NAME);
StorageService.instance.initServer(0);
}
@Before
public void setup() throws Exception
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
// All tests suppose a 2 node ring, with the other peer having the tokens 0, 200, 400
// Initially, the local node has no tokens so when indivividual test set owned tokens or
// pending ranges for the local node, they're always in relation to this.
// e.g. test calls setLocalTokens(100, 300) the ring now looks like
// peer -> (min, 0], (100, 200], (300, 400]
// local -> (0, 100], (200, 300], (400, max]
//
// Pending ranges are set in test using start/end pairs.
// Ring is initialised:
// peer -> (min, max]
// local -> (,]
// e.g. test calls setPendingRanges(0, 100, 200, 300)
// the pending ranges for local would be calculated as:
// local -> (0, 100], (200, 300]
StorageService.instance.getTokenMetadata().updateNormalTokens(Lists.newArrayList(token(0),
token(200),
token(400)),
node1);
}
@Test
public void testReceiveWithNoOwnedRanges() throws Exception
{
int[] tokens = {10, 20};
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSingleOwnedRangeReceivingTableWithRangeContained() throws Exception
{
int[] tokens = {10, 20};
setLocalTokens(100);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithSingleOwnedRangeReceivedTableLowestKeyBoundsExclusive() throws Exception
{
// verify that ranges are left exclusive
int[] tokens = {0, 10};
setLocalTokens(100);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSingleOwnedRangeReceivingTableRangeWithExactMatch() throws Exception
{
// Because ranges are left exlusive, for the range (0, 100] the lowest permissable key is 1
int[] tokens = {1, 100};
setLocalTokens(100);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithSingleOwnedRangeReceivingTableRangeLessThanOwned() throws Exception
{
int[] tokens = {-100, 0};
setLocalTokens(100);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSingleOwnedRangeReceivingTableRangeGreaterThanOwned() throws Exception
{
int[] tokens = {101, 200};
setLocalTokens(100);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSingleOwnedRangeReceivingTableRangeOverlappingOwned() throws Exception
{
int[] tokens = {80, 120};
setLocalTokens(100);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSingleOwnedWrappingRangeReceivingTableContainedBeforeMax() throws Exception
{
int[] tokens = {110, 120};
// local node owns (min, 0] & (100, max], peer owns (0, 100]
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress);
StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithSingleOwnedWrappingRangeReceivingTableContainedAfterMin() throws Exception
{
int[] tokens = {-150, -140};
// local node owns (min, 0] & (100, max], peer owns (0, 100]
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress);
StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithSingleOwnedWrappingRangeReceivingTableOverlappingUpward() throws Exception
{
int[] tokens = {-10, 10};
// local node owns (min, 0] & (100, max], peer owns (0, 100]
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress);
StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSingleOwnedWrappingRangeReceivingTableOverlappingDownward() throws Exception
{
int[] tokens = {90, 110};
// local node owns (min, 0] & (100, max], peer owns (0, 100]
StorageService.instance.getTokenMetadata().clearUnsafe();
StorageService.instance.getTokenMetadata().updateNormalToken(token(0), broadcastAddress);
StorageService.instance.getTokenMetadata().updateNormalToken(token(100), node1);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithMultipleOwnedRangesReceivingTableRangeContainedInFirstOwned() throws Exception
{
int[] tokens = {10, 20};
setLocalTokens(100, 300, 500);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithMultipleOwnedRangesReceivingTableRangeContainedInLastOwned() throws Exception
{
int[] tokens = {450, 460};
setLocalTokens(100, 300, 500);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithMultipleOwnedRangesReceivingTableRangeLessThanOwned() throws Exception
{
int[] tokens = {510, 520};
setLocalTokens(100, 300, 500);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithMultipleOwnedRangesReceivingTableRangeGreaterThanOwned() throws Exception
{
int[] tokens = {-20, -10};
setLocalTokens(100, 300, 500);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithMultipleOwnedRangesReceivingTableRangeOverlappingOwned() throws Exception
{
int[] tokens = {80, 120};
setLocalTokens(100, 300, 500);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithMultipleOwnedRangesAllDisjointFromReceivingTableRange() throws Exception
{
int[] tokens = {310, 320};
setLocalTokens(100, 300, 500);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithMultipleOwnedRangesDisjointSpannedByReceivingTableRange() throws Exception
{
int[] tokens = {80, 320};
setLocalTokens(100, 300, 500);
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithMultipleOwnedRangesReceivedTableRangeExactMatch() throws Exception
{
// bacause ranges are left exclusive, for the range (200, 300] the lowest permissable key is 201
int[] tokens = {201, 300};
setLocalTokens(100, 300, 500);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithOwnedRangeWrappingAndReceivedFileWhollyContained() throws Exception
{
// peer -> (-100, 0], (100, 200], (300, 400]
// local -> (min, -100], (400, max]
setLocalTokens(-100);
int[] tokens = {-200, 500};
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithOwnedRangeWrappingAndReceivedFilePartiallyContained() throws Exception
{
// peer -> (-100, 0], (100, 200], (300, 400]
// local -> (min, -100], (400, max]
setLocalTokens(-100);
int[] tokens = {-200, 300};
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithOwnedRangeWrappingAndReceivedFileNotContained() throws Exception
{
// peer -> (-100, 0], (100, 200], (300, 400]
// local -> (min, -100], (400, max]
setLocalTokens(-100);
int[] tokens = {0, 300};
tryReceiveExpectingFailure(tokens);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, true);
}
@Test
public void testReceiveWithSinglePendingRangeReceivingTableWithRangeContained() throws Exception
{
int[] tokens = {10, 20};
setPendingRanges(KEYSPACE, 0, 100);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveWithMultiplePendingRangesReceivingTableRangeContainedInFirstOwned() throws Exception
{
int[] tokens = {10, 20};
setPendingRanges(KEYSPACE, 0, 100, 200, 300, 400, 500);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
@Test
public void testReceiveNormalizesOwnedAndPendingRanges() throws Exception
{
// Incoming file is not covered by either a single owned or pending range,
// but it is covered by the normalized set of both
int[] tokens = {90, 110};
setLocalTokens(100);
setPendingRanges(KEYSPACE, 100, 120);
tryReceiveExpectingSuccess(tokens, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryReceiveExpectingSuccess(tokens, false);
}
public static StreamSession setupStreamingSessionForTest()
{
StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.REPAIR, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE);
StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.REPAIR, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
streamCoordinator.addSessionInfo(new SessionInfo(peer, 0, peer, Collections.emptyList(), Collections.emptyList(), StreamSession.State.INITIALIZED));
StreamSession session = streamCoordinator.getOrCreateNextSession(peer);
session.init(future);
return session;
}
private static void tryReceiveExpectingSuccess(int[] tokens,
boolean isOutOfRange) throws IOException
{
StreamSession session = setupStreamingSessionForTest();
StreamMessageHeader header = streamHeader();
CassandraStreamHeader streamHeader = streamMessageHeader(tokens);
long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
IStreamReader reader = streamReader(header, streamHeader, session);
StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, 1, 0);
session.prepareReceiving(streamSummary);
reader.read(incomingStream(tokens));
assertEquals(isOutOfRange, StorageMetrics.totalOpsForInvalidToken.getCount() > startMetricCount);
}
private static void tryReceiveExpectingFailure(int[] tokens) throws IOException
{
StreamSession session = setupStreamingSessionForTest();
StreamMessageHeader header = streamHeader();
CassandraStreamHeader streamHeader = streamMessageHeader(tokens);
long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
StreamSummary streamSummary = new StreamSummary(streamHeader.tableId, 1, 0);
session.prepareReceiving(streamSummary);
try
{
IStreamReader reader = streamReader(header, streamHeader, session);
try (SSTableMultiWriter ignore = reader.read(incomingStream(tokens)))
{
fail("Expected StreamReceivedOfTokenRangeException");
}
}
catch (StreamReceivedOutOfTokenRangeException e)
{
// expected
}
assertTrue(StorageMetrics.totalOpsForInvalidToken.getCount() > startMetricCount);
}
private static class BufferSupplier implements AsyncStreamingOutputPlus.BufferSupplier
{
ByteBuffer supplied;
@Override
public ByteBuffer get(int capacity)
{
supplied = ByteBuffer.allocateDirect(capacity);
return supplied;
}
public ByteBuffer getSupplied()
{
return supplied;
}
}
private static DataInputPlus incomingStream(int...tokens) throws IOException
{
final net.jpountz.lz4.LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
DataOutputBuffer out = new DataOutputBuffer();
for (int token : tokens)
ByteBufferUtil.writeWithShortLength(ByteBufferUtil.bytes((long)token), out);
int current_version = MessagingService.current_version;
BufferSupplier bufferSupplier = new BufferSupplier();
StreamCompressionSerializer.serialize(compressor, out.buffer(), current_version).write(bufferSupplier);
return new DataInputBuffer(bufferSupplier.getSupplied(), false);
}
private static IStreamReader streamReader(StreamMessageHeader header, CassandraStreamHeader streamHeader, StreamSession session)
{
return new KeysOnlyStreamReader(header, streamHeader, session);
}
private static StreamMessageHeader streamHeader()
{
TableMetadata tmd = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata();
int fakeSession = randomInt(9);
int fakeSeq = randomInt(9);
return new StreamMessageHeader(tmd.id,
null,
null,
true,
fakeSession,
fakeSeq,
System.currentTimeMillis(),
null);
}
private static CassandraStreamHeader streamMessageHeader(int...tokens)
{
TableMetadata tmd = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).metadata();
Version version = BigFormat.latestVersion;
List<SSTableReader.PartitionPositionBounds> fakeSections = new ArrayList<>();
// each decorated key takes up (2 + 8) bytes, so this enables the
// StreamReader to calculate the expected number of bytes to read
fakeSections.add(new SSTableReader.PartitionPositionBounds(0L, (tokens.length * 10L) - 1));
return CassandraStreamHeader.builder()
.withTableId(tmd.id)
.withSerializationHeader(SerializationHeader.makeWithoutStats(tmd).toComponent())
.withSSTableFormat(SSTableFormat.Type.BIG)
.withSSTableVersion(version)
.withSections(fakeSections)
.build();
}
// Simplifies generating test data as token == key (expects key to be an encoded long)
private static class FakeMurmur3Partitioner extends Murmur3Partitioner
{
public DecoratedKey decorateKey(ByteBuffer key)
{
return new BufferDecoratedKey(new LongToken(ByteBufferUtil.toLong(key)), key);
}
}
// Stream reader which no-ops the reading/writing of the actual partition data.
// As we only care about keys/tokens here, we don't need to generate the rest
// of the sstable data to simulate a stream
private static class KeysOnlyStreamReader extends CassandraStreamReader
{
public KeysOnlyStreamReader(StreamMessageHeader header, CassandraStreamHeader streamHeader, StreamSession session)
{
super(header, streamHeader, session);
}
protected SSTableMultiWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt, UUID pendingRepair, SSTableFormat.Type format) throws IOException
{
return super.createWriter(cfs, totalSize, repairedAt, pendingRepair, format);
}
@Override
protected StreamDeserializer getDeserializer(TableMetadata metadata, TrackedDataInputPlus in, Version inputVersion, StreamSession session, SSTableMultiWriter writer) throws IOException
{
return new TestStreamDeserializer(metadata, in, inputVersion, getHeader(metadata), session, writer);
}
private static class TestStreamDeserializer extends CassandraStreamReader.StreamDeserializer
{
TestStreamDeserializer(TableMetadata metadata, DataInputPlus in, Version version, SerializationHeader header, StreamSession session, SSTableMultiWriter writer) throws IOException
{
super(metadata.unbuild().partitioner(new FakeMurmur3Partitioner()).build(), in, version, header, session, writer);
}
@Override
protected void readPartition()
{
// no-op, our dummy stream contains only decorated keys
partitionLevelDeletion = DeletionTime.LIVE;
iterator = new SSTableSimpleIterator.EmptySSTableSimpleIterator(metadata());
}
@Override
public Row staticRow()
{
return Rows.EMPTY_STATIC_ROW;
}
@Override
public DeletionTime partitionLevelDeletion()
{
return DeletionTime.LIVE;
}
}
}
}

187
test/unit/org/apache/cassandra/streaming/StreamSessionOwnedRangesTest.java Normal file
View File

@ -0,0 +1,187 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.streaming;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import com.google.common.collect.Lists;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.SchemaLoader;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.metrics.StorageMetrics;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.messages.StreamMessage;
import org.apache.cassandra.utils.FBUtilities;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.apache.cassandra.streaming.StreamTestUtils.session;
import static org.apache.cassandra.streaming.messages.StreamMessage.Type.COMPLETE;
import static org.apache.cassandra.streaming.messages.StreamMessage.Type.PREPARE_SYNACK;
import static org.apache.cassandra.utils.TokenRangeTestUtil.generateRangesAtEndpoint;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
import static org.apache.cassandra.utils.TokenRangeTestUtil.setLocalTokens;
import static org.apache.cassandra.utils.TokenRangeTestUtil.token;
public class StreamSessionOwnedRangesTest
{
private static final String TEST_NAME = "strmsn_owned_ranges_test_";
private static final String KEYSPACE = TEST_NAME + "cql_keyspace";
private static final String TABLE = "table1";
@BeforeClass
public static void setupClass() throws Exception
{
DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
SchemaLoader.loadSchema();
SchemaLoader.schemaDefinition(TEST_NAME);
StorageService.instance.initServer(0);
}
@Before
public void setup() throws Exception
{
DatabaseDescriptor.setLogOutOfTokenRangeRequests(true);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(true);
StorageService.instance.getTokenMetadata().clearUnsafe();
// All tests suppose a 2 node ring, with the other peer having the tokens 0, 200, 300
// Initially, the local node has no tokens so when indivividual test set owned tokens or
// pending ranges for the local node, they're always in relation to this.
// e.g. test calls setLocalTokens(100, 300) the ring now looks like
// peer -> (min, 0], (100, 200], (300, 400]
// local -> (0, 100], (200, 300], (400, max]
//
// Pending ranges are set in test using start/end pairs.
// Ring is initialised:
// peer -> (min, max]
// local -> (,]
// e.g. test calls setPendingRanges(0, 100, 200, 300)
// the pending ranges for local would be calculated as:
// local -> (0, 100], (200, 300]
StorageService.instance.getTokenMetadata().updateNormalTokens(Lists.newArrayList(token(0),
token(200),
token(400)),
node1);
}
@Test
public void testPrepareWithAllRequestedRangesWithinOwned()
{
setLocalTokens(100);
InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
Collection<StreamRequest> requests = streamRequests(generateRangesAtEndpoint(endpoint, 0, 10, 70, 80),
RangesAtEndpoint.empty(endpoint));
// prepare request should succeed with or without rejection enabled
tryPrepareExpectingSuccess(requests, false);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryPrepareExpectingSuccess(requests, false);
}
@Test
public void testPrepareWithAllRequestedRangesOutsideOwned()
{
setLocalTokens(100);
InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
Collection<StreamRequest> requests = streamRequests(generateRangesAtEndpoint(endpoint, -20, -10, 110, 120, 310, 320),
RangesAtEndpoint.empty(endpoint));
tryPrepareExpectingFailure(requests);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryPrepareExpectingSuccess(requests, true);
}
@Test
public void testPrepareWithSomeRequestedRangesOutsideOwned()
{
setLocalTokens(100);
InetAddressAndPort endpoint = FBUtilities.getBroadcastAddressAndPort();
Collection<StreamRequest> requests = streamRequests(generateRangesAtEndpoint(endpoint, -20, -10, 30, 40, 310, 320),
RangesAtEndpoint.empty(endpoint));
tryPrepareExpectingFailure(requests);
DatabaseDescriptor.setRejectOutOfTokenRangeRequests(false);
tryPrepareExpectingSuccess(requests, true);
}
private static void tryPrepareExpectingSuccess(Collection<StreamRequest> requests, boolean isOutOfRange)
{
StreamSession session = session();
StreamTestUtils.StubMessageSender handler = (StreamTestUtils.StubMessageSender) session.getMessageSender();
handler.reset();
long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
session.state(StreamSession.State.PREPARING);
session.prepareAsync(requests, Collections.emptySet());
List<StreamMessage> sent = handler.sentMessages;
assertEquals(2, sent.size());
assertEquals(PREPARE_SYNACK, sent.get(0).type);
assertEquals(COMPLETE, sent.get(1).type);
assertEquals(startMetricCount + (isOutOfRange ? 1 : 0), StorageMetrics.totalOpsForInvalidToken.getCount());
}
private static void tryPrepareExpectingFailure(Collection<StreamRequest> requests)
{
StreamSession session = session();
StreamTestUtils.StubMessageSender handler = (StreamTestUtils.StubMessageSender) session.getMessageSender();
handler.reset();
long startMetricCount = StorageMetrics.totalOpsForInvalidToken.getCount();
try
{
session.state(StreamSession.State.PREPARING);
session.prepareAsync(requests, Collections.emptySet());
fail("Expected StreamRequestOfTokenRangeException");
}
catch (StreamRequestOutOfTokenRangeException e)
{
// expected
}
assertTrue(handler.sentMessages.isEmpty());
assertEquals(startMetricCount + 1, StorageMetrics.totalOpsForInvalidToken.getCount());
}
private static Collection<StreamRequest> streamRequests(RangesAtEndpoint fullRanges,
RangesAtEndpoint transientRanges)
{
return Collections.singleton(new StreamRequest(KEYSPACE,
fullRanges,
transientRanges,
Collections.singleton(TABLE)));
}
}

102
test/unit/org/apache/cassandra/streaming/StreamTestUtils.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.streaming;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.apache.cassandra.net.OutboundConnectionSettings;
import org.apache.cassandra.streaming.messages.StreamMessage;
import static org.apache.cassandra.utils.TokenRangeTestUtil.node1;
public class StreamTestUtils
{
static StreamSession session()
{
StubMessageSender messageSender = new StubMessageSender();
StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.REPAIR, 1, new DefaultConnectionFactory(), false, false, null, PreviewKind.NONE);
StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.REPAIR, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
StreamSession session = new StreamSession(StreamOperation.REPAIR,
node1,
connectionFactory,
true,
0,
null,
PreviewKind.NONE)
{
@Override
public void progress(String filename, ProgressInfo.Direction direction, long bytes, long total)
{
//no-op
}
@Override
public StreamingMessageSender getMessageSender()
{
return messageSender;
}
};
session.init(future);
return session;
}
static class StubMessageSender implements StreamingMessageSender
{
final List<StreamMessage> sentMessages = new ArrayList<>();
StubMessageSender()
{
}
public void sendMessage(StreamMessage message)
{
sentMessages.add(message);
}
void reset()
{
sentMessages.clear();
}
public void initialize() throws IOException
{
}
public boolean connected()
{
return false;
}
public void close()
{
}
}
static StreamConnectionFactory connectionFactory = (OutboundConnectionSettings template, int messagingVersion) -> {
throw new UnsupportedOperationException();
};
}

4
test/unit/org/apache/cassandra/streaming/async/NettyStreamingMessageSenderTest.java
View File

@ -68,7 +68,7 @@ public class NettyStreamingMessageSenderTest
session.init(future);
session.attachOutbound(channel);
sender = session.getMessageSender();
sender = (NettyStreamingMessageSender) session.getMessageSender();
sender.setControlMessageChannel(channel);
}
@ -194,7 +194,7 @@ public class NettyStreamingMessageSenderTest
Assert.assertTrue(sender.connected());
ChannelPromise promise = channel.newPromise();
promise.setFailure(new RuntimeException("this is just a testing exception"));
Future f = sender.onControlMessageComplete(promise, new CompleteMessage());
Future<?> f = sender.onControlMessageComplete(promise, new CompleteMessage());
f.get(5, TimeUnit.SECONDS);

166
test/unit/org/apache/cassandra/utils/TokenRangeTestUtil.java Normal file
View File

@ -0,0 +1,166 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.net.UnknownHostException;
import java.util.*;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import org.apache.cassandra.dht.ByteOrderedPartitioner;
import org.apache.cassandra.dht.Murmur3Partitioner;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.RangesAtEndpoint;
import org.apache.cassandra.locator.Replica;
import org.apache.cassandra.net.Message;
import org.apache.cassandra.net.MessagingService;
import org.apache.cassandra.net.Verb;
import org.apache.cassandra.service.StorageService;
public class TokenRangeTestUtil
{
public static final InetAddressAndPort node1;
public static final InetAddressAndPort broadcastAddress;
static
{
try
{
node1 = InetAddressAndPort.getByName("127.0.1.99");
}
catch (UnknownHostException e)
{
throw new RuntimeException("Error initializing InetAddressAndPort");
}
broadcastAddress = FBUtilities.getBroadcastAddressAndPort();
}
public static UUID uuid()
{
return UUIDGen.getTimeUUID();
}
public static void setPendingRanges(String keyspace, int... tokens)
{
Multimap<Range<Token>, Replica> pending = HashMultimap.create();
for (Range<Token> range : generateRanges(tokens))
pending.put(range, new Replica(broadcastAddress, range, true));
StorageService.instance.getTokenMetadata().setPendingRangesUnsafe(keyspace, pending);
}
public static void setLocalTokens(int... tokens)
{
List<Token> tokenList = new ArrayList<>();
for (int token : tokens)
tokenList.add(token(token));
StorageService.instance.getTokenMetadata().updateNormalTokens(tokenList, broadcastAddress);
}
private static final Random random = new Random();
public static int randomInt()
{
return randomInt(Integer.MAX_VALUE);
}
public static int randomInt(int max)
{
return random.nextInt(max);
}
public static RangesAtEndpoint generateRangesAtEndpoint(InetAddressAndPort endpoint, int... rangePairs)
{
if (rangePairs.length % 2 == 1)
throw new RuntimeException("generateRangesAtEndpoint argument count should be even");
RangesAtEndpoint.Builder builder = RangesAtEndpoint.builder(endpoint);
for (int i = 0; i < rangePairs.length; i += 2)
{
builder.add(Replica.fullReplica(endpoint, generateRange(rangePairs[i], rangePairs[i + 1])));
}
return builder.build();
}
public static List<Range<Token>> generateRanges(int... rangePairs)
{
if (rangePairs.length % 2 == 1)
throw new RuntimeException("generateRanges argument count should be even");
List<Range<Token>> ranges = new ArrayList<>();
for (int i = 0; i < rangePairs.length; i += 2)
{
ranges.add(generateRange(rangePairs[i], rangePairs[i + 1]));
}
return ranges;
}
public static Range<Token> generateRange(int left, int right)
{
return new Range<>(token(left), token(right));
}
public static Token token(int token)
{
return new Murmur3Partitioner.LongToken(token);
}
public static Token bytesToken(int token)
{
return new ByteOrderedPartitioner.BytesToken(ByteBufferUtil.bytes(token));
}
public static ListenableFuture<MessageDelivery> registerOutgoingMessageSink(Verb... ignored)
{
SettableFuture<MessageDelivery> future = SettableFuture.create();
Set<Verb> ignore = Sets.newHashSet(ignored);
MessagingService.instance().outboundSink.clear();
MessagingService.instance().outboundSink.add((Message<?> message, InetAddressAndPort to) -> {
if (!ignore.contains(message.verb()))
future.set(new MessageDelivery(message, to));
return true;
});
MessagingService.instance().inboundSink.clear();
MessagingService.instance().inboundSink.add((Message<?> message) -> false);
return future;
}
public static class MessageDelivery
{
public final Message<?> message;
public final InetAddressAndPort to;
MessageDelivery(Message<?> message, InetAddressAndPort to)
{
this.message = message;
this.to = to;
}
}
}