natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-3.0' into trunk

This commit is contained in:
Aleksey Yeschenko 2016-04-11 19:18:21 +01:00
parent c2acf47168 5dbeef3f51
commit cb1a63474d
5 changed files with 51 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -170,6 +170,7 @@ Merged from 2.2:
* (cqlsh) Support timezone conversion using pytz (CASSANDRA-10397)
* cqlsh: change default encoding to UTF-8 (CASSANDRA-11124)
Merged from 2.1:
* Checking if an unlogged batch is local is inefficient (CASSANDRA-11529)
* Fix out-of-space error treatment in memtable flushing (CASSANDRA-11448).
* Don't do defragmentation if reading from repaired sstables (CASSANDRA-10342)
* Fix streaming_socket_timeout_in_ms not enforced (CASSANDRA-11286)

62
conf/cassandra.yaml
View File

@ -649,18 +649,6 @@ snapshot_before_compaction: false
# lose data on truncation or drop.
auto_snapshot: true
# When executing a scan, within or across a partition, we need to keep the
# tombstones seen in memory so we can return them to the coordinator, which
# will use them to make sure other replicas also know about the deleted rows.
# With workloads that generate a lot of tombstones, this can cause performance
# problems and even exaust the server heap.
# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
# Adjust the thresholds here if you understand the dangers and want to
# scan more tombstones anyway. These thresholds may also be adjusted at runtime
# using the StorageService mbean.
tombstone_warn_threshold: 1000
tombstone_failure_threshold: 100000
# Granularity of the collation index of rows within a partition.
# Increase if your rows are large, or if you have a very large
# number of rows per partition. The competing goals are these:
@ -672,14 +660,6 @@ tombstone_failure_threshold: 100000
# you can cache more hot rows
column_index_size_in_kb: 64
# Log WARN on any batch size exceeding this value. 5kb per batch by default.
# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
batch_size_warn_threshold_in_kb: 5
# Fail any batch exceeding this value. 50kb (10x warn threshold) by default.
batch_size_fail_threshold_in_kb: 50
# Number of simultaneous compactions to allow, NOT including
# validation "compactions" for anti-entropy repair. Simultaneous
# compactions can help preserve read performance in a mixed read/write
@ -704,9 +684,6 @@ batch_size_fail_threshold_in_kb: 50
# of compaction, including validation compaction.
compaction_throughput_mb_per_sec: 16
# Log a warning when compacting partitions larger than this value
compaction_large_partition_warning_threshold_mb: 100
# When compacting, the replacement sstable(s) can be opened before they
# are completely written, and used in place of the prior sstables for
# any range that has been written. This helps to smoothly transfer reads
@ -942,11 +919,6 @@ inter_dc_tcp_nodelay: false
tracetype_query_ttl: 86400
tracetype_repair_ttl: 604800
# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
# Adjust the threshold based on your application throughput requirement
# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
gc_warn_threshold_in_ms: 1000
# UDFs (user defined functions) are disabled by default.
# As of Cassandra 3.0 there is a sandbox in place that should prevent execution of evil code.
enable_user_defined_functions: false
@ -992,3 +964,37 @@ transparent_data_encryption_options:
store_type: JCEKS
key_password: cassandra
#####################
# SAFETY THRESHOLDS #
#####################
# When executing a scan, within or across a partition, we need to keep the
# tombstones seen in memory so we can return them to the coordinator, which
# will use them to make sure other replicas also know about the deleted rows.
# With workloads that generate a lot of tombstones, this can cause performance
# problems and even exaust the server heap.
# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets)
# Adjust the thresholds here if you understand the dangers and want to
# scan more tombstones anyway. These thresholds may also be adjusted at runtime
# using the StorageService mbean.
tombstone_warn_threshold: 1000
tombstone_failure_threshold: 100000
# Log WARN on any batch size exceeding this value. 5kb per batch by default.
# Caution should be taken on increasing the size of this threshold as it can lead to node instability.
batch_size_warn_threshold_in_kb: 5
# Fail any batch exceeding this value. 50kb (10x warn threshold) by default.
batch_size_fail_threshold_in_kb: 50
# Log WARN on any batches not of type LOGGED than span across more partitions than this limit
unlogged_batch_across_partitions_warn_threshold: 10
# Log a warning when compacting partitions larger than this value
compaction_large_partition_warning_threshold_mb: 100
# GC Pauses greater than gc_warn_threshold_in_ms will be logged at WARN level
# Adjust the threshold based on your application throughput requirement
# By default, Cassandra logs GC Pauses greater than 200 ms at INFO level
gc_warn_threshold_in_ms: 1000

1
src/java/org/apache/cassandra/config/Config.java
View File

@ -164,6 +164,7 @@ public class Config
public Integer column_index_size_in_kb = 64;
public volatile int batch_size_warn_threshold_in_kb = 5;
public volatile int batch_size_fail_threshold_in_kb = 50;
public Integer unlogged_batch_across_partitions_warn_threshold = 10;
public Integer concurrent_compactors;
public volatile Integer compaction_throughput_mb_per_sec = 16;
public volatile Integer compaction_large_partition_warning_threshold_mb = 100;

5
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -977,6 +977,11 @@ public class DatabaseDescriptor
return conf.batch_size_fail_threshold_in_kb;
}
public static int getUnloggedBatchAcrossPartitionsWarnThreshold()
{
return conf.unlogged_batch_across_partitions_warn_threshold;
}
public static void setBatchSizeWarnThresholdInKB(int threshold)
{
conf.batch_size_warn_threshold_in_kb = threshold;

39
src/java/org/apache/cassandra/cql3/statements/BatchStatement.java
View File

@ -21,7 +21,6 @@ import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -34,8 +33,6 @@ import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.*;
import org.apache.cassandra.tracing.Tracing;
@ -71,7 +68,7 @@ public class BatchStatement implements CQLStatement
private final boolean hasConditions;
private static final Logger logger = LoggerFactory.getLogger(BatchStatement.class);
private static final String UNLOGGED_BATCH_WARNING = "Unlogged batch covering {} partition{} detected " +
private static final String UNLOGGED_BATCH_WARNING = "Unlogged batch covering {} partitions detected " +
"against table{} {}. You should use a logged batch for " +
"atomicity, or asynchronous writes for performance.";
@ -313,45 +310,29 @@ public class BatchStatement implements CQLStatement
Set<DecoratedKey> keySet = new HashSet<>();
Set<String> tableNames = new HashSet<>();
Map<String, Collection<Range<Token>>> localTokensByKs = new HashMap<>();
boolean localPartitionsOnly = true;
for (IMutation mutation : mutations)
{
for (PartitionUpdate update : mutation.getPartitionUpdates())
{
keySet.add(update.partitionKey());
tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName));
}
if (localPartitionsOnly)
localPartitionsOnly &= isPartitionLocal(localTokensByKs, mutation);
}
// CASSANDRA-9303: If we only have local mutations we do not warn
if (localPartitionsOnly)
return;
// CASSANDRA-11529: log only if we have more than a threshold of keys, this was also suggested in the
// original ticket that introduced this warning, CASSANDRA-9282
if (keySet.size() > DatabaseDescriptor.getUnloggedBatchAcrossPartitionsWarnThreshold())
{
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, UNLOGGED_BATCH_WARNING,
keySet.size(), tableNames.size() == 1 ? "" : "s", tableNames);
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, UNLOGGED_BATCH_WARNING,
keySet.size(), keySet.size() == 1 ? "" : "s",
tableNames.size() == 1 ? "" : "s", tableNames);
ClientWarn.instance.warn(MessageFormatter.arrayFormat(UNLOGGED_BATCH_WARNING, new Object[]{keySet.size(), keySet.size() == 1 ? "" : "s",
ClientWarn.instance.warn(MessageFormatter.arrayFormat(UNLOGGED_BATCH_WARNING, new Object[]{keySet.size(),
tableNames.size() == 1 ? "" : "s", tableNames}).getMessage());
}
}
}
private boolean isPartitionLocal(Map<String, Collection<Range<Token>>> localTokensByKs, IMutation mutation)
{
String ksName = mutation.getKeyspaceName();
Collection<Range<Token>> localRanges = localTokensByKs.get(ksName);
if (localRanges == null)
{
localRanges = StorageService.instance.getLocalRanges(ksName);
localTokensByKs.put(ksName, localRanges);
}
return Range.isInRanges(mutation.key().getToken(), localRanges);
}
public ResultMessage execute(QueryState queryState, QueryOptions options) throws RequestExecutionException, RequestValidationException
{