merge from 1.2

CHANGES.txt

@@ -7,6 +7,7 @@ Merged from 1.2:
  * Allow where clause conditions to be in parenthesis (CASSANDRA-6037)
  * Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916)
  * Improve memory usage of metadata min/max column names (CASSANDRA-6077)
+ * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057)
 
 
 2.0.1

conf/cassandra.yaml

@@ -405,6 +405,11 @@ snapshot_before_compaction: false
 # lose data on truncation or drop.
 auto_snapshot: true
 
+# Log a debug message if more than this many tombstones are scanned
+# in a single-partition query.  Set the threshold on SliceQueryFilter
+# to debug to enable.
+tombstone_debug_threshold: 10000
+
 # Add column indexes to a row after its contents reach this size.
 # Increase if your column values are large, or if you have a very large
 # number of columns.  The competing causes are, Cassandra has to

src/java/org/apache/cassandra/config/Config.java

@@ -179,6 +179,8 @@ public class Config
 
     private static boolean outboundBindAny = false;
 
+    public volatile int tombstone_debug_threshold = 10000;
+
     public static boolean getOutboundBindAny()
     {
         return outboundBindAny;

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -879,6 +879,20 @@ public class DatabaseDescriptor
         return conf.commitlog_directory;
     }
 
+    /**
+     * How many tombstones need to be scanned before we log a
+     * debug message
+     */
+    public static int getTombstoneDebugThreshold()
+    {
+        return conf.tombstone_debug_threshold;
+    }
+
+    public static void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+    {
+        conf.tombstone_debug_threshold = tombstoneDebugThreshold;
+    }
+
     /**
      * size of commitlog segments to allocate
      */

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -33,6 +33,7 @@ import com.google.common.base.Function;
 import com.google.common.collect.*;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.cliffc.high_scale_lib.NonBlockingHashMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -49,11 +50,7 @@ import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.db.compaction.*;
-import org.apache.cassandra.db.filter.ColumnSlice;
-import org.apache.cassandra.db.filter.ExtendedFilter;
-import org.apache.cassandra.db.filter.IDiskAtomFilter;
-import org.apache.cassandra.db.filter.QueryFilter;
-import org.apache.cassandra.db.filter.SliceQueryFilter;
+import org.apache.cassandra.db.filter.*;
 import org.apache.cassandra.db.index.SecondaryIndex;
 import org.apache.cassandra.db.index.SecondaryIndexManager;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -72,7 +69,6 @@ import org.apache.cassandra.service.StorageService;
 import org.apache.cassandra.thrift.IndexExpression;
 import org.apache.cassandra.tracing.Tracing;
 import org.apache.cassandra.utils.*;
-import org.cliffc.high_scale_lib.NonBlockingHashMap;
 
 import static org.apache.cassandra.config.CFMetaData.Caching;
 
@@ -1300,6 +1296,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             }
 
             removeDroppedColumns(result);
+
+            if (filter.filter instanceof SliceQueryFilter)
+            {
+                // Log the number of tombstones scanned on single key queries
+                metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored());
+                metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive());
+            }
         }
         finally
         {
@@ -2173,6 +2176,22 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
                     "is deprecated, set the compaction strategy option 'enabled' to 'false' instead or use the nodetool command 'disableautocompaction'.");
     }
 
+    public long getTombstonesPerLastRead()
+    {
+        return metric.tombstoneScannedHistogram.count();
+    }
+
+    public float getPercentageTombstonesPerLastRead()
+    {
+        long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count();
+        return (metric.tombstoneScannedHistogram.count() / total);
+    }
+
+    public long getLiveCellsPerLastRead()
+    {
+        return metric.liveScannedHistogram.count();
+    }
+
     // End JMX get/set.
 
     public long estimateKeys()

src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java

@@ -283,6 +283,15 @@ public interface ColumnFamilyStoreMBean
 
     public boolean isAutoCompactionDisabled();
 
+    /** Number of tombstoned cells retreived during the last slicequery */
+    public long getTombstonesPerLastRead();
+
+    /** Percentage of tombstoned cells retreived during the last slicequery */
+    public float getPercentageTombstonesPerLastRead();
+
+    /** Number of live cells retreived during the last slicequery */
+    public long getLiveCellsPerLastRead();
+
     public long estimateKeys();
 
     /**

src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java

@@ -21,16 +21,12 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 import org.apache.cassandra.db.columniterator.SSTableSliceIterator;
@@ -202,6 +198,8 @@ public class SliceQueryFilter implements IDiskAtomFilter
         }
 
         Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
+        if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold())
+            logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored());
     }
 
     public int getLiveCount(ColumnFamily cf, long now)
@@ -267,6 +265,16 @@ public class SliceQueryFilter implements IDiskAtomFilter
         return columnCounter == null ? 0 : columnCounter.live();
     }
 
+    public int lastIgnored()
+    {
+        return columnCounter == null ? 0 : columnCounter.ignored();
+    }
+
+    public int lastLive()
+    {
+        return columnCounter == null ? 0 : columnCounter.live();
+    }
+
     @Override
     public String toString()
     {

src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java

@@ -79,6 +79,10 @@ public class ColumnFamilyMetrics
     public final Gauge<Long> bloomFilterDiskSpaceUsed;
     /** Key cache hit rate  for this CF */
     public final Gauge<Double> keyCacheHitRate;
+    /** Tombstones scanned in queries on this CF */
+    public final Histogram tombstoneScannedHistogram;
+    /** Live cells scanned in queries on this CF */
+    public final Histogram liveScannedHistogram;
 
     private final MetricNameFactory factory;
 
@@ -298,6 +302,8 @@ public class ColumnFamilyMetrics
                 return Math.max(requests, 1); // to avoid NaN.
             }
         });
+        tombstoneScannedHistogram = Metrics.newHistogram(factory.createMetricName("TombstoneScannedHistogram"));
+        liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"));
     }
 
     public void updateSSTableIterated(int count)
@@ -335,6 +341,8 @@ public class ColumnFamilyMetrics
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate"));
         Metrics.defaultRegistry().removeMetric(factory.createMetricName("SpeculativeRetry"));
+        Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram"));
+        Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram"));
     }
 
     class ColumnFamilyMetricNameFactory implements MetricNameFactory

src/java/org/apache/cassandra/service/StorageService.java

@@ -3653,4 +3653,14 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     {
         return DatabaseDescriptor.getPartitionerName();
     }
+
+    public int getTombstoneDebugThreshold()
+    {
+        return DatabaseDescriptor.getTombstoneDebugThreshold();
+    }
+
+    public void setTombstoneDebugThreshold(int tombstoneDebugThreshold)
+    {
+        DatabaseDescriptor.setTombstoneDebugThreshold(tombstoneDebugThreshold);
+    }
 }

src/java/org/apache/cassandra/service/StorageServiceMBean.java

@@ -476,4 +476,9 @@ public interface StorageServiceMBean extends NotificationEmitter
     public String getClusterName();
     /** Returns the cluster partitioner */
     public String getPartitionerName();
+
+    /** Returns the threshold for returning debugging queries with many tombstones */
+    public int getTombstoneDebugThreshold();
+    /** Sets the threshold for returning debugging queries with many tombstones */
+    public void setTombstoneDebugThreshold(int tombstoneDebugThreshold);
 }