diff --git a/CHANGES.txt b/CHANGES.txt index a7e9313f82..f8f548480d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -52,6 +52,7 @@ dev * force anti-entropy service to stream files on the stream stage to avoid sending streams out of order (CASSANDRA-1169) * remove inactive stream managers after AES streams files (CASSANDRA-1169) + * add JMX metrics for row-level bloom filter false positives (CASSANDRA-1212) * added a redhat init script to contrib (CASSANDRA-1201) diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 301dc94061..ddb4fa2f12 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -32,9 +32,11 @@ import java.util.regex.Pattern; import javax.management.MBeanServer; import javax.management.ObjectName; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; import org.apache.commons.collections.IteratorUtils; -import com.google.common.collect.Iterables; import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor; import org.apache.cassandra.concurrent.NamedThreadFactory; import org.apache.cassandra.concurrent.StageManager; @@ -1168,6 +1170,53 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean }; return commitLogUpdater_.submit(runnable); - } + } + public long getBloomFilterFalsePositives() + { + long count = 0L; + for (SSTableReader sstable: getSSTables()) + { + count += sstable.getBloomFilterFalsePositiveCount(); + } + return count; + } + + public long getRecentBloomFilterFalsePositives() + { + long count = 0L; + for (SSTableReader sstable: getSSTables()) + { + count += sstable.getRecentBloomFilterFalsePositiveCount(); + } + return count; + } + + public double getBloomFilterFalseRatio() + { + Long falseCount = 0L; + Long trueCount = 0L; + for (SSTableReader sstable: getSSTables()) + { + falseCount += sstable.getBloomFilterFalsePositiveCount(); + trueCount += sstable.getBloomFilterTruePositiveCount(); + } + if (falseCount.equals(0L) && trueCount.equals(0L)) + return 0d; + return falseCount.doubleValue() / (trueCount.doubleValue() + falseCount.doubleValue()); + } + + public double getRecentBloomFilterFalseRatio() + { + Long falseCount = 0L; + Long trueCount = 0L; + for (SSTableReader sstable: getSSTables()) + { + falseCount += sstable.getRecentBloomFilterFalsePositiveCount(); + trueCount += sstable.getRecentBloomFilterTruePositiveCount(); + } + if (falseCount.equals(0L) && trueCount.equals(0L)) + return 0d; + return falseCount.doubleValue() / (trueCount.doubleValue() + falseCount.doubleValue()); + } } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java index 0e4a25f086..600cb4efe3 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java @@ -157,4 +157,12 @@ public interface ColumnFamilyStoreMBean * @return */ public long getMeanRowCompactedSize(); + + public long getBloomFilterFalsePositives(); + + public long getRecentBloomFilterFalsePositives(); + + public double getBloomFilterFalseRatio(); + + public double getRecentBloomFilterFalseRatio(); } diff --git a/src/java/org/apache/cassandra/io/BloomFilterTracker.java b/src/java/org/apache/cassandra/io/BloomFilterTracker.java new file mode 100644 index 0000000000..d96126712f --- /dev/null +++ b/src/java/org/apache/cassandra/io/BloomFilterTracker.java @@ -0,0 +1,57 @@ +package org.apache.cassandra.io; + +import java.util.concurrent.atomic.AtomicLong; + +public class BloomFilterTracker +{ + private AtomicLong falsePositiveCount = new AtomicLong(0); + private AtomicLong truePositiveCount = new AtomicLong(0); + private long lastFalsePositiveCount = 0L; + private long lastTruePositiveCount = 0L; + + public void addFalsePositive() + { + falsePositiveCount.incrementAndGet(); + } + + public void addTruePositive() + { + truePositiveCount.incrementAndGet(); + } + + public long getFalsePositiveCount() + { + return falsePositiveCount.get(); + } + + public long getRecentFalsePositiveCount() + { + long fpc = getFalsePositiveCount(); + try + { + return (fpc - lastFalsePositiveCount); + } + finally + { + lastFalsePositiveCount = fpc; + } + } + + public long getTruePositiveCount() + { + return truePositiveCount.get(); + } + + public long getRecentTruePositiveCount() + { + long tpc = getTruePositiveCount(); + try + { + return (tpc - lastTruePositiveCount); + } + finally + { + lastTruePositiveCount = tpc; + } + } +} diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java index c96174100e..edd39404ad 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java @@ -44,6 +44,7 @@ import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; +import org.apache.cassandra.io.BloomFilterTracker; import org.apache.cassandra.io.ICompactSerializer2; import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.utils.BloomFilter; @@ -118,6 +119,8 @@ public class SSTableReader extends SSTable implements Comparable private InstrumentedCache, Long> keyCache; + private BloomFilterTracker bloomFilterTracker = new BloomFilterTracker(); + private volatile SSTableDeletingReference phantomReference; public static int indexInterval() @@ -397,8 +400,12 @@ public class SSTableReader extends SSTable implements Comparable // next, see if the sampled index says it's impossible for the key to be present IndexSummary.KeyPosition sampledPosition = getIndexScanPosition(decoratedKey); if (sampledPosition == null) + { + if (op == Operator.EQ) + bloomFilterTracker.addFalsePositive(); // we matched the -1th position: if the operator might match forward, return the 0th position return op.apply(1) >= 0 ? 0 : -1; + } // scan the on-disk index, starting at the nearest sampled position Iterator segments = ifile.iterator(sampledPosition.indexPosition, INDEX_FILE_BUFFER_BYTES); @@ -418,12 +425,20 @@ public class SSTableReader extends SSTable implements Comparable if (v == 0) { if (comparison == 0 && keyCache != null && keyCache.getCapacity() > 0) + { + if (op == Operator.EQ) + bloomFilterTracker.addTruePositive(); // store exact match for the key keyCache.put(unifiedKey, Long.valueOf(dataPosition)); + } return dataPosition; } if (v < 0) + { + if (op == Operator.EQ) + bloomFilterTracker.addFalsePositive(); return -1; + } } } catch (IOException e) @@ -442,6 +457,9 @@ public class SSTableReader extends SSTable implements Comparable } } } + + if (op == Operator.EQ) + bloomFilterTracker.addFalsePositive(); return -1; } @@ -571,4 +589,24 @@ public class SSTableReader extends SSTable implements Comparable public int apply(int comparison) { return comparison > 0 ? 0 : 1; } } } + + public long getBloomFilterFalsePositiveCount() + { + return bloomFilterTracker.getFalsePositiveCount(); + } + + public long getRecentBloomFilterFalsePositiveCount() + { + return bloomFilterTracker.getRecentFalsePositiveCount(); + } + + public long getBloomFilterTruePositiveCount() + { + return bloomFilterTracker.getTruePositiveCount(); + } + + public long getRecentBloomFilterTruePositiveCount() + { + return bloomFilterTracker.getRecentTruePositiveCount(); + } } diff --git a/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java b/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java new file mode 100644 index 0000000000..6ce8555edc --- /dev/null +++ b/test/unit/org/apache/cassandra/io/BloomFilterTrackerTest.java @@ -0,0 +1,50 @@ +package org.apache.cassandra.io; + +import org.junit.Test; + +import org.apache.cassandra.CleanupHelper; + +import static org.junit.Assert.assertEquals; + +public class BloomFilterTrackerTest extends CleanupHelper +{ + @Test + public void testAddingFalsePositives() + { + BloomFilterTracker bft = new BloomFilterTracker(); + assertEquals(0L, bft.getFalsePositiveCount()); + assertEquals(0L, bft.getRecentFalsePositiveCount()); + bft.addFalsePositive(); + bft.addFalsePositive(); + assertEquals(2L, bft.getFalsePositiveCount()); + assertEquals(2L, bft.getRecentFalsePositiveCount()); + assertEquals(0L, bft.getRecentFalsePositiveCount()); + assertEquals(2L, bft.getFalsePositiveCount()); // sanity check + } + + @Test + public void testAddingTruePositives() + { + BloomFilterTracker bft = new BloomFilterTracker(); + assertEquals(0L, bft.getTruePositiveCount()); + assertEquals(0L, bft.getRecentTruePositiveCount()); + bft.addTruePositive(); + bft.addTruePositive(); + assertEquals(2L, bft.getTruePositiveCount()); + assertEquals(2L, bft.getRecentTruePositiveCount()); + assertEquals(0L, bft.getRecentTruePositiveCount()); + assertEquals(2L, bft.getTruePositiveCount()); // sanity check + } + + @Test + public void testAddingToOneLeavesTheOtherAlone() + { + BloomFilterTracker bft = new BloomFilterTracker(); + bft.addFalsePositive(); + assertEquals(0L, bft.getTruePositiveCount()); + assertEquals(0L, bft.getRecentTruePositiveCount()); + bft.addTruePositive(); + assertEquals(1L, bft.getFalsePositiveCount()); + assertEquals(1L, bft.getRecentFalsePositiveCount()); + } +}