diff --git a/CHANGES.txt b/CHANGES.txt index e184ecc8fe..64b9f77b83 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -32,6 +32,7 @@ * optimize memtable iteration during range scan (CASSANDRA-3638) 1.0.7 + * allow configuring bloom_filter_fp_chance (CASSANDRA-3497) * attempt hint delivery every ten minutes, or when failure detector notifies us that a node is back up, whichever comes first. hint handoff throttle delay default changed to 1ms, from 50 (CASSANDRA-3554) diff --git a/interface/cassandra.thrift b/interface/cassandra.thrift index ce804a41d9..8553f55c8d 100644 --- a/interface/cassandra.thrift +++ b/interface/cassandra.thrift @@ -46,7 +46,7 @@ namespace rb CassandraThrift # for every edit that doesn't result in a change to major/minor. # # See the Semantic Versioning Specification (SemVer) http://semver.org. -const string VERSION = "19.23.1" +const string VERSION = "19.24.0" # @@ -408,6 +408,7 @@ struct CfDef { 29: optional string compaction_strategy, 30: optional map compaction_strategy_options, 32: optional map compression_options, + 33: optional double bloom_filter_fp_chance, } /* describes a keyspace. */ diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java index a4a15ba7af..b0896aa650 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java @@ -17315,6 +17315,8 @@ public class Cassandra { private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java index d05318e596..c3a01a2a6a 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java @@ -71,6 +71,7 @@ public class CfDef implements org.apache.thrift.TBase, jav private static final org.apache.thrift.protocol.TField COMPACTION_STRATEGY_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compaction_strategy_options", org.apache.thrift.protocol.TType.MAP, (short)30); private static final org.apache.thrift.protocol.TField ROW_CACHE_KEYS_TO_SAVE_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_keys_to_save", org.apache.thrift.protocol.TType.I32, (short)31); private static final org.apache.thrift.protocol.TField COMPRESSION_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compression_options", org.apache.thrift.protocol.TType.MAP, (short)32); + private static final org.apache.thrift.protocol.TField BLOOM_FILTER_FP_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("bloom_filter_fp_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)33); public String keyspace; public String name; @@ -98,6 +99,7 @@ public class CfDef implements org.apache.thrift.TBase, jav public Map compaction_strategy_options; public int row_cache_keys_to_save; public Map compression_options; + public double bloom_filter_fp_chance; /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -126,7 +128,8 @@ public class CfDef implements org.apache.thrift.TBase, jav COMPACTION_STRATEGY((short)29, "compaction_strategy"), COMPACTION_STRATEGY_OPTIONS((short)30, "compaction_strategy_options"), ROW_CACHE_KEYS_TO_SAVE((short)31, "row_cache_keys_to_save"), - COMPRESSION_OPTIONS((short)32, "compression_options"); + COMPRESSION_OPTIONS((short)32, "compression_options"), + BLOOM_FILTER_FP_CHANCE((short)33, "bloom_filter_fp_chance"); private static final Map byName = new HashMap(); @@ -193,6 +196,8 @@ public class CfDef implements org.apache.thrift.TBase, jav return ROW_CACHE_KEYS_TO_SAVE; case 32: // COMPRESSION_OPTIONS return COMPRESSION_OPTIONS; + case 33: // BLOOM_FILTER_FP_CHANCE + return BLOOM_FILTER_FP_CHANCE; default: return null; } @@ -245,7 +250,8 @@ public class CfDef implements org.apache.thrift.TBase, jav private static final int __REPLICATE_ON_WRITE_ISSET_ID = 9; private static final int __MERGE_SHARDS_CHANCE_ISSET_ID = 10; private static final int __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID = 11; - private BitSet __isset_bit_vector = new BitSet(12); + private static final int __BLOOM_FILTER_FP_CHANCE_ISSET_ID = 12; + private BitSet __isset_bit_vector = new BitSet(13); public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { @@ -307,6 +313,8 @@ public class CfDef implements org.apache.thrift.TBase, jav new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.BLOOM_FILTER_FP_CHANCE, new org.apache.thrift.meta_data.FieldMetaData("bloom_filter_fp_chance", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CfDef.class, metaDataMap); } @@ -422,6 +430,7 @@ public class CfDef implements org.apache.thrift.TBase, jav } this.compression_options = __this__compression_options; } + this.bloom_filter_fp_chance = other.bloom_filter_fp_chance; } public CfDef deepCopy() { @@ -470,6 +479,8 @@ public class CfDef implements org.apache.thrift.TBase, jav setRow_cache_keys_to_saveIsSet(false); this.row_cache_keys_to_save = 0; this.compression_options = null; + setBloom_filter_fp_chanceIsSet(false); + this.bloom_filter_fp_chance = 0.0; } public String getKeyspace() { @@ -1131,6 +1142,29 @@ public class CfDef implements org.apache.thrift.TBase, jav } } + public double getBloom_filter_fp_chance() { + return this.bloom_filter_fp_chance; + } + + public CfDef setBloom_filter_fp_chance(double bloom_filter_fp_chance) { + this.bloom_filter_fp_chance = bloom_filter_fp_chance; + setBloom_filter_fp_chanceIsSet(true); + return this; + } + + public void unsetBloom_filter_fp_chance() { + __isset_bit_vector.clear(__BLOOM_FILTER_FP_CHANCE_ISSET_ID); + } + + /** Returns true if field bloom_filter_fp_chance is set (has been assigned a value) and false otherwise */ + public boolean isSetBloom_filter_fp_chance() { + return __isset_bit_vector.get(__BLOOM_FILTER_FP_CHANCE_ISSET_ID); + } + + public void setBloom_filter_fp_chanceIsSet(boolean value) { + __isset_bit_vector.set(__BLOOM_FILTER_FP_CHANCE_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case KEYSPACE: @@ -1341,6 +1375,14 @@ public class CfDef implements org.apache.thrift.TBase, jav } break; + case BLOOM_FILTER_FP_CHANCE: + if (value == null) { + unsetBloom_filter_fp_chance(); + } else { + setBloom_filter_fp_chance((Double)value); + } + break; + } } @@ -1424,6 +1466,9 @@ public class CfDef implements org.apache.thrift.TBase, jav case COMPRESSION_OPTIONS: return getCompression_options(); + case BLOOM_FILTER_FP_CHANCE: + return new Double(getBloom_filter_fp_chance()); + } throw new IllegalStateException(); } @@ -1487,6 +1532,8 @@ public class CfDef implements org.apache.thrift.TBase, jav return isSetRow_cache_keys_to_save(); case COMPRESSION_OPTIONS: return isSetCompression_options(); + case BLOOM_FILTER_FP_CHANCE: + return isSetBloom_filter_fp_chance(); } throw new IllegalStateException(); } @@ -1738,6 +1785,15 @@ public class CfDef implements org.apache.thrift.TBase, jav return false; } + boolean this_present_bloom_filter_fp_chance = true && this.isSetBloom_filter_fp_chance(); + boolean that_present_bloom_filter_fp_chance = true && that.isSetBloom_filter_fp_chance(); + if (this_present_bloom_filter_fp_chance || that_present_bloom_filter_fp_chance) { + if (!(this_present_bloom_filter_fp_chance && that_present_bloom_filter_fp_chance)) + return false; + if (this.bloom_filter_fp_chance != that.bloom_filter_fp_chance) + return false; + } + return true; } @@ -1875,6 +1931,11 @@ public class CfDef implements org.apache.thrift.TBase, jav if (present_compression_options) builder.append(compression_options); + boolean present_bloom_filter_fp_chance = true && (isSetBloom_filter_fp_chance()); + builder.append(present_bloom_filter_fp_chance); + if (present_bloom_filter_fp_chance) + builder.append(bloom_filter_fp_chance); + return builder.toHashCode(); } @@ -2146,6 +2207,16 @@ public class CfDef implements org.apache.thrift.TBase, jav return lastComparison; } } + lastComparison = Boolean.valueOf(isSetBloom_filter_fp_chance()).compareTo(typedOther.isSetBloom_filter_fp_chance()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBloom_filter_fp_chance()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bloom_filter_fp_chance, typedOther.bloom_filter_fp_chance); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -2392,6 +2463,14 @@ public class CfDef implements org.apache.thrift.TBase, jav org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); } break; + case 33: // BLOOM_FILTER_FP_CHANCE + if (field.type == org.apache.thrift.protocol.TType.DOUBLE) { + this.bloom_filter_fp_chance = iprot.readDouble(); + setBloom_filter_fp_chanceIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); } @@ -2584,6 +2663,11 @@ public class CfDef implements org.apache.thrift.TBase, jav oprot.writeFieldEnd(); } } + if (isSetBloom_filter_fp_chance()) { + oprot.writeFieldBegin(BLOOM_FILTER_FP_CHANCE_FIELD_DESC); + oprot.writeDouble(this.bloom_filter_fp_chance); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -2800,6 +2884,12 @@ public class CfDef implements org.apache.thrift.TBase, jav } first = false; } + if (isSetBloom_filter_fp_chance()) { + if (!first) sb.append(", "); + sb.append("bloom_filter_fp_chance:"); + sb.append(this.bloom_filter_fp_chance); + first = false; + } sb.append(")"); return sb.toString(); } diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java index 96635c1475..22ab877aae 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java @@ -44,6 +44,6 @@ import org.slf4j.LoggerFactory; public class Constants { - public static final String VERSION = "19.22.1"; + public static final String VERSION = "19.24.0"; } diff --git a/src/avro/internode.genavro b/src/avro/internode.genavro index 4eb10996f1..534d25662f 100644 --- a/src/avro/internode.genavro +++ b/src/avro/internode.genavro @@ -65,6 +65,7 @@ protocol InterNode { union { null, string } compaction_strategy = null; union { null, map } compaction_strategy_options = null; union { null, map } compression_options = null; + union { double, null } bloom_filter_fp_chance; } @aliases(["org.apache.cassandra.config.avro.KsDef"]) diff --git a/src/java/org/apache/cassandra/cli/CliClient.java b/src/java/org/apache/cassandra/cli/CliClient.java index 96f628638f..40c9b967c3 100644 --- a/src/java/org/apache/cassandra/cli/CliClient.java +++ b/src/java/org/apache/cassandra/cli/CliClient.java @@ -139,6 +139,7 @@ public class CliClient COMPACTION_STRATEGY, COMPACTION_STRATEGY_OPTIONS, COMPRESSION_OPTIONS, + BLOOM_FILTER_FP_CHANCE } private static final String DEFAULT_PLACEMENT_STRATEGY = "org.apache.cassandra.locator.NetworkTopologyStrategy"; @@ -1244,6 +1245,9 @@ public class CliClient case COMPRESSION_OPTIONS: cfDef.setCompression_options(getStrategyOptionsFromTree(statement.getChild(i+1))); break; + case BLOOM_FILTER_FP_CHANCE: + cfDef.setBloom_filter_fp_chance(Double.parseDouble(mValue)); + break; default: //must match one of the above or we'd throw an exception at the valueOf statement above. assert(false); diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index 7a5096f75c..d0abf09356 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -107,6 +107,7 @@ public final class CFMetaData // thrift compatibility private double mergeShardsChance; // default 0.1, chance [0.0, 1.0] of merging old shards during replication private ByteBuffer keyAlias; // default NULL + private Double bloomFilterFpChance; // default NULL private Map column_metadata; public Class compactionStrategyClass; @@ -128,6 +129,8 @@ public final class CFMetaData public CFMetaData compactionStrategyClass(Class prop) {compactionStrategyClass = prop; return this;} public CFMetaData compactionStrategyOptions(Map prop) {compactionStrategyOptions = prop; return this;} public CFMetaData compressionParameters(CompressionParameters prop) {compressionParameters = prop; return this;} + public CFMetaData bloomFilterFpChance(Double prop) { + bloomFilterFpChance = prop; return this;} public CFMetaData(String keyspace, String name, ColumnFamilyType type, AbstractType comp, AbstractType subcc) { @@ -235,7 +238,8 @@ public final class CFMetaData .columnMetadata(oldCFMD.column_metadata) .compactionStrategyClass(oldCFMD.compactionStrategyClass) .compactionStrategyOptions(oldCFMD.compactionStrategyOptions) - .compressionParameters(oldCFMD.compressionParameters); + .compressionParameters(oldCFMD.compressionParameters) + .bloomFilterFpChance(oldCFMD.bloomFilterFpChance); } /** @@ -288,6 +292,7 @@ public final class CFMetaData cf.compaction_strategy_options.put(new Utf8(e.getKey()), new Utf8(e.getValue())); } cf.compression_options = compressionParameters.asAvroOptions(); + cf.bloom_filter_fp_chance = bloomFilterFpChance; return cf; } @@ -368,7 +373,8 @@ public final class CFMetaData .defaultValidator(validator) .keyValidator(keyValidator) .columnMetadata(column_metadata) - .compressionParameters(cp); + .compressionParameters(cp) + .bloomFilterFpChance(cf.bloom_filter_fp_chance); } public String getComment() @@ -435,7 +441,12 @@ public final class CFMetaData { return superColumnName == null ? comparator : subcolumnComparator; } - + + public Double getBloomFilterFpChance() + { + return bloomFilterFpChance; + } + public boolean equals(Object obj) { if (obj == this) @@ -469,6 +480,7 @@ public final class CFMetaData .append(compactionStrategyClass, rhs.compactionStrategyClass) .append(compactionStrategyOptions, rhs.compactionStrategyOptions) .append(compressionParameters, rhs.compressionParameters) + .append(bloomFilterFpChance, rhs.bloomFilterFpChance) .isEquals(); } @@ -495,6 +507,7 @@ public final class CFMetaData .append(compactionStrategyClass) .append(compactionStrategyOptions) .append(compressionParameters) + .append(bloomFilterFpChance) .toHashCode(); } @@ -556,6 +569,8 @@ public final class CFMetaData newCFMD.compactionStrategyClass = createCompactionStrategy(cf_def.compaction_strategy); if (cf_def.isSetCompaction_strategy_options()) newCFMD.compactionStrategyOptions(new HashMap(cf_def.compaction_strategy_options)); + if (cf_def.isSetBloom_filter_fp_chance()) + newCFMD.bloomFilterFpChance(cf_def.bloom_filter_fp_chance); CompressionParameters cp = CompressionParameters.create(cf_def.compression_options); @@ -609,6 +624,8 @@ public final class CFMetaData maxCompactionThreshold = cf_def.max_compaction_threshold; mergeShardsChance = cf_def.merge_shards_chance; keyAlias = cf_def.key_alias; + if (cf_def.bloom_filter_fp_chance != null) + bloomFilterFpChance = cf_def.bloom_filter_fp_chance; // adjust column definitions. figure out who is coming and going. Set toRemove = new HashSet(); @@ -748,6 +765,8 @@ public final class CFMetaData def.setCompaction_strategy(compactionStrategyClass.getName()); def.setCompaction_strategy_options(new HashMap(compactionStrategyOptions)); def.setCompression_options(compressionParameters.asThriftOptions()); + if (bloomFilterFpChance != null) + def.setBloom_filter_fp_chance(bloomFilterFpChance); return def; } @@ -884,6 +903,7 @@ public final class CFMetaData .append("compactionStrategyClass", compactionStrategyClass) .append("compactionStrategyOptions", compactionStrategyOptions) .append("compressionOptions", compressionParameters.asThriftOptions()) + .append("bloomFilterFpChance", bloomFilterFpChance) .toString(); } } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java index ef2f1d4dd1..53980ed73d 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java @@ -83,7 +83,7 @@ public class SSTableWriter extends SSTable components(metadata), metadata, partitioner); - iwriter = new IndexWriter(descriptor, partitioner, keyCount); + iwriter = new IndexWriter(keyCount); if (compression) { @@ -385,24 +385,30 @@ public class SSTableWriter extends SSTable /** * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed. */ - static class IndexWriter implements Closeable + class IndexWriter implements Closeable { private final SequentialWriter indexFile; - public final Descriptor desc; - public final IPartitioner partitioner; public final SegmentedFile.Builder builder; public final IndexSummary summary; public final BloomFilter bf; private FileMark mark; - IndexWriter(Descriptor desc, IPartitioner part, long keyCount) throws IOException + IndexWriter(long keyCount) throws IOException { - this.desc = desc; - this.partitioner = part; - indexFile = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_INDEX)), true); + indexFile = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_INDEX)), true); builder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode()); summary = new IndexSummary(keyCount); - bf = BloomFilter.getFilter(keyCount, 15); + + Double fpChance = metadata.getBloomFilterFpChance(); + if (fpChance != null && fpChance == 0) + { + // paranoia -- we've had bugs in the thrift <-> avro <-> CfDef dance before, let's not let that break things + logger.error("Bloom filter FP chance of zero isn't supposed to happen"); + fpChance = null; + } + bf = fpChance == null + ? BloomFilter.getFilter(keyCount, 15) + : BloomFilter.getFilter(keyCount, fpChance); } public void afterAppend(DecoratedKey key, long dataPosition) throws IOException @@ -424,7 +430,7 @@ public class SSTableWriter extends SSTable public void close() throws IOException { // bloom filter - FileOutputStream fos = new FileOutputStream(desc.filenameFor(SSTable.COMPONENT_FILTER)); + FileOutputStream fos = new FileOutputStream(descriptor.filenameFor(SSTable.COMPONENT_FILTER)); DataOutputStream stream = new DataOutputStream(fos); BloomFilter.serializer().serialize(bf, stream); stream.flush(); @@ -456,7 +462,7 @@ public class SSTableWriter extends SSTable @Override public String toString() { - return "IndexWriter(" + desc + ")"; + return "IndexWriter(" + descriptor + ")"; } } } diff --git a/src/resources/org/apache/cassandra/cli/CliHelp.yaml b/src/resources/org/apache/cassandra/cli/CliHelp.yaml index 83a428e970..873cc414ce 100644 --- a/src/resources/org/apache/cassandra/cli/CliHelp.yaml +++ b/src/resources/org/apache/cassandra/cli/CliHelp.yaml @@ -385,6 +385,9 @@ commands: - index_options: Optional additional options for index_type. Options have the form {key:value}. + - bloom_filter_fp_chance: Desired false positive probability for + sstable row bloom filters. Default is 0.000744. + - column_type: Type of columns this column family holds, valid values are Standard and Super. Default is Standard. @@ -587,6 +590,9 @@ commands: - index_options: Optional additional options for index_type. Options have the form {key:value}. + - bloom_filter_fp_chance: Desired false positive probability for + sstable row bloom filters. Default is 0.000744. + - column_type: Type of columns this column family holds, valid values are Standard and Super. Default is Standard.