natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge from 1.0 

git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@1222768 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Ellis 2011-12-23 17:57:08 +00:00
parent 6c28169e16 d38aea1433
commit c7d6904813
10 changed files with 149 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -32,6 +32,7 @@
* optimize memtable iteration during range scan (CASSANDRA-3638)
1.0.7
* allow configuring bloom_filter_fp_chance (CASSANDRA-3497)
* attempt hint delivery every ten minutes, or when failure detector
notifies us that a node is back up, whichever comes first. hint
handoff throttle delay default changed to 1ms, from 50 (CASSANDRA-3554)

3
interface/cassandra.thrift
View File

@ -46,7 +46,7 @@ namespace rb CassandraThrift
# for every edit that doesn't result in a change to major/minor.
#
# See the Semantic Versioning Specification (SemVer) http://semver.org.
const string VERSION = "19.23.1"
const string VERSION = "19.24.0"
#
@ -408,6 +408,7 @@ struct CfDef {
29: optional string compaction_strategy,
30: optional map<string,string> compaction_strategy_options,
32: optional map<string,string> compression_options,
33: optional double bloom_filter_fp_chance,
}
/* describes a keyspace. */

2
interface/thrift/gen-java/org/apache/cassandra/thrift/Cassandra.java
View File

@ -17315,6 +17315,8 @@ public class Cassandra {
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
// it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
__isset_bit_vector = new BitSet(1);
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);

94
interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java
View File

@ -71,6 +71,7 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
private static final org.apache.thrift.protocol.TField COMPACTION_STRATEGY_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compaction_strategy_options", org.apache.thrift.protocol.TType.MAP, (short)30);
private static final org.apache.thrift.protocol.TField ROW_CACHE_KEYS_TO_SAVE_FIELD_DESC = new org.apache.thrift.protocol.TField("row_cache_keys_to_save", org.apache.thrift.protocol.TType.I32, (short)31);
private static final org.apache.thrift.protocol.TField COMPRESSION_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("compression_options", org.apache.thrift.protocol.TType.MAP, (short)32);
private static final org.apache.thrift.protocol.TField BLOOM_FILTER_FP_CHANCE_FIELD_DESC = new org.apache.thrift.protocol.TField("bloom_filter_fp_chance", org.apache.thrift.protocol.TType.DOUBLE, (short)33);
public String keyspace;
public String name;
@ -98,6 +99,7 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
public Map<String,String> compaction_strategy_options;
public int row_cache_keys_to_save;
public Map<String,String> compression_options;
public double bloom_filter_fp_chance;
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@ -126,7 +128,8 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
COMPACTION_STRATEGY((short)29, "compaction_strategy"),
COMPACTION_STRATEGY_OPTIONS((short)30, "compaction_strategy_options"),
ROW_CACHE_KEYS_TO_SAVE((short)31, "row_cache_keys_to_save"),
COMPRESSION_OPTIONS((short)32, "compression_options");
COMPRESSION_OPTIONS((short)32, "compression_options"),
BLOOM_FILTER_FP_CHANCE((short)33, "bloom_filter_fp_chance");
private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
@ -193,6 +196,8 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
return ROW_CACHE_KEYS_TO_SAVE;
case 32: // COMPRESSION_OPTIONS
return COMPRESSION_OPTIONS;
case 33: // BLOOM_FILTER_FP_CHANCE
return BLOOM_FILTER_FP_CHANCE;
default:
return null;
}
@ -245,7 +250,8 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
private static final int __REPLICATE_ON_WRITE_ISSET_ID = 9;
private static final int __MERGE_SHARDS_CHANCE_ISSET_ID = 10;
private static final int __ROW_CACHE_KEYS_TO_SAVE_ISSET_ID = 11;
private BitSet __isset_bit_vector = new BitSet(12);
private static final int __BLOOM_FILTER_FP_CHANCE_ISSET_ID = 12;
private BitSet __isset_bit_vector = new BitSet(13);
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
@ -307,6 +313,8 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING),
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
tmpMap.put(_Fields.BLOOM_FILTER_FP_CHANCE, new org.apache.thrift.meta_data.FieldMetaData("bloom_filter_fp_chance", org.apache.thrift.TFieldRequirementType.OPTIONAL,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.DOUBLE)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(CfDef.class, metaDataMap);
}
@ -422,6 +430,7 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
}
this.compression_options = __this__compression_options;
}
this.bloom_filter_fp_chance = other.bloom_filter_fp_chance;
}
public CfDef deepCopy() {
@ -470,6 +479,8 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
setRow_cache_keys_to_saveIsSet(false);
this.row_cache_keys_to_save = 0;
this.compression_options = null;
setBloom_filter_fp_chanceIsSet(false);
this.bloom_filter_fp_chance = 0.0;
}
public String getKeyspace() {
@ -1131,6 +1142,29 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
}
}
public double getBloom_filter_fp_chance() {
return this.bloom_filter_fp_chance;
}
public CfDef setBloom_filter_fp_chance(double bloom_filter_fp_chance) {
this.bloom_filter_fp_chance = bloom_filter_fp_chance;
setBloom_filter_fp_chanceIsSet(true);
return this;
}
public void unsetBloom_filter_fp_chance() {
__isset_bit_vector.clear(__BLOOM_FILTER_FP_CHANCE_ISSET_ID);
}
/** Returns true if field bloom_filter_fp_chance is set (has been assigned a value) and false otherwise */
public boolean isSetBloom_filter_fp_chance() {
return __isset_bit_vector.get(__BLOOM_FILTER_FP_CHANCE_ISSET_ID);
}
public void setBloom_filter_fp_chanceIsSet(boolean value) {
__isset_bit_vector.set(__BLOOM_FILTER_FP_CHANCE_ISSET_ID, value);
}
public void setFieldValue(_Fields field, Object value) {
switch (field) {
case KEYSPACE:
@ -1341,6 +1375,14 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
}
break;
case BLOOM_FILTER_FP_CHANCE:
if (value == null) {
unsetBloom_filter_fp_chance();
} else {
setBloom_filter_fp_chance((Double)value);
}
break;
}
}
@ -1424,6 +1466,9 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
case COMPRESSION_OPTIONS:
return getCompression_options();
case BLOOM_FILTER_FP_CHANCE:
return new Double(getBloom_filter_fp_chance());
}
throw new IllegalStateException();
}
@ -1487,6 +1532,8 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
return isSetRow_cache_keys_to_save();
case COMPRESSION_OPTIONS:
return isSetCompression_options();
case BLOOM_FILTER_FP_CHANCE:
return isSetBloom_filter_fp_chance();
}
throw new IllegalStateException();
}
@ -1738,6 +1785,15 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
return false;
}
boolean this_present_bloom_filter_fp_chance = true && this.isSetBloom_filter_fp_chance();
boolean that_present_bloom_filter_fp_chance = true && that.isSetBloom_filter_fp_chance();
if (this_present_bloom_filter_fp_chance || that_present_bloom_filter_fp_chance) {
if (!(this_present_bloom_filter_fp_chance && that_present_bloom_filter_fp_chance))
return false;
if (this.bloom_filter_fp_chance != that.bloom_filter_fp_chance)
return false;
}
return true;
}
@ -1875,6 +1931,11 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
if (present_compression_options)
builder.append(compression_options);
boolean present_bloom_filter_fp_chance = true && (isSetBloom_filter_fp_chance());
builder.append(present_bloom_filter_fp_chance);
if (present_bloom_filter_fp_chance)
builder.append(bloom_filter_fp_chance);
return builder.toHashCode();
}
@ -2146,6 +2207,16 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
return lastComparison;
}
}
lastComparison = Boolean.valueOf(isSetBloom_filter_fp_chance()).compareTo(typedOther.isSetBloom_filter_fp_chance());
if (lastComparison != 0) {
return lastComparison;
}
if (isSetBloom_filter_fp_chance()) {
lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bloom_filter_fp_chance, typedOther.bloom_filter_fp_chance);
if (lastComparison != 0) {
return lastComparison;
}
}
return 0;
}
@ -2392,6 +2463,14 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
break;
case 33: // BLOOM_FILTER_FP_CHANCE
if (field.type == org.apache.thrift.protocol.TType.DOUBLE) {
this.bloom_filter_fp_chance = iprot.readDouble();
setBloom_filter_fp_chanceIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
}
@ -2584,6 +2663,11 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
oprot.writeFieldEnd();
}
}
if (isSetBloom_filter_fp_chance()) {
oprot.writeFieldBegin(BLOOM_FILTER_FP_CHANCE_FIELD_DESC);
oprot.writeDouble(this.bloom_filter_fp_chance);
oprot.writeFieldEnd();
}
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@ -2800,6 +2884,12 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
}
first = false;
}
if (isSetBloom_filter_fp_chance()) {
if (!first) sb.append(", ");
sb.append("bloom_filter_fp_chance:");
sb.append(this.bloom_filter_fp_chance);
first = false;
}
sb.append(")");
return sb.toString();
}

2
interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java
View File

@ -44,6 +44,6 @@ import org.slf4j.LoggerFactory;
public class Constants {
public static final String VERSION = "19.22.1";
public static final String VERSION = "19.24.0";
}

1
src/avro/internode.genavro
View File

@ -65,6 +65,7 @@ protocol InterNode {
union { null, string } compaction_strategy = null;
union { null, map<string> } compaction_strategy_options = null;
union { null, map<string> } compression_options = null;
union { double, null } bloom_filter_fp_chance;
}
@aliases(["org.apache.cassandra.config.avro.KsDef"])

4
src/java/org/apache/cassandra/cli/CliClient.java
View File

@ -139,6 +139,7 @@ public class CliClient
COMPACTION_STRATEGY,
COMPACTION_STRATEGY_OPTIONS,
COMPRESSION_OPTIONS,
BLOOM_FILTER_FP_CHANCE
}
private static final String DEFAULT_PLACEMENT_STRATEGY = "org.apache.cassandra.locator.NetworkTopologyStrategy";
@ -1244,6 +1245,9 @@ public class CliClient
case COMPRESSION_OPTIONS:
cfDef.setCompression_options(getStrategyOptionsFromTree(statement.getChild(i+1)));
break;
case BLOOM_FILTER_FP_CHANCE:
cfDef.setBloom_filter_fp_chance(Double.parseDouble(mValue));
break;
default:
//must match one of the above or we'd throw an exception at the valueOf statement above.
assert(false);

26
src/java/org/apache/cassandra/config/CFMetaData.java
View File

@ -107,6 +107,7 @@ public final class CFMetaData
// thrift compatibility
private double mergeShardsChance; // default 0.1, chance [0.0, 1.0] of merging old shards during replication
private ByteBuffer keyAlias; // default NULL
private Double bloomFilterFpChance; // default NULL
private Map<ByteBuffer, ColumnDefinition> column_metadata;
public Class<? extends AbstractCompactionStrategy> compactionStrategyClass;
@ -128,6 +129,8 @@ public final class CFMetaData
public CFMetaData compactionStrategyClass(Class<? extends AbstractCompactionStrategy> prop) {compactionStrategyClass = prop; return this;}
public CFMetaData compactionStrategyOptions(Map<String, String> prop) {compactionStrategyOptions = prop; return this;}
public CFMetaData compressionParameters(CompressionParameters prop) {compressionParameters = prop; return this;}
public CFMetaData bloomFilterFpChance(Double prop) {
bloomFilterFpChance = prop; return this;}
public CFMetaData(String keyspace, String name, ColumnFamilyType type, AbstractType comp, AbstractType subcc)
{
@ -235,7 +238,8 @@ public final class CFMetaData
.columnMetadata(oldCFMD.column_metadata)
.compactionStrategyClass(oldCFMD.compactionStrategyClass)
.compactionStrategyOptions(oldCFMD.compactionStrategyOptions)
.compressionParameters(oldCFMD.compressionParameters);
.compressionParameters(oldCFMD.compressionParameters)
.bloomFilterFpChance(oldCFMD.bloomFilterFpChance);
}
/**
@ -288,6 +292,7 @@ public final class CFMetaData
cf.compaction_strategy_options.put(new Utf8(e.getKey()), new Utf8(e.getValue()));
}
cf.compression_options = compressionParameters.asAvroOptions();
cf.bloom_filter_fp_chance = bloomFilterFpChance;
return cf;
}
@ -368,7 +373,8 @@ public final class CFMetaData
.defaultValidator(validator)
.keyValidator(keyValidator)
.columnMetadata(column_metadata)
.compressionParameters(cp);
.compressionParameters(cp)
.bloomFilterFpChance(cf.bloom_filter_fp_chance);
}
public String getComment()
@ -435,7 +441,12 @@ public final class CFMetaData
{
return superColumnName == null ? comparator : subcolumnComparator;
}
public Double getBloomFilterFpChance()
{
return bloomFilterFpChance;
}
public boolean equals(Object obj)
{
if (obj == this)
@ -469,6 +480,7 @@ public final class CFMetaData
.append(compactionStrategyClass, rhs.compactionStrategyClass)
.append(compactionStrategyOptions, rhs.compactionStrategyOptions)
.append(compressionParameters, rhs.compressionParameters)
.append(bloomFilterFpChance, rhs.bloomFilterFpChance)
.isEquals();
}
@ -495,6 +507,7 @@ public final class CFMetaData
.append(compactionStrategyClass)
.append(compactionStrategyOptions)
.append(compressionParameters)
.append(bloomFilterFpChance)
.toHashCode();
}
@ -556,6 +569,8 @@ public final class CFMetaData
newCFMD.compactionStrategyClass = createCompactionStrategy(cf_def.compaction_strategy);
if (cf_def.isSetCompaction_strategy_options())
newCFMD.compactionStrategyOptions(new HashMap<String, String>(cf_def.compaction_strategy_options));
if (cf_def.isSetBloom_filter_fp_chance())
newCFMD.bloomFilterFpChance(cf_def.bloom_filter_fp_chance);
CompressionParameters cp = CompressionParameters.create(cf_def.compression_options);
@ -609,6 +624,8 @@ public final class CFMetaData
maxCompactionThreshold = cf_def.max_compaction_threshold;
mergeShardsChance = cf_def.merge_shards_chance;
keyAlias = cf_def.key_alias;
if (cf_def.bloom_filter_fp_chance != null)
bloomFilterFpChance = cf_def.bloom_filter_fp_chance;
// adjust column definitions. figure out who is coming and going.
Set<ByteBuffer> toRemove = new HashSet<ByteBuffer>();
@ -748,6 +765,8 @@ public final class CFMetaData
def.setCompaction_strategy(compactionStrategyClass.getName());
def.setCompaction_strategy_options(new HashMap<String, String>(compactionStrategyOptions));
def.setCompression_options(compressionParameters.asThriftOptions());
if (bloomFilterFpChance != null)
def.setBloom_filter_fp_chance(bloomFilterFpChance);
return def;
}
@ -884,6 +903,7 @@ public final class CFMetaData
.append("compactionStrategyClass", compactionStrategyClass)
.append("compactionStrategyOptions", compactionStrategyOptions)
.append("compressionOptions", compressionParameters.asThriftOptions())
.append("bloomFilterFpChance", bloomFilterFpChance)
.toString();
}
}

28
src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
View File

@ -83,7 +83,7 @@ public class SSTableWriter extends SSTable
components(metadata),
metadata,
partitioner);
iwriter = new IndexWriter(descriptor, partitioner, keyCount);
iwriter = new IndexWriter(keyCount);
if (compression)
{
@ -385,24 +385,30 @@ public class SSTableWriter extends SSTable
/**
* Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
*/
static class IndexWriter implements Closeable
class IndexWriter implements Closeable
{
private final SequentialWriter indexFile;
public final Descriptor desc;
public final IPartitioner<?> partitioner;
public final SegmentedFile.Builder builder;
public final IndexSummary summary;
public final BloomFilter bf;
private FileMark mark;
IndexWriter(Descriptor desc, IPartitioner<?> part, long keyCount) throws IOException
IndexWriter(long keyCount) throws IOException
{
this.desc = desc;
this.partitioner = part;
indexFile = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_INDEX)), true);
indexFile = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_INDEX)), true);
builder = SegmentedFile.getBuilder(DatabaseDescriptor.getIndexAccessMode());
summary = new IndexSummary(keyCount);
bf = BloomFilter.getFilter(keyCount, 15);
Double fpChance = metadata.getBloomFilterFpChance();
if (fpChance != null && fpChance == 0)
{
// paranoia -- we've had bugs in the thrift <-> avro <-> CfDef dance before, let's not let that break things
logger.error("Bloom filter FP chance of zero isn't supposed to happen");
fpChance = null;
}
bf = fpChance == null
? BloomFilter.getFilter(keyCount, 15)
: BloomFilter.getFilter(keyCount, fpChance);
}
public void afterAppend(DecoratedKey<?> key, long dataPosition) throws IOException
@ -424,7 +430,7 @@ public class SSTableWriter extends SSTable
public void close() throws IOException
{
// bloom filter
FileOutputStream fos = new FileOutputStream(desc.filenameFor(SSTable.COMPONENT_FILTER));
FileOutputStream fos = new FileOutputStream(descriptor.filenameFor(SSTable.COMPONENT_FILTER));
DataOutputStream stream = new DataOutputStream(fos);
BloomFilter.serializer().serialize(bf, stream);
stream.flush();
@ -456,7 +462,7 @@ public class SSTableWriter extends SSTable
@Override
public String toString()
{
return "IndexWriter(" + desc + ")";
return "IndexWriter(" + descriptor + ")";
}
}
}

6
src/resources/org/apache/cassandra/cli/CliHelp.yaml
View File

@ -385,6 +385,9 @@ commands:
- index_options: Optional additional options for index_type.
Options have the form {key:value}.
- bloom_filter_fp_chance: Desired false positive probability for
sstable row bloom filters. Default is 0.000744.
- column_type: Type of columns this column family holds, valid values are
Standard and Super. Default is Standard.
@ -587,6 +590,9 @@ commands:
- index_options: Optional additional options for index_type.
Options have the form {key:value}.
- bloom_filter_fp_chance: Desired false positive probability for
sstable row bloom filters. Default is 0.000744.
- column_type: Type of columns this column family holds, valid values are
Standard and Super. Default is Standard.