natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge from 1.0 

git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@1170027 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Ellis 2011-09-13 04:00:57 +00:00
parent 9fb8d6cf2b ca468a4fe5
commit a37cb2fff5
8 changed files with 52 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -74,6 +74,8 @@
* Arena-based allocation for memtables (CASSANDRA-2252, 3162, 3163, 3168)
* Default RR chance to 0.1 (CASSANDRA-3169)
* Add RowLevel support to secondary index API (CASSANDRA-3147)
* Make SerializingCacheProvider the default if JNA is available (CASSANDRA-3183)
* Fix backwards compatibilty for CQL memtable properties (CASSANDRA-3190)
0.8.6
* avoid trying to watch cassandra-topology.properties when loaded from jar

4
doc/cql/CQL.textile
View File

@ -292,6 +292,7 @@ A number of optional keyword arguments can be supplied to control the configurat
|_. keyword|_. default|_. description|
|comparator|text|Determines sorting and validation of column names. Valid values are identical to the types listed in "Specifying Column Type":#columntypes above.|
|comment|none|A free-form, human-readable comment.|
|row_cache_provider|SerializingCacheProvider if JNA is present, otherwise ConcurrentHashMapCacheProvider|A factory for the cache with which to back the row cache.|
|row_cache_size|0|Number of rows whose entire contents to cache in memory.|
|key_cache_size|200000|Number of keys per SSTable whose locations are kept in memory in "mostly LRU" order.|
|read_repair_chance|1.0|The probability with which read repairs should be invoked on non-quorum reads.|
@ -301,9 +302,6 @@ A number of optional keyword arguments can be supplied to control the configurat
|max_compaction_threshold|32|Maximum number of SSTables allowed before a minor compaction is forced.|
|row_cache_save_period_in_seconds|0|Number of seconds between saving row caches.|
|key_cache_save_period_in_seconds|14400|Number of seconds between saving key caches.|
|memtable_flush_after_mins|60|Maximum time to leave a dirty table unflushed.|
|memtable_throughput_in_mb|dynamic|Maximum size of the memtable before it is flushed.|
|memtable_operations_in_millions|dynamic|Number of operations in millions before the memtable is flushed.|
|replicate_on_write|false| |
h2. CREATE INDEX

2
interface/cassandra.thrift
View File

@ -399,7 +399,7 @@ struct CfDef {
24: optional bool replicate_on_write,
25: optional double merge_shards_chance,
26: optional string key_validation_class,
27: optional string row_cache_provider="org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider",
27: optional string row_cache_provider,
28: optional binary key_alias,
29: optional string compaction_strategy,
30: optional map<string,string> compaction_strategy_options,

5
interface/thrift/gen-java/org/apache/cassandra/thrift/CfDef.java
View File

@ -322,8 +322,6 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
this.read_repair_chance = 1;
this.row_cache_provider = "org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider";
}
public CfDef(
@ -465,8 +463,7 @@ public class CfDef implements org.apache.thrift.TBase<CfDef, CfDef._Fields>, jav
setMerge_shards_chanceIsSet(false);
this.merge_shards_chance = 0.0;
this.key_validation_class = null;
this.row_cache_provider = "org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider";
this.row_cache_provider = null;
this.key_alias = null;
this.compaction_strategy = null;
this.compaction_strategy_options = null;

3
src/java/org/apache/cassandra/cache/SerializingCacheProvider.java vendored
View File

@ -20,7 +20,6 @@ package org.apache.cassandra.cache;
*
*/
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.ColumnFamily;
import org.apache.cassandra.db.DecoratedKey;
@ -37,7 +36,7 @@ public class SerializingCacheProvider implements IRowCacheProvider
}
catch (NoClassDefFoundError e)
{
throw new ConfigurationException("Cannot intialize SerializationCache without JNA in the class path");
throw new ConfigurationException("Cannot initialize SerializationCache without JNA in the class path");
}
}

35
src/java/org/apache/cassandra/config/CFMetaData.java
View File

@ -37,6 +37,9 @@ import org.apache.cassandra.db.migration.avro.ColumnDef;
import org.apache.cassandra.io.IColumnSerializer;
import org.apache.cassandra.io.compress.CompressionParameters;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider;
import org.apache.cassandra.cache.SerializingCacheProvider;
import org.apache.cassandra.utils.CLibrary;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
@ -64,7 +67,7 @@ public final class CFMetaData
public final static int DEFAULT_MIN_COMPACTION_THRESHOLD = 4;
public final static int DEFAULT_MAX_COMPACTION_THRESHOLD = 32;
public final static double DEFAULT_MERGE_SHARDS_CHANCE = 0.1;
public final static String DEFAULT_ROW_CACHE_PROVIDER = "org.apache.cassandra.cache.ConcurrentLinkedHashCacheProvider";
public final static IRowCacheProvider DEFAULT_ROW_CACHE_PROVIDER = initDefaultRowCacheProvider();
public final static String DEFAULT_COMPACTION_STRATEGY_CLASS = "SizeTieredCompactionStrategy";
public final static ByteBuffer DEFAULT_KEY_NAME = ByteBufferUtil.bytes("KEY");
@ -75,6 +78,18 @@ public final class CFMetaData
public static final CFMetaData IndexCf = newSystemMetadata(SystemTable.INDEX_CF, 5, "indexes that have been completed", UTF8Type.instance, null);
public static final CFMetaData NodeIdCf = newSystemMetadata(SystemTable.NODE_ID_CF, 6, "nodeId and their metadata", TimeUUIDType.instance, null);
private static IRowCacheProvider initDefaultRowCacheProvider()
{
try
{
return new SerializingCacheProvider();
}
catch (ConfigurationException e)
{
return new ConcurrentLinkedHashCacheProvider();
}
}
//REQUIRED
public final Integer cfId; // internal id, never exposed to user
public final String ksName; // name of keyspace
@ -171,14 +186,7 @@ public final class CFMetaData
minCompactionThreshold = DEFAULT_MIN_COMPACTION_THRESHOLD;
maxCompactionThreshold = DEFAULT_MAX_COMPACTION_THRESHOLD;
mergeShardsChance = DEFAULT_MERGE_SHARDS_CHANCE;
try
{
rowCacheProvider = FBUtilities.newCacheProvider(DEFAULT_ROW_CACHE_PROVIDER);
}
catch (ConfigurationException e)
{
throw new AssertionError(e); // the default provider should not error out
}
rowCacheProvider = DEFAULT_ROW_CACHE_PROVIDER;
// Defaults strange or simple enough to not need a DEFAULT_T for
defaultValidator = BytesType.instance;
@ -368,7 +376,10 @@ public final class CFMetaData
}
catch (ConfigurationException e)
{
throw new RuntimeException(e);
// default was already set upon newCFMD init
logger.warn("Unable to instantiate cache provider {}; using default {} instead",
cf.row_cache_provider,
DEFAULT_ROW_CACHE_PROVIDER);
}
}
if (cf.key_alias != null) { newCFMD.keyAlias(cf.key_alias); }
@ -609,8 +620,6 @@ public final class CFMetaData
cf_def.setRow_cache_keys_to_save(CFMetaData.DEFAULT_ROW_CACHE_KEYS_TO_SAVE);
if (!cf_def.isSetMerge_shards_chance())
cf_def.setMerge_shards_chance(CFMetaData.DEFAULT_MERGE_SHARDS_CHANCE);
if (!cf_def.isSetRow_cache_provider())
cf_def.setRow_cache_provider(CFMetaData.DEFAULT_ROW_CACHE_PROVIDER);
if (null == cf_def.compaction_strategy)
cf_def.compaction_strategy = DEFAULT_COMPACTION_STRATEGY_CLASS;
if (null == cf_def.compaction_strategy_options)
@ -836,6 +845,7 @@ public final class CFMetaData
def.setRow_cache_save_period_in_seconds(rowCacheSavePeriodInSeconds);
def.setKey_cache_save_period_in_seconds(keyCacheSavePeriodInSeconds);
def.setRow_cache_keys_to_save(rowCacheKeysToSave);
def.setRow_cache_provider(rowCacheProvider.getClass().getName());
def.setMerge_shards_chance(mergeShardsChance);
def.setKey_alias(getKeyName());
List<org.apache.cassandra.thrift.ColumnDef> column_meta = new ArrayList<org.apache.cassandra.thrift.ColumnDef>(column_metadata.size());
@ -962,6 +972,7 @@ public final class CFMetaData
.append("rowCacheSavePeriodInSeconds", rowCacheSavePeriodInSeconds)
.append("keyCacheSavePeriodInSeconds", keyCacheSavePeriodInSeconds)
.append("rowCacheKeysToSave", rowCacheKeysToSave)
.append("rowCacheProvider", rowCacheProvider)
.append("mergeShardsChance", mergeShardsChance)
.append("keyAlias", keyAlias)
.append("column_metadata", column_metadata)

36
src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java
View File

@ -28,6 +28,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.common.collect.Sets;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.ColumnDefinition;
import org.apache.cassandra.config.ConfigurationException;
@ -41,6 +46,8 @@ import org.apache.cassandra.utils.FBUtilities;
/** A <code>CREATE COLUMNFAMILY</code> parsed from a CQL query statement. */
public class CreateColumnFamilyStatement
{
private static Logger logger = LoggerFactory.getLogger(CreateColumnFamilyStatement.class);
private static final String KW_COMPARATOR = "comparator";
private static final String KW_COMMENT = "comment";
private static final String KW_ROWCACHESIZE = "row_cache_size";
@ -52,14 +59,13 @@ public class CreateColumnFamilyStatement
private static final String KW_MAXCOMPACTIONTHRESHOLD = "max_compaction_threshold";
private static final String KW_ROWCACHESAVEPERIODSECS = "row_cache_save_period_in_seconds";
private static final String KW_KEYCACHESAVEPERIODSECS = "key_cache_save_period_in_seconds";
private static final String KW_MEMTABLESIZEINMB = "memtable_throughput_in_mb";
private static final String KW_MEMTABLEOPSINMILLIONS = "memtable_operations_in_millions";
private static final String KW_REPLICATEONWRITE = "replicate_on_write";
private static final String KW_ROW_CACHE_PROVIDER = "row_cache_provider";
// Maps CQL short names to the respective Cassandra comparator/validator class names
public static final Map<String, String> comparators = new HashMap<String, String>();
private static final Set<String> keywords = new HashSet<String>();
private static final Set<String> obsoleteKeywords = new HashSet<String>();
static
{
@ -89,10 +95,12 @@ public class CreateColumnFamilyStatement
keywords.add(KW_MAXCOMPACTIONTHRESHOLD);
keywords.add(KW_ROWCACHESAVEPERIODSECS);
keywords.add(KW_KEYCACHESAVEPERIODSECS);
keywords.add(KW_MEMTABLESIZEINMB);
keywords.add(KW_MEMTABLEOPSINMILLIONS);
keywords.add(KW_REPLICATEONWRITE);
keywords.add(KW_ROW_CACHE_PROVIDER);
obsoleteKeywords.add("memtable_throughput_in_mb");
obsoleteKeywords.add("memtable_operations_in_millions");
obsoleteKeywords.add("memtable_flush_after_mins");
}
private final String name;
@ -114,11 +122,10 @@ public class CreateColumnFamilyStatement
throw new InvalidRequestException(String.format("\"%s\" is not a valid column family name", name));
// Catch the case where someone passed a kwarg that is not recognized.
Set<String> keywordsFound = new HashSet<String>(properties.keySet());
keywordsFound.removeAll(keywords);
for (String bogus : keywordsFound)
for (String bogus : Sets.difference(properties.keySet(), Sets.union(keywords, obsoleteKeywords)))
throw new InvalidRequestException(bogus + " is not a valid keyword argument for CREATE COLUMNFAMILY");
for (String obsolete : Sets.intersection(properties.keySet(), obsoleteKeywords))
logger.warn("Ignoring obsolete property {}", obsolete);
// Validate min/max compaction thresholds
Integer minCompaction = getPropertyInt(KW_MINCOMPACTIONTHRESHOLD, null);
@ -148,17 +155,6 @@ public class CreateColumnFamilyStatement
CFMetaData.DEFAULT_MIN_COMPACTION_THRESHOLD));
}
// Validate memtable settings
Integer memMb = getPropertyInt(KW_MEMTABLESIZEINMB, null);
Double memOps = getPropertyDouble(KW_MEMTABLEOPSINMILLIONS, null);
if ((memMb != null) && (memMb <= 0))
throw new InvalidRequestException(String.format("%s must be non-negative and greater than zero",
KW_MEMTABLESIZEINMB));
if ((memOps != null) && (memOps <=0))
throw new InvalidRequestException(String.format("%s must be non-negative and greater than zero",
KW_MEMTABLEOPSINMILLIONS));
// Ensure that exactly one key has been specified.
if (keyValidator.size() < 1)
throw new InvalidRequestException("You must specify a PRIMARY KEY");
@ -304,7 +300,7 @@ public class CreateColumnFamilyStatement
.mergeShardsChance(0.0)
.columnMetadata(getColumns(comparator))
.keyValidator(TypeParser.parse(comparators.get(getKeyType())))
.rowCacheProvider(FBUtilities.newCacheProvider(getPropertyString(KW_ROW_CACHE_PROVIDER, CFMetaData.DEFAULT_ROW_CACHE_PROVIDER)))
.rowCacheProvider(FBUtilities.newCacheProvider(getPropertyString(KW_ROW_CACHE_PROVIDER, CFMetaData.DEFAULT_ROW_CACHE_PROVIDER.getClass().getName())))
.keyAlias(keyAlias);
}
catch (ConfigurationException e)

15
src/resources/org/apache/cassandra/cli/CliHelp.yaml
View File

@ -555,14 +555,13 @@ commands:
It is also valid to specify the fully-qualified class name to a class
that implements org.apache.cassandra.cache.IRowCacheProvider.
row_cache_provider defaults to ConcurrentLinkedHashCacheProvider,
but if you have JNA installed you should usually use
SerializingCacheProvider, which serialises the contents of the
row and stores it in native memory, i.e., off the JVM
Heap. Serialized rows take significantly less memory than
"live" rows in the JVM, so you can cache more rows in a given
memory footprint. And storing the cache off-heap means you
can use smaller heap sizes, reducing the impact of GC pauses.
row_cache_provider defaults to SerializingCacheProvider if you have JNA
enabled, otherwise ConcurrentLinkedHashCacheProvider.
SerializingCacheProvider serialises the contents of the row and stores
it in native memory, i.e., off the JVM Heap. Serialized rows take
significantly less memory than "live" rows in the JVM, so you can cache
more rows in a given memory footprint. And storing the cache off-heap
means you can use smaller heap sizes, reducing the impact of GC pauses.
- compression: Use compression for SSTable data files.