natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge from 1.0 

git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@1224998 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Ellis 2011-12-27 20:03:37 +00:00
parent 2eacaf7378 c68e90b417
commit 1bd7b42102
9 changed files with 87 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CHANGES.txt
View File

@ -45,11 +45,13 @@
* Avoid creating empty and non cleaned writer during compaction (CASSANDRA-3616)
* stop thrift service in shutdown hook so we can quiesce MessagingService
(CASSANDRA-3335)
* (CQL) compaction_strategy_options and compression_parameters for
CREATE COLUMNFAMILY statement (CASSANDRA-3374)
Merged from 0.8:
* avoid logging (harmless) exception when GC takes < 1ms (CASSANDRA-3656)
* prevent new nodes from thinking down nodes are up forever (CASSANDRA-3626)
* Flush non-cfs backed secondary indexes (CASSANDRA-3659)
* Secondary Indexes should report memory consumption (CASSANDRA-3155)
1.0.6
* (CQL) fix cqlsh support for replicate_on_write (CASSANDRA-3596)

11
doc/cql/CQL.textile
View File

@ -488,9 +488,14 @@ bc(syntax).
<createColumnFamilyStatement> ::= "CREATE" "COLUMNFAMILY" <name>
"(" <term> <storageType> "PRIMARY" "KEY"
( "," <term> <storageType> )* ")"
( "WITH" <identifier> "=" <cfOptionVal>
( "AND" <identifier> "=" <cfOptionVal> )* )?
( "WITH" <optionName> "=" <cfOptionVal>
( "AND" <optionName> "=" <cfOptionVal> )* )?
;
<optionName> ::= <identifier>
| <optionName> ":" <identifier>
| <optionName> ":" <integer>
;
<cfOptionVal> ::= <storageType>
| <identifier>
| <stringLiteral>
@ -544,6 +549,8 @@ A number of optional keyword arguments can be supplied to control the configurat
|min_compaction_threshold|4|Minimum number of SSTables needed to start a minor compaction.|
|max_compaction_threshold|32|Maximum number of SSTables allowed before a minor compaction is forced.|
|replicate_on_write|false| |
|compaction_strategy_options|none|CompactionStrategy specific options such as "sstable_size_in_mb" for LeveledCompactionStrategy and "min_sstable_size" for SizeTieredCompactionStrategy|
|compression_parameters|none|Compression parameters such as "sstable_compressor" and "chunk_length_kb"|
h2. CREATE INDEX

2
src/avro/internode.genavro
View File

@ -65,7 +65,7 @@ protocol InterNode {
union { null, string } compaction_strategy = null;
union { null, map<string> } compaction_strategy_options = null;
union { null, map<string> } compression_options = null;
union { double, null } bloom_filter_fp_chance;
union { null, double } bloom_filter_fp_chance = null;
}
@aliases(["org.apache.cassandra.config.avro.KsDef"])

4
src/java/org/apache/cassandra/cql/Cql.g
View File

@ -379,8 +379,8 @@ createKeyspaceStatement returns [CreateKeyspaceStatement expr]
createColumnFamilyStatement returns [CreateColumnFamilyStatement expr]
: K_CREATE K_COLUMNFAMILY name=( IDENT | STRING_LITERAL | INTEGER ) { $expr = new CreateColumnFamilyStatement($name.text); }
( '(' createCfamColumns[expr] ( ',' createCfamColumns[expr] )* ')' )?
( K_WITH prop1=IDENT '=' arg1=createCfamKeywordArgument { $expr.addProperty($prop1.text, $arg1.arg); }
( K_AND propN=IDENT '=' argN=createCfamKeywordArgument { $expr.addProperty($propN.text, $argN.arg); } )*
( K_WITH prop1=(COMPIDENT | IDENT) '=' arg1=createCfamKeywordArgument { $expr.addProperty($prop1.text, $arg1.arg); }
( K_AND propN=(COMPIDENT | IDENT) '=' argN=createCfamKeywordArgument { $expr.addProperty($propN.text, $argN.arg); } )*
)?
endStmnt
;

41
src/java/org/apache/cassandra/cql/CreateColumnFamilyStatement.java
View File

@ -42,6 +42,7 @@ import org.apache.cassandra.db.marshal.TypeParser;
import org.apache.cassandra.thrift.InvalidRequestException;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.io.compress.CompressionParameters;
/** A <code>CREATE COLUMNFAMILY</code> parsed from a CQL query statement. */
public class CreateColumnFamilyStatement
@ -57,11 +58,16 @@ public class CreateColumnFamilyStatement
private static final String KW_MAXCOMPACTIONTHRESHOLD = "max_compaction_threshold";
private static final String KW_REPLICATEONWRITE = "replicate_on_write";
private static final String KW_COMPACTION_STRATEGY_CLASS = "compaction_strategy_class";
// Maps CQL short names to the respective Cassandra comparator/validator class names
public static final Map<String, String> comparators = new HashMap<String, String>();
private static final Set<String> keywords = new HashSet<String>();
private static final Set<String> obsoleteKeywords = new HashSet<String>();
private static final String COMPACTION_OPTIONS_PREFIX = "compaction_strategy_options";
private static final String COMPRESSION_PARAMETERS_PREFIX = "compression_parameters";
static
{
comparators.put("ascii", "AsciiType");
@ -87,6 +93,7 @@ public class CreateColumnFamilyStatement
keywords.add(KW_MINCOMPACTIONTHRESHOLD);
keywords.add(KW_MAXCOMPACTIONTHRESHOLD);
keywords.add(KW_REPLICATEONWRITE);
keywords.add(KW_COMPACTION_STRATEGY_CLASS);
obsoleteKeywords.add("row_cache_size");
obsoleteKeywords.add("key_cache_size");
@ -103,6 +110,8 @@ public class CreateColumnFamilyStatement
private final Map<String, String> properties = new HashMap<String, String>();
private List<String> keyValidator = new ArrayList<String>();
private ByteBuffer keyAlias = null;
private final Map<String, String> compactionStrategyOptions = new HashMap<String, String>();
private final Map<String, String> compressionParameters = new HashMap<String, String>();
public CreateColumnFamilyStatement(String name)
{
@ -112,6 +121,34 @@ public class CreateColumnFamilyStatement
/** Perform validation of parsed params */
private void validate(List<String> variables) throws InvalidRequestException
{
// we need to remove parent:key = value pairs from the main properties
Set<String> propsToRemove = new HashSet<String>();
// check if we have compaction/compression options
for (String property : properties.keySet())
{
if (!property.contains(":"))
continue;
String key = property.split(":")[1];
String val = properties.get(property);
if (property.startsWith(COMPACTION_OPTIONS_PREFIX))
{
compactionStrategyOptions.put(key, val);
propsToRemove.add(property);
}
if (property.startsWith(COMPRESSION_PARAMETERS_PREFIX))
{
compressionParameters.put(key, val);
propsToRemove.add(property);
}
}
for (String property : propsToRemove)
properties.remove(property);
// Column family name
if (!name.matches("\\w+"))
throw new InvalidRequestException(String.format("\"%s\" is not a valid column family name", name));
@ -292,6 +329,8 @@ public class CreateColumnFamilyStatement
.columnMetadata(getColumns(comparator))
.keyValidator(TypeParser.parse(comparators.get(getKeyType())))
.keyAlias(keyAlias)
.compactionStrategyOptions(compactionStrategyOptions)
.compressionParameters(CompressionParameters.create(compressionParameters))
.validate();
}
catch (ConfigurationException e)

5
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -977,10 +977,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
public long getTotalMemtableLiveSize()
{
long total = 0;
for (ColumnFamilyStore cfs : concatWithIndexes())
total += cfs.getMemtableThreadSafe().getLiveSize();
return total;
return getMemtableDataSize() + indexManager.getTotalLiveSize();
}
public int getMemtableSwitchCount()

5
src/java/org/apache/cassandra/db/index/SecondaryIndex.java
View File

@ -111,6 +111,11 @@ public abstract class SecondaryIndex
*/
public abstract void forceBlockingFlush() throws IOException;
/**
* Get current amount of memory this index is consuming (in bytes)
*/
public abstract long getLiveSize();
/**
* Allow access to the underlying column family store if there is one
* @return the underlying column family store or null

21
src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
View File

@ -328,6 +328,27 @@ public class SecondaryIndexManager
return indexList.keySet();
}
/**
* @return total current ram size of all indexes
*/
public long getTotalLiveSize()
{
long total = 0;
// we use identity map because per row indexes use same instance
// across many columns
IdentityHashMap<SecondaryIndex, Object> indexList = new IdentityHashMap<SecondaryIndex, Object>();
for (Map.Entry<ByteBuffer, SecondaryIndex> entry : indexesByColumn.entrySet())
{
SecondaryIndex index = entry.getValue();
if (indexList.put(index, index) == null)
total += index.getLiveSize();
}
return total;
}
/**
* Removes obsolete index entries and creates new ones for the given row key

5
src/java/org/apache/cassandra/db/index/keys/KeysIndex.java
View File

@ -152,4 +152,9 @@ public class KeysIndex extends PerColumnSecondaryIndex
{
// no options used
}
public long getLiveSize()
{
return indexCfs.getMemtableDataSize();
}
}