diff --git a/CHANGES.txt b/CHANGES.txt index 4430de6f84..b3c1a80b17 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,9 +8,22 @@ * add CQL TTL support (CASSANDRA-2476) * optimize batches containing multiple updates to the same row (CASSANDRA-2583) + * adjust hinted handoff page size to avoid OOM with large columns + (CASSANDRA-2652) + * mark BRAF buffer invalid post-flush so we don't re-flush partial + buffers again, especially on CL writes (CASSANDRA-2660) + * add DROP INDEX support to CLI (CASSANDRA-2616) + * don't perform HH to client-mode [storageproxy] nodes (CASSANDRA-2668) -0.8.0-rc1 +0.8.0-final + * fix CQL grammar warning and cqlsh regression from CASSANDRA-2622 + * add ant generate-cql-html target (CASSANDRA-2526) + * update CQL consistency levels (CASSANDRA-2566) + * debian packaging fixes (CASSANDRA-2481, 2647) + + +0.8.0-rc1 * faster flushes and compaction from fixing excessively pessimistic rebuffering in BRAF (CASSANDRA-2581) * fix returning null column values in the python cql driver (CASSANDRA-2593) @@ -37,8 +50,10 @@ * initialize local ep state prior to gossip startup if needed (CASSANDRA-2638) * fix counter increment lost after restart (CASSANDRA-2642) * add quote-escaping via backslash to CLI (CASSANDRA-2623) - * fig pig example script (CASSANDRA-2487) + * fix pig example script (CASSANDRA-2487) * fix dynamic snitch race in adding latencies (CASSANDRA-2618) + * Start/stop cassandra after more important services such as mdadm in + debian packaging (CASSANDRA-2481) 0.8.0-beta2 diff --git a/NEWS.txt b/NEWS.txt index 4f7bcff887..0b2c57e553 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -62,6 +62,15 @@ JMX - By default, JMX now listens on port 7199. +0.7.6 +===== + +Upgrading +--------- + - Nothing specific to 0.7.6, but see 0.7.3 Upgrading if upgrading + from earlier than 0.7.1. + + 0.7.5 ===== diff --git a/build.xml b/build.xml index 0b91a511ea..268e4f11ef 100644 --- a/build.xml +++ b/build.xml @@ -221,6 +221,19 @@ + + + + + + + + + + + + + diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 346a9c45aa..04df3fac32 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -102,6 +102,7 @@ seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: # seeds is actually a comma-delimited list of addresses. + # Ex: ",," - seeds: "127.0.0.1" # emergency pressure valve: each time heap usage after a full (CMS) diff --git a/debian/cassandra.in.sh b/debian/cassandra.in.sh index d17b6c1948..5e45b520d4 100644 --- a/debian/cassandra.in.sh +++ b/debian/cassandra.in.sh @@ -2,9 +2,15 @@ # The directory where Cassandra's configs live (required) CASSANDRA_CONF=/etc/cassandra +CASSANDRA_HOME=/usr/share/cassandra + # The java classpath (required) CLASSPATH=$CASSANDRA_CONF +for jar in /usr/share/cassandra/lib/*.jar; do + CLASSPATH=$CLASSPATH:$jar +done + for jar in /usr/share/cassandra/*.jar; do CLASSPATH=$CLASSPATH:$jar done diff --git a/debian/cassandra.install b/debian/cassandra.install index fb2ea72052..5f54cd3415 100644 --- a/debian/cassandra.install +++ b/debian/cassandra.install @@ -9,5 +9,5 @@ bin/nodetool usr/bin bin/json2sstable usr/bin bin/sstable2json usr/bin bin/sstablekeys usr/bin -lib/*.jar usr/share/cassandra +lib/*.jar usr/share/cassandra/lib lib/licenses usr/share/doc/cassandra diff --git a/debian/changelog b/debian/changelog index 47aa3cae4e..d65ccbb421 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,7 +2,7 @@ cassandra (0.8.0~rc1) unstable; urgency=low * Release candidate - -- Eric Evans Tue, 10 May 2011 19:09:27 -0500 + -- Eric Evans Thu, 12 May 2011 18:35:26 -0500 cassandra (0.8.0~beta2) unstable; urgency=low diff --git a/debian/dirs b/debian/dirs index 68b939d3af..e79846baca 100644 --- a/debian/dirs +++ b/debian/dirs @@ -1,4 +1,5 @@ usr/share/cassandra +usr/share/cassandra/lib usr/sbin usr/bin etc/cassandra diff --git a/debian/init b/debian/init index 709bb6a455..7425c61fdd 100644 --- a/debian/init +++ b/debian/init @@ -1,8 +1,10 @@ #! /bin/sh ### BEGIN INIT INFO # Provides: cassandra -# Required-Start: $remote_fs -# Required-Stop: $remote_fs +# Required-Start: $remote_fs $network $named $time +# Required-Stop: $remote_fs $network $named $time +# Should-Start: ntp mdadm +# Should-Stop: ntp mdadm # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: distributed storage system for structured data @@ -19,6 +21,7 @@ SCRIPTNAME=/etc/init.d/$NAME CONFDIR=/etc/cassandra JSVC=/usr/bin/jsvc WAIT_FOR_START=10 +CASSANDRA_HOME=/usr/share/cassandra # The first existing directory is used for JAVA_HOME if needed. JVM_SEARCH_DIRS="/usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-sun" @@ -78,6 +81,9 @@ fi classpath() { cp="$EXTRA_CLASSPATH" + for j in /usr/share/$NAME/lib/*.jar; do + [ "x$cp" = "x" ] && cp=$j || cp=$cp:$j + done for j in /usr/share/$NAME/*.jar; do [ "x$cp" = "x" ] && cp=$j || cp=$cp:$j done diff --git a/debian/rules b/debian/rules index 4bab97ed8a..a7190ffd78 100755 --- a/debian/rules +++ b/debian/rules @@ -51,7 +51,7 @@ binary-indep: build install dh_testdir dh_testroot dh_installchangelogs - dh_installinit + dh_installinit -u'start 50 2 3 4 5 . stop 50 0 1 6 .' dh_installdocs README.txt CHANGES.txt NEWS.txt dh_compress dh_fixperms diff --git a/doc/cql/CQL.html b/doc/cql/CQL.html deleted file mode 100644 index a29227929a..0000000000 --- a/doc/cql/CQL.html +++ /dev/null @@ -1,63 +0,0 @@ - - -

Cassandra Query Language (CQL) v1.0.0

Table of Contents

  1. Cassandra Query Language (CQL) v1.0.0
    1. Table of Contents
    2. USE
    3. SELECT
      1. Specifying Columns
      2. Column Family
      3. Consistency Level
      4. Filtering rows
      5. Limits
    4. UPDATE
      1. Column Family
      2. Consistency Level
      3. Specifying Columns and Row
    5. DELETE
      1. Specifying Columns
      2. Column Family
      3. Consistency Level
      4. Specifying Rows
    6. TRUNCATE
    7. CREATE KEYSPACE
    8. CREATE COLUMNFAMILY
      1. Specifying Key Type
      2. Specifying Column Type (optional)
      3. Column Family Options (optional)
    9. CREATE INDEX
    10. DROP
    11. Common Idioms
      1. Specifying Consistency
      2. Term specification
  2. Versioning
  3. Changes

USE

Synopsis:

USE <KEYSPACE>;
-

A USE statement consists of the USE keyword, followed by a valid keyspace name. Its purpose is to assign the per-connection, current working keyspace. All subsequent keyspace-specific actions will be performed in the context of the supplied value.

SELECT

Synopsis:

SELECT [FIRST N] [REVERSED] <SELECT EXPR> FROM <COLUMN FAMILY> [USING <CONSISTENCY>]
-        [WHERE <CLAUSE>] [LIMIT N];
-

A SELECT is used to read one or more records from a Cassandra column family. It returns a result-set of rows, where each row consists of a key and a collection of columns corresponding to the query.

Specifying Columns

SELECT [FIRST N] [REVERSED] name1, name2, name3 FROM ...
-SELECT [FIRST N] [REVERSED] name1..nameN FROM ...
-

The SELECT expression determines which columns will appear in the results and takes the form of either a comma separated list of names, or a range. The range notation consists of a start and end column name separated by two periods (..). The set of columns returned for a range is start and end inclusive.

The FIRST option accepts an integer argument and can be used to apply a limit to the number of columns returned per row. When this limit is left unset it defaults to 10,000 columns.

The REVERSED option causes the sort order of the results to be reversed.

It is worth noting that unlike the projection in a SQL SELECT, there is no guarantee that the results will contain all of the columns specified. This is because Cassandra is schema-less and there are no guarantees that a given column exists.

Column Family

SELECT ... FROM <COLUMN FAMILY> ...
-

The FROM clause is used to specify the Cassandra column family applicable to a SELECT query.

Consistency Level

SELECT ... [USING <CONSISTENCY>] ...
-

Following the column family clause is an optional consistency level specification.

Filtering rows

SELECT ... WHERE KEY = keyname AND name1 = value1
-SELECT ... WHERE KEY >= startkey and KEY =< endkey AND name1 = value1
-

The WHERE clause provides for filtering the rows that appear in results. The clause can filter on a key name, or range of keys, and in the case of indexed columns, on column values. Key filters are specified using the KEY keyword, a relational operator, (one of =, >, >=, <, and <=), and a term value. When terms appear on both sides of a relational operator it is assumed the filter applies to an indexed column. With column index filters, the term on the left of the operator is the name, the term on the right is the value to filter on.

Note: The greater-than and less-than operators (> and <) result in key ranges that are inclusive of the terms. There is no supported notion of “strictly” greater-than or less-than; these operators are merely supported as aliases to >= and <=.

Limits

SELECT ... WHERE <CLAUSE> [LIMIT N] ...
-

Limiting the number of rows returned can be achieved by adding the LIMIT option to a SELECT expression. LIMIT defaults to 10,000 when left unset.

UPDATE

Synopsis:

UPDATE <COLUMN FAMILY> [USING CONSISTENCY <CL>]
-        SET name1 = value1, name2 = value2 WHERE KEY = keyname;
-

An UPDATE is used to write one or more columns to a record in a Cassandra column family. No results are returned.

Column Family

UPDATE <COLUMN FAMILY> ...
-

Statements begin with the UPDATE keyword followed by a Cassandra column family name.

Consistency Level

UPDATE ... [USING <CONSISTENCY>] ...
-

Following the column family identifier is an optional consistency level specification.

Specifying Columns and Row

UPDATE ... SET name1 = value1, name2 = value2 WHERE KEY = keyname;
-

Rows are created or updated by supplying column names and values in term assignment format. Multiple columns can be set by separating the name/value pairs using commas. Each update statement requires exactly one key to be specified using a WHERE clause and the KEY keyword.

Additionally, it is also possible to send multiple UPDATES to a node at once using a batch syntax:

BEGIN BATCH [USING <CONSISTENCY>]
-UPDATE CF1 SET name1 = value1, name2 = value2 WHERE KEY = keyname1;
-UPDATE CF1 SET name3 = value3 WHERE KEY = keyname2;
-UPDATE CF2 SET name4 = value4, name5 = value5 WHERE KEY = keyname3;
-APPLY BATCH
-

When batching UPDATEs, a single consistency level is used for the entire batch, it appears after the BEGIN BATCH statement, and uses the standard consistency level specification. Batch UPDATEs default to CONSISTENCY.ONE when left unspecified.

NOTE: While there are no isolation guarantees, UPDATE queries are atomic within a give record.

DELETE

Synopsis:

DELETE [COLUMNS] FROM <COLUMN FAMILY> [USING <CONSISTENCY>] WHERE KEY = keyname1
-DELETE [COLUMNS] FROM <COLUMN FAMILY> [USING <CONSISTENCY>] WHERE KEY IN (keyname1, keyname2);
-

A DELETE is used to perform the removal of one or more columns from one or more rows.

Specifying Columns

DELETE [COLUMNS] ...
-

Following the DELETE keyword is an optional comma-delimited list of column name terms. When no column names are specified, the remove applies to the entire row(s) matched by the WHERE clause

Column Family

DELETE ... FROM <COLUMN FAMILY> ...
-

The column family name follows the list of column names.

Consistency Level

UPDATE ... [USING <CONSISTENCY>] ...
-

Following the column family identifier is an optional consistency level specification.

Specifying Rows

UPDATE ... WHERE KEY = keyname1
-UPDATE ... WHERE KEY IN (keyname1, keyname2)
-

The WHERE clause is used to determine which row(s) a DELETE applies to. The first form allows the specification of a single keyname using the KEY keyword and the = operator. The second form allows a list of keyname terms to be specified using the IN notation and a parenthesized list of comma-delimited keyname terms.

TRUNCATE

Synopsis:

TRUNCATE <COLUMN FAMILY>
-

Accepts a single argument for the column family name, and permanently removes all data from said column family.

CREATE KEYSPACE

Synopsis:

CREATE KEYSPACE <NAME> WITH replication_factor = <NUM> AND strategy_class = <STRATEGY>
-    [AND strategy_options.<OPTION> = <VALUE> [AND strategy_options.<OPTION> = <VALUE>]];
-

The CREATE KEYSPACE statement creates a new top-level namespace (aka “keyspace”). Valid names are any string constructed of alphanumeric characters and underscores, but must begin with a letter. Properties such as replication strategy and count are specified during creation using the following accepted keyword arguments:

keywordrequireddescription
replication_factoryesNumeric argument that specifies the number of replicas for this keyspace.
strategy_classyesClass name to use for managing replica placement. Any of the shipped strategies can be used by specifying the class name relative to org.apache.cassandra.locator, others will need to be fully-qualified and located on the classpath.
strategy_optionsnoSome strategies require additional arguments which can be supplied by appending the option name to the strategy_options keyword, separated by a colon (:). For example, a strategy option of “DC1” with a value of “1” would be specified as strategy_options:DC1 = 1.

CREATE COLUMNFAMILY

Synopsis:

CREATE COLUMNFAMILY <COLUMN FAMILY> (KEY <type> PRIMARY KEY [, name1 type, name2 type, ...]);
-CREATE COLUMNFAMILY <COLUMN FAMILY> (KEY <type> PRIMARY KEY [, name1 type, name2 type, ...])
-    [WITH keyword1 = arg1 [AND keyword2 = arg2 [AND ...]]];
-

CREATE COLUMNFAMILY statements create new column family namespaces under the current keyspace. Valid column family names are strings of alphanumeric characters and underscores, which begin with a letter.

Specifying Key Type

CREATE ... (KEY <type> PRIMARY KEY) ...
-

When creating a new column family, you must specify key type. The list of possible key types is identical to column comparators/validators, (see Specifying Column Type). It’s important to note that the key type must be compatible with the partitioner in use, for example OrderPreservingPartitioner and CollatingOrderPreservingPartitioner both require UTF-8 keys.

Specifying Column Type (optional)

CREATE ... (KEY <type> PRIMARY KEY, name1 type, name2 type) ...
-

It is possible to assign columns a type during column family creation. Columns configured with a type are validated accordingly when a write occurs. Column types are specified as a parenthesized, comma-separated list of column term and type pairs. The list of recognized types are:

typedescription
byteaArbitrary bytes (no validation)
asciiASCII character string
textUTF8 encoded string
varcharUTF8 encoded string
uuidType 1, or type 4 UUID
varint4-byte integer
bigint8-byte long

Note: In addition to the recognized types listed above, it is also possible to supply a string containing the name of a class (a sub-class of AbstractType), either fully qualified, or relative to the org.apache.cassandra.db.marshal package.

Column Family Options (optional)

CREATE COLUMNFAMILY ... WITH keyword1 = arg1 AND keyword2 = arg2;
-

A number of optional keyword arguments can be supplied to control the configuration of a new column family.

keyworddefaultdescription
comparatortextDetermines sorting and validation of column names. Valid values are identical to the types listed in Specifying Column Type above.
commentnoneA free-form, human-readable comment.
row_cache_size0Number of rows whose entire contents to cache in memory.
key_cache_size200000Number of keys per SSTable whose locations are kept in memory in “mostly LRU” order.
read_repair_chance1.0The probability with which read repairs should be invoked on non-quorum reads.
gc_grace_seconds864000Time to wait before garbage collecting tombstones (deletion markers).
default_validationtextDetermines validation of column values. Valid values are identical to the types listed in Specifying Column Type above.
min_compaction_threshold4Minimum number of SSTables needed to start a minor compaction.
max_compaction_threshold32Maximum number of SSTables allowed before a minor compaction is forced.
row_cache_save_period_in_seconds0Number of seconds between saving row caches.
key_cache_save_period_in_seconds14400Number of seconds between saving key caches.
memtable_flush_after_mins60Maximum time to leave a dirty table unflushed.
memtable_throughput_in_mbdynamicMaximum size of the memtable before it is flushed.
memtable_operations_in_millionsdynamicNumber of operations in millions before the memtable is flushed.
replicate_on_writefalse

CREATE INDEX

Synopsis:

CREATE INDEX [index_name] ON <column_family> (column_name);
-

A CREATE INDEX statement is used to create a new, automatic secondary index for the named column.

DROP

Synopsis:

DROP <KEYSPACE|COLUMNFAMILY> namespace;
-

DROP statements result in the immediate, irreversible removal of keyspace and column family namespaces.

Common Idioms

Specifying Consistency

... USING <CONSISTENCY> ...
-

Consistency level specifications are made up the keyword USING, followed by a consistency level identifier. Valid consistency levels are as follows:

  • CONSISTENCY ZERO
  • CONSISTENCY ONE (default)
  • CONSISTENCY QUORUM
  • CONSISTENCY ALL
  • CONSISTENCY DCQUORUM
  • CONSISTENCY DCQUORUMSYNC

Term specification

Terms are used in statements to specify things such as keyspaces, column families, indexes, column names and values, and keyword arguments. The rules governing term specification are as follows:

  • Any single quoted string literal (example: 'apple').
  • Unquoted alpha-numeric strings that begin with a letter (example: carrot).
  • Unquoted numeric literals (example: 100).
  • UUID strings in hyphen-delimited hex notation (example: 1438fc5c-4ff6-11e0-b97f-0026c650d722).

Terms which do not conform to these rules result in an exception.

How column name/value terms are interpreted is determined by the configured type.

typeterm
asciiAny string which can be decoded using ASCII charset
text / varcharAny string which can be decoded using UTF8 charset
uuidStandard UUID string format (hyphen-delimited hex notation)
uuidStandard UUID string format (hyphen-delimited hex notation)
uuidThe string now, to represent a type-1 (time-based) UUID with a date-time component based on the current time
uuidNumeric value representing milliseconds since epoch
uuidAn iso8601 timestamp
bigintNumeric value capable of fitting in 8 bytes
varintNumeric value of arbitrary size
byteaHex-encoded strings (converted directly to the corresponding bytes)

Versioning

Versioning of the CQL language adheres to the Semantic Versioning guidelines. Versions take the form X.Y.Z where X, Y, and Z are integer values representing major, minor, and patch level respectively. There is no correlation between Cassandra release versions and the CQL language version.

versiondescription
PatchThe patch version is incremented when bugs are fixed.
MinorMinor version increments occur when new, but backward compatible, functionality is introduced.
MajorThe major version must be bumped when backward incompatible changes are introduced. This should rarely (if ever) occur.

Changes

Tue, 22 Mar 2011 18:10:28 -0700 - Eric Evans <eevans@rackspace.com>
- * Initial version, 1.0.0
-
diff --git a/doc/cql/CQL.textile b/doc/cql/CQL.textile index 6769cfff70..6f33d99e26 100644 --- a/doc/cql/CQL.textile +++ b/doc/cql/CQL.textile @@ -299,12 +299,12 @@ bc. Consistency level specifications are made up the keyword @USING@, followed by a consistency level identifier. Valid consistency levels are as follows: -* @CONSISTENCY ZERO@ +* @CONSISTENCY ANY@ * @CONSISTENCY ONE@ (default) * @CONSISTENCY QUORUM@ * @CONSISTENCY ALL@ -* @CONSISTENCY DCQUORUM@ -* @CONSISTENCY DCQUORUMSYNC@ +* @CONSISTENCY LOCAL_QUORUM@ +* @CONSISTENCY EACH_QUORUM@ h3(#terms). Term specification diff --git a/drivers/py/cqlsh b/drivers/py/cqlsh index 0d28926316..338aeae784 100755 --- a/drivers/py/cqlsh +++ b/drivers/py/cqlsh @@ -30,7 +30,7 @@ try: except ImportError: sys.path.append(os.path.abspath(os.path.dirname(__file__))) import cql -from cql.results import ResultSet +from cql.cursor import _COUNT_DESCRIPTION HISTORY = os.path.join(os.path.expanduser('~'), '.cqlsh') CQLTYPES = ("bytes", "ascii", "utf8", "timeuuid", "uuid", "long", "int") @@ -131,7 +131,9 @@ class Shell(cmd.Cmd): self.cursor.execute(statement) - if isinstance(self.cursor.result, ResultSet): + if self.cursor.description is _COUNT_DESCRIPTION: + if self.cursor.result: print self.cursor.result[0] + else: for x in range(self.cursor.rowcount): row = self.cursor.fetchone() self.printout(repr(row[0]), BLUE, False) @@ -142,8 +144,6 @@ class Shell(cmd.Cmd): self.printout(",", newline=False) self.printout(repr(value), YELLOW, False) self.printout("") - else: - if self.cursor.result: print self.cursor.result[0] def emptyline(self): pass diff --git a/drivers/py/setup.py b/drivers/py/setup.py index 4b1879cc37..d26e3c6aa6 100755 --- a/drivers/py/setup.py +++ b/drivers/py/setup.py @@ -20,7 +20,7 @@ from os.path import abspath, join, dirname setup( name="cql", - version="1.0.1", + version="1.0.2", description="Cassandra Query Language driver", long_description=open(abspath(join(dirname(__file__), 'README'))).read(), url="http://cassandra.apache.org", diff --git a/src/java/org/apache/cassandra/cli/Cli.g b/src/java/org/apache/cassandra/cli/Cli.g index ed22b38386..66f199f4c7 100644 --- a/src/java/org/apache/cassandra/cli/Cli.g +++ b/src/java/org/apache/cassandra/cli/Cli.g @@ -61,6 +61,7 @@ tokens { NODE_TRUNCATE; NODE_ASSUME; NODE_CONSISTENCY_LEVEL; + NODE_DROP_INDEX; // Internal Nodes. NODE_COLUMN_ACCESS; @@ -160,6 +161,7 @@ statement | truncateStatement | assumeStatement | consistencyLevelStatement + | dropIndex | -> ^(NODE_NO_OP) ; @@ -203,6 +205,8 @@ helpStatement -> ^(NODE_HELP NODE_DEL_KEYSPACE) | HELP DROP COLUMN FAMILY -> ^(NODE_HELP NODE_DEL_COLUMN_FAMILY) + | HELP DROP INDEX + -> ^(NODE_HELP NODE_DROP_INDEX) | HELP GET -> ^(NODE_HELP NODE_THRIFT_GET) | HELP SET @@ -337,6 +341,11 @@ delColumnFamily -> ^(NODE_DEL_COLUMN_FAMILY columnFamily) ; +dropIndex + : DROP INDEX ON columnFamily '.' columnName + -> ^(NODE_DROP_INDEX columnFamily columnName) + ; + showVersion : SHOW API_VERSION -> ^(NODE_SHOW_VERSION) @@ -562,6 +571,8 @@ TRUNCATE: 'TRUNCATE'; ASSUME: 'ASSUME'; TTL: 'TTL'; CONSISTENCYLEVEL: 'CONSISTENCYLEVEL'; +INDEX: 'INDEX'; +ON: 'ON'; IP_ADDRESS : IntegerPositiveLiteral '.' IntegerPositiveLiteral '.' IntegerPositiveLiteral '.' IntegerPositiveLiteral @@ -603,10 +614,57 @@ Identifier // literals StringLiteral - : - '\'' (~'\'')* '\'' ( '\'' (~'\'')* '\'' )* + : '\'' SingleStringCharacter* '\'' ; +fragment SingleStringCharacter + : ~('\'' | '\\') + | '\\' EscapeSequence + ; + +fragment EscapeSequence + : CharacterEscapeSequence + | '0' + | HexEscapeSequence + | UnicodeEscapeSequence + ; + +fragment CharacterEscapeSequence + : SingleEscapeCharacter + | NonEscapeCharacter + ; + +fragment NonEscapeCharacter + : ~(EscapeCharacter) + ; + +fragment SingleEscapeCharacter + : '\'' | '"' | '\\' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' + ; + +fragment EscapeCharacter + : SingleEscapeCharacter + | DecimalDigit + | 'x' + | 'u' + ; + +fragment HexEscapeSequence + : 'x' HexDigit HexDigit + ; + +fragment UnicodeEscapeSequence + : 'u' HexDigit HexDigit HexDigit HexDigit + ; + +fragment HexDigit + : DecimalDigit | ('a'..'f') | ('A'..'F') + ; + +fragment DecimalDigit + : ('0'..'9') + ; + // // syntactic elements // diff --git a/src/java/org/apache/cassandra/cli/CliClient.java b/src/java/org/apache/cassandra/cli/CliClient.java index 3e2fcacc16..00132cd6c7 100644 --- a/src/java/org/apache/cassandra/cli/CliClient.java +++ b/src/java/org/apache/cassandra/cli/CliClient.java @@ -61,13 +61,14 @@ public class CliClient */ public enum Function { - BYTES (BytesType.instance), - INTEGER (IntegerType.instance), - LONG (LongType.instance), - LEXICALUUID (LexicalUUIDType.instance), - TIMEUUID (TimeUUIDType.instance), - UTF8 (UTF8Type.instance), - ASCII (AsciiType.instance); + BYTES (BytesType.instance), + INTEGER (IntegerType.instance), + LONG (LongType.instance), + LEXICALUUID (LexicalUUIDType.instance), + TIMEUUID (TimeUUIDType.instance), + UTF8 (UTF8Type.instance), + ASCII (AsciiType.instance), + COUNTERCOLUMN (CounterColumnType.instance); private AbstractType validator; @@ -268,6 +269,10 @@ public class CliClient case CliParser.NODE_THRIFT_DECR: executeIncr(tree, -1L); break; + case CliParser.NODE_DROP_INDEX: + executeDropIndex(tree); + break; + case CliParser.NODE_NO_OP: // comment lines come here; they are treated as no ops. break; @@ -1321,6 +1326,58 @@ public class CliClient printSliceList(columnFamilyDef, keySlices); } + // DROP INDEX ON . + private void executeDropIndex(Tree statement) + { + if (!CliMain.isConnected() || !hasKeySpace()) + return; + + // getColumnFamily will check if CF exists for us + String columnFamily = CliCompiler.getColumnFamily(statement, keyspacesMap.get(keySpace).cf_defs); + String rawColumName = statement.getChild(1).getText(); + + CfDef cfDef = getCfDef(columnFamily); + + ByteBuffer columnName = columnNameAsBytes(rawColumName, cfDef); + + boolean foundColumn = false; + + for (ColumnDef column : cfDef.getColumn_metadata()) + { + if (column.name.equals(columnName)) + { + foundColumn = true; + + if (column.getIndex_type() == null) + throw new RuntimeException(String.format("Column '%s' does not have an index.", rawColumName)); + + column.setIndex_name(null); + column.setIndex_type(null); + } + } + + if (!foundColumn) + throw new RuntimeException(String.format("Column '%s' definition was not found in ColumnFamily '%s'.", + rawColumName, + columnFamily)); + + try + { + String mySchemaVersion = thriftClient.system_update_column_family(cfDef); + sessionState.out.println(mySchemaVersion); + validateSchemaIsSettled(mySchemaVersion); + keyspacesMap.put(keySpace, thriftClient.describe_keyspace(keySpace)); + } + catch (InvalidRequestException e) + { + System.err.println(e.why); + } + catch (Exception e) + { + throw new RuntimeException(e.getMessage(), e); + } + } + // TRUNCATE private void executeTruncate(String columnFamily) { @@ -1403,6 +1460,9 @@ public class CliClient return; } + // making string representation look property e.g. o.a.c.db.marshal.UTF8Type + defaultType = comparator.getClass().getName(); + if (assumptionElement.equals("COMPARATOR")) { columnFamily.setComparator_type(defaultType); diff --git a/src/java/org/apache/cassandra/cql/Cql.g b/src/java/org/apache/cassandra/cql/Cql.g index d30a19bf8f..b4e73609dd 100644 --- a/src/java/org/apache/cassandra/cql/Cql.g +++ b/src/java/org/apache/cassandra/cql/Cql.g @@ -412,7 +412,7 @@ termPair[Map columns] // Note: ranges are inclusive so >= and >, and < and <= all have the same semantics. relation returns [Relation rel] : { Term entity = new Term("KEY", STRING_LITERAL); } - (K_KEY | name=term { entity = $name.item; } ) type=('=' | '<' | '<=' | '>=' | '>') t=term + (name=term { entity = $name.item; } ) type=('=' | '<' | '<=' | '>=' | '>') t=term { return new Relation(entity, $type.text, $t.item); } ; @@ -440,9 +440,10 @@ K_USING: U S I N G; K_CONSISTENCY: C O N S I S T E N C Y; K_LEVEL: ( O N E | Q U O R U M - | A L L - | D C Q U O R U M - | D C Q U O R U M S Y N C + | A L L + | A N Y + | L O C A L '_' Q U O R U M + | E A C H '_' Q U O R U M ) ; K_USE: U S E; diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 465779705b..0032f5d2af 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -461,6 +461,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean */ public static void scrubDataDirectories(String table, String columnFamily) { + logger.info("Removing compacted SSTable files (see http://wiki.apache.org/cassandra/MemtableSSTable)"); for (Map.Entry> sstableFiles : files(table, columnFamily, true).entrySet()) { Descriptor desc = sstableFiles.getKey(); diff --git a/src/java/org/apache/cassandra/db/CompactionManager.java b/src/java/org/apache/cassandra/db/CompactionManager.java index 929d16339d..11c9ac89e5 100644 --- a/src/java/org/apache/cassandra/db/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/CompactionManager.java @@ -845,19 +845,19 @@ public class CompactionManager implements CompactionManagerMBean totalkeysWritten++; } else - { - cfs.invalidateCachedRow(row.getKey()); - if (!indexedColumns.isEmpty() || isCommutative) + { + cfs.invalidateCachedRow(row.getKey()); + if (!indexedColumns.isEmpty() || isCommutative) { while (row.hasNext()) { IColumn column = row.next(); if (column instanceof CounterColumn) - renewer.maybeRenew((CounterColumn)column); + renewer.maybeRenew((CounterColumn) column); if (indexedColumns.contains(column.name())) Table.cleanupIndexEntry(cfs, row.getKey().key, column); } - } + } } } } @@ -1169,31 +1169,6 @@ public class CompactionManager implements CompactionManagerMBean } } - public void checkAllColumnFamilies() throws IOException - { - // perform estimates - for (final ColumnFamilyStore cfs : ColumnFamilyStore.all()) - { - Runnable runnable = new Runnable() - { - public void run () - { - logger.debug("Estimating compactions for " + cfs.columnFamily); - final Set> buckets = getBuckets(convertSSTablesToPairs(cfs.getSSTables()), 50L * 1024L * 1024L); - updateEstimateFor(cfs, buckets); - } - }; - executor.submit(runnable); - } - - // actually schedule compactions. done in a second pass so all the estimates occur before we - // bog down the executor in actual compactions. - for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) - { - submitMinorIfNeeded(cfs); - } - } - public int getActiveCompactions() { return executor.getActiveCount(); diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java index 8733b6c7fc..3a5b13f728 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java @@ -115,7 +115,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean logger_.debug("Created HHOM instance, registered MBean."); } - private static boolean sendMessage(InetAddress endpoint, String tableName, String cfName, ByteBuffer key) throws IOException + private static boolean sendRow(InetAddress endpoint, String tableName, String cfName, ByteBuffer key) throws IOException { if (!Gossiper.instance.isKnownEndpoint(endpoint)) { @@ -130,10 +130,21 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean Table table = Table.open(tableName); DecoratedKey dkey = StorageService.getPartitioner().decorateKey(key); ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName); + + int pageSize = PAGE_SIZE; + // send less columns per page if they are very large + if (cfs.getMeanColumns() > 0) + { + int averageColumnSize = (int) (cfs.getMeanRowSize() / cfs.getMeanColumns()); + pageSize = Math.min(PAGE_SIZE, DatabaseDescriptor.getInMemoryCompactionLimit() / averageColumnSize); + pageSize = Math.max(2, pageSize); // page size of 1 does not allow actual paging b/c of >= behavior on startColumn + logger_.debug("average hinted-row column size is {}; using pageSize of {}", averageColumnSize, pageSize); + } + ByteBuffer startColumn = ByteBufferUtil.EMPTY_BYTE_BUFFER; while (true) { - QueryFilter filter = QueryFilter.getSliceFilter(dkey, new QueryPath(cfs.getColumnFamilyName()), startColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, PAGE_SIZE); + QueryFilter filter = QueryFilter.getSliceFilter(dkey, new QueryPath(cfs.getColumnFamilyName()), startColumn, ByteBufferUtil.EMPTY_BYTE_BUFFER, false, pageSize); ColumnFamily cf = cfs.getColumnFamily(filter); if (pagingFinished(cf, startColumn)) break; @@ -326,7 +337,7 @@ public class HintedHandOffManager implements HintedHandOffManagerMBean for (IColumn tableCF : tableCFs) { String[] parts = getTableAndCFNames(tableCF.name()); - if (sendMessage(endpoint, parts[0], parts[1], keyColumn.name())) + if (sendRow(endpoint, parts[0], parts[1], keyColumn.name())) { deleteHintKey(endpointAsUTF8, keyColumn.name(), tableCF.name(), tableCF.timestamp()); rowsReplayed++; diff --git a/src/java/org/apache/cassandra/gms/EndpointState.java b/src/java/org/apache/cassandra/gms/EndpointState.java index 0715b51ea2..3e05c83008 100644 --- a/src/java/org/apache/cassandra/gms/EndpointState.java +++ b/src/java/org/apache/cassandra/gms/EndpointState.java @@ -128,7 +128,7 @@ public class EndpointState hasToken = value; } - public boolean getHasToken() + public boolean hasToken() { return hasToken; } diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java index 12cbcf6e5f..a3333fac0c 100644 --- a/src/java/org/apache/cassandra/gms/Gossiper.java +++ b/src/java/org/apache/cassandra/gms/Gossiper.java @@ -452,7 +452,7 @@ public class Gossiper implements IFailureDetectionEventListener // check if this is a fat client. fat clients are removed automatically from // gosip after FatClientTimeout - if (!epState.getHasToken() && !epState.isAlive() && (duration > FatClientTimeout)) + if (!epState.hasToken() && !epState.isAlive() && (duration > FatClientTimeout)) { if (StorageService.instance.getTokenMetadata().isMember(endpoint)) epState.setHasToken(true); @@ -475,7 +475,7 @@ public class Gossiper implements IFailureDetectionEventListener if (!justRemovedEndpoints.isEmpty()) { - for (Map.Entry entry : justRemovedEndpoints.entrySet()) + for (Entry entry : justRemovedEndpoints.entrySet()) { if ((now - entry.getValue()) > QUARANTINE_DELAY) { diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index b4ff1b7d96..39a6a3a5f6 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -156,7 +156,7 @@ public abstract class SSTable { throw new IOError(e); } - logger.info("Deleted " + desc); + logger.debug("Deleted {}", desc); return true; } diff --git a/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java b/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java index 9a51a9f40c..e9f17a9a74 100644 --- a/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java +++ b/src/java/org/apache/cassandra/io/util/BufferedRandomAccessFile.java @@ -128,6 +128,9 @@ public class BufferedRandomAccessFile extends RandomAccessFile implements FileDa fd = CLibrary.getfd(this.getFD()); } + /** + * Flush (flush()) whatever writes are pending, and block until the data has been persistently committed (fsync()). + */ public void sync() throws IOException { if (syncNeeded) @@ -150,6 +153,11 @@ public class BufferedRandomAccessFile extends RandomAccessFile implements FileDa } } + /** + * If we are dirty, flush dirty contents to the operating system. Does not imply fsync(). + * + * Currently, for implementation reasons, this also invalidates the buffer. + */ public void flush() throws IOException { if (isDirty) @@ -181,6 +189,9 @@ public class BufferedRandomAccessFile extends RandomAccessFile implements FileDa } + // Remember that we wrote, so we don't write it again on next flush(). + resetBuffer(); + isDirty = false; } } @@ -220,16 +231,18 @@ public class BufferedRandomAccessFile extends RandomAccessFile implements FileDa reBuffer(); } + private void resetBuffer() + { + bufferOffset = current; + validBufferBytes = 0; + } + private void reBuffer() throws IOException { flush(); // synchronizing buffer and file on disk - - bufferOffset = current; + resetBuffer(); if (bufferOffset >= channel.size()) - { - validBufferBytes = 0; return; - } if (bufferOffset < minBufferOffset) minBufferOffset = bufferOffset; diff --git a/src/java/org/apache/cassandra/locator/RackInferringSnitch.java b/src/java/org/apache/cassandra/locator/RackInferringSnitch.java index 26a2d6f4e9..cde1dcdf7e 100644 --- a/src/java/org/apache/cassandra/locator/RackInferringSnitch.java +++ b/src/java/org/apache/cassandra/locator/RackInferringSnitch.java @@ -28,11 +28,11 @@ public class RackInferringSnitch extends AbstractNetworkTopologySnitch { public String getRack(InetAddress endpoint) { - return Byte.toString(endpoint.getAddress()[2]); + return Integer.toString(endpoint.getAddress()[2] & 0xFF, 10); } public String getDatacenter(InetAddress endpoint) { - return Byte.toString(endpoint.getAddress()[1]); + return Integer.toString(endpoint.getAddress()[1] & 0xFF, 10); } } diff --git a/src/java/org/apache/cassandra/net/MessagingService.java b/src/java/org/apache/cassandra/net/MessagingService.java index 7eb1ffb70a..1eab78e99f 100644 --- a/src/java/org/apache/cassandra/net/MessagingService.java +++ b/src/java/org/apache/cassandra/net/MessagingService.java @@ -83,6 +83,7 @@ public final class MessagingService implements MessagingServiceMBean private final SimpleCondition listenGate; private final Map droppedMessages = new EnumMap(StorageService.Verb.class); private final List subscribers = new ArrayList(); + private static final long DEFAULT_CALLBACK_TIMEOUT = (long) (1.1 * DatabaseDescriptor.getRpcTimeout()); { for (StorageService.Verb verb : StorageService.Verb.values()) @@ -121,7 +122,7 @@ public final class MessagingService implements MessagingServiceMBean return null; } }; - callbacks = new ExpiringMap>((long) (1.1 * DatabaseDescriptor.getRpcTimeout()), timeoutReporter); + callbacks = new ExpiringMap>(DEFAULT_CALLBACK_TIMEOUT, timeoutReporter); MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); try @@ -256,7 +257,12 @@ public final class MessagingService implements MessagingServiceMBean private void addCallback(IMessageCallback cb, String messageId, InetAddress to) { - Pair previous = callbacks.put(messageId, new Pair(to, cb)); + addCallback(cb, messageId, to, DEFAULT_CALLBACK_TIMEOUT); + } + + private void addCallback(IMessageCallback cb, String messageId, InetAddress to, long timeout) + { + Pair previous = callbacks.put(messageId, new Pair(to, cb), timeout); assert previous == null; } @@ -267,6 +273,14 @@ public final class MessagingService implements MessagingServiceMBean return Integer.toString(idGen.incrementAndGet()); } + /* + * @see #sendRR(Message message, InetAddress to, IMessageCallback cb, long timeout) + */ + public String sendRR(Message message, InetAddress to, IMessageCallback cb) + { + return sendRR(message, to, cb, DEFAULT_CALLBACK_TIMEOUT); + } + /** * Send a message to a given endpoint. This method specifies a callback * which is invoked with the actual response. @@ -275,12 +289,13 @@ public final class MessagingService implements MessagingServiceMBean * @param cb callback interface which is used to pass the responses or * suggest that a timeout occurred to the invoker of the send(). * suggest that a timeout occurred to the invoker of the send(). + * @param timeout the timeout used for expiration * @return an reference to message id used to match with the result */ - public String sendRR(Message message, InetAddress to, IMessageCallback cb) + public String sendRR(Message message, InetAddress to, IMessageCallback cb, long timeout) { String id = nextId(); - addCallback(cb, id, to); + addCallback(cb, id, to, timeout); sendOneWay(message, id, to); return id; } @@ -624,4 +639,9 @@ public final class MessagingService implements MessagingServiceMBean completedTasks.put(entry.getKey().getHostAddress(), entry.getValue().ackCon.getCompletedMesssages()); return completedTasks; } + + public static long getDefaultCallbackTimeout() + { + return DEFAULT_CALLBACK_TIMEOUT; + } } diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java index 8a82d2cb65..5e899dde4c 100644 --- a/src/java/org/apache/cassandra/service/StorageService.java +++ b/src/java/org/apache/cassandra/service/StorageService.java @@ -1184,7 +1184,7 @@ public class StorageService implements IEndpointStateChangeSubscriber, StorageSe public void onAlive(InetAddress endpoint, EndpointState state) { - if (!isClientMode) + if (!isClientMode && state.hasToken()) deliverHints(endpoint); } diff --git a/src/java/org/apache/cassandra/utils/ExpiringMap.java b/src/java/org/apache/cassandra/utils/ExpiringMap.java index a9e1953da4..ed56e1cbf3 100644 --- a/src/java/org/apache/cassandra/utils/ExpiringMap.java +++ b/src/java/org/apache/cassandra/utils/ExpiringMap.java @@ -32,11 +32,13 @@ public class ExpiringMap { private final T value; private final long age; + private final long expiration; - CacheableObject(T o) + CacheableObject(T o, long e) { assert o != null; value = o; + expiration = e; age = System.currentTimeMillis(); } @@ -45,26 +47,21 @@ public class ExpiringMap return value; } - boolean isReadyToDie(long expiration) + boolean isReadyToDie(long start) { - return ((System.currentTimeMillis() - age) > expiration); + return ((start - age) > expiration); } } private class CacheMonitor extends TimerTask { - private final long expiration; - - CacheMonitor(long expiration) - { - this.expiration = expiration; - } public void run() { + long start = System.currentTimeMillis(); for (Map.Entry> entry : cache.entrySet()) { - if (entry.getValue().isReadyToDie(expiration)) + if (entry.getValue().isReadyToDie(start)) { cache.remove(entry.getKey()); if (postExpireHook != null) @@ -77,6 +74,7 @@ public class ExpiringMap private final NonBlockingHashMap> cache = new NonBlockingHashMap>(); private final Timer timer; private static int counter = 0; + private final long expiration; public ExpiringMap(long expiration) { @@ -90,13 +88,15 @@ public class ExpiringMap public ExpiringMap(long expiration, Function, ?> postExpireHook) { this.postExpireHook = postExpireHook; + this.expiration = expiration; + if (expiration <= 0) { throw new IllegalArgumentException("Argument specified must be a positive number"); } timer = new Timer("EXPIRING-MAP-TIMER-" + (++counter), true); - timer.schedule(new CacheMonitor(expiration), expiration / 2, expiration / 2); + timer.schedule(new CacheMonitor(), expiration / 2, expiration / 2); } public void shutdown() @@ -106,7 +106,12 @@ public class ExpiringMap public V put(K key, V value) { - CacheableObject previous = cache.put(key, new CacheableObject(value)); + return put(key, value, this.expiration); + } + + public V put(K key, V value, long timeout) + { + CacheableObject previous = cache.put(key, new CacheableObject(value, timeout)); return (previous == null) ? null : previous.getValue(); } diff --git a/src/resources/org/apache/cassandra/cli/CliHelp.yaml b/src/resources/org/apache/cassandra/cli/CliHelp.yaml index 95924edeb3..e1d570a675 100644 --- a/src/resources/org/apache/cassandra/cli/CliHelp.yaml +++ b/src/resources/org/apache/cassandra/cli/CliHelp.yaml @@ -44,6 +44,7 @@ help: | describe keyspace Describe a keyspace and it's column families. drop column family Remove a column family and it's data. drop keyspace Remove a keyspace and it's data. + drop index Remove an existing index from specific column. get Get rows and columns. incr Increments a counter column. list List rows in a column family. @@ -827,6 +828,18 @@ commands: Example: drop column family Standard2; + - name: NODE_DROP_INDEX + help: | + drop index on .; + + Drops index on specified column of the column family. + + Required Parameters: + - cf: Name of the column family. + - column: Name of the column to delete index on. + + Example: + drop index on Users.name; - name: NODE_THRIFT_GET help: | get ['']; @@ -1074,13 +1087,13 @@ commands: - type: Validator type to use when processing values. Supported values are: - - AsciiType - - BytesType - - CounterColumnType (distributed counter column) - - IntegerType (a generic variable-length integer type) - - LexicalUUIDType - - LongType - - UTF8Type + - ascii + - bytes + - counterColumn (distributed counter column) + - integer (a generic variable-length integer type) + - lexicalUUID + - long + - utf8 It is also valid to specify the fully-qualified class name to a class that extends org.apache.Cassandra.db.marshal.AbstractType. diff --git a/test/unit/org/apache/cassandra/cli/CliTest.java b/test/unit/org/apache/cassandra/cli/CliTest.java index 5b5a19ff1f..4e83b3e850 100644 --- a/test/unit/org/apache/cassandra/cli/CliTest.java +++ b/test/unit/org/apache/cassandra/cli/CliTest.java @@ -40,9 +40,14 @@ public class CliTest extends CleanupHelper "create column family CF1 with comparator=UTF8Type and column_metadata=[{ column_name:world, validation_class:IntegerType, index_type:0, index_name:IdxName }, { column_name:world2, validation_class:LongType, index_type:KEYS, index_name:LongIdxName}];", "assume CF1 keys as utf8;", "set CF1[hello][world] = 123848374878933948398384;", + "set CF1[hello][test_quote] = 'value\\'';", + "set CF1['k\\'ey'][VALUE] = 'VAL';", + "set CF1['k\\'ey'][VALUE] = 'VAL\\'';", "set CF1[hello][-31337] = 'some string value';", "get CF1[hello][-31337];", "get CF1[hello][world];", + "get CF1[hello][test_quote];", + "get CF1['k\\'ey'][VALUE]", "set CF1[hello][-31337] = -23876;", "set CF1[hello][-31337] = long(-23876);", "set CF1[hello][world2] = 15;", @@ -105,6 +110,7 @@ public class CliTest extends CleanupHelper "get Counter1['hello']['cassandra'];", "get Counter1['hello'];", "truncate CF1;", + "drop index on CF1.world2;", "update keyspace TestKeySpace with placement_strategy='org.apache.cassandra.locator.LocalStrategy';", "update keyspace TestKeySpace with strategy_options=[{DC1:3, DC2:4, DC5:1}];", "assume CF1 comparator as utf8;",