mirror of https://github.com/apache/cassandra
Merge branch 'cassandra-1.1' into trunk
This commit is contained in:
Brandon Williams
commit
0471062910
|
|
@ -29,6 +29,7 @@
|
|||
* Move CfDef and KsDef validation out of thrift (CASSANDRA-4037)
|
||||
* Expose repairing by a user provided range (CASSANDRA-3912)
|
||||
* Add way to force the cassandra-cli to refresh it's schema (CASSANDRA-4052)
|
||||
* Avoids having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889)
|
||||
Merged from 1.0:
|
||||
* Fix super columns bug where cache is not updated (CASSANDRA-4190)
|
||||
|
||||
|
|
|
|||
146
bin/cqlsh
146
bin/cqlsh
|
|
@ -50,6 +50,7 @@ import ConfigParser
|
|||
import codecs
|
||||
import re
|
||||
import platform
|
||||
import warnings
|
||||
|
||||
# cqlsh should run correctly when run out of a Cassandra source tree,
|
||||
# out of an unpacked Cassandra tarball, and after a proper package install.
|
||||
|
|
@ -57,9 +58,11 @@ cqlshlibdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'p
|
|||
if os.path.isdir(cqlshlibdir):
|
||||
sys.path.insert(0, cqlshlibdir)
|
||||
|
||||
from cqlshlib import cqlhandling, pylexotron, wcwidth
|
||||
from cqlshlib import cqlhandling, cql3handling, pylexotron, wcwidth
|
||||
from cqlshlib.cqlhandling import (token_dequote, cql_dequote, cql_escape,
|
||||
maybe_cql_escape, cql_typename)
|
||||
from cqlshlib.cql3handling import (CqlTableDef, maybe_cql3_escape_name,
|
||||
cql3_escape_value)
|
||||
|
||||
try:
|
||||
import readline
|
||||
|
|
@ -167,6 +170,7 @@ cqlhandling.commands_end_with_newline.update((
|
|||
'assume',
|
||||
'source',
|
||||
'capture',
|
||||
'debug',
|
||||
'exit',
|
||||
'quit'
|
||||
))
|
||||
|
|
@ -180,6 +184,7 @@ cqlhandling.CqlRuleSet.append_rules(r'''
|
|||
| <assumeCommand>
|
||||
| <sourceCommand>
|
||||
| <captureCommand>
|
||||
| <debugCommand>
|
||||
| <helpCommand>
|
||||
| <exitCommand>
|
||||
;
|
||||
|
|
@ -209,6 +214,9 @@ cqlhandling.CqlRuleSet.append_rules(r'''
|
|||
<captureCommand> ::= "CAPTURE" ( fname=( <stringLiteral> | "OFF" ) )?
|
||||
;
|
||||
|
||||
<debugCommand> ::= "DEBUG"
|
||||
;
|
||||
|
||||
<helpCommand> ::= ( "HELP" | "?" ) [topic]=( <identifier> | <stringLiteral> )*
|
||||
;
|
||||
|
||||
|
|
@ -456,6 +464,16 @@ def format_value(val, casstype, output_encoding, addcolor=False, time_format='',
|
|||
|
||||
return FormattedValue(bval, coloredval, displaywidth)
|
||||
|
||||
def show_warning_without_quoting_line(message, category, filename, lineno, file=None, line=None):
|
||||
if file is None:
|
||||
file = sys.stderr
|
||||
try:
|
||||
file.write(warnings.formatwarning(message, category, filename, lineno, line=''))
|
||||
except IOError:
|
||||
pass
|
||||
warnings.showwarning = show_warning_without_quoting_line
|
||||
warnings.filterwarnings('always', category=cql3handling.UnexpectedTableStructure)
|
||||
|
||||
class Shell(cmd.Cmd):
|
||||
default_prompt = "cqlsh> "
|
||||
continue_prompt = " ... "
|
||||
|
|
@ -589,6 +607,18 @@ class Shell(cmd.Cmd):
|
|||
return {'build': 'unknown', 'cql': 'unknown', 'thrift': thrift_ver}
|
||||
return vers
|
||||
|
||||
def fetchdict(self):
|
||||
row = self.cursor.fetchone()
|
||||
desc = self.cursor.description
|
||||
return dict(zip([d[0] for d in desc], row))
|
||||
|
||||
def fetchdict_all(self):
|
||||
dicts = []
|
||||
for row in self.cursor:
|
||||
desc = self.cursor.description
|
||||
dicts.append(dict(zip([d[0] for d in desc], row)))
|
||||
return dicts
|
||||
|
||||
def get_keyspace_names(self):
|
||||
return [k.name for k in self.get_keyspaces()]
|
||||
|
||||
|
|
@ -674,6 +704,21 @@ class Shell(cmd.Cmd):
|
|||
|
||||
# ===== end thrift-dependent parts =====
|
||||
|
||||
# ===== cql3-dependent parts =====
|
||||
|
||||
def get_columnfamily_layout(self, ksname, cfname):
|
||||
self.cursor.execute("""select * from system.schema_columnfamilies
|
||||
where "keyspace"=:ks and "columnfamily"=:cf""",
|
||||
{'ks': ksname, 'cf': cfname})
|
||||
layout = self.fetchdict()
|
||||
self.cursor.execute("""select * from system.schema_columns
|
||||
where "keyspace"=:ks and "columnfamily"=:cf""",
|
||||
{'ks': ksname, 'cf': cfname})
|
||||
cols = self.fetchdict_all()
|
||||
return CqlTableDef.from_layout(layout, cols)
|
||||
|
||||
# ===== end cql3-dependent parts =====
|
||||
|
||||
def reset_statement(self):
|
||||
self.reset_prompt()
|
||||
self.statement.truncate(0)
|
||||
|
|
@ -1030,12 +1075,45 @@ class Shell(cmd.Cmd):
|
|||
out.write('\nUSE %s;\n' % ksname)
|
||||
for cf in ksdef.cf_defs:
|
||||
out.write('\n')
|
||||
self.print_recreate_columnfamily(cf, out)
|
||||
# yes, cf might be looked up again. oh well.
|
||||
self.print_recreate_columnfamily(ksname, cf.name, out)
|
||||
|
||||
def print_recreate_columnfamily(self, cfdef, out):
|
||||
def print_recreate_columnfamily(self, ksname, cfname, out):
|
||||
"""
|
||||
Output CQL commands which should be pasteable back into a CQL session
|
||||
to recreate the given table. Can change based on CQL version in use;
|
||||
CQL 3 syntax will not be output when in CQL 2 mode, and properties
|
||||
which are deprecated with CQL 3 use (like default_validation) will not
|
||||
be output when in CQL 3 mode.
|
||||
|
||||
Writes output to the given out stream.
|
||||
"""
|
||||
|
||||
# no metainfo available from system.schema_* for system CFs, so we have
|
||||
# to use cfdef-based description for those. also, use cfdef-based
|
||||
# description when the CF doesn't have a composite key. that seems like
|
||||
# an ok compromise between hiding "comparator",
|
||||
# "default_validation_class", etc for cql3, and still allowing users
|
||||
# to work with old cql2-style wide tables.
|
||||
|
||||
if cfname != 'system' \
|
||||
and self.cqlver_atleast(3):
|
||||
try:
|
||||
layout = self.get_columnfamily_layout(ksname, cfname)
|
||||
except CQL_ERRORS:
|
||||
# most likely a 1.1 beta where cql3 is supported, but not system.schema_*
|
||||
pass
|
||||
else:
|
||||
if len(layout.key_components) > 1:
|
||||
return self.print_recreate_columnfamily_from_layout(layout, out)
|
||||
|
||||
cfdef = self.get_columnfamily(cfname, ksname=ksname)
|
||||
return self.print_recreate_columnfamily_from_cfdef(cfdef, out)
|
||||
|
||||
def print_recreate_columnfamily_from_cfdef(self, cfdef, out):
|
||||
cfname = maybe_cql_escape(cfdef.name)
|
||||
out.write("CREATE COLUMNFAMILY %s (\n" % cfname)
|
||||
alias = cfdef.key_alias if cfdef.key_alias else 'KEY'
|
||||
alias = maybe_cql_escape(cfdef.key_alias) if cfdef.key_alias else 'KEY'
|
||||
keytype = cql_typename(cfdef.key_validation_class)
|
||||
out.write(" %s %s PRIMARY KEY" % (alias, keytype))
|
||||
indexed_columns = []
|
||||
|
|
@ -1061,6 +1139,8 @@ class Shell(cmd.Cmd):
|
|||
for option, thriftname, _ in cqlhandling.columnfamily_map_options:
|
||||
optmap = getattr(cfdef, thriftname or option, {})
|
||||
for k, v in optmap.items():
|
||||
if option == 'compression_parameters' and k == 'sstable_compression':
|
||||
v = trim_if_present(v, 'org.apache.cassandra.io.compress.')
|
||||
notable_columns.append(('%s:%s' % (option, k), cql_escape(v)))
|
||||
out.write('\n)')
|
||||
if notable_columns:
|
||||
|
|
@ -1076,6 +1156,58 @@ class Shell(cmd.Cmd):
|
|||
out.write('CREATE INDEX %s ON %s (%s);\n'
|
||||
% (col.index_name, cfname, maybe_cql_escape(col.name)))
|
||||
|
||||
def print_recreate_columnfamily_from_layout(self, layout, out):
|
||||
cfname = maybe_cql3_escape_name(layout.name)
|
||||
out.write("CREATE COLUMNFAMILY %s (\n" % cfname)
|
||||
keycol = layout.columns[0]
|
||||
out.write(" %s %s" % (maybe_cql3_escape_name(keycol.name), keycol.cqltype))
|
||||
if len(layout.key_components) == 1:
|
||||
out.write(" PRIMARY KEY")
|
||||
|
||||
indexed_columns = []
|
||||
for col in layout.columns[1:]:
|
||||
colname = maybe_cql3_escape_name(col.name)
|
||||
out.write(",\n %s %s" % (colname, col.cqltype))
|
||||
if col.index_name is not None:
|
||||
indexed_columns.append(col)
|
||||
|
||||
if len(layout.key_components) > 1:
|
||||
out.write(",\n PRIMARY KEY (%s)" % ', '.join(map(maybe_cql3_escape_name, layout.key_components)))
|
||||
out.write("\n)")
|
||||
joiner = 'WITH'
|
||||
|
||||
if layout.compact_storage:
|
||||
out.write(' WITH COMPACT STORAGE')
|
||||
joiner = 'AND'
|
||||
|
||||
cf_opts = []
|
||||
for option in cql3handling.columnfamily_options:
|
||||
optval = getattr(layout, option, None)
|
||||
if optval is None:
|
||||
continue
|
||||
if option == 'row_cache_provider':
|
||||
optval = trim_if_present(optval, 'org.apache.cassandra.cache.')
|
||||
elif option == 'compaction_strategy_class':
|
||||
optval = trim_if_present(optval, 'org.apache.cassandra.db.compaction.')
|
||||
cf_opts.append((option, cql3_escape_value(optval)))
|
||||
for option, _ in cql3handling.columnfamily_map_options:
|
||||
optmap = getattr(layout, option, {})
|
||||
for k, v in optmap.items():
|
||||
if option == 'compression_parameters' and k == 'sstable_compression':
|
||||
v = trim_if_present(v, 'org.apache.cassandra.io.compress.')
|
||||
cf_opts.append(('%s:%s' % (option, k.encode('ascii')), cql3_escape_value(v)))
|
||||
if cf_opts:
|
||||
for optname, optval in cf_opts:
|
||||
out.write(" %s\n %s=%s" % (joiner, optname, optval))
|
||||
joiner = 'AND'
|
||||
out.write(";\n")
|
||||
|
||||
for col in indexed_columns:
|
||||
out.write('\n')
|
||||
# guess CQL can't represent index_type or index_options
|
||||
out.write('CREATE INDEX %s ON %s (%s);\n'
|
||||
% (col.index_name, cfname, maybe_cql3_escape_name(col.name)))
|
||||
|
||||
def describe_keyspace(self, ksname):
|
||||
print
|
||||
self.print_recreate_keyspace(self.get_keyspace(ksname), sys.stdout)
|
||||
|
|
@ -1083,7 +1215,7 @@ class Shell(cmd.Cmd):
|
|||
|
||||
def describe_columnfamily(self, cfname):
|
||||
print
|
||||
self.print_recreate_columnfamily(self.get_columnfamily(cfname), sys.stdout)
|
||||
self.print_recreate_columnfamily(ksname, cfname, sys.stdout)
|
||||
print
|
||||
|
||||
def describe_columnfamilies(self, ksname):
|
||||
|
|
@ -1386,6 +1518,10 @@ class Shell(cmd.Cmd):
|
|||
self.stop = True
|
||||
do_quit = do_exit
|
||||
|
||||
def do_debug(self, parsed):
|
||||
import pdb
|
||||
pdb.set_trace()
|
||||
|
||||
def get_names(self):
|
||||
names = cmd.Cmd.get_names(self)
|
||||
for hide_from_help in ('do_quit',):
|
||||
|
|
|
|||
|
|
@ -1077,7 +1077,6 @@
|
|||
</fileset>
|
||||
<fileset dir="${build.lib}">
|
||||
<include name="**/guava*.jar" />
|
||||
<include name="**/commons-lang*.jar" />
|
||||
</fileset>
|
||||
</classpath>
|
||||
</junit>
|
||||
|
|
|
|||
|
|
@ -23,6 +23,16 @@ from itertools import izip
|
|||
|
||||
Hint = pylexotron.Hint
|
||||
|
||||
keywords = set((
|
||||
'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with',
|
||||
'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any',
|
||||
'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set',
|
||||
'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create',
|
||||
'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop',
|
||||
'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type',
|
||||
'first', 'reversed'
|
||||
))
|
||||
|
||||
columnfamily_options = (
|
||||
# (CQL option name, Thrift option name (or None if same))
|
||||
('comment', None),
|
||||
|
|
@ -109,7 +119,7 @@ consistency_levels = (
|
|||
valid_cql_word_re = re.compile(r"^(?:[a-z][a-z0-9_]*|-?[0-9][0-9.]*)$", re.I)
|
||||
|
||||
def is_valid_cql_word(s):
|
||||
return valid_cql_word_re.match(s) is not None
|
||||
return valid_cql_word_re.match(s) is not None and s not in keywords
|
||||
|
||||
def tokenize_cql(cql_text):
|
||||
return CqlLexotron.scan(cql_text)[0]
|
||||
|
|
@ -146,9 +156,11 @@ def token_is_word(tok):
|
|||
def cql_escape(value):
|
||||
if value is None:
|
||||
return 'NULL' # this totally won't work
|
||||
if isinstance(value, float):
|
||||
if isinstance(value, bool):
|
||||
value = str(value).lower()
|
||||
elif isinstance(value, float):
|
||||
return '%f' % value
|
||||
if isinstance(value, int):
|
||||
elif isinstance(value, int):
|
||||
return str(value)
|
||||
return "'%s'" % value.replace("'", "''")
|
||||
|
||||
|
|
|
|||
|
|
@ -36,13 +36,15 @@ public class StageManager
|
|||
|
||||
public static final long KEEPALIVE = 60; // seconds to keep "extra" threads alive for when idle
|
||||
|
||||
public static final int MAX_REPLICATE_ON_WRITE_TASKS = 1024 * Runtime.getRuntime().availableProcessors();
|
||||
|
||||
static
|
||||
{
|
||||
stages.put(Stage.MUTATION, multiThreadedConfigurableStage(Stage.MUTATION, getConcurrentWriters()));
|
||||
stages.put(Stage.READ, multiThreadedConfigurableStage(Stage.READ, getConcurrentReaders()));
|
||||
stages.put(Stage.REQUEST_RESPONSE, multiThreadedStage(Stage.REQUEST_RESPONSE, Runtime.getRuntime().availableProcessors()));
|
||||
stages.put(Stage.INTERNAL_RESPONSE, multiThreadedStage(Stage.INTERNAL_RESPONSE, Runtime.getRuntime().availableProcessors()));
|
||||
stages.put(Stage.REPLICATE_ON_WRITE, multiThreadedConfigurableStage(Stage.REPLICATE_ON_WRITE, getConcurrentReplicators()));
|
||||
stages.put(Stage.REPLICATE_ON_WRITE, multiThreadedConfigurableStage(Stage.REPLICATE_ON_WRITE, getConcurrentReplicators(), MAX_REPLICATE_ON_WRITE_TASKS));
|
||||
// the rest are all single-threaded
|
||||
stages.put(Stage.STREAM, new JMXEnabledThreadPoolExecutor(Stage.STREAM));
|
||||
stages.put(Stage.GOSSIP, new JMXEnabledThreadPoolExecutor(Stage.GOSSIP));
|
||||
|
|
@ -72,6 +74,16 @@ public class StageManager
|
|||
stage.getJmxType());
|
||||
}
|
||||
|
||||
private static ThreadPoolExecutor multiThreadedConfigurableStage(Stage stage, int numThreads, int maxTasksBeforeBlock)
|
||||
{
|
||||
return new JMXConfigurableThreadPoolExecutor(numThreads,
|
||||
KEEPALIVE,
|
||||
TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(maxTasksBeforeBlock),
|
||||
new NamedThreadFactory(stage.getJmxName()),
|
||||
stage.getJmxType());
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a stage from the StageManager
|
||||
* @param stage name of the stage to be retrieved.
|
||||
|
|
|
|||
|
|
@ -410,8 +410,8 @@ truncateStatement returns [TruncateStatement stmt]
|
|||
|
||||
// Column Identifiers
|
||||
cident returns [ColumnIdentifier id]
|
||||
: t=( IDENT | UUID | INTEGER ) { $id = new ColumnIdentifier($t.text, false); }
|
||||
| t=QUOTED_NAME { $id = new ColumnIdentifier($t.text, true); }
|
||||
: t=IDENT { $id = new ColumnIdentifier($t.text, false); }
|
||||
| t=QUOTED_NAME { $id = new ColumnIdentifier($t.text, true); }
|
||||
;
|
||||
|
||||
// Keyspace & Column family names
|
||||
|
|
@ -437,8 +437,8 @@ cidentList returns [List<ColumnIdentifier> items]
|
|||
|
||||
// Values (includes prepared statement markers)
|
||||
term returns [Term term]
|
||||
: t=(STRING_LITERAL | UUID | IDENT | INTEGER | FLOAT ) { $term = new Term($t.text, $t.type); }
|
||||
| t=QMARK { $term = new Term($t.text, $t.type, ++currentBindMarkerIdx); }
|
||||
: t=(STRING_LITERAL | UUID | INTEGER | FLOAT ) { $term = new Term($t.text, $t.type); }
|
||||
| t=QMARK { $term = new Term($t.text, $t.type, ++currentBindMarkerIdx); }
|
||||
;
|
||||
|
||||
intTerm returns [Term integer]
|
||||
|
|
|
|||
|
|
@ -17,6 +17,12 @@
|
|||
*/
|
||||
package org.apache.cassandra.utils;
|
||||
|
||||
/*
|
||||
* BE ADVISED: New imports added here might introduce new dependencies for
|
||||
* the clientutil jar. If in doubt, run the `ant test-clientutil-jar' target
|
||||
* afterward, and ensure the tests still pass.
|
||||
*/
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.CharacterCodingException;
|
||||
|
|
@ -28,8 +34,6 @@ import static com.google.common.base.Charsets.UTF_8;
|
|||
import org.apache.cassandra.io.util.FileDataInput;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
|
||||
/**
|
||||
* Utility methods to make ByteBuffers less painful
|
||||
* The following should illustrate the different ways byte buffers can be used
|
||||
|
|
@ -70,7 +74,7 @@ import org.apache.commons.lang.ArrayUtils;
|
|||
*/
|
||||
public class ByteBufferUtil
|
||||
{
|
||||
public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(ArrayUtils.EMPTY_BYTE_ARRAY);
|
||||
public static final ByteBuffer EMPTY_BYTE_BUFFER = ByteBuffer.wrap(new byte[0]);
|
||||
|
||||
public static int compareUnsigned(ByteBuffer o1, ByteBuffer o2)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue