diff --git a/bin/cqlsh b/bin/cqlsh index 85e1228f11..cf22aef15f 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -60,7 +60,6 @@ if os.path.isdir(cqlshlibdir): sys.path.insert(0, cqlshlibdir) from cqlshlib import cqlhandling, cql3handling, pylexotron, wcwidth -from cqlshlib.cqlhandling import cql_typename try: import readline @@ -159,7 +158,7 @@ else: debug_completion = bool(os.environ.get('CQLSH_DEBUG_COMPLETION', '') == 'YES') # we want the cql parser to understand our cqlsh-specific commands too -cqlhandling.commands_end_with_newline.update(( +my_commands_ending_with_newline = ( 'help', '?', 'describe', @@ -171,9 +170,16 @@ cqlhandling.commands_end_with_newline.update(( 'debug', 'exit', 'quit' -)) +) -cqlhandling.CqlRuleSet.append_rules(r''' +cqlsh_syntax_completers = [] +def cqlsh_syntax_completer(rulename, termname): + def registrator(f): + cqlsh_syntax_completers.append((rulename, termname, f)) + return f + return registrator + +cqlsh_extra_syntax_rules = r''' ::= | ( ";" | "\n" ) ; @@ -187,9 +193,10 @@ cqlhandling.CqlRuleSet.append_rules(r''' | ; - ::= ( "DESCRIBE" | "DESC" ) ( "KEYSPACE" ksname=? - | "COLUMNFAMILY" cfname= - | "COLUMNFAMILIES" + ::= ( "DESCRIBE" | "DESC" ) + ( "KEYSPACE" ksname=? + | ( "COLUMNFAMILY" | "TABLE" ) cf= + | ( "COLUMNFAMILIES" | "TABLES" ) | "SCHEMA" | "CLUSTER" ) ; @@ -197,13 +204,13 @@ cqlhandling.CqlRuleSet.append_rules(r''' ::= "SHOW" what=( "VERSION" | "HOST" | "ASSUMPTIONS" ) ; - ::= "ASSUME" ( ks= "." )? cf= - ( "," )* + ::= "ASSUME" cf= + ( "," )* ; ::= "NAMES" "ARE" names= | "VALUES" "ARE" values= - | "(" colname= ")" "VALUES" "ARE" colvalues= + | "(" colname= ")" "VALUES" "ARE" colvalues= ; ::= "SOURCE" fname= @@ -212,7 +219,8 @@ cqlhandling.CqlRuleSet.append_rules(r''' ::= "CAPTURE" ( fname=( | "OFF" ) )? ; - ::= "DEBUG" +# avoiding just "DEBUG" so that this rule doesn't get treated as a terminal + ::= "DEBUG" "THINGS"? ; ::= ( "HELP" | "?" ) [topic]=( | )* @@ -222,35 +230,16 @@ cqlhandling.CqlRuleSet.append_rules(r''' ; ::= "?" ; -''') +''' -@cqlhandling.cql_add_completer('helpCommand', 'topic') +@cqlsh_syntax_completer('helpCommand', 'topic') def complete_help(ctxt, cqlsh): helpfuncs = [n[5:].upper() for n in cqlsh.get_names() if n.startswith('help_')] funcs_with_docstrings = [n[3:].upper() for n in cqlsh.get_names() if n.startswith('do_') and getattr(cqlsh, n, None).__doc__] return sorted(helpfuncs + funcs_with_docstrings) -@cqlhandling.cql_add_completer('describeCommand', 'ksname') -def complete_describe_ks(ctxt, cqlsh): - return map(cqlsh.cql_protect_name, cqlsh.get_keyspace_names()) - -@cqlhandling.cql_add_completer('describeCommand', 'cfname') -def complete_describe_cf(ctxt, cqlsh): - return map(cqlsh.cql_protect_name, cqlsh.get_columnfamily_names()) - -@cqlhandling.cql_add_completer('assumeCommand', 'ks') -def complete_assume_ks(ctxt, cqlsh): - return [cqlsh.cql_protect_name(ks) + '.' for ks in cqlsh.get_keyspace_names()] - -@cqlhandling.cql_add_completer('assumeCommand', 'cf') -def complete_assume_cf(ctxt, cqlsh): - ks = ctxt.get_binding('ks', None) - if ks is not None: - ks = cqlsh.cql_unprotect_name(ks) - return map(cqlsh.cql_protect_name, cqlsh.get_columnfamily_names(ks)) - -@cqlhandling.cql_add_completer('assumeTypeDef', 'colname') +@cqlsh_syntax_completer('assumeTypeDef', 'colname') def complete_assume_col(ctxt, cqlsh): ks = ctxt.get_binding('ks', None) ks = cqlsh.cql_unprotect_name(ks) if ks is not None else None @@ -277,9 +266,9 @@ def complete_source_quoted_filename(ctxt, cqlsh): annotated.append(match) return annotated -cqlhandling.cql_add_completer('sourceCommand', 'fname') \ +cqlsh_syntax_completer('sourceCommand', 'fname') \ (complete_source_quoted_filename) -cqlhandling.cql_add_completer('captureCommand', 'fname') \ +cqlsh_syntax_completer('captureCommand', 'fname') \ (complete_source_quoted_filename) class NoKeyspaceError(Exception): @@ -310,6 +299,13 @@ class DecodeError(Exception): def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.message()) +def full_cql_version(ver): + while ver.count('.') < 2: + ver += '.0' + ver_parts = ver.split('-', 1) + [''] + vertuple = tuple(map(int, ver_parts[0].split('.')) + [ver_parts[1]]) + return ver, vertuple + def trim_if_present(s, prefix): if s.startswith(prefix): return s[len(prefix):] @@ -441,7 +437,7 @@ def format_value(val, casstype, output_encoding, addcolor=False, time_format='', elif casstype in ('DecimalType', 'UUIDType', 'BooleanType'): # let python do these for us bval = str(val) - color = colormap[cql_typename(casstype)] + color = colormap[cqlruleset.cql_typename(casstype)] elif casstype == 'BytesType': bval = ''.join('%02x' % ord(c) for c in val) color = colormap['hex'] @@ -527,12 +523,10 @@ class Shell(cmd.Cmd): self.query_out = sys.stdout def set_expanded_cql_version(self, ver): - while ver.count('.') < 2: - ver += '.0' + ver, vertuple = full_cql_version(ver) self.set_cql_version(ver) self.cql_version = ver - ver_parts = ver.split('-', 1) + [''] - self.cql_ver_tuple = tuple(map(int, ver_parts[0].split('.')) + [ver_parts[1]]) + self.cql_ver_tuple = vertuple def cqlver_atleast(self, major, minor=0, patch=0): return self.cql_ver_tuple[:3] >= (major, minor, patch) @@ -581,14 +575,14 @@ class Shell(cmd.Cmd): cf = self.cql_protect_name(cf) if override.default_name_type: print 'ASSUME %s NAMES ARE %s;' \ - % (cf, cql_typename(override.default_name_type)) + % (cf, cqlruleset.cql_typename(override.default_name_type)) if override.default_value_type: print 'ASSUME %s VALUES ARE %s;' \ - % (cf, cql_typename(override.default_value_type)) + % (cf, cqlruleset.cql_typename(override.default_value_type)) for colname, vtype in override.value_types.items(): colname = self.cql_protect_name(colname) print 'ASSUME %s(%s) VALUES ARE %s;' \ - % (cf, colname, cql_typename(vtype)) + % (cf, colname, cqlruleset.cql_typename(vtype)) print def get_cluster_versions(self): @@ -647,8 +641,7 @@ class Shell(cmd.Cmd): indnames.append(md.index_name) return indnames - def filterable_column_names(self, cfname, ksname=None): - cfdef = self.get_columnfamily(cfname, ksname=ksname) + def filterable_column_names(self, cfdef): filterable = set() if cfdef.key_alias is not None and cfdef.key_alias != 'KEY': filterable.add(cfdef.key_alias) @@ -705,6 +698,8 @@ class Shell(cmd.Cmd): # ===== cql3-dependent parts ===== def get_columnfamily_layout(self, ksname, cfname): + if ksname is None: + ksname = self.current_keyspace self.cursor.execute("""select * from system.schema_columnfamilies where "keyspace"=:ks and "columnfamily"=:cf""", {'ks': ksname, 'cf': cfname}) @@ -792,7 +787,7 @@ class Shell(cmd.Cmd): """ try: - statements, in_batch = cqlhandling.cql_split_statements(statementtext) + statements, in_batch = cqlruleset.cql_split_statements(statementtext) except pylexotron.LexingError, e: if self.show_line_nums: self.printerr('Invalid syntax at char %d' % (e.charnum,)) @@ -837,20 +832,20 @@ class Shell(cmd.Cmd): cmdword = 'help' custom_handler = getattr(self, 'do_' + cmdword.lower(), None) if custom_handler: - parsed = cqlhandling.cql_whole_parse_tokens(tokens, srcstr=srcstr, - startsymbol='cqlshCommand') + parsed = cqlruleset.cql_whole_parse_tokens(tokens, srcstr=srcstr, + startsymbol='cqlshCommand') if parsed and not parsed.remainder: # successful complete parse return custom_handler(parsed) else: return self.handle_parse_error(cmdword, tokens, parsed, srcstr) - return self.perform_statement(cqlhandling.cql_extract_orig(tokens, srcstr)) + return self.perform_statement(cqlruleset.cql_extract_orig(tokens, srcstr)) def handle_parse_error(self, cmdword, tokens, parsed, srcstr): if cmdword.lower() == 'select': # hey, maybe they know about some new syntax we don't. type # assumptions won't work, but maybe the query will. - return self.perform_statement(cqlhandling.cql_extract_orig(tokens, srcstr)) + return self.perform_statement(cqlruleset.cql_extract_orig(tokens, srcstr)) if parsed: self.printerr('Improper %s command (problem at %r).' % (cmdword, parsed.remainder[0])) else: @@ -861,13 +856,14 @@ class Shell(cmd.Cmd): USE ; Tells cqlsh and the connected Cassandra instance that you will be - working in the given keyspace. All subsequent operations on column - families or indexes will be in the context of this keyspace, unless - otherwise specified, until another USE command is issued or the - connection terminates. + working in the given keyspace. All subsequent operations on tables + or indexes will be in the context of this keyspace, unless otherwise + specified, until another USE command is issued or the connection + terminates. As always, when a keyspace name does not work as a normal identifier or - number, it can be enclosed in quotes and expressed as a string literal. + number, it can be quoted using single quotes (CQL 2) or double quotes + (CQL 3). """ ksname = parsed.get_binding('ksname') if self.perform_statement(parsed.extract_orig()): @@ -876,19 +872,22 @@ class Shell(cmd.Cmd): def do_select(self, parsed): """ SELECT [FIRST n] [REVERSED] - FROM [.] + FROM [.] [USING CONSISTENCY ] [WHERE ] + [ORDER BY [DESC]] [LIMIT m]; - SELECT is used to read one or more records from a Cassandra column - family. It returns a result-set of rows, where each row consists of a - key and a collection of columns corresponding to the query. + SELECT is used to read one or more records from a CQL table. It returns + a set of rows matching the selection criteria specified. + + Note that FIRST and REVERSED are only supported in CQL 2, and ORDER BY + is only supported in CQL 3 and higher. For more information, see one of the following: HELP SELECT_EXPR - HELP SELECT_COLUMNFAMILY + HELP SELECT_TABLE HELP SELECT_WHERE HELP SELECT_LIMIT HELP CONSISTENCYLEVEL @@ -1053,29 +1052,27 @@ class Shell(cmd.Cmd): begidx = readline.get_begidx() + len(prevlines) endidx = readline.get_endidx() + len(prevlines) stuff_to_complete = wholestmt[:begidx] - return cqlhandling.cql_complete(stuff_to_complete, text, cassandra_conn=self, - debug=debug_completion, startsymbol='cqlshCommand') + return cqlruleset.cql_complete(stuff_to_complete, text, cassandra_conn=self, + debug=debug_completion, startsymbol='cqlshCommand') def set_prompt(self, prompt): if self.prompt != '': self.prompt = prompt def cql_protect_name(self, name): - if self.cqlver_atleast(3): - return cql3handling.maybe_cql3_escape_name(name) - else: - return cqlhandling.maybe_cql_escape(name) + return cqlruleset.maybe_escape_name(name) def cql_protect_value(self, value): - return cqlhandling.cql_escape(value) + return cqlruleset.escape_value(value) def cql_unprotect_name(self, namestr): - if namestr is not None: - return cql3handling.cql3_dequote_name(namestr) + if namestr is None: + return + return cqlruleset.dequote_name(namestr) def cql_unprotect_value(self, valstr): if valstr is not None: - return cqlhandling.cql_dequote(valstr) + return cqlruleset.dequote_value(valstr) def print_recreate_keyspace(self, ksdef, out): stratclass = trim_if_present(ksdef.strategy_class, 'org.apache.cassandra.locator.') @@ -1127,23 +1124,23 @@ class Shell(cmd.Cmd): def print_recreate_columnfamily_from_cfdef(self, cfdef, out): cfname = self.cql_protect_name(cfdef.name) - out.write("CREATE COLUMNFAMILY %s (\n" % cfname) + out.write("CREATE TABLE %s (\n" % cfname) alias = self.cql_protect_name(cfdef.key_alias) if cfdef.key_alias else 'KEY' - keytype = cql_typename(cfdef.key_validation_class) + keytype = cqlruleset.cql_typename(cfdef.key_validation_class) out.write(" %s %s PRIMARY KEY" % (alias, keytype)) indexed_columns = [] for col in cfdef.column_metadata: colname = self.cql_protect_name(col.name) - out.write(",\n %s %s" % (colname, cql_typename(col.validation_class))) + out.write(",\n %s %s" % (colname, cqlruleset.cql_typename(col.validation_class))) if col.index_name is not None: indexed_columns.append(col) cf_opts = [] - for (option, thriftname) in cqlhandling.columnfamily_options: + for (option, thriftname) in cqlruleset.columnfamily_options: optval = getattr(cfdef, thriftname or option, None) if optval is None: continue if option in ('comparator', 'default_validation'): - optval = cql_typename(optval) + optval = cqlruleset.cql_typename(optval) else: if option == 'row_cache_provider': optval = trim_if_present(optval, 'org.apache.cassandra.cache.') @@ -1151,7 +1148,7 @@ class Shell(cmd.Cmd): optval = trim_if_present(optval, 'org.apache.cassandra.db.compaction.') optval = self.cql_protect_value(optval) cf_opts.append((option, optval)) - for option, thriftname, _ in cqlhandling.columnfamily_map_options: + for option, thriftname, _ in cqlruleset.columnfamily_map_options: optmap = getattr(cfdef, thriftname or option, {}) for k, v in optmap.items(): if option == 'compression_parameters' and k == 'sstable_compression': @@ -1173,7 +1170,7 @@ class Shell(cmd.Cmd): def print_recreate_columnfamily_from_layout(self, layout, out): cfname = self.cql_protect_name(layout.name) - out.write("CREATE COLUMNFAMILY %s (\n" % cfname) + out.write("CREATE TABLE %s (\n" % cfname) keycol = layout.columns[0] out.write(" %s %s" % (self.cql_protect_name(keycol.name), keycol.cqltype)) if len(layout.key_components) == 1: @@ -1196,7 +1193,7 @@ class Shell(cmd.Cmd): joiner = 'AND' cf_opts = [] - for option in cql3handling.columnfamily_options: + for option in cqlruleset.columnfamily_layout_options: optval = getattr(layout, option, None) if optval is None: continue @@ -1205,7 +1202,7 @@ class Shell(cmd.Cmd): elif option == 'compaction_strategy_class': optval = trim_if_present(optval, 'org.apache.cassandra.db.compaction.') cf_opts.append((option, self.cql_protect_value(optval))) - for option, _ in cql3handling.columnfamily_map_options: + for option, _ in cqlruleset.columnfamily_layout_map_options: optmap = getattr(layout, option, {}) for k, v in optmap.items(): if option == 'compression_parameters' and k == 'sstable_compression': @@ -1228,9 +1225,11 @@ class Shell(cmd.Cmd): self.print_recreate_keyspace(self.get_keyspace(ksname), sys.stdout) print - def describe_columnfamily(self, cfname): + def describe_columnfamily(self, ksname, cfname): + if ksname is None: + ksname = self.current_keyspace print - self.print_recreate_columnfamily(self.current_keyspace, cfname, sys.stdout) + self.print_recreate_columnfamily(ksname, cfname, sys.stdout) print def describe_columnfamilies(self, ksname): @@ -1277,23 +1276,23 @@ class Shell(cmd.Cmd): DESCRIBE KEYSPACE [] Output CQL commands that could be used to recreate the given - keyspace, and the columnfamilies in it. In some cases, as the CQL - interface matures, there will be some metadata about a keyspace that - is not representable with CQL. That metadata will not be shown. + keyspace, and the tables in it. In some cases, as the CQL interface + matures, there will be some metadata about a keyspace that is not + representable with CQL. That metadata will not be shown. The '' argument may be omitted when using a non-system keyspace; in that case, the current keyspace will be described. - DESCRIBE COLUMNFAMILIES + DESCRIBE TABLES - Output the names of all column families in the current keyspace, or - in all keyspaces if there is no current keyspace. + Output the names of all tables in the current keyspace, or in all + keyspaces if there is no current keyspace. - DESCRIBE COLUMNFAMILY + DESCRIBE TABLE - Output CQL commands that could be used to recreate the given - columnfamily. In some cases, as above, there may be columnfamily - metadata which is not representable and which will not be shown. + Output CQL commands that could be used to recreate the given table. + In some cases, as above, there may be table metadata which is not + representable and which will not be shown. DESCRIBE CLUSTER @@ -1318,10 +1317,11 @@ class Shell(cmd.Cmd): self.printerr('Not in any keyspace.') return self.describe_keyspace(ksname) - elif what == 'columnfamily': - cfname = self.cql_unprotect_name(parsed.get_binding('cfname')) - self.describe_columnfamily(cfname) - elif what == 'columnfamilies': + elif what in ('columnfamily', 'table'): + ks = self.cql_unprotect_name(parsed.get_binding('ksname', None)) + cf = self.cql_unprotect_name(parsed.get_binding('cfname')) + self.describe_columnfamily(ks, cf) + elif what in ('columnfamilies', 'tables'): self.describe_columnfamilies(self.current_keyspace) elif what == 'cluster': self.describe_cluster() @@ -1369,62 +1369,62 @@ class Shell(cmd.Cmd): Instruct cqlsh to consider certain column names or values to be of a specified type, even if that type information is not specified in - the columnfamily's metadata. Data will be deserialized according to - the given type, and displayed appropriately when retrieved. + the table's metadata. Data will be deserialized according to the + given type, and displayed appropriately when retrieved. Use thus: - ASSUME [.] NAMES ARE ; + ASSUME [.] NAMES ARE ; - Treat all column names in the given columnfamily as being of the + Treat all column names in the given table as being of the given type. - ASSUME [.] VALUES ARE ; + ASSUME [.] VALUES ARE ; - Treat all column values in the given columnfamily as being of the + Treat all column values in the given table as being of the given type, unless there is more information about the specific column being deserialized. That is, a column-specific ASSUME will take precedence here, as will column-specific metadata in the - columnfamily's definition. + table's definition. - ASSUME [.]() VALUES ARE ; + ASSUME [.]() VALUES ARE ; - Treat all values in the given column in the given columnfamily as + Treat all values in the given column in the given table as being of the specified type. This overrides any other information about the type of a value. - Assign multiple overrides at once for the same columnfamily by + Assign multiple overrides at once for the same table by separating with commas: - ASSUME ks.cf NAMES ARE uuid, VALUES ARE int, (col) VALUES ARE ascii + ASSUME ks.table NAMES ARE uuid, VALUES ARE int, (col) VALUES ARE ascii See HELP TYPES for information on the supported data storage types. """ + ks = self.cql_unprotect_name(parsed.get_binding('ksname', None)) + cf = self.cql_unprotect_name(parsed.get_binding('cfname')) + colname = self.cql_unprotect_name(parsed.get_binding('colname', None)) + params = {} - for paramname in ('ks', 'cf', 'colname'): - val = parsed.get_binding(paramname, None) - params[paramname] = self.cql_unprotect_name(val) for paramname in ('names', 'values', 'colvalues'): val = parsed.get_binding(paramname, None) params[paramname] = self.cql_unprotect_value(val) - if params['ks'] is None: + if ks is None: if self.current_keyspace is None: self.printerr('Error: not in any keyspace.') return - params['ks'] = self.current_keyspace + ks = self.current_keyspace for overridetype in ('names', 'values', 'colvalues'): cqltype = params[overridetype] if cqltype is None: continue try: - validator_class = cqlhandling.find_validator_class(cqltype) + validator_class = cqlruleset.find_validator_class(cqltype) except KeyError: self.printerr('Error: validator type %s not found.' % cqltype) else: - self.add_assumption(params['ks'], params['cf'], params['colname'], - overridetype, validator_class) + self.add_assumption(ks, cf, colname, overridetype, validator_class) def do_source(self, parsed): """ @@ -1441,7 +1441,7 @@ class Shell(cmd.Cmd): That is, the path to the file to be executed must be given inside a string literal. The path is interpreted relative to the current working - directory. The tilde shorthand notation ("~/mydir") is supported for + directory. The tilde shorthand notation ('~/mydir') is supported for referring to $HOME. See also the --file option to cqlsh. @@ -1473,9 +1473,9 @@ class Shell(cmd.Cmd): CAPTURE OFF; CAPTURE; - That is, the path to the file to be executed must be given inside a + That is, the path to the file to be appended to must be given inside a string literal. The path is interpreted relative to the current working - directory. The tilde shorthand notation ("~/mydir") is supported for + directory. The tilde shorthand notation ('~/mydir') is supported for referring to $HOME. Only query result output is captured. Errors and output from cqlsh-only @@ -1562,7 +1562,7 @@ class Shell(cmd.Cmd): def help_types(self): print "\n CQL types recognized by this version of cqlsh:\n" - for t in cqlhandling.cql_types: + for t in cqlruleset.cql_types: print ' ' + t print """ For information on the various recognizable input formats for these @@ -1696,20 +1696,22 @@ class Shell(cmd.Cmd): value is the number of rows from the pre-aggregation resultset. Currently, COUNT is the only function supported by CQL. + + ** [FIRST n] and [REVERSED] are no longer supported in CQL 3. """ - def help_select_columnfamily(self): + def help_select_table(self): print """ - SELECT: Specifying Column Family + SELECT: Specifying Table - SELECT ... FROM [.] ... + SELECT ... FROM [.] ... - The FROM clause is used to specify the Cassandra column family - applicable to a SELECT query. The keyspace in which the column family - exists can optionally be specified along with the column family name, - separated by a dot (.). This will not change the current keyspace of - the session (see HELP USE). + The FROM clause is used to specify the CQL table applicable to a SELECT + query. The keyspace in which the table exists can optionally be + specified along with the table name, separated by a dot (.). This will + not change the current keyspace of the session (see HELP USE). """ + help_select_columnfamily = help_select_table def help_select_where(self): print """ @@ -1758,6 +1760,8 @@ class Shell(cmd.Cmd): * ANY * ONE (default) + * TWO + * THREE * QUORUM * ALL * LOCAL_QUORUM @@ -1769,18 +1773,18 @@ class Shell(cmd.Cmd): def help_insert(self): print """ - INSERT INTO - ( , [, [, ...]] ) - VALUES ( , [, [, ...]] ) + INSERT INTO [.] + ( , [, [, ...]] ) + VALUES ( , [, [, ...]] ) [USING CONSISTENCY [AND TIMESTAMP ] [AND TTL [USING CONSISTENCY + UPDATE [.] + [USING CONSISTENCY [AND TIMESTAMP ] [AND TTL ]] - SET name1 = value1, name2 = value2 WHERE = keyname; + SET name1 = value1, name2 = value2 WHERE = keyval; - An UPDATE is used to write one or more columns to a record in a - Cassandra column family. No results are returned. Key can be given - using the KEY keyword or by an alias set per columnfamily. + An UPDATE is used to write one or more columns to a record in a table. + No results are returned. The record's primary key must be completely + and uniquely specified; that is, if the primary key includes multiple + columns, all must be explicitly given in the WHERE clause. - Statements begin with the UPDATE keyword followed by a Cassandra - column family name. + Statements begin with the UPDATE keyword followed by the name of the + table to be updated. For more information, see one of the following: @@ -1832,10 +1838,11 @@ class Shell(cmd.Cmd): UPDATE ... USING TTL 43200 AND CONSISTENCY LOCAL_QUORUM; defines the optional timestamp for the new column value(s). - It must be an integer. + It must be an integer. Cassandra timestamps are generally specified + using milliseconds since the Unix epoch (1970-01-01 00:00:00 UTC). - defines the optional time to live (TTL) for the new column - value(s). It must be an integer. + defines the optional time to live (TTL) in seconds for the + new column value(s). It must be an integer. """ def help_update_set(self): @@ -1870,17 +1877,18 @@ class Shell(cmd.Cmd): UPDATE ... WHERE = ; UPDATE ... WHERE IN (, , ...); + UPDATE ... WHERE = AND = ; Each update statement requires a precise set of keys to be specified using a WHERE clause. - can be the keyword KEY or the key alias for the column - family. + If the table's primary key consists of multiple columns, an explicit + value must be given for each for the UPDATE statement to make sense. """ def help_delete(self): print """ - DELETE [ [, , ...] FROM + DELETE [ [, , ...] FROM [.] [USING CONSISTENCY [AND TIMESTAMP ]] WHERE = ; @@ -1911,33 +1919,39 @@ class Shell(cmd.Cmd): DELETE ... CONSISTENCY LOCAL_QUORUM AND TIMESTAMP 1318452291034; defines the optional timestamp for the new tombstone - record. It must be an integer. + record. It must be an integer. Cassandra timestamps are generally + specified using milliseconds since the Unix epoch (1970-01-01 00:00:00 + UTC). """ def help_delete_columns(self): print """ DELETE: specifying columns - DELETE col1, 'col2 name', 3 FROM ... + DELETE col1, col2, col3 FROM ... Following the DELETE keyword is an optional comma-delimited list of column name terms. When no column names are given, the remove applies to the entire row(s) matched by the WHERE clause. + + When column names do not parse as valid CQL identifiers, they can be + quoted in single quotes (CQL 2) or double quotes (CQL 3). """ def help_delete_where(self): print """ DELETE: specifying rows - DELETE ... WHERE KEY = 'some_key_value'; - DELETE ... WHERE keyalias IN (key1, key2); + DELETE ... WHERE keycol = 'some_key_value'; + DELETE ... WHERE keycol1 = 'val1' AND keycol2 = 'val2'; + DELETE ... WHERE keycol IN (key1, key2); The WHERE clause is used to determine to which row(s) a DELETE - applies. The first form allows the specification of a single keyname - using the KEY keyword (or the key alias) and the = operator. The - second form allows a list of keyname terms to be specified using the - IN operator and a parenthesized list of comma-delimited keyname - terms. + applies. The first form allows the specification of a precise row + by specifying a particular primary key value (if the primary key has + multiple columns, values for each must be given). The second form + allows a list of key values to be specified using the IN operator + and a parenthesized list of comma-delimited key values. """ def help_create(self): @@ -1946,7 +1960,7 @@ class Shell(cmd.Cmd): one of the following: HELP CREATE_KEYSPACE; - HELP CREATE_COLUMNFAMILY; + HELP CREATE_TABLE; HELP CREATE_INDEX; """ @@ -1974,69 +1988,74 @@ class Shell(cmd.Cmd): "strategy_options:replication_factor=3". """ - def help_create_columnfamily(self): + def help_create_table(self): print """ - CREATE COLUMNFAMILY ( PRIMARY KEY [, - [, ...]] ) + CREATE TABLE ( PRIMARY KEY [, + [, ...]] ) [WITH = [AND = [...]]]; - CREATE COLUMNFAMILY statements create a new column family under the - current keyspace. Valid column family names are strings of alphanumeric - characters and underscores, which begin with a letter. + CREATE TABLE statements create a new CQL table under the current + keyspace. Valid table names are strings of alphanumeric characters and + underscores, which begin with a letter. - Each columnfamily requires at least one column definition and type, - which will correspond to the columnfamily key and key validator. It's - important to note that the key type you use must be compatible with the - partitioner in use. For example, OrderPreservingPartitioner and - CollatingOrderPreservingPartitioner both require UTF-8 keys. If you - use an identifier for the primary key name, instead of the KEY - keyword, a key alias will be set automatically. + Each table requires a primary key, which will correspond to the + underlying columnfamily key and key validator. It's important to + note that the key type you use must be compatible with the partitioner + in use. For example, OrderPreservingPartitioner and + CollatingOrderPreservingPartitioner both require UTF-8 keys. + + In cql3 mode, a table can have multiple columns composing the primary + key (see HELP COMPOSITE_PRIMARY_KEYS). For more information, see one of the following: - HELP CREATE_COLUMNFAMILY_TYPES; - HELP CREATE_COLUMNFAMILY_OPTIONS; + HELP CREATE_TABLE_TYPES; + HELP CREATE_TABLE_OPTIONS; """ + help_create_columnfamily = help_create_table - def help_create_columnfamily_types(self): + def help_create_table_types(self): print """ - CREATE COLUMNFAMILY: Specifying column types + CREATE TABLE: Specifying column types CREATE ... (KEY PRIMARY KEY, othercol ) ... - It is possible to assign columns a type during column family creation. - Columns configured with a type are validated accordingly when a write - occurs, and intelligent CQL drivers and interfaces will be able to - decode the column values correctly when receiving them. Column types - are specified as a parenthesized, comma-separated list of column term - and type pairs. See HELP TYPES; for the list of recognized types. + It is possible to assign columns a type during table creation. Columns + configured with a type are validated accordingly when a write occurs, + and intelligent CQL drivers and interfaces will be able to decode the + column values correctly when receiving them. Column types are specified + as a parenthesized, comma-separated list of column term and type pairs. + See HELP TYPES; for the list of recognized types. """ + help_create_columnfamily_types = help_create_table_types - def help_create_columnfamily_options(self): + def help_create_table_options(self): print """ - CREATE COLUMNFAMILY: Specifying columnfamily options + CREATE TABLE: Specifying columnfamily options - CREATE COLUMNFAMILY blah (...) + CREATE TABLE blah (...) WITH optionname = val AND otheroption = val2; A number of optional keyword arguments can be supplied to control the - configuration of a new column family, such as the size of the - associated row and key caches. Consult your CQL reference for the - complete list of options and possible values. + configuration of a new CQL table, such as the size of the associated + row and key caches for the underlying Cassandra columnfamily. Consult + your CQL reference for the complete list of options and possible + values. """ + help_create_columnfamily_options = help_create_table_options def help_create_index(self): print """ CREATE INDEX [] ON ( ); A CREATE INDEX statement is used to create a new, automatic secondary - index on the given column family, for the named column. A name for the + index on the given CQL table, for the named column. A name for the index itself can be specified before the ON keyword, if desired. A single column name must be specified inside the parentheses. It is not necessary for the column to exist on any current rows (Cassandra is - schemaless), but the column must already have a type (specified during - the CREATE COLUMNFAMILY, or added afterwards with ALTER COLUMNFAMILY. + schema-optional), but the column must already have a type (specified + during the CREATE TABLE, or added afterwards with ALTER TABLE). """ def help_drop(self): @@ -2045,7 +2064,7 @@ class Shell(cmd.Cmd): one of the following: HELP DROP_KEYSPACE; - HELP DROP_COLUMNFAMILY; + HELP DROP_TABLE; HELP DROP_INDEX; """ @@ -2058,13 +2077,15 @@ class Shell(cmd.Cmd): data contained in those column families. """ - def help_drop_columnfamily(self): + def help_drop_table(self): print """ - DROP COLUMNFAMILY ; + DROP TABLE ; - A DROP COLUMNFAMILY statement results in the immediate, irreversible - removal of a column family, including all data contained in it. + A DROP TABLE statement results in the immediate, irreversible + removal of a CQL table and the underlying column family, including all + data contained in it. """ + help_drop_columnfamily = help_drop_table def help_drop_index(self): print """ @@ -2075,10 +2096,10 @@ class Shell(cmd.Cmd): def help_truncate(self): print """ - TRUNCATE ; + TRUNCATE ; - TRUNCATE accepts a single argument for the column family name, and - permanently removes all data from said column family. + TRUNCATE accepts a single argument for the table name, and permanently + removes all data from it. """ def help_begin(self): @@ -2109,15 +2130,15 @@ class Shell(cmd.Cmd): def help_alter(self): print """ - ALTER COLUMNFAMILY ALTER TYPE ; - ALTER COLUMNFAMILY ADD ; - ALTER COLUMNFAMILY DROP ; - ALTER COLUMNFAMILY WITH = [AND = [...]]; + ALTER TABLE ALTER TYPE ; + ALTER TABLE ADD ; + ALTER TABLE DROP ; + ALTER TABLE WITH = [AND = [...]]; - An ALTER statement is used to manipulate column family column - metadata. It allows you to add new columns, drop existing columns, - change the data storage type of existing columns, or change column - family properties. No results are returned. + An ALTER statement is used to manipulate table metadata. It allows you + to add new typed columns, drop existing columns, change the data + storage type of existing columns, or change table properties. + No results are returned. See one of the following for more information: @@ -2129,26 +2150,26 @@ class Shell(cmd.Cmd): def help_alter_alter(self): print """ - ALTER COLUMNFAMILY: altering existing typed columns + ALTER TABLE: altering existing typed columns - ALTER COLUMNFAMILY addamsFamily ALTER lastKnownLocation TYPE uuid; + ALTER TABLE addamsFamily ALTER lastKnownLocation TYPE uuid; - ALTER COLUMNFAMILY ... ALTER changes the expected storage type for a - column. The column must either be the key alias or already have a type - in the column family metadata. The column may or may not already exist - in current rows-- but be aware that no validation of existing data is - done. The bytes stored in values for that column will remain unchanged, - and if existing data is not deserializable according to the new type, - this may cause your CQL driver or interface to report errors. + ALTER TABLE ... ALTER changes the expected storage type for a column. + The column must already have a type in the column family metadata. The + column may or may not already exist in current rows-- but be aware that + no validation of existing data is done. The bytes stored in values for + that column will remain unchanged, and if existing data is not + deserializable according to the new type, this may cause your CQL + driver or interface to report errors. """ def help_alter_add(self): print """ - ALTER COLUMNFAMILY: adding a typed column + ALTER TABLE: adding a typed column - ALTER COLUMNFAMILY addamsFamily ADD gravesite varchar; + ALTER TABLE addamsFamily ADD gravesite varchar; - The ALTER COLUMNFAMILY ... ADD variant adds a typed column to a column + The ALTER TABLE ... ADD variant adds a typed column to a column family. The column must not already have a type in the column family metadata. See the warnings on HELP ALTER_ALTER regarding the lack of validation of existing data; they apply here as well. @@ -2156,11 +2177,11 @@ class Shell(cmd.Cmd): def help_alter_drop(self): print """ - ALTER COLUMNFAMILY: dropping a typed column + ALTER TABLE: dropping a typed column - ALTER COLUMNFAMILY addamsFamily DROP gender; + ALTER TABLE addamsFamily DROP gender; - An ALTER COLUMNFAMILY ... DROP statement removes the type of a column + An ALTER TABLE ... DROP statement removes the type of a column from the column family metadata. Note that this does _not_ remove the column from current rows; it just removes the metadata saying that the bytes stored under that column are expected to be deserializable @@ -2169,15 +2190,15 @@ class Shell(cmd.Cmd): def help_alter_with(self): print """ - ALTER COLUMNFAMILY: changing column family properties + ALTER TABLE: changing column family properties - ALTER COLUMNFAMILY addamsFamily WITH comment = 'Glad to be here!' - AND read_repair_chance = 0.2; + ALTER TABLE addamsFamily WITH comment = 'Glad to be here!' + AND read_repair_chance = 0.2; - An ALTER COLUMNFAMILY ... WITH statement makes adjustments to the - column family properties, as defined when the column family was created - (see HELP CREATE_COLUMNFAMILY_OPTIONS, and your Cassandra documentation - for information about the supported parameter names and values). + An ALTER TABLE ... WITH statement makes adjustments to the + table properties, as defined when the table was created (see + HELP CREATE_TABLE_OPTIONS and your Cassandra documentation for + information about the supported parameter names and values). """ def applycolor(self, text, color=None): @@ -2297,6 +2318,12 @@ def read_options(cmdlineargs, environment): options.color = False options.tty = False + options.cqlversion, cqlvertup = full_cql_version(options.cqlversion) + if cqlvertup[0] < 3: + options.cqlmodule = cqlhandling + else: + options.cqlmodule = cql3handling + try: port = int(port) except ValueError: @@ -2304,7 +2331,17 @@ def read_options(cmdlineargs, environment): return options, hostname, port +def setup_cqlruleset(cqlmodule): + global cqlruleset + cqlruleset = cqlmodule.CqlRuleSet + cqlruleset.append_rules(cqlsh_extra_syntax_rules) + for rulename, termname, func in cqlsh_syntax_completers: + cqlruleset.completer_for(rulename, termname)(func) + cqlruleset.commands_end_with_newline.update(my_commands_ending_with_newline) + def main(options, hostname, port): + setup_cqlruleset(options.cqlmodule) + if os.path.exists(HISTORY) and readline is not None: readline.read_history_file(HISTORY) delims = readline.get_completer_delims() diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index a8828ab65d..8785a65412 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -16,7 +16,7 @@ import re from warnings import warn -from .cqlhandling import cql_typename, cql_escape, cql_dequote +from .cqlhandling import CqlParsingRuleSet, Hint try: import json @@ -30,62 +30,638 @@ class UnexpectedTableStructure(UserWarning): def __str__(self): return 'Unexpected table structure; may not translate correctly to CQL. ' + self.msg -keywords = set(( - 'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with', - 'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any', - 'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set', - 'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create', - 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop', - 'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type', - 'compact', 'storage', 'order', 'by', 'asc', 'desc' -)) +class Cql3ParsingRuleSet(CqlParsingRuleSet): + keywords = set(( + 'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with', + 'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any', + 'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set', + 'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create', + 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop', + 'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type', + 'compact', 'storage', 'order', 'by', 'asc', 'desc', 'clustering', 'token' + )) -columnfamily_options = ( - 'comment', - 'bloom_filter_fp_chance', - 'caching', - 'read_repair_chance', - # 'local_read_repair_chance', -- not yet a valid cql option - 'gc_grace_seconds', - 'min_compaction_threshold', - 'max_compaction_threshold', - 'replicate_on_write', - 'compaction_strategy_class', -) + columnfamily_layout_options = ( + 'comment', + 'bloom_filter_fp_chance', + 'caching', + 'read_repair_chance', + # 'local_read_repair_chance', -- not yet a valid cql option + 'gc_grace_seconds', + 'min_compaction_threshold', + 'max_compaction_threshold', + 'replicate_on_write', + 'compaction_strategy_class', + ) -columnfamily_map_options = ( - ('compaction_strategy_options', - ()), - ('compression_parameters', - ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')), -) + columnfamily_layout_map_options = ( + ('compaction_strategy_options', + ()), + ('compression_parameters', + ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')), + ) -def cql3_escape_value(value): - return cql_escape(value) + @staticmethod + def token_dequote(tok): + if tok[0] == 'unclosedName': + # strip one quote + return tok[1][1:].replace('""', '"') + # cql2 version knows how to do everything else + return CqlParsingRuleSet.token_dequote(tok) -def cql3_escape_name(name): - return '"%s"' % name.replace('"', '""') + @classmethod + def cql3_dequote_value(cls, value): + return cls.cql2_dequote_value(value) -def cql3_dequote_value(value): - return cql_dequote(value) - -def cql3_dequote_name(name): - name = name.strip() - if name == '': + @staticmethod + def cql3_dequote_name(name): + name = name.strip() + if name == '': + return name + if name[0] == '"': + name = name[1:-1].replace('""', '"') return name - if name[0] == '"': - name = name[1:-1].replace('""', '"') - return name -valid_cql3_word_re = re.compile(r'^[a-z][0-9a-z_]*$', re.I) + @classmethod + def cql3_escape_value(cls, value): + return cls.cql2_escape_value(value) + + @staticmethod + def cql3_escape_name(name): + return '"%s"' % name.replace('"', '""') + + valid_cql3_word_re = re.compile(r'^[a-z][0-9a-z_]*$', re.I) + + @classmethod + def is_valid_cql3_name(cls, s): + if s is None or s.lower() in cls.keywords: + return False + return cls.valid_cql3_word_re.match(s) is not None + + @classmethod + def cql3_maybe_escape_name(cls, name): + if cls.is_valid_cql3_name(name): + return name + return cls.cql3_escape_name(name) + + @classmethod + def dequote_any(cls, t): + if t[0] == '"': + return cls.cql3_dequote_name(t) + return CqlParsingRuleSet.dequote_any(t) + + dequote_value = cql3_dequote_value + dequote_name = cql3_dequote_name + escape_value = cql3_escape_value + escape_name = cql3_escape_name + maybe_escape_name = cql3_maybe_escape_name + +CqlRuleSet = Cql3ParsingRuleSet() + +# convenience for remainder of module +shorthands = ('completer_for', 'explain_completion', + 'dequote_value', 'dequote_name', + 'escape_value', 'escape_name', + 'maybe_escape_name', 'cql_typename') + +for shorthand in shorthands: + globals()[shorthand] = getattr(CqlRuleSet, shorthand) + + + +# BEGIN SYNTAX/COMPLETION RULE DEFINITIONS + +syntax_rules = r''' + ::= * + ; + + ::= [statements]= ";" + ; + +# the order of these terminal productions is significant: + ::= /\n/ ; + +JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; + + ::= /'([^']|'')*'/ ; + ::= /"([^"]|"")*"/ ; + ::= /-?[0-9]+\.[0-9]+/ ; + ::= /[0-9]+/ ; + ::= /-?[0-9]+/ ; + ::= /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ ; + ::= /[a-z][a-z0-9_]*/ ; + ::= ":" ; + ::= "*" ; + ::= ";" ; + ::= /[-+=,().]/ ; + ::= /[<>]=?/ ; + + ::= /'([^']|'')*/ ; + ::= /"([^"]|"")*/ ; + ::= /[/][*][^\n]*$/ ; + + ::= + | + | + | + ; + ::= token="TOKEN" "(" ")" + | + ; + ::= + | + | + ; + ::= ; # just an alias + + ::= + | + | + | + ; + + ::= + | + | + | + | + ; + + ::= + | + | + | + | + | + | + ; + + ::= + | + | + | + | + | + | + | + ; + + ::= typename=( | ) ; + + ::= ( ksname= "." )? cfname= ; + + ::= ksname= ; + + ::= + | + | ; + + ::= nocomplete= + ( + | + | + # | -- to get count(*) completion, treat count as reserved + | + | + | + | + | + | ) + ; +''' + +@completer_for('extendedTerm', 'token') +def token_word_completer(ctxt, cass): + return ['TOKEN('] + +@completer_for('storageType', 'typename') +def storagetype_completer(ctxt, cass): + return CqlRuleSet.cql_types + +@completer_for('keyspaceName', 'ksname') +def ks_name_completer(ctxt, cass): + return map(maybe_escape_name, cass.get_keyspace_names()) + +@completer_for('columnFamilyName', 'ksname') +def cf_ks_name_completer(ctxt, cass): + return [maybe_escape_name(ks) + '.' for ks in cass.get_keyspace_names()] + +@completer_for('columnFamilyName', 'cfname') +def cf_name_completer(ctxt, cass): + ks = ctxt.get_binding('ksname', None) + if ks is not None: + ks = dequote_name(ks) + try: + cfnames = cass.get_columnfamily_names(ks) + except Exception: + if ks is None: + return () + raise + return map(maybe_escape_name, cfnames) + +@completer_for('unreservedKeyword', 'nocomplete') +def unreserved_keyword_completer(ctxt, cass): + # we never want to provide completions through this production; + # this is always just to allow use of some keywords as column + # names, CF names, property values, etc. + return () + +def get_cf_layout(ctxt, cass): + ks = ctxt.get_binding('ksname', None) + cf = ctxt.get_binding('cfname') + return cass.get_columnfamily_layout(ks, cf) + +syntax_rules += r''' + ::= "USE" + ; + ::= "SELECT" + "FROM" cf= + ("USING" "CONSISTENCY" )? + ("WHERE" )? + ("ORDER" "BY" ( "," )* )? + ("LIMIT" )? + ; + ::= ("AND" )* + ; + ::= [rel_lhs]= ("=" | "<" | ">" | "<=" | ">=") + | token="TOKEN" "(" rel_tokname= ")" ("=" | "<" | ">" | "<=" | ">=") + | [rel_lhs]= "IN" "(" ( "," )* ")" + ; + ::= colname= ("," colname=)* + | "*" + | "COUNT" "(" star=( "*" | "1" ) ")" + ; + ::= [ordercol]= ( "ASC" | "DESC" )? + ; +''' + +@completer_for('orderByClause', 'ordercol') +def select_order_column_completer(ctxt, cass): + prev_order_cols = ctxt.get_binding('ordercol', ()) + keyname = ctxt.get_binding('keyname') + if keyname is None: + keyname = ctxt.get_binding('rel_lhs', ()) + if not keyname: + return [Hint("Can't ORDER BY here: need to specify partition key in WHERE clause")] + layout = get_cf_layout(ctxt, cass) + order_by_candidates = layout.key_components[1:] # can't order by first part of key + if len(order_by_candidates) > len(prev_order_cols): + return [maybe_escape_name(order_by_candidates[len(prev_order_cols)])] + return [Hint('No more orderable columns here.')] + +@completer_for('relation', 'token') +def relation_token_word_completer(ctxt, cass): + return ['TOKEN('] + +@completer_for('relation', 'rel_tokname') +def relation_token_subject_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + return [layout.key_components[0]] + +@completer_for('relation', 'rel_lhs') +def select_relation_lhs_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + filterable = set(layout.key_components[:2]) + already_filtered_on = ctxt.get_binding('rel_lhs') + for num in range(1, len(layout.key_components)): + if layout.key_components[num - 1] in already_filtered_on: + filterable.add(layout.key_components[num]) + else: + break + for cd in layout.columns: + if cd.index_name is not None: + filterable.add(cd.name) + return map(maybe_escape_name, filterable) + +@completer_for('selectClause', 'star') +def select_count_star_completer(ctxt, cass): + return ['*'] + +explain_completion('selectClause', 'colname') + +syntax_rules += r''' + ::= "INSERT" "INTO" cf= + "(" keyname= "," + [colname]= ( "," [colname]= )* ")" + "VALUES" "(" "," ( "," )* ")" + ( "USING" [insertopt]= + ( "AND" [insertopt]= )* )? + ; + ::= "CONSISTENCY" + | "TIMESTAMP" + | "TTL" + ; +''' + +@completer_for('insertStatement', 'keyname') +def insert_keyname_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + return [layout.key_components[0]] + +explain_completion('insertStatement', 'colname') + +@completer_for('insertStatement', 'insertopt') +def insert_option_completer(ctxt, cass): + opts = set('CONSISTENCY TIMESTAMP TTL'.split()) + for opt in ctxt.get_binding('insertopt', ()): + opts.discard(opt.split()[0]) + return opts + +syntax_rules += r''' + ::= "UPDATE" cf= + ( "USING" [updateopt]= + ( "AND" [updateopt]= )* )? + "SET" ( "," )* + "WHERE" + ; + ::= updatecol= "=" update_rhs= + ( counterop=( "+" | "-" ) )? + ; +''' + +@completer_for('updateStatement', 'updateopt') +def insert_option_completer(ctxt, cass): + opts = set('CONSISTENCY TIMESTAMP TTL'.split()) + for opt in ctxt.get_binding('updateopt', ()): + opts.discard(opt.split()[0]) + return opts + +@completer_for('assignment', 'updatecol') +def update_col_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + return map(maybe_escape_name, [cm.name for cm in layout.columns]) + +@completer_for('assignment', 'update_rhs') +def update_countername_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + curcol = dequote_name(ctxt.get_binding('updatecol', '')) + return [maybe_escape_name(curcol)] if layout.is_counter_col(curcol) else [Hint('')] + +@completer_for('assignment', 'counterop') +def update_counterop_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + curcol = dequote_name(ctxt.get_binding('updatecol', '')) + return ['+', '-'] if layout.is_counter_col(curcol) else [] + +syntax_rules += r''' + ::= "DELETE" ( [delcol]= ( "," [delcol]= )* )? + "FROM" cf= + ( "USING" [delopt]= ( "AND" [delopt]= )* )? + "WHERE" + ; + ::= "CONSISTENCY" + | "TIMESTAMP" + ; +''' + +@completer_for('deleteStatement', 'delopt') +def delete_opt_completer(ctxt, cass): + opts = set('CONSISTENCY TIMESTAMP'.split()) + for opt in ctxt.get_binding('delopt', ()): + opts.discard(opt.split()[0]) + return opts + +explain_completion('deleteStatement', 'delcol', '') + +syntax_rules += r''' + ::= "BEGIN" "BATCH" + ( "USING" [batchopt]= + ( "AND" [batchopt]= )* )? + [batchstmt]= ";" + ( [batchstmt]= ";" )* + "APPLY" "BATCH" + ; + ::= + | + | + ; +''' + +@completer_for('batchStatement', 'batchopt') +def batch_opt_completer(ctxt, cass): + opts = set('CONSISTENCY TIMESTAMP'.split()) + for opt in ctxt.get_binding('batchopt', ()): + opts.discard(opt.split()[0]) + return opts + +syntax_rules += r''' + ::= "TRUNCATE" cf= + ; +''' + +syntax_rules += r''' + ::= "CREATE" "KEYSPACE" ksname= + "WITH" [optname]= "=" [optval]= + ( "AND" [optname]= "=" [optval]= )* + ; + ::= ( ":" ( | ) )? + ; + ::= + | + | + ; +''' + +explain_completion('createKeyspaceStatement', 'ksname', '') + +@completer_for('createKeyspaceStatement', 'optname') +def create_ks_opt_completer(ctxt, cass): + exist_opts = ctxt.get_binding('optname', ()) + try: + stratopt = exist_opts.index('strategy_class') + except ValueError: + return ['strategy_class ='] + vals = ctxt.get_binding('optval') + stratclass = dequote_value(vals[stratopt]) + if stratclass in ('SimpleStrategy', 'OldNetworkTopologyStrategy'): + return ['strategy_options:replication_factor ='] + return [Hint('')] + +@completer_for('createKeyspaceStatement', 'optval') +def create_ks_optval_completer(ctxt, cass): + exist_opts = ctxt.get_binding('optname', (None,)) + if exist_opts[-1] == 'strategy_class': + return map(escape_value, CqlRuleSet.replication_strategies) + return [Hint('')] + +syntax_rules += r''' + ::= "CREATE" ( "COLUMNFAMILY" | "TABLE" ) + ( ks= "." )? cf= + "(" ( | ) ")" + ( "WITH" [cfopt]= "=" [optval]= + ( "AND" [cfopt]= "=" [optval]= )* )? + ; + + ::= keyalias= "PRIMARY" "KEY" + ( "," colname= )* + ; + + ::= [newcolname]= + "," [newcolname]= + ( "," [newcolname]= )* + "," "PRIMARY" k="KEY" p="(" [pkey]= + ( c="," [pkey]= )* ")" + ; + + ::= cfoptname= ( cfoptsep=":" cfsubopt=( | ) )? + ; + + ::= + | + | + | + ; +''' + +explain_completion('createColumnFamilyStatement', 'cf', '') +explain_completion('singleKeyCfSpec', 'keyalias', '') +explain_completion('singleKeyCfSpec', 'colname', '') +explain_completion('compositeKeyCfSpec', 'newcolname', '') + +@completer_for('compositeKeyCfSpec', 'pkey') +def create_cf_composite_key_declaration(ctxt, cass): + cols_declared = ctxt.get_binding('newcolname') + pieces_already = ctxt.get_binding('pkey', ()) + while cols_declared[0] in pieces_already: + cols_declared = cols_declared[1:] + if len(cols_declared) < 2: + return () + return [maybe_escape_name(cols_declared[0])] + +@completer_for('compositeKeyCfSpec', 'k') +def create_cf_composite_primary_key_keyword_completer(ctxt, cass): + return ['KEY ('] + +@completer_for('compositeKeyCfSpec', 'p') +def create_cf_composite_primary_key_paren_completer(ctxt, cass): + return ['('] + +@completer_for('compositeKeyCfSpec', 'c') +def create_cf_composite_primary_key_comma_completer(ctxt, cass): + cols_declared = ctxt.get_binding('newcolname') + pieces_already = ctxt.get_binding('pkey', ()) + if len(pieces_already) >= len(cols_declared) - 1: + return () + return [','] + +@completer_for('cfOptionName', 'cfoptname') +def create_cf_option_completer(ctxt, cass): + return list(CqlRuleSet.columnfamily_layout_options) + \ + [c[0] + ':' for c in CqlRuleSet.columnfamily_map_options] + +@completer_for('cfOptionName', 'cfoptsep') +def create_cf_suboption_separator(ctxt, cass): + opt = ctxt.get_binding('cfoptname') + if any(opt == c[0] for c in CqlRuleSet.columnfamily_map_options): + return [':'] + return () + +@completer_for('cfOptionName', 'cfsubopt') +def create_cf_suboption_completer(ctxt, cass): + opt = ctxt.get_binding('cfoptname') + if opt == 'compaction_strategy_options': + # try to determine the strategy class in use + prevopts = ctxt.get_binding('cfopt', ()) + prevvals = ctxt.get_binding('optval', ()) + for prevopt, prevval in zip(prevopts, prevvals): + if prevopt == 'compaction_strategy_class': + csc = dequote_value(prevval) + break + else: + layout = get_cf_layout(ctxt, cass) + try: + csc = layout.compaction_strategy + except Exception: + csc = '' + csc = csc.split('.')[-1] + if csc == 'SizeTieredCompactionStrategy': + return ['min_sstable_size'] + elif csc == 'LeveledCompactionStrategy': + return ['sstable_size_in_mb'] + for optname, _, subopts in CqlRuleSet.columnfamily_map_options: + if opt == optname: + return subopts + return () + +def create_cf_option_val_completer(ctxt, cass): + exist_opts = ctxt.get_binding('cfopt') + this_opt = exist_opts[-1] + if this_opt == 'compression_parameters:sstable_compression': + return map(escape_value, CqlRuleSet.available_compression_classes) + if this_opt == 'compaction_strategy_class': + return map(escape_value, CqlRuleSet.available_compaction_classes) + if any(this_opt == opt[0] for opt in CqlRuleSet.obsolete_cf_options): + return ["''"] + if this_opt in ('comparator', 'default_validation'): + return CqlRuleSet.cql_types + if this_opt in ('read_repair_chance', 'bloom_filter_fp_chance'): + return [Hint('')] + if this_opt == 'replicate_on_write': + return [Hint('')] + if this_opt in ('min_compaction_threshold', 'max_compaction_threshold', 'gc_grace_seconds'): + return [Hint('')] + return [Hint('')] + +completer_for('createColumnFamilyStatement', 'optval') \ + (create_cf_option_val_completer) + +syntax_rules += r''' + ::= "CREATE" "INDEX" indexname=? "ON" + cf= "(" col= ")" + ; +''' + +explain_completion('createIndexStatement', 'indexname', '') + +@completer_for('createIndexStatement', 'col') +def create_index_col_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + colnames = [cd.name for cd in layout.columns if cd.index_name is None] + return map(maybe_escape_name, colnames) + +syntax_rules += r''' + ::= "DROP" "KEYSPACE" ksname= + ; + + ::= "DROP" ( "COLUMNFAMILY" | "TABLE" ) cf= + ; + + ::= "DROP" "INDEX" indexname= + ; +''' + +@completer_for('dropIndexStatement', 'indexname') +def drop_index_completer(ctxt, cass): + return map(maybe_escape_name, cass.get_index_names()) + +syntax_rules += r''' + ::= "ALTER" ( "COLUMNFAMILY" | "TABLE" ) cf= + + ; + ::= "ALTER" existcol= "TYPE" + | "ADD" newcol= + | "DROP" existcol= + | "WITH" [cfopt]= "=" [optval]= + ( "AND" [cfopt]= "=" [optval]= )* + ; +''' + +@completer_for('alterInstructions', 'existcol') +def alter_table_col_completer(ctxt, cass): + layout = get_cf_layout(ctxt, cass) + cols = [md.name for md in layout.columns] + return map(maybe_escape_name, cols) + +explain_completion('alterInstructions', 'newcol', '') + +completer_for('alterInstructions', 'optval') \ + (create_cf_option_val_completer) + +# END SYNTAX/COMPLETION RULE DEFINITIONS + +CqlRuleSet.append_rules(syntax_rules) -def is_valid_cql3_name(s): - return valid_cql3_word_re.match(s) is not None and s not in keywords -def maybe_cql3_escape_name(name): - if is_valid_cql3_name(name): - return name - return cql3_escape_name(name) class CqlColumnDef: index_name = None @@ -93,6 +669,7 @@ class CqlColumnDef: def __init__(self, name, cqltype): self.name = name self.cqltype = cqltype + assert name is not None @classmethod def from_layout(cls, layout): @@ -128,6 +705,8 @@ class CqlTableDef: setattr(cf, attr, json.loads(getattr(cf, attr))) except AttributeError: pass + if cf.key_alias is None: + cf.key_alias = 'KEY' cf.key_components = [cf.key_alias.decode('ascii')] + list(cf.column_aliases) cf.key_validator = cql_typename(cf.key_validator) cf.default_validator = cql_typename(cf.default_validator) @@ -212,10 +791,15 @@ class CqlTableDef: self.compact_storage = True value_cols = [self.column_class(self.value_alias, self.default_validator)] + subtypes = subtypes[:len(self.key_components)] keycols = map(self.column_class, self.key_components, subtypes) normal_cols = map(self.column_class.from_layout, self.coldefs) self.columns = keycols + value_cols + normal_cols + def is_counter_col(self, colname): + col_info = [cm for cm in self.columns if cm.name == colname] + return bool(col_info and col_info[0].cqltype == 'counter') + def __str__(self): return '<%s %s.%s>' % (self.__class__.__name__, self.keyspace, self.name) __repr__ = __str__ diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py index 09a2980d45..3866f3c426 100644 --- a/pylib/cqlshlib/cqlhandling.py +++ b/pylib/cqlshlib/cqlhandling.py @@ -18,191 +18,435 @@ # i.e., stuff that's not necessarily cqlsh-specific import re -from . import pylexotron -from itertools import izip +import traceback +from . import pylexotron, util Hint = pylexotron.Hint -keywords = set(( - 'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with', - 'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any', - 'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set', - 'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create', - 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop', - 'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type', - 'first', 'reversed' -)) +class CqlParsingRuleSet(pylexotron.ParsingRuleSet): + keywords = set(( + 'select', 'from', 'where', 'and', 'key', 'insert', 'update', 'with', + 'limit', 'using', 'consistency', 'one', 'quorum', 'all', 'any', + 'local_quorum', 'each_quorum', 'two', 'three', 'use', 'count', 'set', + 'begin', 'apply', 'batch', 'truncate', 'delete', 'in', 'create', + 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop', + 'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type', + 'first', 'reversed' + )) -columnfamily_options = ( - # (CQL option name, Thrift option name (or None if same)) - ('comment', None), - ('comparator', 'comparator_type'), - ('read_repair_chance', None), - ('gc_grace_seconds', None), - ('default_validation', 'default_validation_class'), - ('min_compaction_threshold', None), - ('max_compaction_threshold', None), - ('replicate_on_write', None), - ('compaction_strategy_class', 'compaction_strategy'), -) + columnfamily_options = ( + # (CQL option name, Thrift option name (or None if same)) + ('comment', None), + ('comparator', 'comparator_type'), + ('read_repair_chance', None), + ('gc_grace_seconds', None), + ('default_validation', 'default_validation_class'), + ('min_compaction_threshold', None), + ('max_compaction_threshold', None), + ('replicate_on_write', None), + ('compaction_strategy_class', 'compaction_strategy'), + ) -obsolete_cf_options = ( - ('key_cache_size', None), - ('row_cache_size', None), - ('row_cache_save_period_in_seconds', None), - ('key_cache_save_period_in_seconds', None), - ('memtable_throughput_in_mb', None), - ('memtable_operations_in_millions', None), - ('memtable_flush_after_mins', None), - ('row_cache_provider', None), -) + obsolete_cf_options = ( + ('key_cache_size', None), + ('row_cache_size', None), + ('row_cache_save_period_in_seconds', None), + ('key_cache_save_period_in_seconds', None), + ('memtable_throughput_in_mb', None), + ('memtable_operations_in_millions', None), + ('memtable_flush_after_mins', None), + ('row_cache_provider', None), + ) -all_columnfamily_options = columnfamily_options + obsolete_cf_options + all_columnfamily_options = columnfamily_options + obsolete_cf_options -columnfamily_map_options = ( - ('compaction_strategy_options', None, - ()), - ('compression_parameters', 'compression_options', - ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')), -) + columnfamily_map_options = ( + ('compaction_strategy_options', None, + ()), + ('compression_parameters', 'compression_options', + ('sstable_compression', 'chunk_length_kb', 'crc_check_chance')), + ) -available_compression_classes = ( - 'DeflateCompressor', - 'SnappyCompressor', -) + available_compression_classes = ( + 'DeflateCompressor', + 'SnappyCompressor', + ) -available_compaction_classes = ( - 'LeveledCompactionStrategy', - 'SizeTieredCompactionStrategy' -) + available_compaction_classes = ( + 'LeveledCompactionStrategy', + 'SizeTieredCompactionStrategy' + ) -cql_type_to_apache_class = { - 'ascii': 'AsciiType', - 'bigint': 'LongType', - 'blob': 'BytesType', - 'boolean': 'BooleanType', - 'counter': 'CounterColumnType', - 'decimal': 'DecimalType', - 'double': 'DoubleType', - 'float': 'FloatType', - 'int': 'Int32Type', - 'text': 'UTF8Type', - 'timestamp': 'DateType', - 'uuid': 'UUIDType', - 'varchar': 'UTF8Type', - 'varint': 'IntegerType' -} + cql_type_to_apache_class = { + 'ascii': 'AsciiType', + 'bigint': 'LongType', + 'blob': 'BytesType', + 'boolean': 'BooleanType', + 'counter': 'CounterColumnType', + 'decimal': 'DecimalType', + 'double': 'DoubleType', + 'float': 'FloatType', + 'int': 'Int32Type', + 'text': 'UTF8Type', + 'timestamp': 'DateType', + 'uuid': 'UUIDType', + 'varchar': 'UTF8Type', + 'varint': 'IntegerType' + } -apache_class_to_cql_type = dict((v,k) for (k,v) in cql_type_to_apache_class.items()) + apache_class_to_cql_type = dict((v,k) for (k,v) in cql_type_to_apache_class.items()) -cql_types = sorted(cql_type_to_apache_class.keys()) + cql_types = sorted(cql_type_to_apache_class.keys()) -def find_validator_class(cqlname): - return cql_type_to_apache_class[cqlname] + replication_strategies = ( + 'SimpleStrategy', + 'OldNetworkTopologyStrategy', + 'NetworkTopologyStrategy' + ) -replication_strategies = ( - 'SimpleStrategy', - 'OldNetworkTopologyStrategy', - 'NetworkTopologyStrategy' -) + consistency_levels = ( + 'ANY', + 'ONE', + 'TWO', + 'THREE', + 'QUORUM', + 'ALL', + 'LOCAL_QUORUM', + 'EACH_QUORUM' + ) -consistency_levels = ( - 'ANY', - 'ONE', - 'QUORUM', - 'ALL', - 'LOCAL_QUORUM', - 'EACH_QUORUM' -) + # if a term matches this, it shouldn't need to be quoted to be valid cql + valid_cql_word_re = re.compile(r"^(?:[a-z][a-z0-9_]*|-?[0-9][0-9.]*)$", re.I) -# if a term matches this, it shouldn't need to be quoted to be valid cql -valid_cql_word_re = re.compile(r"^(?:[a-z][a-z0-9_]*|-?[0-9][0-9.]*)$", re.I) + def __init__(self, *args, **kwargs): + pylexotron.ParsingRuleSet.__init__(self, *args, **kwargs) -def is_valid_cql_word(s): - return valid_cql_word_re.match(s) is not None and s not in keywords + # note: commands_end_with_newline may be extended by callers. + self.commands_end_with_newline = set() + self.set_keywords_as_syntax() -def tokenize_cql(cql_text): - return CqlLexotron.scan(cql_text)[0] + def completer_for(self, rulename, symname): + def registrator(f): + def completerwrapper(ctxt): + cass = ctxt.get_binding('cassandra_conn', None) + if cass is None: + return () + return f(ctxt, cass) + completerwrapper.func_name = 'completerwrapper_on_' + f.func_name + self.register_completer(completerwrapper, rulename, symname) + return completerwrapper + return registrator -def cql_extract_orig(toklist, srcstr): - # low end of span for first token, to high end of span for last token - return srcstr[toklist[0][2][0]:toklist[-1][2][1]] + def explain_completion(self, rulename, symname, explanation=None): + if explanation is None: + explanation = '<%s>' % (symname,) + @self.completer_for(rulename, symname) + def explainer(ctxt, cass): + return [Hint(explanation)] + return explainer -# note: commands_end_with_newline may be extended by an importing module. -commands_end_with_newline = set() + def set_keywords_as_syntax(self): + syntax = [] + for k in self.keywords: + syntax.append(' ::= "%s" ;' % (k.upper(), k)) + self.append_rules('\n'.join(syntax)) -def token_dequote(tok): - if tok[0] == 'stringLiteral': - # strip quotes - return tok[1][1:-1].replace("''", "'") - if tok[0] == 'unclosedString': - # strip one quote - return tok[1][1:].replace("''", "'") - if tok[0] == 'unclosedComment': - return '' - return tok[1] + def cql_massage_tokens(self, toklist): + curstmt = [] + output = [] -def cql_dequote(cqlword): - cqlword = cqlword.strip() - if cqlword == '': + term_on_nl = False + + for t in toklist: + if t[0] == 'endline': + if term_on_nl: + t = ('endtoken',) + t[1:] + else: + # don't put any 'endline' tokens in output + continue + curstmt.append(t) + if t[0] == 'endtoken': + term_on_nl = False + output.extend(curstmt) + curstmt = [] + else: + if len(curstmt) == 1: + # first token in statement; command word + cmd = t[1].lower() + term_on_nl = bool(cmd in self.commands_end_with_newline) + + output.extend(curstmt) + return output + + def cql_parse(self, text, startsymbol='Start'): + tokens = self.lex(text) + tokens = self.cql_massage_tokens(tokens) + return self.parse(startsymbol, tokens, init_bindings={'*SRC*': text}) + + def cql_whole_parse_tokens(self, toklist, srcstr=None, startsymbol='Start'): + return self.whole_match(startsymbol, toklist, srcstr=srcstr) + + def cql_split_statements(self, text): + tokens = self.lex(text) + tokens = self.cql_massage_tokens(tokens) + stmts = util.split_list(tokens, lambda t: t[0] == 'endtoken') + output = [] + in_batch = False + for stmt in stmts: + if in_batch: + output[-1].extend(stmt) + else: + output.append(stmt) + if len(stmt) > 1 \ + and stmt[0][0] == 'identifier' and stmt[1][0] == 'identifier' \ + and stmt[1][1].lower() == 'batch': + if stmt[0][1].lower() == 'begin': + in_batch = True + elif stmt[0][1].lower() == 'apply': + in_batch = False + return output, in_batch + + def cql_complete_single(self, text, partial, init_bindings={}, ignore_case=True, + startsymbol='Start'): + tokens = (self.cql_split_statements(text)[0] or [[]])[-1] + bindings = init_bindings.copy() + + # handle some different completion scenarios- in particular, completing + # inside a string literal + prefix = None + dequoter = util.identity + if tokens: + if tokens[-1][0] == 'unclosedString': + prefix = self.token_dequote(tokens[-1]) + tokens = tokens[:-1] + partial = prefix + partial + dequoter = self.dequote_value + requoter = self.escape_value + elif tokens[-1][0] == 'unclosedName': + prefix = self.token_dequote(tokens[-1]) + tokens = tokens[:-1] + partial = prefix + partial + dequoter = self.dequote_name + requoter = self.escape_name + elif tokens[-1][0] == 'unclosedComment': + return [] + bindings['partial'] = partial + bindings['*SRC*'] = text + + # find completions for the position + completions = self.complete(startsymbol, tokens, bindings) + + hints, strcompletes = util.list_bifilter(pylexotron.is_hint, completions) + + # it's possible to get a newline token from completion; of course, we + # don't want to actually have that be a candidate, we just want to hint + if '\n' in strcompletes: + strcompletes.remove('\n') + if partial == '': + hints.append(Hint('')) + + # find matches with the partial word under completion + if ignore_case: + partial = partial.lower() + f = lambda s: s and dequoter(s).lower().startswith(partial) + else: + f = lambda s: s and dequoter(s).startswith(partial) + candidates = filter(f, strcompletes) + + if prefix is not None: + # dequote, re-escape, strip quotes: gets us the right quoted text + # for completion. the opening quote is already there on the command + # line and not part of the word under completion, and readline + # fills in the closing quote for us. + candidates = [requoter(dequoter(c))[len(prefix)+1:-1] for c in candidates] + + # the above process can result in an empty string; this doesn't help for + # completions + candidates = filter(None, candidates) + + # prefix a space when desirable for pleasant cql formatting + if tokens: + newcandidates = [] + for c in candidates: + if self.want_space_between(tokens[-1], c) \ + and prefix is None \ + and not text[-1].isspace() \ + and not c[0].isspace(): + c = ' ' + c + newcandidates.append(c) + candidates = newcandidates + + # append a space for single, complete identifiers + if len(candidates) == 1 and candidates[0][-1].isalnum(): + candidates[0] += ' ' + return candidates, hints + + @staticmethod + def want_space_between(tok, following): + if following in (',', ')', ':'): + return False + if tok[0] == 'op' and tok[1] in (',', ')', '='): + return True + if tok[0] == 'stringLiteral' and following[0] != ';': + return True + if tok[0] == 'star' and following[0] != ')': + return True + if tok[0] == 'endtoken': + return True + if tok[1][-1].isalnum() and following[0] != ',': + return True + return False + + def cql_complete(self, text, partial, cassandra_conn=None, ignore_case=True, debug=False, + startsymbol='Start'): + init_bindings = {'cassandra_conn': cassandra_conn} + if debug: + init_bindings['*DEBUG*'] = True + + completions, hints = self.cql_complete_single(text, partial, init_bindings, + startsymbol=startsymbol) + + if hints: + hints = [h.text for h in hints] + hints.append('') + + if len(completions) == 1 and len(hints) == 0: + c = completions[0] + if debug: + print "** Got one completion: %r. Checking for further matches...\n" % (c,) + if not c.isspace(): + new_c = self.cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol) + completions = [new_c] + if debug: + print "** New list of completions: %r" % (completions,) + + return hints + completions + + def cql_complete_multiple(self, text, first, init_bindings, startsymbol='Start'): + debug = init_bindings.get('*DEBUG*', False) + try: + completions, hints = self.cql_complete_single(text + first, '', init_bindings, + startsymbol=startsymbol) + except Exception: + if debug: + print "** completion expansion had a problem:" + traceback.print_exc() + return first + if hints: + if not first[-1].isspace(): + first += ' ' + if debug: + print "** completion expansion found hints: %r" % (hints,) + return first + if len(completions) == 1 and completions[0] != '': + if debug: + print "** Got another completion: %r." % (completions[0],) + if completions[0][0] in (',', ')', ':') and first[-1] == ' ': + first = first[:-1] + first += completions[0] + else: + common_prefix = util.find_common_prefix(completions) + if common_prefix == '': + return first + if common_prefix[0] in (',', ')', ':') and first[-1] == ' ': + first = first[:-1] + if debug: + print "** Got a partial completion: %r." % (common_prefix,) + first += common_prefix + if debug: + print "** New total completion: %r. Checking for further matches...\n" % (first,) + return self.cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol) + + @classmethod + def cql_typename(cls, classname): + fq_classname = 'org.apache.cassandra.db.marshal.' + if classname.startswith(fq_classname): + classname = classname[len(fq_classname):] + try: + return cls.apache_class_to_cql_type[classname] + except KeyError: + return cls.escape_value(classname) + + @classmethod + def find_validator_class(cls, cqlname): + return cls.cql_type_to_apache_class[cqlname] + + @classmethod + def is_valid_cql_word(cls, s): + return cls.valid_cql_word_re.match(s) is not None and s.lower() not in cls.keywords + + @staticmethod + def cql_extract_orig(toklist, srcstr): + # low end of span for first token, to high end of span for last token + return srcstr[toklist[0][2][0]:toklist[-1][2][1]] + + @staticmethod + def token_dequote(tok): + if tok[0] == 'stringLiteral': + # strip quotes + return tok[1][1:-1].replace("''", "'") + if tok[0] == 'unclosedString': + # strip one quote + return tok[1][1:].replace("''", "'") + if tok[0] == 'unclosedComment': + return '' + return tok[1] + + @staticmethod + def token_is_word(tok): + return tok[0] == 'identifier' + + @classmethod + def cql2_maybe_escape_name(cls, name): + if cls.is_valid_cql_word(name): + return name + return cls.cql2_escape_name(name) + + # XXX: this doesn't really belong here. + @classmethod + def is_counter_col(cls, cfdef, colname): + col_info = [cm for cm in cfdef.column_metadata if cm.name == colname] + return bool(col_info and cls.cql_typename(col_info[0].validation_class) == 'counter') + + @staticmethod + def cql2_dequote_value(cqlword): + cqlword = cqlword.strip() + if cqlword == '': + return cqlword + if cqlword[0] == "'": + cqlword = cqlword[1:-1].replace("''", "'") return cqlword - if cqlword[0] == "'": - cqlword = cqlword[1:-1].replace("''", "'") - return cqlword -def token_is_word(tok): - return tok[0] == 'identifier' + @staticmethod + def cql2_escape_value(value): + if value is None: + return 'NULL' # this totally won't work + if isinstance(value, bool): + value = str(value).lower() + elif isinstance(value, float): + return '%f' % value + elif isinstance(value, int): + return str(value) + return "'%s'" % value.replace("'", "''") -def cql_escape(value): - if value is None: - return 'NULL' # this totally won't work - if isinstance(value, bool): - value = str(value).lower() - elif isinstance(value, float): - return '%f' % value - elif isinstance(value, int): - return str(value) - return "'%s'" % value.replace("'", "''") + # use _name for keyspace, cf, and column names, and _value otherwise. + # also use the cql2_ prefix when dealing with cql2, or leave it off to + # get whatever behavior is default for this CqlParsingRuleSet. + cql2_dequote_name = dequote_name = dequote_value = cql2_dequote_value + cql2_escape_name = escape_name = escape_value = cql2_escape_value + maybe_escape_name = cql2_maybe_escape_name + dequote_any = cql2_dequote_value -def maybe_cql_escape(value): - if is_valid_cql_word(value): - return value - return cql_escape(value) +CqlRuleSet = CqlParsingRuleSet() -def cql_typename(classname): - fq_classname = 'org.apache.cassandra.db.marshal.' - if classname.startswith(fq_classname): - classname = classname[len(fq_classname):] - try: - return apache_class_to_cql_type[classname] - except KeyError: - return cql_escape(classname) +# convenience for remainder of module +shorthands = ('completer_for', 'explain_completion', + 'dequote_value', 'dequote_name', + 'escape_value', 'escape_name', + 'maybe_escape_name') -special_completers = [] - -def completer_for(rulename, symname): - def registrator(f): - def completerwrapper(ctxt): - cass = ctxt.get_binding('cassandra_conn', None) - if cass is None: - return () - return f(ctxt, cass) - completerwrapper.func_name = 'completerwrapper_on_' + f.func_name - special_completers.append((rulename, symname, completerwrapper)) - return completerwrapper - return registrator - -def explain_completion(rulename, symname, explanation=None): - if explanation is None: - explanation = '<%s>' % (symname,) - @completer_for(rulename, symname) - def explainer(ctxt, cass): - return [Hint(explanation)] - return explainer - -def is_counter_col(cfdef, colname): - col_info = [cm for cm in cfdef.column_metadata if cm.name == colname] - return bool(col_info and cql_typename(col_info[0].validation_class) == 'counter') +for shorthand in shorthands: + globals()[shorthand] = getattr(CqlRuleSet, shorthand) @@ -221,7 +465,6 @@ syntax_rules = r''' JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; ::= /'([^']|'')*'/ ; - ::= /"([^"]|"")*"/ ; ::= /-?[0-9]+\.[0-9]+/ ; ::= /-?[0-9]+/ ; ::= /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ ; @@ -243,7 +486,6 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; ; ::= | - | | ; ::= @@ -279,29 +521,52 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; ::= cl= ; - ::= typename=( | ); + ::= typename=( | ) ; + + ::= ksname= ; + + ::= ( ksname= "." )? cfname= ; ''' @completer_for('consistencylevel', 'cl') def cl_completer(ctxt, cass): - return consistency_levels + return CqlRuleSet.consistency_levels @completer_for('storageType', 'typename') def storagetype_completer(ctxt, cass): - return cql_types + return CqlRuleSet.cql_types + +@completer_for('keyspaceName', 'ksname') +def ks_name_completer(ctxt, cass): + return map(maybe_escape_name, cass.get_keyspace_names()) + +@completer_for('columnFamilyName', 'ksname') +def cf_ks_name_completer(ctxt, cass): + return [maybe_escape_name(ks) + '.' for ks in cass.get_keyspace_names()] + +@completer_for('columnFamilyName', 'cfname') +def cf_name_completer(ctxt, cass): + ks = ctxt.get_binding('ksname', None) + if ks is not None: + ks = dequote_name(ks) + try: + cfnames = cass.get_columnfamily_names(ks) + except Exception: + if ks is None: + return () + raise + return map(maybe_escape_name, cfnames) + +def get_cfdef(ctxt, cass): + ks = ctxt.get_binding('ksname', None) + cf = ctxt.get_binding('cfname') + return cass.get_columnfamily(cf, ksname=ks) syntax_rules += r''' - ::= "USE" ksname= + ::= "USE" ksname= ; -''' - -@completer_for('useStatement', 'ksname') -def use_ks_completer(ctxt, cass): - return map(maybe_cql_escape, cass.get_keyspace_names()) - -syntax_rules += r''' ::= "SELECT" - "FROM" ( selectks= "." )? selectsource= + "FROM" cf= ("USING" "CONSISTENCY" )? ("WHERE" )? ("LIMIT" )? @@ -318,39 +583,15 @@ syntax_rules += r''' ; ''' -@completer_for('selectStatement', 'selectsource') -def select_source_completer(ctxt, cass): - ks = ctxt.get_binding('selectks', None) - if ks is not None: - ks = cql_dequote(ks) - try: - cfnames = cass.get_columnfamily_names(ks) - except Exception: - if ks is None: - return () - raise - return map(maybe_cql_escape, cfnames) - -@completer_for('selectStatement', 'selectks') -def select_keyspace_completer(ctxt, cass): - return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()] - @completer_for('selectWhereClause', 'keyname') def select_where_keyname_completer(ctxt, cass): - ksname = ctxt.get_binding('selectks') - if ksname is not None: - ksname = cql_dequote(ksname) - selectsource = cql_dequote(ctxt.get_binding('selectsource')) - cfdef = cass.get_columnfamily(selectsource, ksname=ksname) + cfdef = get_cfdef(ctxt, cass) return [cfdef.key_alias if cfdef.key_alias is not None else 'KEY'] @completer_for('relation', 'rel_lhs') def select_relation_lhs_completer(ctxt, cass): - ksname = ctxt.get_binding('selectks') - if ksname is not None: - ksname = cql_dequote(ksname) - selectsource = cql_dequote(ctxt.get_binding('selectsource')) - return map(maybe_cql_escape, cass.filterable_column_names(selectsource, ksname=ksname)) + cfdef = get_cfdef(ctxt, cass) + return map(maybe_escape_name, cass.filterable_column_names(cfdef)) @completer_for('whatToSelect', 'countparens') def select_count_parens_completer(ctxt, cass): @@ -361,7 +602,7 @@ explain_completion('whatToSelect', 'rangestart', '') explain_completion('whatToSelect', 'rangeend', '') syntax_rules += r''' - ::= "INSERT" "INTO" ( insertks= "." )? insertcf= + ::= "INSERT" "INTO" cf= "(" keyname= "," [colname]= ( "," [colname]= )* ")" "VALUES" "(" "," ( "," )* ")" @@ -374,27 +615,9 @@ syntax_rules += r''' ; ''' -@completer_for('insertStatement', 'insertks') -def insert_ks_completer(ctxt, cass): - return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()] - -@completer_for('insertStatement', 'insertcf') -def insert_cf_completer(ctxt, cass): - ks = ctxt.get_binding('insertks', None) - if ks is not None: - ks = cql_dequote(ks) - try: - cfnames = cass.get_columnfamily_names(ks) - except Exception: - if ks is None: - return () - raise - return map(maybe_cql_escape, cfnames) - @completer_for('insertStatement', 'keyname') def insert_keyname_completer(ctxt, cass): - insertcf = ctxt.get_binding('insertcf') - cfdef = cass.get_columnfamily(cql_dequote(insertcf)) + cfdef = get_cfdef(ctxt, cass) return [cfdef.key_alias if cfdef.key_alias is not None else 'KEY'] explain_completion('insertStatement', 'colname') @@ -407,7 +630,7 @@ def insert_option_completer(ctxt, cass): return opts syntax_rules += r''' - ::= "UPDATE" ( updateks= "." )? updatecf= + ::= "UPDATE" cf= ( "USING" [updateopt]= ( "AND" [updateopt]= )* )? "SET" ( "," )* @@ -421,23 +644,6 @@ syntax_rules += r''' ; ''' -@completer_for('updateStatement', 'updateks') -def update_cf_completer(ctxt, cass): - return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()] - -@completer_for('updateStatement', 'updatecf') -def update_cf_completer(ctxt, cass): - ks = ctxt.get_binding('updateks', None) - if ks is not None: - ks = cql_dequote(ks) - try: - cfnames = cass.get_columnfamily_names(ks) - except Exception: - if ks is None: - return () - raise - return map(maybe_cql_escape, cfnames) - @completer_for('updateStatement', 'updateopt') def insert_option_completer(ctxt, cass): opts = set('CONSISTENCY TIMESTAMP TTL'.split()) @@ -447,41 +653,41 @@ def insert_option_completer(ctxt, cass): @completer_for('assignment', 'updatecol') def update_col_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) - colnames = map(maybe_cql_escape, [cm.name for cm in cfdef.column_metadata]) + cfdef = get_cfdef(ctxt, cass) + colnames = map(maybe_escape_name, [cm.name for cm in cfdef.column_metadata]) return colnames + [Hint('')] @completer_for('assignment', 'update_rhs') def update_countername_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) - curcol = cql_dequote(ctxt.get_binding('updatecol', '')) - return [maybe_cql_escape(curcol)] if is_counter_col(cfdef, curcol) else [Hint('')] + cfdef = get_cfdef(ctxt, cass) + curcol = dequote_name(ctxt.get_binding('updatecol', '')) + return [maybe_escape_name(curcol)] if CqlRuleSet.is_counter_col(cfdef, curcol) else [Hint('')] @completer_for('assignment', 'counterop') def update_counterop_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) - curcol = cql_dequote(ctxt.get_binding('updatecol', '')) - return ['+', '-'] if is_counter_col(cfdef, curcol) else [] + cfdef = get_cfdef(ctxt, cass) + curcol = dequote_name(ctxt.get_binding('updatecol', '')) + return ['+', '-'] if CqlRuleSet.is_counter_col(cfdef, curcol) else [] @completer_for('updateWhereClause', 'updatefiltercol') def update_filtercol_completer(ctxt, cass): - cfname = cql_dequote(ctxt.get_binding('cf')) - return map(maybe_cql_escape, cass.filterable_column_names(cfname)) + cfdef = get_cfdef(ctxt, cass) + return map(maybe_escape_name, cass.filterable_column_names(cfdef)) @completer_for('updateWhereClause', 'updatefilterkey') def update_filterkey_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) + cfdef = get_cfdef(ctxt, cass) return [cfdef.key_alias if cfdef.key_alias is not None else 'KEY'] @completer_for('updateWhereClause', 'filter_in') def update_filter_in_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) + cfdef = get_cfdef(ctxt, cass) fk = ctxt.get_binding('updatefilterkey') return ['IN'] if fk in ('KEY', cfdef.key_alias) else [] syntax_rules += r''' ::= "DELETE" ( [delcol]= ( "," [delcol]= )* )? - "FROM" ( deleteks= "." )? deletecf= + "FROM" cf= ( "USING" [delopt]= ( "AND" [delopt]= )* )? "WHERE" ; @@ -490,23 +696,6 @@ syntax_rules += r''' ; ''' -@completer_for('deleteStatement', 'deleteks') -def update_cf_completer(ctxt, cass): - return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()] - -@completer_for('deleteStatement', 'deletecf') -def delete_cf_completer(ctxt, cass): - ks = ctxt.get_binding('deleteks', None) - if ks is not None: - ks = cql_dequote(ks) - try: - cfnames = cass.get_columnfamily_names(ks) - except Exception: - if ks is None: - return () - raise - return map(maybe_cql_escape, cfnames) - @completer_for('deleteStatement', 'delopt') def delete_opt_completer(ctxt, cass): opts = set('CONSISTENCY TIMESTAMP'.split()) @@ -538,27 +727,10 @@ def batch_opt_completer(ctxt, cass): return opts syntax_rules += r''' - ::= "TRUNCATE" ( truncateks= "." )? truncatecf= + ::= "TRUNCATE" cf= ; ''' -@completer_for('truncateStatement', 'truncateks') -def update_cf_completer(ctxt, cass): - return [maybe_cql_escape(ks) + '.' for ks in cass.get_keyspace_names()] - -@completer_for('truncateStatement', 'truncatecf') -def truncate_cf_completer(ctxt, cass): - ks = ctxt.get_binding('truncateks', None) - if ks is not None: - ks = cql_dequote(ks) - try: - cfnames = cass.get_columnfamily_names(ks) - except Exception: - if ks is None: - return () - raise - return map(maybe_cql_escape, cfnames) - syntax_rules += r''' ::= "CREATE" "KEYSPACE" ksname= "WITH" [optname]= "=" [optval]= @@ -582,7 +754,7 @@ def create_ks_opt_completer(ctxt, cass): except ValueError: return ['strategy_class ='] vals = ctxt.get_binding('optval') - stratclass = cql_dequote(vals[stratopt]) + stratclass = dequote_value(vals[stratopt]) if stratclass in ('SimpleStrategy', 'OldNetworkTopologyStrategy'): return ['strategy_options:replication_factor ='] return [Hint('')] @@ -591,11 +763,11 @@ def create_ks_opt_completer(ctxt, cass): def create_ks_optval_completer(ctxt, cass): exist_opts = ctxt.get_binding('optname', (None,)) if exist_opts[-1] == 'strategy_class': - return map(cql_escape, replication_strategies) + return map(escape_value, CqlRuleSet.replication_strategies) return [Hint('')] syntax_rules += r''' - ::= "CREATE" "COLUMNFAMILY" cf= + ::= "CREATE" ( "COLUMNFAMILY" | "TABLE" ) cf= "(" keyalias= "PRIMARY" "KEY" ( "," colname= )* ")" ( "WITH" [cfopt]= "=" [optval]= @@ -612,19 +784,19 @@ syntax_rules += r''' ; ''' -explain_completion('createColumnFamilyStatement', 'keyalias', '') -explain_completion('createColumnFamilyStatement', 'cf', '') +explain_completion('createColumnFamilyStatement', 'keyalias', '') +explain_completion('createColumnFamilyStatement', 'cf', '') explain_completion('createColumnFamilyStatement', 'colname', '') @completer_for('cfOptionName', 'cfoptname') def create_cf_option_completer(ctxt, cass): - return [c[0] for c in columnfamily_options] + \ - [c[0] + ':' for c in columnfamily_map_options] + return [c[0] for c in CqlRuleSet.columnfamily_options] + \ + [c[0] + ':' for c in CqlRuleSet.columnfamily_map_options] @completer_for('cfOptionName', 'cfoptsep') def create_cf_suboption_separator(ctxt, cass): opt = ctxt.get_binding('cfoptname') - if any(opt == c[0] for c in columnfamily_map_options): + if any(opt == c[0] for c in CqlRuleSet.columnfamily_map_options): return [':'] return () @@ -637,7 +809,7 @@ def create_cf_suboption_completer(ctxt, cass): prevvals = ctxt.get_binding('optval', ()) for prevopt, prevval in zip(prevopts, prevvals): if prevopt == 'compaction_strategy_class': - csc = cql_dequote(prevval) + csc = dequote_value(prevval) break else: cf = ctxt.get_binding('cf') @@ -650,7 +822,7 @@ def create_cf_suboption_completer(ctxt, cass): return ['min_sstable_size'] elif csc == 'LeveledCompactionStrategy': return ['sstable_size_in_mb'] - for optname, _, subopts in columnfamily_map_options: + for optname, _, subopts in CqlRuleSet.columnfamily_map_options: if opt == optname: return subopts return () @@ -659,13 +831,13 @@ def create_cf_option_val_completer(ctxt, cass): exist_opts = ctxt.get_binding('cfopt') this_opt = exist_opts[-1] if this_opt == 'compression_parameters:sstable_compression': - return map(cql_escape, available_compression_classes) + return map(escape_value, CqlRuleSet.available_compression_classes) if this_opt == 'compaction_strategy_class': - return map(cql_escape, available_compaction_classes) - if any(this_opt == opt[0] for opt in obsolete_cf_options): + return map(escape_value, CqlRuleSet.available_compaction_classes) + if any(this_opt == opt[0] for opt in CqlRuleSet.obsolete_cf_options): return ["''"] if this_opt in ('comparator', 'default_validation'): - return cql_types + return CqlRuleSet.cql_types if this_opt == 'read_repair_chance': return [Hint('')] if this_opt == 'replicate_on_write': @@ -687,43 +859,43 @@ explain_completion('createIndexStatement', 'indexname', '') @completer_for('createIndexStatement', 'cf') def create_index_cf_completer(ctxt, cass): - return map(maybe_cql_escape, cass.get_columnfamily_names()) + return map(maybe_escape_name, cass.get_columnfamily_names()) @completer_for('createIndexStatement', 'col') def create_index_col_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) + cfdef = cass.get_columnfamily(dequote_name(ctxt.get_binding('cf'))) colnames = [md.name for md in cfdef.column_metadata if md.index_name is None] - return map(maybe_cql_escape, colnames) + return map(maybe_escape_name, colnames) syntax_rules += r''' - ::= "DROP" "KEYSPACE" ksname= + ::= "DROP" "KEYSPACE" ksname= ; ''' @completer_for('dropKeyspaceStatement', 'ksname') def drop_ks_completer(ctxt, cass): - return map(maybe_cql_escape, cass.get_keyspace_names()) + return map(maybe_escape_name, cass.get_keyspace_names()) syntax_rules += r''' - ::= "DROP" "COLUMNFAMILY" cf= + ::= "DROP" ( "COLUMNFAMILY" | "TABLE" ) cf= ; ''' @completer_for('dropColumnFamilyStatement', 'cf') def drop_cf_completer(ctxt, cass): - return map(maybe_cql_escape, cass.get_columnfamily_names()) + return map(maybe_escape_name, cass.get_columnfamily_names()) syntax_rules += r''' ::= "DROP" "INDEX" indexname= ; ''' -@completer_for('dropIndexStatement', 'cf') +@completer_for('dropIndexStatement', 'indexname') def drop_index_completer(ctxt, cass): - return map(maybe_cql_escape, cass.get_index_names()) + return map(maybe_escape_name, cass.get_index_names()) syntax_rules += r''' - ::= "ALTER" "COLUMNFAMILY" cf= + ::= "ALTER" ( "COLUMNFAMILY" | "TABLE" ) cf= ; ::= "ALTER" existcol= "TYPE" | "ADD" newcol= @@ -735,15 +907,15 @@ syntax_rules += r''' @completer_for('alterTableStatement', 'cf') def alter_table_cf_completer(ctxt, cass): - return map(maybe_cql_escape, cass.get_columnfamily_names()) + return map(maybe_escape_name, cass.get_columnfamily_names()) @completer_for('alterInstructions', 'existcol') def alter_table_col_completer(ctxt, cass): - cfdef = cass.get_columnfamily(cql_dequote(ctxt.get_binding('cf'))) + cfdef = cass.get_columnfamily(dequote_name(ctxt.get_binding('cf'))) cols = [md.name for md in cfdef.column_metadata] if cfdef.key_alias is not None: cols.append(cfdef.key_alias) - return map(maybe_cql_escape, cols) + return map(maybe_escape_name, cols) explain_completion('alterInstructions', 'newcol', '') @@ -752,211 +924,4 @@ completer_for('alterInstructions', 'optval') \ # END SYNTAX/COMPLETION RULE DEFINITIONS - - -CqlRuleSet = pylexotron.ParsingRuleSet.from_rule_defs(syntax_rules) -for rulename, symname, compf in special_completers: - CqlRuleSet.register_completer(compf, rulename, symname) - -def cql_add_completer(rulename, symname): - registrator = completer_for(rulename, symname) - def more_registration(f): - f = registrator(f) - CqlRuleSet.register_completer(f, rulename, symname) - return f - return more_registration - -def cql_parse(text, startsymbol='Start'): - tokens = CqlRuleSet.lex(text) - tokens = cql_massage_tokens(tokens) - return CqlRuleSet.parse(startsymbol, tokens, init_bindings={'*SRC*': text}) - -def cql_whole_parse_tokens(toklist, srcstr=None, startsymbol='Start'): - return CqlRuleSet.whole_match(startsymbol, toklist, srcstr=srcstr) - -def cql_massage_tokens(toklist): - curstmt = [] - output = [] - - term_on_nl = False - - for t in toklist: - if t[0] == 'endline': - if term_on_nl: - t = ('endtoken',) + t[1:] - else: - # don't put any 'endline' tokens in output - continue - curstmt.append(t) - if t[0] == 'endtoken': - term_on_nl = False - output.extend(curstmt) - curstmt = [] - else: - if len(curstmt) == 1: - # first token in statement; command word - cmd = t[1].lower() - term_on_nl = bool(cmd in commands_end_with_newline) - - output.extend(curstmt) - return output - -def split_list(items, pred): - thisresult = [] - results = [thisresult] - for i in items: - thisresult.append(i) - if pred(i): - thisresult = [] - results.append(thisresult) - return results - -def cql_split_statements(text): - tokens = CqlRuleSet.lex(text) - tokens = cql_massage_tokens(tokens) - stmts = split_list(tokens, lambda t: t[0] == 'endtoken') - output = [] - in_batch = False - for stmt in stmts: - if in_batch: - output[-1].extend(stmt) - else: - output.append(stmt) - if len(stmt) > 1 \ - and stmt[0][0] == 'identifier' and stmt[1][0] == 'identifier' \ - and stmt[1][1].lower() == 'batch': - if stmt[0][1].lower() == 'begin': - in_batch = True - elif stmt[0][1].lower() == 'apply': - in_batch = False - return output, in_batch - -def want_space_between(tok, following): - if tok[0] == 'op' and tok[1] in (',', ')', '='): - return True - if tok[0] == 'stringLiteral' and following[0] != ';': - return True - if tok[0] == 'star' and following[0] != ')': - return True - if tok[0] == 'endtoken': - return True - if tok[1][-1].isalnum() and following[0] != ',': - return True - return False - -def find_common_prefix(strs): - common = [] - for cgroup in izip(*strs): - if all(x == cgroup[0] for x in cgroup[1:]): - common.append(cgroup[0]) - else: - break - return ''.join(common) - -def list_bifilter(pred, iterable): - yes_s = [] - no_s = [] - for i in iterable: - (yes_s if pred(i) else no_s).append(i) - return yes_s, no_s - -def cql_complete_single(text, partial, init_bindings={}, ignore_case=True, startsymbol='Start'): - tokens = (cql_split_statements(text)[0] or [[]])[-1] - bindings = init_bindings.copy() - - # handle some different completion scenarios- in particular, completing - # inside a string literal - prefix = None - if tokens and tokens[-1][0] == 'unclosedString': - prefix = token_dequote(tokens[-1]) - tokens = tokens[:-1] - partial = prefix + partial - if tokens and tokens[-1][0] == 'unclosedComment': - return [] - bindings['partial'] = partial - bindings['*SRC*'] = text - - # find completions for the position - completions = CqlRuleSet.complete(startsymbol, tokens, bindings) - - hints, strcompletes = list_bifilter(pylexotron.is_hint, completions) - - # it's possible to get a newline token from completion; of course, we - # don't want to actually have that be a candidate, we just want to hint - if '\n' in strcompletes: - strcompletes.remove('\n') - if partial == '': - hints.append(Hint('')) - - # find matches with the partial word under completion - if ignore_case: - partial = partial.lower() - f = lambda s: s and cql_dequote(s).lower().startswith(partial) - else: - f = lambda s: s and cql_dequote(s).startswith(partial) - candidates = filter(f, strcompletes) - - if prefix is not None: - # dequote, re-escape, strip quotes: gets us the right quoted text - # for completion. the opening quote is already there on the command - # line and not part of the word under completion, and readline - # fills in the closing quote for us. - candidates = [cql_escape(cql_dequote(c))[len(prefix)+1:-1] for c in candidates] - - # the above process can result in an empty string; this doesn't help for - # completions - candidates = filter(None, candidates) - - # prefix a space when desirable for pleasant cql formatting - if tokens: - newcandidates = [] - for c in candidates: - if want_space_between(tokens[-1], c) \ - and prefix is None \ - and not text[-1].isspace() \ - and not c[0].isspace(): - c = ' ' + c - newcandidates.append(c) - candidates = newcandidates - - return candidates, hints - -def cql_complete(text, partial, cassandra_conn=None, ignore_case=True, debug=False, - startsymbol='Start'): - init_bindings = {'cassandra_conn': cassandra_conn} - if debug: - init_bindings['*DEBUG*'] = True - - completions, hints = cql_complete_single(text, partial, init_bindings, startsymbol=startsymbol) - - if hints: - hints = [h.text for h in hints] - hints.append('') - - if len(completions) == 1 and len(hints) == 0: - c = completions[0] - if not c.isspace(): - new_c = cql_complete_multiple(text, c, init_bindings, startsymbol=startsymbol) - completions = [new_c] - - return hints + completions - -def cql_complete_multiple(text, first, init_bindings, startsymbol='Start'): - try: - completions, hints = cql_complete_single(text + first, '', init_bindings, - startsymbol=startsymbol) - except Exception: - return first - if hints: - if not first[-1].isspace(): - first += ' ' - return first - if len(completions) == 1 and completions[0] != '': - first += completions[0] - else: - common_prefix = find_common_prefix(completions) - if common_prefix != '': - first += common_prefix - else: - return first - return cql_complete_multiple(text, first, init_bindings, startsymbol=startsymbol) +CqlRuleSet.append_rules(syntax_rules) diff --git a/pylib/cqlshlib/pylexotron.py b/pylib/cqlshlib/pylexotron.py index 7482abac3c..0534c87042 100644 --- a/pylib/cqlshlib/pylexotron.py +++ b/pylib/cqlshlib/pylexotron.py @@ -269,6 +269,28 @@ class case_match(text_match): def pattern(self): return re.escape(self.arg) +class word_match(text_match): + def pattern(self): + return r'\b' + text_match.pattern(self) + r'\b' + +class case_word_match(case_match): + def pattern(self): + return r'\b' + case_match.pattern(self) + r'\b' + +class terminal_type_matcher(matcher): + def __init__(self, tokentype, submatcher): + matcher.__init__(self, tokentype) + self.tokentype = tokentype + self.submatcher = submatcher + + def match(self, ctxt, completions): + if ctxt.remainder: + if ctxt.remainder[0][0] == self.tokentype: + return [ctxt.with_match(1)] + elif completions is not None: + self.submatcher.match(ctxt, completions) + return [] + class ParsingRuleSet: RuleSpecScanner = SaferScanner([ (r'::=', lambda s,t: t), @@ -309,9 +331,10 @@ class ParsingRuleSet: raise ValueError('Unexpected token %r; expected "::="' % (assign,)) name = t[1] production = cls.read_rule_tokens_until(';', tokeniter) - rules[name] = production if isinstance(production, terminal_matcher): terminals.append((name, production)) + production = terminal_type_matcher(name, production) + rules[name] = production else: raise ValueError('Unexpected token %r; expected name' % (t,)) return rules, terminals @@ -345,7 +368,10 @@ class ParsingRuleSet: if t[0] == 'reference': t = rule_reference(t[1]) elif t[0] == 'litstring': - t = text_match(t[1]) + if t[1][1].isalnum() or t[1][1] == '_': + t = word_match(t[1]) + else: + t = text_match(t[1]) elif t[0] == 'regex': t = regex_rule(t[1]) elif t[0] == 'named_collector': diff --git a/pylib/cqlshlib/util.py b/pylib/cqlshlib/util.py new file mode 100644 index 0000000000..ea0fbf45c3 --- /dev/null +++ b/pylib/cqlshlib/util.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from itertools import izip + +def split_list(items, pred): + """ + Split up a list (or other iterable) on the elements which satisfy the + given predicate 'pred'. Elements for which 'pred' returns true start a new + sublist for subsequent elements, which will accumulate in the new sublist + until the next satisfying element. + + >>> split_list([0, 1, 2, 5, 99, 8], lambda n: (n % 2) == 0) + [[0], [1, 2], [5, 99, 8], []] + """ + + thisresult = [] + results = [thisresult] + for i in items: + thisresult.append(i) + if pred(i): + thisresult = [] + results.append(thisresult) + return results + +def find_common_prefix(strs): + """ + Given a list (iterable) of strings, return the longest common prefix. + + >>> find_common_prefix(['abracadabra', 'abracadero', 'abranch']) + 'abra' + >>> find_common_prefix(['abracadabra', 'abracadero', 'mt. fuji']) + '' + """ + + common = [] + for cgroup in izip(*strs): + if all(x == cgroup[0] for x in cgroup[1:]): + common.append(cgroup[0]) + else: + break + return ''.join(common) + +def list_bifilter(pred, iterable): + """ + Filter an iterable into two output lists: the first containing all + elements of the iterable for which 'pred' returns true, and the second + containing all others. Order of the elements is otherwise retained. + + >>> list_bifilter(lambda x: isinstance(x, int), (4, 'bingo', 1.2, 6, True)) + ([4, 6], ['bingo', 1.2, True]) + """ + + yes_s = [] + no_s = [] + for i in iterable: + (yes_s if pred(i) else no_s).append(i) + return yes_s, no_s + +def identity(x): + return x