User types support for cqlsh.

Patch by Mikhail Stepura; reviewed by Aleksey Yeschenko for CASSANDRA-6305
This commit is contained in:
Mikhail Stepura 2014-01-06 18:51:50 -08:00
parent 4b08be49a3
commit e5199eeac1
4 changed files with 279 additions and 10 deletions

101
bin/cqlsh
View File

@ -32,7 +32,7 @@ exit 1
from __future__ import with_statement
description = "CQL Shell for Apache Cassandra"
version = "4.1.1"
version = "4.1.2"
from StringIO import StringIO
from itertools import groupby
@ -112,7 +112,7 @@ cqlshlibdir = os.path.join(CASSANDRA_PATH, 'pylib')
if os.path.isdir(cqlshlibdir):
sys.path.insert(0, cqlshlibdir)
from cqlshlib import cqlhandling, cql3handling, pylexotron
from cqlshlib import cqlhandling, cql3handling, pylexotron, usertypes
from cqlshlib.displaying import (RED, BLUE, ANSI_RESET, COLUMN_NAME_COLORS,
FormattedValue, colorme)
from cqlshlib.formatting import format_by_type
@ -230,7 +230,9 @@ cqlsh_extra_syntax_rules = r'''
| ( "COLUMNFAMILY" | "TABLE" ) cf=<columnFamilyName>
| ( "COLUMNFAMILIES" | "TABLES" )
| "FULL"? "SCHEMA"
| "CLUSTER" )
| "CLUSTER"
| "TYPES"
| "TYPE" ut=<userTypeName>)
;
<consistencyCommand> ::= "CONSISTENCY" ( level=<consistencyLevel> )?
@ -366,6 +368,9 @@ class ColumnFamilyNotFound(Exception):
class VersionNotSupported(Exception):
pass
class UserTypeNotFound(Exception):
pass
class DecodeError(Exception):
verb = 'decode'
@ -455,6 +460,7 @@ class Shell(cmd.Cmd):
shunted_query_out = None
csv_dialect_defaults = dict(delimiter=',', doublequote=False,
escapechar='\\', quotechar='"')
utypes_meta = None
def __init__(self, hostname, port, transport_factory, color=False,
username=None, password=None, encoding=None, stdin=None, tty=True,
@ -514,6 +520,9 @@ class Shell(cmd.Cmd):
self.show_line_nums = True
self.stdin = stdin
self.query_out = sys.stdout
self.refresh_utypes_meta()
#UserTypes will use it to get theirs field names
usertypes.get_field_names = lambda k, t: self.utypes_meta.get_field_names(k, t)
def set_expanded_cql_version(self, ver):
ver, vertuple = full_cql_version(ver)
@ -626,6 +635,21 @@ class Shell(cmd.Cmd):
layout = self.get_columnfamily_layout(ksname, cfname)
return [col.name for col in layout.columns]
def get_usertype_names(self, ksname=None):
if ksname is None:
ksname = self.current_keyspace
return self.utypes_meta.get_usertypes_names(ksname)
def get_usertype_layout(self, ksname, typename):
if ksname is None:
ksname = self.current_keyspace
layout = self.utypes_meta.get_fields_with_types(ksname, typename)
if not layout:
raise UserTypeNotFound("User type %r not found" % typename)
return layout
# ===== thrift-dependent parts =====
def get_cluster_name(self):
@ -693,6 +717,15 @@ class Shell(cmd.Cmd):
cols = self.fetchdict_all()
return cql3handling.CqlTableDef.from_layout(layout, cols)
def build_usertypes_meta(self):
ut_q = """select * from system.schema_usertypes"""
self.cursor.execute(ut_q, consistency_level='ONE')
data = self.fetchdict_all()
if not data:
return None
return cql3handling.UserTypesMeta.from_layout(data)
# ===== end cql3-dependent parts =====
def reset_statement(self):
@ -755,6 +788,15 @@ class Shell(cmd.Cmd):
return
yield newline
def refresh_utypes_meta(self):
try:
self.utypes_meta = self.build_usertypes_meta()
except Exception, e:
if self.debug:
import traceback
traceback.print_exc()
def cmdloop(self):
"""
Adapted from cmd.Cmd's version, because there is literally no way with
@ -926,6 +968,8 @@ class Shell(cmd.Cmd):
self.writeresult("")
self.print_static_result(self.cursor)
self.flush_output()
if self.could_change_schema(statement):
self.refresh_utypes_meta()
return True
def get_nametype(self, cursor, num):
@ -1082,6 +1126,14 @@ class Shell(cmd.Cmd):
out.write(" AND durable_writes = 'false'")
out.write(';\n')
#print usertypes first
uts = self.get_usertype_names(ksname)
if uts:
out.write('\nUSE %s;\n' % ksname)
for ut in uts:
out.write('\n')
self.print_recreate_usertype(ksdef.name, ut, out)
cfs = self.get_columnfamily_names(ksname)
if cfs:
out.write('\nUSE %s;\n' % ksname)
@ -1090,6 +1142,16 @@ class Shell(cmd.Cmd):
# yes, cf might be looked up again. oh well.
self.print_recreate_columnfamily(ksdef.name, cf, out)
def print_recreate_usertype(self, ksname, utname, out):
layout = self.get_usertype_layout(ksname, utname)
out.write("CREATE TYPE %s (\n" % utname)
for (index,(tname,ttype)) in enumerate(layout, 1):
out.write(" %s %s" % (tname, ttype))
if index < len(layout):
out.write(",\n")
else:
out.write("\n); \n")
def print_recreate_columnfamily(self, ksname, cfname, out):
"""
Output CQL commands which should be pasteable back into a CQL session
@ -1099,6 +1161,7 @@ class Shell(cmd.Cmd):
"""
layout = self.get_columnfamily_layout(ksname, cfname)
cfname = self.cql_protect_name(layout.name)
out.write("CREATE TABLE %s (\n" % cfname)
for col in layout.columns:
@ -1238,6 +1301,26 @@ class Shell(cmd.Cmd):
cmd.Cmd.columnize(self, self.get_columnfamily_names(ksname))
print
def describe_usertypes(self, ksname):
print
if ksname is None:
for k in self.get_keyspaces():
name = self.cql_protect_name(k.name)
print 'Keyspace %s' % (name,)
print '---------%s' % ('-' * len(name))
cmd.Cmd.columnize(self, self.get_usertype_names(k.name))
print
else:
cmd.Cmd.columnize(self, self.get_usertype_names(ksname))
print
def describe_usertype(self, ksname, typename):
if ksname is None:
ksname = self.current_keyspace
print
self.print_recreate_usertype(ksname, typename, sys.stdout)
print
def describe_cluster(self):
print '\nCluster: %s' % self.get_cluster_name()
p = trim_if_present(self.get_partitioner(), 'org.apache.cassandra.dht.')
@ -1323,6 +1406,12 @@ class Shell(cmd.Cmd):
self.describe_columnfamily(ks, cf)
elif what in ('columnfamilies', 'tables'):
self.describe_columnfamilies(self.current_keyspace)
elif what == 'types':
self.describe_usertypes(self.current_keyspace)
elif what == 'type':
ks = self.cql_unprotect_name(parsed.get_binding('ksname', None))
ut = self.cql_unprotect_name(parsed.get_binding('utname'))
self.describe_usertype(ks, ut)
elif what == 'cluster':
self.describe_cluster()
elif what == 'schema':
@ -1850,6 +1939,12 @@ class Shell(cmd.Cmd):
text = '%s:%d:%s' % (self.stdin.name, self.lineno, text)
self.writeresult(text, color, newline=newline, out=sys.stderr)
@staticmethod
def could_change_schema(statement):
import re
return re.match(r"(alter|create|drop)\s+type", statement.lower())
class ErrorHandlingSchemaDecoder(cql.decoders.SchemaDecoder):
def name_decode_error(self, err, namebytes, expectedtype):
return DecodeError(namebytes, err, expectedtype)

View File

@ -245,11 +245,14 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
<schemaChangeStatement> ::= <createKeyspaceStatement>
| <createColumnFamilyStatement>
| <createIndexStatement>
| <createUserTypeStatement>
| <dropKeyspaceStatement>
| <dropColumnFamilyStatement>
| <dropIndexStatement>
| <dropUserTypeStatement>
| <alterTableStatement>
| <alterKeyspaceStatement>
| <alterUserTypeStatement>
;
<authenticationStatement> ::= <createUserStatement>
@ -266,15 +269,19 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ;
# timestamp is included here, since it's also a keyword
<simpleStorageType> ::= typename=( <identifier> | <stringLiteral> | <K_TIMESTAMP> ) ;
<storageType> ::= <simpleStorageType> | <collectionType> ;
<userType> ::= utname=<cfOrKsName> ;
<collectionType> ::= "map" "<" <simpleStorageType> "," <simpleStorageType> ">"
| "list" "<" <simpleStorageType> ">"
| "set" "<" <simpleStorageType> ">"
<storageType> ::= <simpleStorageType> | <collectionType> | <userType> ;
<collectionType> ::= "map" "<" <simpleStorageType> "," ( <simpleStorageType> | <userType> ) ">"
| "list" "<" ( <simpleStorageType> | <userType> ) ">"
| "set" "<" ( <simpleStorageType> | <userType> ) ">"
;
<columnFamilyName> ::= ( ksname=<cfOrKsName> dot="." )? cfname=<cfOrKsName> ;
<userTypeName> ::= ( ksname=<cfOrKsName> dot="." )? utname=<cfOrKsName> ;
<keyspaceName> ::= ksname=<cfOrKsName> ;
<nonSystemKeyspaceName> ::= ksname=<cfOrKsName> ;
@ -508,17 +515,19 @@ def ks_name_completer(ctxt, cass):
ksnames = [n for n in cass.get_keyspace_names() if n not in NONALTERBALE_KEYSPACES]
return map(maybe_escape_name, ksnames)
@completer_for('columnFamilyName', 'ksname')
def cf_ks_name_completer(ctxt, cass):
return [maybe_escape_name(ks) + '.' for ks in cass.get_keyspace_names()]
@completer_for('columnFamilyName', 'dot')
completer_for('columnFamilyName', 'ksname')(cf_ks_name_completer)
def cf_ks_dot_completer(ctxt, cass):
name = dequote_name(ctxt.get_binding('ksname'))
if name in cass.get_keyspace_names():
return ['.']
return []
completer_for('columnFamilyName', 'dot')(cf_ks_dot_completer)
@completer_for('columnFamilyName', 'cfname')
def cf_name_completer(ctxt, cass):
ks = ctxt.get_binding('ksname', None)
@ -532,6 +541,26 @@ def cf_name_completer(ctxt, cass):
raise
return map(maybe_escape_name, cfnames)
completer_for('userTypeName', 'ksname')(cf_ks_name_completer)
completer_for('userTypeName', 'dot')(cf_ks_dot_completer)
def ut_name_completer(ctxt, cass):
ks = ctxt.get_binding('ksname', None)
if ks is not None:
ks = dequote_name(ks)
try:
utnames = cass.get_usertype_names(ks)
except Exception:
if ks is None:
return ()
raise
return map(maybe_escape_name, utnames)
completer_for('userTypeName', 'utname')(ut_name_completer)
completer_for('userType', 'utname')(ut_name_completer)
@completer_for('unreservedKeyword', 'nocomplete')
def unreserved_keyword_completer(ctxt, cass):
# we never want to provide completions through this production;
@ -546,6 +575,13 @@ def get_cf_layout(ctxt, cass):
cf = dequote_name(ctxt.get_binding('cfname'))
return cass.get_columnfamily_layout(ks, cf)
def get_ut_layout(ctxt, cass):
ks = ctxt.get_binding('ksname', None)
if ks is not None:
ks = dequote_name(ks)
ut = dequote_name(ctxt.get_binding('utname'))
return cass.get_usertype_layout(ks, ut)
def working_on_keyspace(ctxt):
wat = ctxt.get_binding('wat').upper()
if wat in ('KEYSPACE', 'SCHEMA'):
@ -930,9 +966,16 @@ syntax_rules += r'''
cf=<columnFamilyName> "(" col=<cident> ")"
( "USING" <stringLiteral> ( "WITH" "OPTIONS" "=" <mapLiteral> )? )?
;
<createUserTypeStatement> ::= "CREATE" "TYPE" ( ks=<nonSystemKeyspaceName> dot="." )? typename=<cfOrKsName> "(" newcol=<cident> <storageType>
( "," [newcolname]=<cident> <storageType> )*
")"
;
'''
explain_completion('createIndexStatement', 'indexname', '<new_index_name>')
explain_completion('createUserTypeStatement', 'typename', '<new_type_name>')
explain_completion('createUserTypeStatement', 'newcol', '<new_field_name>')
@completer_for('createIndexStatement', 'col')
def create_index_col_completer(ctxt, cass):
@ -949,6 +992,10 @@ syntax_rules += r'''
<dropIndexStatement> ::= "DROP" "INDEX" ("IF" "EXISTS")? indexname=<identifier>
;
<dropUserTypeStatement> ::= "DROP" "TYPE" ut=<userTypeName>
;
'''
@completer_for('dropIndexStatement', 'indexname')
@ -966,6 +1013,15 @@ syntax_rules += r'''
| "RENAME" existcol=<cident> "TO" newcol=<cident>
( "AND" existcol=<cident> "TO" newcol=<cident> )*
;
<alterUserTypeStatement> ::= "ALTER" "TYPE" ut=<userTypeName>
<alterTypeInstructions>
;
<alterTypeInstructions> ::= "RENAME" "TO" typename=<cfOrKsName>
| "ALTER" existcol=<cident> "TYPE" <storageType>
| "ADD" newcol=<cident> <storageType>
| "RENAME" existcol=<cident> "TO" newcol=<cident>
;
'''
@completer_for('alterInstructions', 'existcol')
@ -974,7 +1030,16 @@ def alter_table_col_completer(ctxt, cass):
cols = [md.name for md in layout.columns]
return map(maybe_escape_name, cols)
@completer_for('alterTypeInstructions', 'existcol')
def alter_type_field_completer(ctxt, cass):
layout = get_ut_layout(ctxt, cass)
fields = [tuple[0] for tuple in layout]
return map(maybe_escape_name, fields)
explain_completion('alterInstructions', 'newcol', '<new_column_name>')
explain_completion('alterTypeInstructions', 'typename', '<new_type_name>')
explain_completion('alterTypeInstructions', 'newcol', '<new_field_name>')
syntax_rules += r'''
<alterKeyspaceStatement> ::= "ALTER" ( "KEYSPACE" | "SCHEMA" ) ks=<alterableKeyspaceName>
@ -1170,3 +1235,31 @@ class CqlTableDef:
def __str__(self):
return '<%s %s.%s>' % (self.__class__.__name__, self.keyspace, self.name)
__repr__ = __str__
class UserTypesMeta(object):
_meta = {}
def __init__(self, meta):
self._meta = meta
@classmethod
def from_layout(cls, layout):
result = {}
for row in layout:
ksname = row[u'keyspace_name']
if ksname not in result:
result[ksname] = {}
utname = row[u'type_name']
result[ksname][utname] = zip(row[u'column_names'], row[u'column_types'])
return cls(meta=result)
def get_usertypes_names(self, keyspace):
return map(str, self._meta.get(keyspace, {}).keys())
def get_field_names(self, keyspace, type):
return [row[0] for row in self._meta.get(keyspace, {}).get(type, [])]
def get_fields_with_types(self, ksname, typename):
return [(field[0], lookup_casstype(field[1]).cql_parameterized_type()) for field in
self._meta.get(ksname, {}).get(typename, [])]

View File

@ -21,7 +21,6 @@ import math
from collections import defaultdict
from . import wcwidth
from .displaying import colorme, FormattedValue, DEFAULT_VALUE_COLORS
from cql import cqltypes
unicode_controlchars_re = re.compile(r'[\x00-\x31\x7f-\xa0]')
controlchars_re = re.compile(r'[\x00-\x31\x7f-\xff]')
@ -238,3 +237,22 @@ def format_value_map(val, encoding, colormap, time_format, float_precision, subt
+ rb
displaywidth = 4 * len(subs) + sum(k.displaywidth + v.displaywidth for (k, v) in subs)
return FormattedValue(bval, coloredval, displaywidth)
def format_value_utype(val, encoding, colormap, time_format, float_precision, subtypes, nullval, **_):
def format_field_value(v, subtype):
return format_value(subtype, v, encoding=encoding, colormap=colormap,
time_format=time_format, float_precision=float_precision,
nullval=nullval, quote=True)
def format_field_name(name):
return format_value_text(name, encoding=encoding, colormap=colormap, quote=False)
subs = [(format_field_name(k), format_field_value(v, subtypes[index])) for (index, (k, v)) in enumerate(val)]
bval = '{' + ', '.join(k.strval + ': ' + v.strval for (k, v) in subs) + '}'
lb, comma, colon, rb = [colormap['collection'] + s + colormap['reset']
for s in ('{', ', ', ': ', '}')]
coloredval = lb \
+ comma.join(k.coloredval + colon + v.coloredval for (k, v) in subs) \
+ rb
displaywidth = 4 * len(subs) + sum(k.displaywidth + v.displaywidth for (k, v) in subs)
return FormattedValue(bval, coloredval, displaywidth)

View File

@ -0,0 +1,63 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from cql.marshal import uint16_unpack
from cql.cqltypes import CompositeType
from formatting import formatter_for, format_value_utype
def get_field_names(ks_name, ut_name):
"""
UserTypes will use this function to get its fields names from Shell's utypes_dict
"""
raise NotImplementedError("this function shall be overloaded by Shell")
class UserType(CompositeType):
typename = 'UserType'
@classmethod
def apply_parameters(cls, *subtypes):
ksname = subtypes[0].cassname
newname = subtypes[1].cassname.decode("hex")
field_names = get_field_names(ksname, newname)
assert len(field_names) == len(subtypes[2:])
formatter_for(newname)(format_value_utype)
return type(newname, (cls,), {'subtypes': subtypes[2:],
'cassname': cls.cassname, 'typename': newname, 'fieldnames': field_names})
@classmethod
def cql_parameterized_type(cls):
return cls.typename
@classmethod
def deserialize_safe(cls, byts):
p = 0
result = []
for col_name, col_type in zip(cls.fieldnames, cls.subtypes):
if p == len(byts):
break
itemlen = uint16_unpack(byts[p:p + 2])
p += 2
item = byts[p:p + itemlen]
p += itemlen
result.append((str(col_name), col_type.from_binary(item)))
p += 1
return result