Update cqlsh python driver to 2.1

Patch by Mikhail Stepura and Tyler Hobbs; reviewed by Aleksey Yeschenko
for CASSANDRA-7509

CHANGES.txt

@@ -6,6 +6,7 @@
  * Add missing table name to DROP INDEX responses and notifications (CASSANDRA-7539)
  * Bump CQL version to 3.2.0 and update CQL documentation (CASSANDRA-7527)
  * Fix configuration error message when running nodetool ring (CASSANDRA-7508)
+ * Support conditional updates, tuple type, and the v3 protocol in cqlsh (CASSANDRA-7509)
 Merged from 2.0:
  * (Windows) force range-based repair to non-sequential mode (CASSANDRA-7541)
  * Fix range merging when DES scores are zero (CASSANDRA-7535)

bin/cqlsh

@@ -65,6 +65,7 @@ except ImportError:
 
 CQL_LIB_PREFIX = 'cassandra-driver-internal-only-'
 FUTURES_LIB_PREFIX = 'futures-'
+SIX_LIB_PREFIX = 'six-'
 
 CASSANDRA_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
 
@@ -91,6 +92,9 @@ if cql_zip:
 futures_zip = find_zip(FUTURES_LIB_PREFIX)
 if futures_zip:
     sys.path.insert(0, futures_zip)
+six_zip = find_zip(SIX_LIB_PREFIX)
+if six_zip:
+    sys.path.insert(0, six_zip)
 
 warnings.filterwarnings("ignore", r".*blist.*")
 try:
@@ -113,13 +117,12 @@ cqlshlibdir = os.path.join(CASSANDRA_PATH, 'pylib')
 if os.path.isdir(cqlshlibdir):
     sys.path.insert(0, cqlshlibdir)
 
-from cqlshlib import cqlhandling, cql3handling, pylexotron, usertypes, sslhandling
+from cqlshlib import cqlhandling, cql3handling, pylexotron, sslhandling
 from cqlshlib.displaying import (RED, BLUE, ANSI_RESET, COLUMN_NAME_COLORS,
                                  FormattedValue, colorme)
-from cqlshlib.formatting import format_by_type
+from cqlshlib.formatting import format_by_type, formatter_for, format_value_utype
 from cqlshlib.util import trim_if_present
 from cqlshlib.tracing import print_trace_session
-from cqlshlib.usertypes import deserialize_safe_collection, deserialize_safe_map
 
 HISTORY_DIR = os.path.expanduser(os.path.join('~', '.cassandra'))
 CONFIG_FILE = os.path.join(HISTORY_DIR, 'cqlshrc')
@@ -140,6 +143,7 @@ if os.path.exists(OLD_HISTORY):
 DEFAULT_HOST = '127.0.0.1'
 DEFAULT_PORT = 9042
 DEFAULT_CQLVER = '3.2.0'
+DEFAULT_PROTOCOL_VERSION = 3
 
 DEFAULT_TIME_FORMAT = '%Y-%m-%d %H:%M:%S%z'
 DEFAULT_FLOAT_PRECISION = 5
@@ -179,7 +183,7 @@ CQL_ERRORS = (
     cassandra.Timeout, cassandra.Unauthorized, cassandra.OperationTimedOut,
     cassandra.cluster.NoHostAvailable,
     cassandra.connection.ConnectionBusy, cassandra.connection.ProtocolError, cassandra.connection.ConnectionException,
-    cassandra.decoder.ErrorMessage, cassandra.decoder.InternalError, cassandra.query.TraceUnavailable
+    cassandra.protocol.ErrorMessage, cassandra.protocol.InternalError, cassandra.query.TraceUnavailable
 )
 
 debug_completion = bool(os.environ.get('CQLSH_DEBUG_COMPLETION', '') == 'YES')
@@ -443,6 +447,26 @@ def describe_interval(seconds):
         words = desc[0] + ' and ' + words
     return words
 
+
+def auto_format_udts():
+    # when we see a new user defined type, set up the shell formatting for it
+    udt_apply_params = cassandra.cqltypes.UserType.apply_parameters
+    def new_apply_params(cls, *args, **kwargs):
+        udt_class = udt_apply_params(*args, **kwargs)
+        formatter_for(udt_class.typename)(format_value_utype)
+        return udt_class
+
+    cassandra.cqltypes.UserType.udt_apply_parameters = classmethod(new_apply_params)
+
+    make_udt_class = cassandra.cqltypes.UserType.make_udt_class
+    def new_make_udt_class(cls, *args, **kwargs):
+        udt_class = make_udt_class(*args, **kwargs)
+        formatter_for(udt_class.typename)(format_value_utype)
+        return udt_class
+
+    cassandra.cqltypes.UserType.make_udt_class = classmethod(new_make_udt_class)
+
+
 class Shell(cmd.Cmd):
     custom_prompt = os.getenv('CQLSH_PROMPT', '')
     if custom_prompt is not '':
@@ -483,6 +507,7 @@ class Shell(cmd.Cmd):
             self.conn = use_conn
         else:
             self.conn = Cluster(contact_points=(self.hostname,), port=self.port, cql_version=cqlver,
+                                protocol_version=DEFAULT_PROTOCOL_VERSION,
                                 auth_provider=self.auth_provider,
                                 ssl_options=sslhandling.ssl_settings(hostname, CONFIG_FILE) if ssl else None,
                                 load_balancing_policy=WhiteListRoundRobinPolicy([self.hostname]))
@@ -525,17 +550,12 @@ class Shell(cmd.Cmd):
         #Python driver returns BLOBs as string, but we expect them as buffer()
         cassandra.cqltypes.BytesType.deserialize = staticmethod(cassandra.cqltypes.BytesType.validate)
         cassandra.cqltypes.CassandraType.support_empty_values = True
-        # see CASSANDRA-7267
-        cassandra.cqltypes._SimpleParameterizedType.deserialize_safe = classmethod(deserialize_safe_collection)
-        # see CASSANDRA-7267
-        cassandra.cqltypes.MapType.deserialize_safe = classmethod(deserialize_safe_map)
+
+        auto_format_udts()
+
         self.empty_lines = 0
         self.statement_error = False
         self.single_statement = single_statement
-        #see CASSANDRA-7399
-        type_for_composites = lambda cls: "'%s'" % cls.cass_parameterized_type_with(cls.subtypes, True)
-        cassandra.cqltypes.CompositeType.cql_parameterized_type = classmethod(type_for_composites)
-        cassandra.cqltypes.DynamicCompositeType.cql_parameterized_type = classmethod(type_for_composites)
 
     def set_expanded_cql_version(self, ver):
         ver, vertuple = full_cql_version(ver)

lib/licenses/six-1.7.3.txt

@@ -0,0 +1,18 @@
+Copyright (c) 2010-2014 Benjamin Peterson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pylib/cqlshlib/formatting.py

@@ -110,7 +110,7 @@ def formatter_for(typname):
 
 @formatter_for('bytearray')
 def format_value_blob(val, colormap, **_):
-    bval = '0x' + ''.join('%02x' % ord(c) for c in val)
+    bval = '0x' + ''.join('%02x' % c for c in val)
     return colorme(bval, colormap, 'blob')
 formatter_for('buffer')(format_value_blob)
 
@@ -215,7 +215,11 @@ def format_simple_collection(val, lbracket, rbracket, encoding,
 def format_value_list(val, encoding, colormap, time_format, float_precision, nullval, **_):
     return format_simple_collection(val, '[', ']', encoding, colormap,
                                     time_format, float_precision, nullval)
-formatter_for('tuple')(format_value_list)
+
+@formatter_for('tuple')
+def format_value_tuple(val, encoding, colormap, time_format, float_precision, nullval, **_):
+    return format_simple_collection(val, '(', ')', encoding, colormap,
+                                    time_format, float_precision, nullval)
 
 @formatter_for('set')
 def format_value_set(val, encoding, colormap, time_format, float_precision, nullval, **_):

pylib/cqlshlib/test/test_cqlsh_completion.py

@@ -177,7 +177,7 @@ class TestCqlshCompletion(CqlshCompletionCase):
     def test_complete_in_string_literals(self):
         # would be great if we could get a space after this sort of completion,
         # but readline really wants to make things difficult for us
-        self.trycompletions('insert into system."NodeId', 'Info"')
+        self.trycompletions('insert into system."Index', 'Info"')
         self.trycompletions('USE "', choices=('system', self.cqlsh.keyspace),
                             other_choices_ok=True)
         self.trycompletions("create keyspace blah with replication = {'class': 'Sim",

pylib/cqlshlib/test/test_cqlsh_output.py

@@ -661,7 +661,7 @@ class TestCqlshOutput(BaseTestCase):
                 AND max_index_interval = 2048
                 AND memtable_flush_period_in_ms = 0
                 AND min_index_interval = 128
-                AND read_repair_chance = 0.1
+                AND read_repair_chance = 0.0
                 AND speculative_retry = '99.0PERCENTILE';
 
         """ % quote_name(get_test_keyspace()))

pylib/cqlshlib/usertypes.py

@@ -1,117 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from cassandra.marshal import int32_unpack, uint16_unpack
-from cassandra.cqltypes import CompositeType
-import collections
-from formatting import formatter_for, format_value_utype
-
-class UserType(CompositeType):
-    typename = "'org.apache.cassandra.db.marshal.UserType'"
-
-    FIELD_LENGTH = 4
-
-    @classmethod
-    def apply_parameters(cls, subtypes, names):
-        newname = subtypes[1].cassname.decode("hex")
-        field_names = [encoded_name.decode("hex") for encoded_name in names[2:]]
-        assert len(field_names) == len(subtypes[2:])
-        formatter_for(newname)(format_value_utype)
-        return type(newname, (cls,), {'subtypes': subtypes[2:],
-                                      'cassname': cls.cassname, 'typename': newname, 'fieldnames': field_names})
-
-    @classmethod
-    def cql_parameterized_type(cls):
-        return cls.typename
-
-    @classmethod
-    def deserialize_safe(cls, byts):
-        p = 0
-        Result = collections.namedtuple(cls.typename, cls.fieldnames)
-        result = []
-        for col_type in cls.subtypes:
-            if p == len(byts):
-                break
-            itemlen = int32_unpack(byts[p:p + cls.FIELD_LENGTH])
-            p += cls.FIELD_LENGTH
-            if itemlen < 0:
-                result.append(None)
-            else:
-                item = byts[p:p + itemlen]
-                p += itemlen
-                result.append(col_type.from_binary(item))
-
-        if len(result) < len(cls.subtypes):
-            nones = [None] * (len(cls.subtypes) - len(result))
-            result = result + nones
-
-        return Result(*result)
-
-def deserialize_safe_collection(cls, byts):
-    """
-        Temporary work around for CASSANDRA-7267
-    """
-    subtype, = cls.subtypes
-    unpack = uint16_unpack
-    length = 2
-    numelements = unpack(byts[:length])
-    if numelements == 0 and len(byts) > 2 :
-        unpack = int32_unpack
-        length = 4
-        numelements = unpack(byts[:length])
-    p = length
-    result = []
-    for n in xrange(numelements):
-        itemlen = unpack(byts[p:p + length])
-        p += length
-        item = byts[p:p + itemlen]
-        p += itemlen
-        result.append(subtype.from_binary(item))
-    return cls.adapter(result)
-
-try:
-    from collections import OrderedDict
-except ImportError:  # Python <2.7
-    from cassandra.util import OrderedDict
-
-def deserialize_safe_map(cls, byts):
-    """
-        Temporary work around for CASSANDRA-7267
-    """
-    subkeytype, subvaltype = cls.subtypes
-    unpack = uint16_unpack
-    length = 2
-    numelements = unpack(byts[:length])
-    if numelements == 0 and len(byts) > 2:
-        unpack = int32_unpack
-        length = 4
-        numelements = unpack(byts[:length])
-
-    p = length
-    themap = OrderedDict()
-    for n in xrange(numelements):
-        key_len = unpack(byts[p:p + length])
-        p += length
-        keybytes = byts[p:p + key_len]
-        p += key_len
-        val_len = unpack(byts[p:p + length])
-        p += length
-        valbytes = byts[p:p + val_len]
-        p += val_len
-        key = subkeytype.from_binary(keybytes)
-        val = subvaltype.from_binary(valbytes)
-        themap[key] = val
-    return themap