Fix cqlsh encoding error with unicode in multi-line statement

Patch by Adam Holmberg; reviewed by brandonwilliams for CASSANDRA-16539

CHANGES.txt

@@ -1,4 +1,5 @@
 4.0-rc2
+ * Fix cqlsh encoding error with unicode in multi-line statement (CASSANDRA-16539)
  * Fix race in fat client removal (CASSANDRA-16238)
  * Test org.apache.cassandra.net.AsyncPromiseTest FAILED (CASSANDRA-16596)
 Merged from 3.11:

bin/cqlsh.py

@@ -972,7 +972,7 @@ class Shell(cmd.Cmd):
         if readline is not None:
             nl_count = srcstr.count("\n")
 
-            new_hist = srcstr.replace("\n", " ").rstrip()
+            new_hist = ensure_str(srcstr.replace("\n", " ").rstrip())
 
             if nl_count > 1 and self.last_hist != new_hist:
                 readline.add_history(new_hist)

pylib/cqlshlib/test/cassconnect.py

@@ -21,23 +21,28 @@ import os.path
 import random
 import string
 import tempfile
+from nose.tools import nottest
 
 from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, rundir
 from .run_cqlsh import run_cqlsh, call_cqlsh
 
 test_keyspace_init = os.path.join(rundir, 'test_keyspace_init.cql')
 
+
 def get_cassandra_connection(cql_version=None):
     conn = cql((TEST_HOST,), TEST_PORT, cql_version=cql_version, load_balancing_policy=policy)
     # until the cql lib does this for us
     conn.cql_version = cql_version
     return conn
 
+
 def get_cassandra_cursor(cql_version=None):
     return get_cassandra_connection(cql_version=cql_version).cursor()
 
+
 TEST_KEYSPACES_CREATED = []
 
+
 def get_keyspace():
     return None if len(TEST_KEYSPACES_CREATED) == 0 else TEST_KEYSPACES_CREATED[-1]
 
@@ -55,6 +60,7 @@ def make_ks_name():
     _used_ks_names.add(s)
     return s
 
+
 def create_keyspace(cursor):
     ksname = make_ks_name().lower()
     qksname = quote_name(ksname)
@@ -66,6 +72,7 @@ def create_keyspace(cursor):
     TEST_KEYSPACES_CREATED.append(ksname)
     return ksname
 
+
 def split_cql_commands(source):
     ruleset = cql_rule_set()
     statements, endtoken_escaped = ruleset.cql_split_statements(source)
@@ -74,25 +81,30 @@ def split_cql_commands(source):
 
     return [ruleset.cql_extract_orig(toks, source) for toks in statements if toks]
 
+
 def execute_cql_commands(cursor, source, logprefix='INIT: '):
     for cql in split_cql_commands(source):
         cqlshlog.debug((logprefix + cql).encode("utf-8"))
         cursor.execute(cql)
 
+
 def execute_cql_file(cursor, fname):
     with io.open(fname, "r", encoding="utf-8") as f:
         return execute_cql_commands(cursor, f.read())
 
+
 def create_db():
     with cassandra_cursor(ks=None) as c:
         k = create_keyspace(c)
         execute_cql_file(c, test_keyspace_init)
     return k
 
+
 def remove_db():
     with cassandra_cursor(ks=None) as c:
         c.execute('DROP KEYSPACE %s' % quote_name(TEST_KEYSPACES_CREATED.pop(-1)))
 
+
 @contextlib.contextmanager
 def cassandra_connection(cql_version=None):
     """
@@ -133,11 +145,15 @@ def cassandra_cursor(cql_version=None, ks=''):
     finally:
         conn.shutdown()
 
+
 def cql_rule_set():
     return cqlsh.cql3handling.CqlRuleSet
 
+
 class DEFAULTVAL: pass
 
+
+@nottest
 def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs):
     # use a positive default sentinel so that keyspace=None can be used
     # to override the default behavior
@@ -145,9 +161,11 @@ def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs):
         keyspace = get_keyspace()
     return run_cqlsh(keyspace=keyspace, **kwargs)
 
+
+@nottest
 def testcall_cqlsh(keyspace=None, **kwargs):
     if keyspace is None:
         keyspace = get_keyspace()
-    if ('input' in kwargs.keys() and isinstance(kwargs['input'], str)):
+    if 'input' in kwargs.keys() and isinstance(kwargs['input'], str):
         kwargs['input'] = kwargs['input'].encode('utf-8')
     return call_cqlsh(keyspace=keyspace, **kwargs)

pylib/cqlshlib/test/run_cqlsh.py

@@ -306,6 +306,7 @@ class CqlshRunner(ProcRunner):
         if coverage:
             args += ('--coverage',)
         self.keyspace = keyspace
+        env.setdefault('CQLSH_PYTHON', sys.executable)  # run with the same interpreter as the test
         ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs)
         self.prompt = prompt
         if self.prompt is None:

pylib/cqlshlib/test/test_cqlsh_completion.py

@@ -56,8 +56,6 @@ class CqlshCompletionCase(BaseTestCase):
         env['COLUMNS'] = '100000'
         if (locale.getpreferredencoding() != 'UTF-8'):
              env['LC_CTYPE'] = 'en_US.utf8'
-        if ('PATH' in os.environ.keys()):
-            env['PATH'] = os.environ['PATH']
         self.cqlsh_runner = testrun_cqlsh(cqlver=None, env=env)
         self.cqlsh = self.cqlsh_runner.__enter__()
 

pylib/cqlshlib/test/test_cqlsh_output.py

@@ -24,6 +24,7 @@ import os
 import re
 import subprocess
 import sys
+import six
 import unittest
 
 from .basecase import (BaseTestCase, TEST_HOST, TEST_PORT,
@@ -37,8 +38,6 @@ from .ansi_colors import (ColoredText, ansi_seq, lookup_colorcode,
 CONTROL_C = '\x03'
 CONTROL_D = '\x04'
 
-has_python27 = not subprocess.call(['python2.7', '--version'])
-has_python3 = not subprocess.call(['python3', '--version'])
 
 class TestCqlshOutput(BaseTestCase):
 
@@ -58,8 +57,6 @@ class TestCqlshOutput(BaseTestCase):
             env['LC_CTYPE'] = 'en_US.utf8'
         else:
             env['LC_CTYPE'] = os.environ.get('LC_CTYPE', 'en_US.utf8')
-        if ('PATH' in os.environ.keys()):
-            env['PATH'] = os.environ['PATH']
         self.default_env = env
 
     def tearDown(self):
@@ -784,21 +781,20 @@ class TestCqlshOutput(BaseTestCase):
             self.assertRegex(output, '^Connected to .* at %s:%d$'
                                              % (re.escape(TEST_HOST), TEST_PORT))
 
-    @unittest.skipIf(not has_python27, 'Python 2.7 not available to test warning')
+    @unittest.skipIf(six.PY3, 'Will not emit warning when running Python 3')
     def test_warn_py2(self):
-        env = self.default_env.copy()
-        env['USER_SPECIFIED_PYTHON'] = 'python2.7'
         # has the warning
-        with testrun_cqlsh(tty=True, env=env) as c:
+        with testrun_cqlsh(tty=True, env=self.default_env) as c:
             self.assertIn('Python 2.7 support is deprecated.', c.output_header, 'cqlsh did not output expected warning.')
 
         # can suppress
+        env = self.default_env.copy()
         env['CQLSH_NO_WARN_PY2'] = '1'
         with testrun_cqlsh(tty=True, env=env) as c:
             self.assertNotIn('Python 2.7 support is deprecated.', c.output_header, 'cqlsh did not output expected warning.')
 
-    @unittest.skipIf(not (has_python27 and has_python3), 'Python 3 and 2.7 not available to test preference')
-    def test_no_warn_both_py_present(self):
+    @unittest.skipIf(six.PY2, 'Warning will be emitted when running Python 2.7')
+    def test_no_warn_py3(self):
         with testrun_cqlsh(tty=True, env=self.default_env) as c:
             self.assertNotIn('Python 2.7 support is deprecated.', c.output_header, 'cqlsh did not output expected warning.')
 

pylib/cqlshlib/test/test_unicode.py

@@ -0,0 +1,65 @@
+# coding=utf-8
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import unicode_literals, with_statement
+
+import os
+import subprocess
+
+from .basecase import BaseTestCase
+from .cassconnect import (get_cassandra_connection, create_keyspace, testrun_cqlsh)
+
+
+class TestCqlshUnicode(BaseTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        s = get_cassandra_connection().connect()
+        s.default_timeout = 60.0
+        create_keyspace(s)
+        s.execute('CREATE TABLE t (k int PRIMARY KEY, v text)')
+
+        env = os.environ.copy()
+        env['LC_CTYPE'] = 'UTF-8'
+        cls.default_env = env
+
+    def test_unicode_value_round_trip(self):
+        with testrun_cqlsh(tty=True, env=self.default_env) as c:
+            value = 'ϑΉӁװڜ'
+            c.cmd_and_response("INSERT INTO t(k, v) VALUES (1, '%s');" % (value,))
+            output = c.cmd_and_response('SELECT * FROM t;')
+            self.assertIn(value, output)
+
+    def test_unicode_identifier(self):
+        col_name = 'テスト'
+        with testrun_cqlsh(tty=True, env=self.default_env) as c:
+            c.cmd_and_response('ALTER TABLE t ADD "%s" int;' % (col_name,))
+            # describe command reproduces name
+            output = c.cmd_and_response('DESC t')
+            self.assertIn('"%s" int' % (col_name,), output)
+            c.cmd_and_response("INSERT INTO t(k, v) VALUES (1, '値');")
+            # results header reproduces name
+            output = c.cmd_and_response('SELECT * FROM t;')
+            self.assertIn(col_name, output)
+
+    def test_multiline_input(self):  # CASSANDRA-16539
+        with testrun_cqlsh(tty=True, env=self.default_env) as c:
+            value = '値'
+            c.send("INSERT INTO t(k, v) VALUES (1, \n'%s');\n" % (value,))
+            c.read_to_next_prompt()
+            output = c.cmd_and_response('SELECT v FROM t;')
+            self.assertIn(value, output)