From 288bb9d9021d297611eb6ce27143de15a10b119d Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Tue, 4 May 2021 08:45:26 -0500 Subject: [PATCH] Fix cqlsh encoding error with unicode in multi-line statement Patch by Adam Holmberg; reviewed by brandonwilliams for CASSANDRA-16539 --- CHANGES.txt | 1 + bin/cqlsh.py | 2 +- pylib/cqlshlib/test/cassconnect.py | 20 +++++- pylib/cqlshlib/test/run_cqlsh.py | 1 + pylib/cqlshlib/test/test_cqlsh_completion.py | 2 - pylib/cqlshlib/test/test_cqlsh_output.py | 16 ++--- pylib/cqlshlib/test/test_unicode.py | 65 ++++++++++++++++++++ 7 files changed, 93 insertions(+), 14 deletions(-) create mode 100644 pylib/cqlshlib/test/test_unicode.py diff --git a/CHANGES.txt b/CHANGES.txt index c4f4d09eef..269338886a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0-rc2 + * Fix cqlsh encoding error with unicode in multi-line statement (CASSANDRA-16539) * Fix race in fat client removal (CASSANDRA-16238) * Test org.apache.cassandra.net.AsyncPromiseTest FAILED (CASSANDRA-16596) Merged from 3.11: diff --git a/bin/cqlsh.py b/bin/cqlsh.py index a92412cbc5..f964fc9834 100755 --- a/bin/cqlsh.py +++ b/bin/cqlsh.py @@ -972,7 +972,7 @@ class Shell(cmd.Cmd): if readline is not None: nl_count = srcstr.count("\n") - new_hist = srcstr.replace("\n", " ").rstrip() + new_hist = ensure_str(srcstr.replace("\n", " ").rstrip()) if nl_count > 1 and self.last_hist != new_hist: readline.add_history(new_hist) diff --git a/pylib/cqlshlib/test/cassconnect.py b/pylib/cqlshlib/test/cassconnect.py index 7391623e31..a7d8e1cbc1 100644 --- a/pylib/cqlshlib/test/cassconnect.py +++ b/pylib/cqlshlib/test/cassconnect.py @@ -21,23 +21,28 @@ import os.path import random import string import tempfile +from nose.tools import nottest from .basecase import TEST_HOST, TEST_PORT, cql, cqlsh, cqlshlog, policy, quote_name, rundir from .run_cqlsh import run_cqlsh, call_cqlsh test_keyspace_init = os.path.join(rundir, 'test_keyspace_init.cql') + def get_cassandra_connection(cql_version=None): conn = cql((TEST_HOST,), TEST_PORT, cql_version=cql_version, load_balancing_policy=policy) # until the cql lib does this for us conn.cql_version = cql_version return conn + def get_cassandra_cursor(cql_version=None): return get_cassandra_connection(cql_version=cql_version).cursor() + TEST_KEYSPACES_CREATED = [] + def get_keyspace(): return None if len(TEST_KEYSPACES_CREATED) == 0 else TEST_KEYSPACES_CREATED[-1] @@ -55,6 +60,7 @@ def make_ks_name(): _used_ks_names.add(s) return s + def create_keyspace(cursor): ksname = make_ks_name().lower() qksname = quote_name(ksname) @@ -66,6 +72,7 @@ def create_keyspace(cursor): TEST_KEYSPACES_CREATED.append(ksname) return ksname + def split_cql_commands(source): ruleset = cql_rule_set() statements, endtoken_escaped = ruleset.cql_split_statements(source) @@ -74,25 +81,30 @@ def split_cql_commands(source): return [ruleset.cql_extract_orig(toks, source) for toks in statements if toks] + def execute_cql_commands(cursor, source, logprefix='INIT: '): for cql in split_cql_commands(source): cqlshlog.debug((logprefix + cql).encode("utf-8")) cursor.execute(cql) + def execute_cql_file(cursor, fname): with io.open(fname, "r", encoding="utf-8") as f: return execute_cql_commands(cursor, f.read()) + def create_db(): with cassandra_cursor(ks=None) as c: k = create_keyspace(c) execute_cql_file(c, test_keyspace_init) return k + def remove_db(): with cassandra_cursor(ks=None) as c: c.execute('DROP KEYSPACE %s' % quote_name(TEST_KEYSPACES_CREATED.pop(-1))) + @contextlib.contextmanager def cassandra_connection(cql_version=None): """ @@ -133,11 +145,15 @@ def cassandra_cursor(cql_version=None, ks=''): finally: conn.shutdown() + def cql_rule_set(): return cqlsh.cql3handling.CqlRuleSet + class DEFAULTVAL: pass + +@nottest def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): # use a positive default sentinel so that keyspace=None can be used # to override the default behavior @@ -145,9 +161,11 @@ def testrun_cqlsh(keyspace=DEFAULTVAL, **kwargs): keyspace = get_keyspace() return run_cqlsh(keyspace=keyspace, **kwargs) + +@nottest def testcall_cqlsh(keyspace=None, **kwargs): if keyspace is None: keyspace = get_keyspace() - if ('input' in kwargs.keys() and isinstance(kwargs['input'], str)): + if 'input' in kwargs.keys() and isinstance(kwargs['input'], str): kwargs['input'] = kwargs['input'].encode('utf-8') return call_cqlsh(keyspace=keyspace, **kwargs) diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py index 134bb383cf..a6e0d68936 100644 --- a/pylib/cqlshlib/test/run_cqlsh.py +++ b/pylib/cqlshlib/test/run_cqlsh.py @@ -306,6 +306,7 @@ class CqlshRunner(ProcRunner): if coverage: args += ('--coverage',) self.keyspace = keyspace + env.setdefault('CQLSH_PYTHON', sys.executable) # run with the same interpreter as the test ProcRunner.__init__(self, path, tty=tty, args=args, env=env, **kwargs) self.prompt = prompt if self.prompt is None: diff --git a/pylib/cqlshlib/test/test_cqlsh_completion.py b/pylib/cqlshlib/test/test_cqlsh_completion.py index 270110afa3..8b296b8f9b 100644 --- a/pylib/cqlshlib/test/test_cqlsh_completion.py +++ b/pylib/cqlshlib/test/test_cqlsh_completion.py @@ -56,8 +56,6 @@ class CqlshCompletionCase(BaseTestCase): env['COLUMNS'] = '100000' if (locale.getpreferredencoding() != 'UTF-8'): env['LC_CTYPE'] = 'en_US.utf8' - if ('PATH' in os.environ.keys()): - env['PATH'] = os.environ['PATH'] self.cqlsh_runner = testrun_cqlsh(cqlver=None, env=env) self.cqlsh = self.cqlsh_runner.__enter__() diff --git a/pylib/cqlshlib/test/test_cqlsh_output.py b/pylib/cqlshlib/test/test_cqlsh_output.py index 4432a54904..5678def303 100644 --- a/pylib/cqlshlib/test/test_cqlsh_output.py +++ b/pylib/cqlshlib/test/test_cqlsh_output.py @@ -24,6 +24,7 @@ import os import re import subprocess import sys +import six import unittest from .basecase import (BaseTestCase, TEST_HOST, TEST_PORT, @@ -37,8 +38,6 @@ from .ansi_colors import (ColoredText, ansi_seq, lookup_colorcode, CONTROL_C = '\x03' CONTROL_D = '\x04' -has_python27 = not subprocess.call(['python2.7', '--version']) -has_python3 = not subprocess.call(['python3', '--version']) class TestCqlshOutput(BaseTestCase): @@ -58,8 +57,6 @@ class TestCqlshOutput(BaseTestCase): env['LC_CTYPE'] = 'en_US.utf8' else: env['LC_CTYPE'] = os.environ.get('LC_CTYPE', 'en_US.utf8') - if ('PATH' in os.environ.keys()): - env['PATH'] = os.environ['PATH'] self.default_env = env def tearDown(self): @@ -784,21 +781,20 @@ class TestCqlshOutput(BaseTestCase): self.assertRegex(output, '^Connected to .* at %s:%d$' % (re.escape(TEST_HOST), TEST_PORT)) - @unittest.skipIf(not has_python27, 'Python 2.7 not available to test warning') + @unittest.skipIf(six.PY3, 'Will not emit warning when running Python 3') def test_warn_py2(self): - env = self.default_env.copy() - env['USER_SPECIFIED_PYTHON'] = 'python2.7' # has the warning - with testrun_cqlsh(tty=True, env=env) as c: + with testrun_cqlsh(tty=True, env=self.default_env) as c: self.assertIn('Python 2.7 support is deprecated.', c.output_header, 'cqlsh did not output expected warning.') # can suppress + env = self.default_env.copy() env['CQLSH_NO_WARN_PY2'] = '1' with testrun_cqlsh(tty=True, env=env) as c: self.assertNotIn('Python 2.7 support is deprecated.', c.output_header, 'cqlsh did not output expected warning.') - @unittest.skipIf(not (has_python27 and has_python3), 'Python 3 and 2.7 not available to test preference') - def test_no_warn_both_py_present(self): + @unittest.skipIf(six.PY2, 'Warning will be emitted when running Python 2.7') + def test_no_warn_py3(self): with testrun_cqlsh(tty=True, env=self.default_env) as c: self.assertNotIn('Python 2.7 support is deprecated.', c.output_header, 'cqlsh did not output expected warning.') diff --git a/pylib/cqlshlib/test/test_unicode.py b/pylib/cqlshlib/test/test_unicode.py new file mode 100644 index 0000000000..b31e81c991 --- /dev/null +++ b/pylib/cqlshlib/test/test_unicode.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import unicode_literals, with_statement + +import os +import subprocess + +from .basecase import BaseTestCase +from .cassconnect import (get_cassandra_connection, create_keyspace, testrun_cqlsh) + + +class TestCqlshUnicode(BaseTestCase): + + @classmethod + def setUpClass(cls): + s = get_cassandra_connection().connect() + s.default_timeout = 60.0 + create_keyspace(s) + s.execute('CREATE TABLE t (k int PRIMARY KEY, v text)') + + env = os.environ.copy() + env['LC_CTYPE'] = 'UTF-8' + cls.default_env = env + + def test_unicode_value_round_trip(self): + with testrun_cqlsh(tty=True, env=self.default_env) as c: + value = 'ϑΉӁװڜ' + c.cmd_and_response("INSERT INTO t(k, v) VALUES (1, '%s');" % (value,)) + output = c.cmd_and_response('SELECT * FROM t;') + self.assertIn(value, output) + + def test_unicode_identifier(self): + col_name = 'テスト' + with testrun_cqlsh(tty=True, env=self.default_env) as c: + c.cmd_and_response('ALTER TABLE t ADD "%s" int;' % (col_name,)) + # describe command reproduces name + output = c.cmd_and_response('DESC t') + self.assertIn('"%s" int' % (col_name,), output) + c.cmd_and_response("INSERT INTO t(k, v) VALUES (1, '値');") + # results header reproduces name + output = c.cmd_and_response('SELECT * FROM t;') + self.assertIn(col_name, output) + + def test_multiline_input(self): # CASSANDRA-16539 + with testrun_cqlsh(tty=True, env=self.default_env) as c: + value = '値' + c.send("INSERT INTO t(k, v) VALUES (1, \n'%s');\n" % (value,)) + c.read_to_next_prompt() + output = c.cmd_and_response('SELECT v FROM t;') + self.assertIn(value, output)