diff --git a/CHANGES.txt b/CHANGES.txt index af51a9dd9e..9aadeffa91 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,7 @@ * Add ability to stop compaction by ID (CASSANDRA-7207) * Let CassandraVersion handle SNAPSHOT version (CASSANDRA-9438) Merged from 2.1: + * (cqlsh) Fix using COPY through SOURCE or -f (CASSANDRA-9083) * Fix occasional lack of `system` keyspace in schema tables (CASSANDRA-8487) * Use ProtocolError code instead of ServerError code for native protocol error responses to unsupported protocol versions (CASSANDRA-9451) diff --git a/pylib/cqlshlib/cqlhandling.py b/pylib/cqlshlib/cqlhandling.py index 6f4c8160a3..e03c418642 100644 --- a/pylib/cqlshlib/cqlhandling.py +++ b/pylib/cqlshlib/cqlhandling.py @@ -22,6 +22,7 @@ from . import pylexotron, util Hint = pylexotron.Hint + class CqlParsingRuleSet(pylexotron.ParsingRuleSet): keywords = set() @@ -72,9 +73,11 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): def explain_completion(self, rulename, symname, explanation=None): if explanation is None: explanation = '<%s>' % (symname,) + @self.completer_for(rulename, symname) def explainer(ctxt, cass): return [Hint(explanation)] + return explainer def set_keywords_as_syntax(self): @@ -96,6 +99,19 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): else: # don't put any 'endline' tokens in output continue + + # Convert all unicode tokens to ascii, where possible. This + # helps avoid problems with performing unicode-incompatible + # operations on tokens (like .lower()). See CASSANDRA-9083 + # for one example of this. + str_token = t[1] + if isinstance(str_token, unicode): + try: + str_token = str_token.encode('ascii') + t = (t[0], str_token) + t[2:] + except UnicodeEncodeError: + pass + curstmt.append(t) if t[0] == 'endtoken': term_on_nl = False @@ -191,7 +207,7 @@ class CqlParsingRuleSet(pylexotron.ParsingRuleSet): # for completion. the opening quote is already there on the command # line and not part of the word under completion, and readline # fills in the closing quote for us. - candidates = [requoter(dequoter(c))[len(prefix)+1:-1] for c in candidates] + candidates = [requoter(dequoter(c))[len(prefix) + 1:-1] for c in candidates] # the above process can result in an empty string; this doesn't help for # completions diff --git a/pylib/cqlshlib/pylexotron.py b/pylib/cqlshlib/pylexotron.py index b4ac36f599..b7558f5372 100644 --- a/pylib/cqlshlib/pylexotron.py +++ b/pylib/cqlshlib/pylexotron.py @@ -100,7 +100,17 @@ class ParseContext: # pretty much just guess return ' '.join([t[1] for t in tokens]) # low end of span for first token, to high end of span for last token - return orig[tokens[0][2][0]:tokens[-1][2][1]] + orig_text = orig[tokens[0][2][0]:tokens[-1][2][1]] + + # Convert all unicode tokens to ascii, where possible. This + # helps avoid problems with performing unicode-incompatible + # operations on tokens (like .lower()). See CASSANDRA-9083 + # for one example of this. + try: + orig_text = orig_text.encode('ascii') + except UnicodeEncodeError: + pass + return orig_text def __repr__(self): return '<%s matched=%r remainder=%r prodname=%r bindings=%r>' \