From 768bdffe5298b937bfafc2eb42fb93454cfca521 Mon Sep 17 00:00:00 2001 From: Brad Schoening <5796692+bschoening@users.noreply.github.com> Date: Thu, 10 Mar 2022 21:18:30 -0500 Subject: [PATCH] change six functions in cqlshlib to native Python 3 ones patch by Brad Schoening; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-17417 --- CHANGES.txt | 1 + bin/cqlsh.py | 45 ++++++++----------- pylib/cqlshlib/copyutil.py | 69 +++++++++++++----------------- pylib/cqlshlib/displaying.py | 2 +- pylib/cqlshlib/formatting.py | 17 +++----- pylib/cqlshlib/helptopics.py | 2 +- pylib/cqlshlib/test/ansi_colors.py | 3 +- pylib/cqlshlib/test/run_cqlsh.py | 2 +- pylib/cqlshlib/test/winpty.py | 4 +- pylib/cqlshlib/tracing.py | 4 +- pylib/cqlshlib/util.py | 2 +- pylib/requirements.txt | 4 -- 12 files changed, 64 insertions(+), 91 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index cd1270d125..cc502990a2 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1 + * change six functions in cqlshlib to native Python 3 (CASSANDRA-17417) * reduce hot-path object allocations required to record local/remote requests against the client request metrics (CASSANDRA-17424) * Disallow removing DC from system_auth while nodes are active in the DC (CASSANDRA-17478) * Add guardrail for the number of fields per UDT (CASSANDRA-17385) diff --git a/bin/cqlsh.py b/bin/cqlsh.py index 9e0e063382..481bb0bf22 100755 --- a/bin/cqlsh.py +++ b/bin/cqlsh.py @@ -110,19 +110,16 @@ if cql_zip: ver = os.path.splitext(os.path.basename(cql_zip))[0][len(CQL_LIB_PREFIX):] sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver)) -third_parties = ('futures-', 'six-') +# the driver needs dependencies +third_parties = ('six-') for lib in third_parties: lib_zip = find_zip(lib) if lib_zip: sys.path.insert(0, lib_zip) -# We cannot import six until we add its location to sys.path so the Python -# interpreter can find it. Do not move this to the top. -import six - -from six.moves import configparser, input -from six import StringIO, ensure_text, ensure_str +import configparser +from io import StringIO warnings.filterwarnings("ignore", r".*blist.*") try: @@ -361,7 +358,7 @@ class DecodeError(Exception): def maybe_ensure_text(val): - return ensure_text(val) if val else val + return str(val) if val else val class FormatError(DecodeError): @@ -426,7 +423,7 @@ def insert_driver_hooks(): class Shell(cmd.Cmd): - custom_prompt = ensure_text(os.getenv('CQLSH_PROMPT', '')) + custom_prompt = os.getenv('CQLSH_PROMPT', '') if custom_prompt != '': custom_prompt += "\n" default_prompt = custom_prompt + "cqlsh> " @@ -860,15 +857,14 @@ class Shell(cmd.Cmd): def get_input_line(self, prompt=''): if self.tty: - self.lastcmd = input(ensure_str(prompt)) - line = ensure_text(self.lastcmd) + '\n' + self.lastcmd = input(str(prompt)) + line = self.lastcmd + '\n' else: - self.lastcmd = ensure_text(self.stdin.readline()) + self.lastcmd = self.stdin.readline() line = self.lastcmd if not len(line): raise EOFError self.lineno += 1 - line = ensure_text(line) return line def use_stdin_reader(self, until='', prompt=''): @@ -929,7 +925,6 @@ class Shell(cmd.Cmd): Returns true if the statement is complete and was handled (meaning it can be reset). """ - statementtext = ensure_text(statementtext) statementtext = self.strip_comment_blocks(statementtext) try: statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext) @@ -975,7 +970,7 @@ class Shell(cmd.Cmd): if readline is not None: nl_count = srcstr.count("\n") - new_hist = ensure_str(srcstr.replace("\n", " ").rstrip()) + new_hist = srcstr.replace("\n", " ").rstrip() if nl_count > 1 and self.last_hist != new_hist: readline.add_history(new_hist) @@ -1026,7 +1021,6 @@ class Shell(cmd.Cmd): self.tracing_enabled = tracing_was_enabled def perform_statement(self, statement): - statement = ensure_text(statement) stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None) success, future = self.perform_simple_statement(stmt) @@ -1082,7 +1076,7 @@ class Shell(cmd.Cmd): try: result = future.result() except CQL_ERRORS as err: - err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err)) + err_msg = err.message if hasattr(err, 'message') else str(err) self.printerr(str(err.__class__.__name__) + ": " + err_msg) except Exception: import traceback @@ -1397,7 +1391,7 @@ class Shell(cmd.Cmd): self.describe_element(result) except CQL_ERRORS as err: - err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err)) + err_msg = err.message if hasattr(err, 'message') else str(err) self.printerr(err_msg.partition("message=")[2].strip('"')) except Exception: import traceback @@ -1413,7 +1407,7 @@ class Shell(cmd.Cmd): """ Print the output for a DESCRIBE KEYSPACES query """ - names = [ensure_str(r['name']) for r in rows] + names = [r['name'] for r in rows] print('') cmd.Cmd.columnize(self, names) @@ -1433,7 +1427,7 @@ class Shell(cmd.Cmd): keyspace = row['keyspace_name'] names = list() - names.append(ensure_str(row['name'])) + names.append(str(row['name'])) if keyspace is not None: self.print_keyspace_element_names(keyspace, names) @@ -1571,7 +1565,7 @@ class Shell(cmd.Cmd): if fname is not None: fname = self.cql_unprotect_value(fname) - copyoptnames = list(map(six.text_type.lower, parsed.get_binding('optnames', ()))) + copyoptnames = list(map(str.lower, parsed.get_binding('optnames', ()))) copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ()))) opts = dict(list(zip(copyoptnames, copyoptvals))) @@ -1992,11 +1986,10 @@ class Shell(cmd.Cmd): out = self.query_out # convert Exceptions, etc to text - if not isinstance(text, six.text_type): - text = "{}".format(text) + if not isinstance(text, str): + text = str(text) to_write = self.applycolor(text, color) + ('\n' if newline else '') - to_write = ensure_str(to_write) out.write(to_write) def flush_output(self): @@ -2128,7 +2121,7 @@ def is_file_secure(filename): def read_options(cmdlineargs, environment): - configs = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser() + configs = configparser.ConfigParser() configs.read(CONFIG_FILE) rawconfigs = configparser.RawConfigParser() @@ -2200,7 +2193,7 @@ def read_options(cmdlineargs, environment): options.credentials = '' # ConfigParser.read() will ignore unreadable files if not options.username: - credentials = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser() + credentials = configparser.ConfigParser() credentials.read(options.credentials) # use the username from credentials file but fallback to cqlshrc if username is absent from the command line parameters diff --git a/pylib/cqlshlib/copyutil.py b/pylib/cqlshlib/copyutil.py index b87340bed8..92af3a3897 100644 --- a/pylib/cqlshlib/copyutil.py +++ b/pylib/cqlshlib/copyutil.py @@ -26,7 +26,6 @@ import platform import random import re import signal -import six import struct import sys import threading @@ -39,15 +38,12 @@ from calendar import timegm from collections import defaultdict, namedtuple from decimal import Decimal from random import randint -from io import BytesIO, StringIO +from io import StringIO from select import select from uuid import UUID -from .util import profile_on, profile_off -from six import ensure_str, ensure_text -from six.moves import configparser -from six.moves import range -from six.moves.queue import Queue +import configparser +from queue import Queue from cassandra import OperationTimedOut from cassandra.cluster import Cluster, DefaultConnection @@ -334,9 +330,9 @@ class CopyTask(object): opts = self.clean_options(self.maybe_read_config_file(opts, direction)) dialect_options = dict() - dialect_options['quotechar'] = ensure_str(opts.pop('quote', '"')) - dialect_options['escapechar'] = ensure_str(opts.pop('escape', '\\')) - dialect_options['delimiter'] = ensure_str(opts.pop('delimiter', ',')) + dialect_options['quotechar'] = opts.pop('quote', '"') + dialect_options['escapechar'] = opts.pop('escape', '\\') + dialect_options['delimiter'] = opts.pop('delimiter', ',') if dialect_options['quotechar'] == dialect_options['escapechar']: dialect_options['doublequote'] = True del dialect_options['escapechar'] @@ -344,7 +340,7 @@ class CopyTask(object): dialect_options['doublequote'] = False copy_options = dict() - copy_options['nullval'] = ensure_str(opts.pop('null', '')) + copy_options['nullval'] = opts.pop('null', '') copy_options['header'] = bool(opts.pop('header', '').lower() == 'true') copy_options['encoding'] = opts.pop('encoding', 'utf8') copy_options['maxrequests'] = int(opts.pop('maxrequests', 6)) @@ -366,7 +362,7 @@ class CopyTask(object): copy_options['consistencylevel'] = shell.consistency_level copy_options['decimalsep'] = opts.pop('decimalsep', '.') copy_options['thousandssep'] = opts.pop('thousandssep', '') - copy_options['boolstyle'] = [ensure_str(s.strip()) for s in opts.pop('boolstyle', 'True, False').split(',')] + copy_options['boolstyle'] = [s.strip() for s in opts.pop('boolstyle', 'True, False').split(',')] copy_options['numprocesses'] = int(opts.pop('numprocesses', self.get_num_processes(16))) copy_options['begintoken'] = opts.pop('begintoken', '') copy_options['endtoken'] = opts.pop('endtoken', '') @@ -569,7 +565,7 @@ class ExportWriter(object): if self.header: writer = csv.writer(self.current_dest.output, **self.options.dialect) - writer.writerow([ensure_str(c) for c in self.columns]) + writer.writerow([str(c) for c in self.columns]) return True @@ -1728,7 +1724,7 @@ class ExportProcess(ChildProcess): return # no rows in this range try: - output = StringIO() if six.PY3 else BytesIO() + output = StringIO() writer = csv.writer(output, **self.options.dialect) for row in rows: @@ -1758,7 +1754,7 @@ class ExportProcess(ChildProcess): float_precision=cqltype.precision, nullval=self.nullval, quote=False, decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep, boolean_styles=self.boolean_styles) - return formatted if six.PY3 else formatted.encode('utf8') + return formatted def close(self): ChildProcess.close(self) @@ -1898,7 +1894,7 @@ class ImportConversion(object): select_query = 'SELECT * FROM %s.%s WHERE %s' % (protect_name(parent.ks), protect_name(parent.table), where_clause) - return parent.session.prepare(ensure_str(select_query)) + return parent.session.prepare(select_query) @staticmethod def unprotect(v): @@ -1934,20 +1930,20 @@ class ImportConversion(object): return BlobType(v[2:].decode("hex")) def convert_text(v, **_): - return ensure_str(v) + return str(v) def convert_uuid(v, **_): return UUID(v) def convert_bool(v, **_): - return True if v.lower() == ensure_str(self.boolean_styles[0]).lower() else False + return True if v.lower() == self.boolean_styles[0].lower() else False def get_convert_integer_fcn(adapter=int): """ Return a slow and a fast integer conversion function depending on self.thousands_sep """ if self.thousands_sep: - return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ensure_str(''))) + return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, '')) else: return lambda v, ct=cql_type: adapter(v) @@ -1955,8 +1951,8 @@ class ImportConversion(object): """ Return a slow and a fast decimal conversion function depending on self.thousands_sep and self.decimal_sep """ - empty_str = ensure_str('') - dot_str = ensure_str('.') + empty_str = '' + dot_str = '.' if self.thousands_sep and self.decimal_sep: return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str).replace(self.decimal_sep, dot_str)) elif self.thousands_sep: @@ -2020,14 +2016,8 @@ class ImportConversion(object): def convert_datetime(val, **_): try: - if six.PY2: - # Python 2 implementation - tval = time.strptime(val, self.date_time_format) - return timegm(tval) * 1e3 # scale seconds to millis for the raw value - else: - # Python 3 implementation - dtval = datetime.datetime.strptime(val, self.date_time_format) - return dtval.timestamp() * 1000 + dtval = datetime.datetime.strptime(val, self.date_time_format) + return dtval.timestamp() * 1000 except ValueError: pass # if it's not in the default format we try CQL formats @@ -2078,8 +2068,8 @@ class ImportConversion(object): """ See ImmutableDict above for a discussion of why a special object is needed here. """ - split_format_str = ensure_str('{%s}') - sep = ensure_str(':') + split_format_str = '{%s}' + sep = ':' return ImmutableDict(frozenset((convert_mandatory(ct.subtypes[0], v[0]), convert(ct.subtypes[1], v[1])) for v in [split(split_format_str % vv, sep=sep) for vv in split(val)])) @@ -2092,8 +2082,8 @@ class ImportConversion(object): Also note that it is possible that the subfield names in the csv are in the wrong order, so we must sort them according to ct.fieldnames, see CASSANDRA-12959. """ - split_format_str = ensure_str('{%s}') - sep = ensure_str(':') + split_format_str = '{%s}' + sep = ':' vals = [v for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]] dict_vals = dict((unprotect(v[0]), v[1]) for v in vals) sorted_converted_vals = [(n, convert(t, dict_vals[n]) if n in dict_vals else self.get_null_val()) @@ -2151,7 +2141,7 @@ class ImportConversion(object): or "NULL" otherwise. Note that for counters we never use prepared statements, so we only check is_counter when use_prepared_statements is false. """ - return None if self.use_prepared_statements else (ensure_str("0") if self.is_counter else ensure_str("NULL")) + return None if self.use_prepared_statements else ("0" if self.is_counter else "NULL") def convert_row(self, row): """ @@ -2436,7 +2426,6 @@ class ImportProcess(ChildProcess): if self.ttl >= 0: query += 'USING TTL %s' % (self.ttl,) make_statement = self.wrap_make_statement(self.make_non_prepared_batch_statement) - query = ensure_str(query) conv = ImportConversion(self, table_meta, prepared_statement) tm = TokenMap(self.ks, self.hostname, self.local_dc, self.session) @@ -2504,12 +2493,12 @@ class ImportProcess(ChildProcess): set_clause = [] for i, value in enumerate(row): if i in conv.primary_key_indexes: - where_clause.append(ensure_text("{}={}").format(self.valid_columns[i], ensure_text(value))) + where_clause.append("{}={}".format(self.valid_columns[i], str(value))) else: - set_clause.append(ensure_text("{}={}+{}").format(self.valid_columns[i], self.valid_columns[i], ensure_text(value))) + set_clause.append("{}={}+{}".format(self.valid_columns[i], self.valid_columns[i], str(value))) - full_query_text = query % (ensure_text(',').join(set_clause), ensure_text(' AND ').join(where_clause)) - statement.add(ensure_str(full_query_text)) + full_query_text = query % (','.join(set_clause), ' AND '.join(where_clause)) + statement.add(full_query_text) return statement def make_prepared_batch_statement(self, query, _, batch, replicas): @@ -2533,7 +2522,7 @@ class ImportProcess(ChildProcess): statement = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=self.consistency_level) statement.replicas = replicas statement.keyspace = self.ks - field_sep = b',' if six.PY2 else ',' + field_sep = ',' statement._statements_and_parameters = [(False, query % (field_sep.join(r),), ()) for r in batch['rows']] return statement diff --git a/pylib/cqlshlib/displaying.py b/pylib/cqlshlib/displaying.py index 7109f11815..424d6334b6 100644 --- a/pylib/cqlshlib/displaying.py +++ b/pylib/cqlshlib/displaying.py @@ -41,7 +41,7 @@ def get_str(val): return val -class FormattedValue(object): +class FormattedValue: def __init__(self, strval, coloredval=None, displaywidth=None): self.strval = strval diff --git a/pylib/cqlshlib/formatting.py b/pylib/cqlshlib/formatting.py index b82486a785..42c9305e76 100644 --- a/pylib/cqlshlib/formatting.py +++ b/pylib/cqlshlib/formatting.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import binascii import calendar import datetime import math @@ -22,8 +21,6 @@ import os import re import sys -from six import ensure_text - from collections import defaultdict from cassandra.cqltypes import EMPTY @@ -121,7 +118,7 @@ class DateTimeFormat: self.milliseconds_only = milliseconds_only # the microseconds part, .NNNNNN, wil be rounded to .NNN -class CqlType(object): +class CqlType: """ A class for converting a string into a cql type name that can match a formatter and a list of its sub-types, if any. @@ -203,7 +200,7 @@ class CqlType(object): def format_value_default(val, colormap, **_): - val = ensure_text(str(val)) + val = str(val) escapedval = val.replace('\\', '\\\\') bval = controlchars_re.sub(_show_control_chars, escapedval) return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap) @@ -235,7 +232,7 @@ def formatter_for(typname): return registrator -class BlobType(object): +class BlobType: def __init__(self, val): self.val = val @@ -245,7 +242,7 @@ class BlobType(object): @formatter_for('BlobType') def format_value_blob(val, colormap, **_): - bval = ensure_text('0x') + ensure_text(binascii.hexlify(val)) + bval = '0x' + val.hex() return colorme(bval, colormap, 'blob') @@ -255,7 +252,7 @@ formatter_for('blob')(format_value_blob) def format_python_formatted_type(val, colormap, color, quote=False): - bval = ensure_text(str(val)) + bval = str(val) if quote: bval = "'%s'" % bval return colorme(bval, colormap, color) @@ -325,7 +322,7 @@ formatter_for('double')(format_floating_point_type) def format_integer_type(val, colormap, thousands_sep=None, **_): # base-10 only for now; support others? bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val) - bval = ensure_text(bval) + bval = str(bval) return colorme(bval, colormap, 'int') @@ -360,7 +357,7 @@ def format_value_timestamp(val, colormap, date_time_format, quote=False, **_): if date_time_format.milliseconds_only: bval = round_microseconds(bval) else: - bval = ensure_text(str(val)) + bval = str(val) if quote: bval = "'%s'" % bval diff --git a/pylib/cqlshlib/helptopics.py b/pylib/cqlshlib/helptopics.py index 46cd1561e6..9be56b9f2e 100644 --- a/pylib/cqlshlib/helptopics.py +++ b/pylib/cqlshlib/helptopics.py @@ -15,7 +15,7 @@ # limitations under the License. -class CQL3HelpTopics(object): +class CQL3HelpTopics: def get_help_topics(self): return [t[5:] for t in dir(self) if t.startswith('help_')] diff --git a/pylib/cqlshlib/test/ansi_colors.py b/pylib/cqlshlib/test/ansi_colors.py index 2bdc736b78..494b7c62ec 100644 --- a/pylib/cqlshlib/test/ansi_colors.py +++ b/pylib/cqlshlib/test/ansi_colors.py @@ -15,7 +15,6 @@ # limitations under the License. import re -import six LIGHT = 0o10 @@ -106,7 +105,7 @@ class ColoredChar(object): class ColoredText(object): def __init__(self, source=''): - if isinstance(source, six.text_type): + if isinstance(source, str): plain, colors = self.parse_ansi_colors(source) self.chars = list(map(ColoredChar, plain, colors)) else: diff --git a/pylib/cqlshlib/test/run_cqlsh.py b/pylib/cqlshlib/test/run_cqlsh.py index 9adb854b4b..180796abad 100644 --- a/pylib/cqlshlib/test/run_cqlsh.py +++ b/pylib/cqlshlib/test/run_cqlsh.py @@ -30,7 +30,7 @@ from os.path import join import pty DEFAULT_PREFIX = os.linesep -DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> ' +DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + r'(\S+@)?cqlsh(:\S+)?> ' DEFAULT_CQLSH_TERM = 'xterm' try: diff --git a/pylib/cqlshlib/test/winpty.py b/pylib/cqlshlib/test/winpty.py index 9381a2f586..02b1981ef0 100644 --- a/pylib/cqlshlib/test/winpty.py +++ b/pylib/cqlshlib/test/winpty.py @@ -15,8 +15,8 @@ # limitations under the License. from threading import Thread -from six import StringIO -from six.moves.queue import Queue, Empty +from io import StringIO +from queue import Queue, Empty class WinPty(object): diff --git a/pylib/cqlshlib/tracing.py b/pylib/cqlshlib/tracing.py index e068250339..3db8e3491d 100644 --- a/pylib/cqlshlib/tracing.py +++ b/pylib/cqlshlib/tracing.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime, timedelta +from datetime import datetime import time from cassandra.query import QueryTrace, TraceUnavailable @@ -73,8 +73,6 @@ def make_trace_rows(trace): if trace.duration: finished_at = (datetime_from_utc_to_local(trace.started_at) + trace.duration) rows.append(['Request complete', str(finished_at), trace.coordinator, total_micro_seconds(trace.duration), trace.client]) - else: - finished_at = trace.duration = "--" return rows diff --git a/pylib/cqlshlib/util.py b/pylib/cqlshlib/util.py index 82a332f85a..f29141d80f 100644 --- a/pylib/cqlshlib/util.py +++ b/pylib/cqlshlib/util.py @@ -21,7 +21,7 @@ import pstats from datetime import timedelta, tzinfo -from six import StringIO +from io import StringIO try: from line_profiler import LineProfiler diff --git a/pylib/requirements.txt b/pylib/requirements.txt index 10084dd2c7..4d66ef2a3f 100644 --- a/pylib/requirements.txt +++ b/pylib/requirements.txt @@ -1,7 +1,3 @@ -# See python driver docs: six have to be installed before -# cythonizing the driver, perhaps only on old pips. -# http://datastax.github.io/python-driver/installation.html#cython-based-extensions -six>=1.12.0 -e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver # Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases. -e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm