natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

change six functions in cqlshlib to native Python 3 ones 

patch by Brad Schoening; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-17417
This commit is contained in:
Brad Schoening 2022-03-10 21:18:30 -05:00 committed by Brandon Williams
parent b9d9f9c7da
commit 768bdffe52
12 changed files with 64 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.1
* change six functions in cqlshlib to native Python 3 (CASSANDRA-17417)
* reduce hot-path object allocations required to record local/remote requests against the client request metrics (CASSANDRA-17424)
* Disallow removing DC from system_auth while nodes are active in the DC (CASSANDRA-17478)
* Add guardrail for the number of fields per UDT (CASSANDRA-17385)

45
bin/cqlsh.py
View File

@ -110,19 +110,16 @@ if cql_zip:
ver = os.path.splitext(os.path.basename(cql_zip))[0][len(CQL_LIB_PREFIX):]
sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
third_parties = ('futures-', 'six-')
# the driver needs dependencies
third_parties = ('six-')
for lib in third_parties:
lib_zip = find_zip(lib)
if lib_zip:
sys.path.insert(0, lib_zip)
# We cannot import six until we add its location to sys.path so the Python
# interpreter can find it. Do not move this to the top.
import six
from six.moves import configparser, input
from six import StringIO, ensure_text, ensure_str
import configparser
from io import StringIO
warnings.filterwarnings("ignore", r".*blist.*")
try:
@ -361,7 +358,7 @@ class DecodeError(Exception):
def maybe_ensure_text(val):
return ensure_text(val) if val else val
return str(val) if val else val
class FormatError(DecodeError):
@ -426,7 +423,7 @@ def insert_driver_hooks():
class Shell(cmd.Cmd):
custom_prompt = ensure_text(os.getenv('CQLSH_PROMPT', ''))
custom_prompt = os.getenv('CQLSH_PROMPT', '')
if custom_prompt != '':
custom_prompt += "\n"
default_prompt = custom_prompt + "cqlsh> "
@ -860,15 +857,14 @@ class Shell(cmd.Cmd):
def get_input_line(self, prompt=''):
if self.tty:
self.lastcmd = input(ensure_str(prompt))
line = ensure_text(self.lastcmd) + '\n'
self.lastcmd = input(str(prompt))
line = self.lastcmd + '\n'
else:
self.lastcmd = ensure_text(self.stdin.readline())
self.lastcmd = self.stdin.readline()
line = self.lastcmd
if not len(line):
raise EOFError
self.lineno += 1
line = ensure_text(line)
return line
def use_stdin_reader(self, until='', prompt=''):
@ -929,7 +925,6 @@ class Shell(cmd.Cmd):
Returns true if the statement is complete and was handled (meaning it
can be reset).
"""
statementtext = ensure_text(statementtext)
statementtext = self.strip_comment_blocks(statementtext)
try:
statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext)
@ -975,7 +970,7 @@ class Shell(cmd.Cmd):
if readline is not None:
nl_count = srcstr.count("\n")
new_hist = ensure_str(srcstr.replace("\n", " ").rstrip())
new_hist = srcstr.replace("\n", " ").rstrip()
if nl_count > 1 and self.last_hist != new_hist:
readline.add_history(new_hist)
@ -1026,7 +1021,6 @@ class Shell(cmd.Cmd):
self.tracing_enabled = tracing_was_enabled
def perform_statement(self, statement):
statement = ensure_text(statement)
stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None)
success, future = self.perform_simple_statement(stmt)
@ -1082,7 +1076,7 @@ class Shell(cmd.Cmd):
try:
result = future.result()
except CQL_ERRORS as err:
err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err))
err_msg = err.message if hasattr(err, 'message') else str(err)
self.printerr(str(err.__class__.__name__) + ": " + err_msg)
except Exception:
import traceback
@ -1397,7 +1391,7 @@ class Shell(cmd.Cmd):
self.describe_element(result)
except CQL_ERRORS as err:
err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err))
err_msg = err.message if hasattr(err, 'message') else str(err)
self.printerr(err_msg.partition("message=")[2].strip('"'))
except Exception:
import traceback
@ -1413,7 +1407,7 @@ class Shell(cmd.Cmd):
"""
Print the output for a DESCRIBE KEYSPACES query
"""
names = [ensure_str(r['name']) for r in rows]
names = [r['name'] for r in rows]
print('')
cmd.Cmd.columnize(self, names)
@ -1433,7 +1427,7 @@ class Shell(cmd.Cmd):
keyspace = row['keyspace_name']
names = list()
names.append(ensure_str(row['name']))
names.append(str(row['name']))
if keyspace is not None:
self.print_keyspace_element_names(keyspace, names)
@ -1571,7 +1565,7 @@ class Shell(cmd.Cmd):
if fname is not None:
fname = self.cql_unprotect_value(fname)
copyoptnames = list(map(six.text_type.lower, parsed.get_binding('optnames', ())))
copyoptnames = list(map(str.lower, parsed.get_binding('optnames', ())))
copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ())))
opts = dict(list(zip(copyoptnames, copyoptvals)))
@ -1992,11 +1986,10 @@ class Shell(cmd.Cmd):
out = self.query_out
# convert Exceptions, etc to text
if not isinstance(text, six.text_type):
text = "{}".format(text)
if not isinstance(text, str):
text = str(text)
to_write = self.applycolor(text, color) + ('\n' if newline else '')
to_write = ensure_str(to_write)
out.write(to_write)
def flush_output(self):
@ -2128,7 +2121,7 @@ def is_file_secure(filename):
def read_options(cmdlineargs, environment):
configs = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser()
configs = configparser.ConfigParser()
configs.read(CONFIG_FILE)
rawconfigs = configparser.RawConfigParser()
@ -2200,7 +2193,7 @@ def read_options(cmdlineargs, environment):
options.credentials = '' # ConfigParser.read() will ignore unreadable files
if not options.username:
credentials = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser()
credentials = configparser.ConfigParser()
credentials.read(options.credentials)
# use the username from credentials file but fallback to cqlshrc if username is absent from the command line parameters

69
pylib/cqlshlib/copyutil.py
View File

@ -26,7 +26,6 @@ import platform
import random
import re
import signal
import six
import struct
import sys
import threading
@ -39,15 +38,12 @@ from calendar import timegm
from collections import defaultdict, namedtuple
from decimal import Decimal
from random import randint
from io import BytesIO, StringIO
from io import StringIO
from select import select
from uuid import UUID
from .util import profile_on, profile_off
from six import ensure_str, ensure_text
from six.moves import configparser
from six.moves import range
from six.moves.queue import Queue
import configparser
from queue import Queue
from cassandra import OperationTimedOut
from cassandra.cluster import Cluster, DefaultConnection
@ -334,9 +330,9 @@ class CopyTask(object):
opts = self.clean_options(self.maybe_read_config_file(opts, direction))
dialect_options = dict()
dialect_options['quotechar'] = ensure_str(opts.pop('quote', '"'))
dialect_options['escapechar'] = ensure_str(opts.pop('escape', '\\'))
dialect_options['delimiter'] = ensure_str(opts.pop('delimiter', ','))
dialect_options['quotechar'] = opts.pop('quote', '"')
dialect_options['escapechar'] = opts.pop('escape', '\\')
dialect_options['delimiter'] = opts.pop('delimiter', ',')
if dialect_options['quotechar'] == dialect_options['escapechar']:
dialect_options['doublequote'] = True
del dialect_options['escapechar']
@ -344,7 +340,7 @@ class CopyTask(object):
dialect_options['doublequote'] = False
copy_options = dict()
copy_options['nullval'] = ensure_str(opts.pop('null', ''))
copy_options['nullval'] = opts.pop('null', '')
copy_options['header'] = bool(opts.pop('header', '').lower() == 'true')
copy_options['encoding'] = opts.pop('encoding', 'utf8')
copy_options['maxrequests'] = int(opts.pop('maxrequests', 6))
@ -366,7 +362,7 @@ class CopyTask(object):
copy_options['consistencylevel'] = shell.consistency_level
copy_options['decimalsep'] = opts.pop('decimalsep', '.')
copy_options['thousandssep'] = opts.pop('thousandssep', '')
copy_options['boolstyle'] = [ensure_str(s.strip()) for s in opts.pop('boolstyle', 'True, False').split(',')]
copy_options['boolstyle'] = [s.strip() for s in opts.pop('boolstyle', 'True, False').split(',')]
copy_options['numprocesses'] = int(opts.pop('numprocesses', self.get_num_processes(16)))
copy_options['begintoken'] = opts.pop('begintoken', '')
copy_options['endtoken'] = opts.pop('endtoken', '')
@ -569,7 +565,7 @@ class ExportWriter(object):
if self.header:
writer = csv.writer(self.current_dest.output, **self.options.dialect)
writer.writerow([ensure_str(c) for c in self.columns])
writer.writerow([str(c) for c in self.columns])
return True
@ -1728,7 +1724,7 @@ class ExportProcess(ChildProcess):
return # no rows in this range
try:
output = StringIO() if six.PY3 else BytesIO()
output = StringIO()
writer = csv.writer(output, **self.options.dialect)
for row in rows:
@ -1758,7 +1754,7 @@ class ExportProcess(ChildProcess):
float_precision=cqltype.precision, nullval=self.nullval, quote=False,
decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep,
boolean_styles=self.boolean_styles)
return formatted if six.PY3 else formatted.encode('utf8')
return formatted
def close(self):
ChildProcess.close(self)
@ -1898,7 +1894,7 @@ class ImportConversion(object):
select_query = 'SELECT * FROM %s.%s WHERE %s' % (protect_name(parent.ks),
protect_name(parent.table),
where_clause)
return parent.session.prepare(ensure_str(select_query))
return parent.session.prepare(select_query)
@staticmethod
def unprotect(v):
@ -1934,20 +1930,20 @@ class ImportConversion(object):
return BlobType(v[2:].decode("hex"))
def convert_text(v, **_):
return ensure_str(v)
return str(v)
def convert_uuid(v, **_):
return UUID(v)
def convert_bool(v, **_):
return True if v.lower() == ensure_str(self.boolean_styles[0]).lower() else False
return True if v.lower() == self.boolean_styles[0].lower() else False
def get_convert_integer_fcn(adapter=int):
"""
Return a slow and a fast integer conversion function depending on self.thousands_sep
"""
if self.thousands_sep:
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ensure_str('')))
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ''))
else:
return lambda v, ct=cql_type: adapter(v)
@ -1955,8 +1951,8 @@ class ImportConversion(object):
"""
Return a slow and a fast decimal conversion function depending on self.thousands_sep and self.decimal_sep
"""
empty_str = ensure_str('')
dot_str = ensure_str('.')
empty_str = ''
dot_str = '.'
if self.thousands_sep and self.decimal_sep:
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str).replace(self.decimal_sep, dot_str))
elif self.thousands_sep:
@ -2020,14 +2016,8 @@ class ImportConversion(object):
def convert_datetime(val, **_):
try:
if six.PY2:
# Python 2 implementation
tval = time.strptime(val, self.date_time_format)
return timegm(tval) * 1e3 # scale seconds to millis for the raw value
else:
# Python 3 implementation
dtval = datetime.datetime.strptime(val, self.date_time_format)
return dtval.timestamp() * 1000
dtval = datetime.datetime.strptime(val, self.date_time_format)
return dtval.timestamp() * 1000
except ValueError:
pass # if it's not in the default format we try CQL formats
@ -2078,8 +2068,8 @@ class ImportConversion(object):
"""
See ImmutableDict above for a discussion of why a special object is needed here.
"""
split_format_str = ensure_str('{%s}')
sep = ensure_str(':')
split_format_str = '{%s}'
sep = ':'
return ImmutableDict(frozenset((convert_mandatory(ct.subtypes[0], v[0]), convert(ct.subtypes[1], v[1]))
for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]))
@ -2092,8 +2082,8 @@ class ImportConversion(object):
Also note that it is possible that the subfield names in the csv are in the
wrong order, so we must sort them according to ct.fieldnames, see CASSANDRA-12959.
"""
split_format_str = ensure_str('{%s}')
sep = ensure_str(':')
split_format_str = '{%s}'
sep = ':'
vals = [v for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]]
dict_vals = dict((unprotect(v[0]), v[1]) for v in vals)
sorted_converted_vals = [(n, convert(t, dict_vals[n]) if n in dict_vals else self.get_null_val())
@ -2151,7 +2141,7 @@ class ImportConversion(object):
or "NULL" otherwise. Note that for counters we never use prepared statements, so we
only check is_counter when use_prepared_statements is false.
"""
return None if self.use_prepared_statements else (ensure_str("0") if self.is_counter else ensure_str("NULL"))
return None if self.use_prepared_statements else ("0" if self.is_counter else "NULL")
def convert_row(self, row):
"""
@ -2436,7 +2426,6 @@ class ImportProcess(ChildProcess):
if self.ttl >= 0:
query += 'USING TTL %s' % (self.ttl,)
make_statement = self.wrap_make_statement(self.make_non_prepared_batch_statement)
query = ensure_str(query)
conv = ImportConversion(self, table_meta, prepared_statement)
tm = TokenMap(self.ks, self.hostname, self.local_dc, self.session)
@ -2504,12 +2493,12 @@ class ImportProcess(ChildProcess):
set_clause = []
for i, value in enumerate(row):
if i in conv.primary_key_indexes:
where_clause.append(ensure_text("{}={}").format(self.valid_columns[i], ensure_text(value)))
where_clause.append("{}={}".format(self.valid_columns[i], str(value)))
else:
set_clause.append(ensure_text("{}={}+{}").format(self.valid_columns[i], self.valid_columns[i], ensure_text(value)))
set_clause.append("{}={}+{}".format(self.valid_columns[i], self.valid_columns[i], str(value)))
full_query_text = query % (ensure_text(',').join(set_clause), ensure_text(' AND ').join(where_clause))
statement.add(ensure_str(full_query_text))
full_query_text = query % (','.join(set_clause), ' AND '.join(where_clause))
statement.add(full_query_text)
return statement
def make_prepared_batch_statement(self, query, _, batch, replicas):
@ -2533,7 +2522,7 @@ class ImportProcess(ChildProcess):
statement = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=self.consistency_level)
statement.replicas = replicas
statement.keyspace = self.ks
field_sep = b',' if six.PY2 else ','
field_sep = ','
statement._statements_and_parameters = [(False, query % (field_sep.join(r),), ()) for r in batch['rows']]
return statement

2
pylib/cqlshlib/displaying.py
View File

@ -41,7 +41,7 @@ def get_str(val):
return val
class FormattedValue(object):
class FormattedValue:
def __init__(self, strval, coloredval=None, displaywidth=None):
self.strval = strval

17
pylib/cqlshlib/formatting.py
View File

@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import binascii
import calendar
import datetime
import math
@ -22,8 +21,6 @@ import os
import re
import sys
from six import ensure_text
from collections import defaultdict
from cassandra.cqltypes import EMPTY
@ -121,7 +118,7 @@ class DateTimeFormat:
self.milliseconds_only = milliseconds_only # the microseconds part, .NNNNNN, wil be rounded to .NNN
class CqlType(object):
class CqlType:
"""
A class for converting a string into a cql type name that can match a formatter
and a list of its sub-types, if any.
@ -203,7 +200,7 @@ class CqlType(object):
def format_value_default(val, colormap, **_):
val = ensure_text(str(val))
val = str(val)
escapedval = val.replace('\\', '\\\\')
bval = controlchars_re.sub(_show_control_chars, escapedval)
return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap)
@ -235,7 +232,7 @@ def formatter_for(typname):
return registrator
class BlobType(object):
class BlobType:
def __init__(self, val):
self.val = val
@ -245,7 +242,7 @@ class BlobType(object):
@formatter_for('BlobType')
def format_value_blob(val, colormap, **_):
bval = ensure_text('0x') + ensure_text(binascii.hexlify(val))
bval = '0x' + val.hex()
return colorme(bval, colormap, 'blob')
@ -255,7 +252,7 @@ formatter_for('blob')(format_value_blob)
def format_python_formatted_type(val, colormap, color, quote=False):
bval = ensure_text(str(val))
bval = str(val)
if quote:
bval = "'%s'" % bval
return colorme(bval, colormap, color)
@ -325,7 +322,7 @@ formatter_for('double')(format_floating_point_type)
def format_integer_type(val, colormap, thousands_sep=None, **_):
# base-10 only for now; support others?
bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
bval = ensure_text(bval)
bval = str(bval)
return colorme(bval, colormap, 'int')
@ -360,7 +357,7 @@ def format_value_timestamp(val, colormap, date_time_format, quote=False, **_):
if date_time_format.milliseconds_only:
bval = round_microseconds(bval)
else:
bval = ensure_text(str(val))
bval = str(val)
if quote:
bval = "'%s'" % bval

2
pylib/cqlshlib/helptopics.py
View File

@ -15,7 +15,7 @@
# limitations under the License.
class CQL3HelpTopics(object):
class CQL3HelpTopics:
def get_help_topics(self):
return [t[5:] for t in dir(self) if t.startswith('help_')]

3
pylib/cqlshlib/test/ansi_colors.py
View File

@ -15,7 +15,6 @@
# limitations under the License.
import re
import six
LIGHT = 0o10
@ -106,7 +105,7 @@ class ColoredChar(object):
class ColoredText(object):
def __init__(self, source=''):
if isinstance(source, six.text_type):
if isinstance(source, str):
plain, colors = self.parse_ansi_colors(source)
self.chars = list(map(ColoredChar, plain, colors))
else:

2
pylib/cqlshlib/test/run_cqlsh.py
View File

@ -30,7 +30,7 @@ from os.path import join
import pty
DEFAULT_PREFIX = os.linesep
DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> '
DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + r'(\S+@)?cqlsh(:\S+)?> '
DEFAULT_CQLSH_TERM = 'xterm'
try:

4
pylib/cqlshlib/test/winpty.py
View File

@ -15,8 +15,8 @@
# limitations under the License.
from threading import Thread
from six import StringIO
from six.moves.queue import Queue, Empty
from io import StringIO
from queue import Queue, Empty
class WinPty(object):

4
pylib/cqlshlib/tracing.py
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime, timedelta
from datetime import datetime
import time
from cassandra.query import QueryTrace, TraceUnavailable
@ -73,8 +73,6 @@ def make_trace_rows(trace):
if trace.duration:
finished_at = (datetime_from_utc_to_local(trace.started_at) + trace.duration)
rows.append(['Request complete', str(finished_at), trace.coordinator, total_micro_seconds(trace.duration), trace.client])
else:
finished_at = trace.duration = "--"
return rows

2
pylib/cqlshlib/util.py
View File

@ -21,7 +21,7 @@ import pstats
from datetime import timedelta, tzinfo
from six import StringIO
from io import StringIO
try:
from line_profiler import LineProfiler

4
pylib/requirements.txt
View File

@ -1,7 +1,3 @@
# See python driver docs: six have to be installed before
# cythonizing the driver, perhaps only on old pips.
# http://datastax.github.io/python-driver/installation.html#cython-based-extensions
six>=1.12.0
-e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver
# Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases.
-e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm