natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "change six functions in cqlshlib to native Python 3 ones" 

This reverts commit 7275443eca.
This commit is contained in:
Ekaterina Dimitrova 2022-03-28 15:59:10 -04:00
parent 3d9365096b
commit 813e6ae4a0
11 changed files with 97 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
bin/cqlsh.py
View File

@ -110,8 +110,19 @@ if cql_zip:
ver = os.path.splitext(os.path.basename(cql_zip))[0][len(CQL_LIB_PREFIX):]
sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
import configparser
from io import StringIO
third_parties = ('futures-', 'six-')
for lib in third_parties:
lib_zip = find_zip(lib)
if lib_zip:
sys.path.insert(0, lib_zip)
# We cannot import six until we add its location to sys.path so the Python
# interpreter can find it. Do not move this to the top.
import six
from six.moves import configparser, input
from six import StringIO, ensure_text, ensure_str
warnings.filterwarnings("ignore", r".*blist.*")
try:
@ -350,7 +361,7 @@ class DecodeError(Exception):
def maybe_ensure_text(val):
return str(val) if val else val
return ensure_text(val) if val else val
class FormatError(DecodeError):
@ -415,7 +426,7 @@ def insert_driver_hooks():
class Shell(cmd.Cmd):
custom_prompt = os.getenv('CQLSH_PROMPT', '')
custom_prompt = ensure_text(os.getenv('CQLSH_PROMPT', ''))
if custom_prompt != '':
custom_prompt += "\n"
default_prompt = custom_prompt + "cqlsh> "
@ -849,14 +860,15 @@ class Shell(cmd.Cmd):
def get_input_line(self, prompt=''):
if self.tty:
self.lastcmd = input(str(prompt))
line = self.lastcmd + '\n'
self.lastcmd = input(ensure_str(prompt))
line = ensure_text(self.lastcmd) + '\n'
else:
self.lastcmd = self.stdin.readline()
self.lastcmd = ensure_text(self.stdin.readline())
line = self.lastcmd
if not len(line):
raise EOFError
self.lineno += 1
line = ensure_text(line)
return line
def use_stdin_reader(self, until='', prompt=''):
@ -917,6 +929,7 @@ class Shell(cmd.Cmd):
Returns true if the statement is complete and was handled (meaning it
can be reset).
"""
statementtext = ensure_text(statementtext)
statementtext = self.strip_comment_blocks(statementtext)
try:
statements, endtoken_escaped = cqlruleset.cql_split_statements(statementtext)
@ -962,7 +975,7 @@ class Shell(cmd.Cmd):
if readline is not None:
nl_count = srcstr.count("\n")
new_hist = srcstr.replace("\n", " ").rstrip()
new_hist = ensure_str(srcstr.replace("\n", " ").rstrip())
if nl_count > 1 and self.last_hist != new_hist:
readline.add_history(new_hist)
@ -1013,6 +1026,7 @@ class Shell(cmd.Cmd):
self.tracing_enabled = tracing_was_enabled
def perform_statement(self, statement):
statement = ensure_text(statement)
stmt = SimpleStatement(statement, consistency_level=self.consistency_level, serial_consistency_level=self.serial_consistency_level, fetch_size=self.page_size if self.use_paging else None)
success, future = self.perform_simple_statement(stmt)
@ -1068,7 +1082,7 @@ class Shell(cmd.Cmd):
try:
result = future.result()
except CQL_ERRORS as err:
err_msg = err.message if hasattr(err, 'message') else str(err)
err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err))
self.printerr(str(err.__class__.__name__) + ": " + err_msg)
except Exception:
import traceback
@ -1383,7 +1397,7 @@ class Shell(cmd.Cmd):
self.describe_element(result)
except CQL_ERRORS as err:
err_msg = err.message if hasattr(err, 'message') else str(err)
err_msg = ensure_text(err.message if hasattr(err, 'message') else str(err))
self.printerr(err_msg.partition("message=")[2].strip('"'))
except Exception:
import traceback
@ -1399,7 +1413,7 @@ class Shell(cmd.Cmd):
"""
Print the output for a DESCRIBE KEYSPACES query
"""
names = [r['name'] for r in rows]
names = [ensure_str(r['name']) for r in rows]
print('')
cmd.Cmd.columnize(self, names)
@ -1419,7 +1433,7 @@ class Shell(cmd.Cmd):
keyspace = row['keyspace_name']
names = list()
names.append(str(row['name']))
names.append(ensure_str(row['name']))
if keyspace is not None:
self.print_keyspace_element_names(keyspace, names)
@ -1557,7 +1571,7 @@ class Shell(cmd.Cmd):
if fname is not None:
fname = self.cql_unprotect_value(fname)
copyoptnames = list(map(str.lower, parsed.get_binding('optnames', ())))
copyoptnames = list(map(six.text_type.lower, parsed.get_binding('optnames', ())))
copyoptvals = list(map(self.cql_unprotect_value, parsed.get_binding('optvals', ())))
opts = dict(list(zip(copyoptnames, copyoptvals)))
@ -1978,10 +1992,11 @@ class Shell(cmd.Cmd):
out = self.query_out
# convert Exceptions, etc to text
if not isinstance(text, str):
text = str(text)
if not isinstance(text, six.text_type):
text = "{}".format(text)
to_write = self.applycolor(text, color) + ('\n' if newline else '')
to_write = ensure_str(to_write)
out.write(to_write)
def flush_output(self):
@ -2113,7 +2128,7 @@ def is_file_secure(filename):
def read_options(cmdlineargs, environment):
configs = configparser.ConfigParser()
configs = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser()
configs.read(CONFIG_FILE)
rawconfigs = configparser.RawConfigParser()
@ -2185,7 +2200,7 @@ def read_options(cmdlineargs, environment):
options.credentials = '' # ConfigParser.read() will ignore unreadable files
if not options.username:
credentials = configparser.ConfigParser()
credentials = configparser.SafeConfigParser() if sys.version_info < (3, 2) else configparser.ConfigParser()
credentials.read(options.credentials)
# use the username from credentials file but fallback to cqlshrc if username is absent from the command line parameters

69
pylib/cqlshlib/copyutil.py
View File

@ -26,6 +26,7 @@ import platform
import random
import re
import signal
import six
import struct
import sys
import threading
@ -38,12 +39,15 @@ from calendar import timegm
from collections import defaultdict, namedtuple
from decimal import Decimal
from random import randint
from io import StringIO
from io import BytesIO, StringIO
from select import select
from uuid import UUID
from .util import profile_on, profile_off
import configparser
from queue import Queue
from six import ensure_str, ensure_text
from six.moves import configparser
from six.moves import range
from six.moves.queue import Queue
from cassandra import OperationTimedOut
from cassandra.cluster import Cluster, DefaultConnection
@ -330,9 +334,9 @@ class CopyTask(object):
opts = self.clean_options(self.maybe_read_config_file(opts, direction))
dialect_options = dict()
dialect_options['quotechar'] = opts.pop('quote', '"')
dialect_options['escapechar'] = opts.pop('escape', '\\')
dialect_options['delimiter'] = opts.pop('delimiter', ',')
dialect_options['quotechar'] = ensure_str(opts.pop('quote', '"'))
dialect_options['escapechar'] = ensure_str(opts.pop('escape', '\\'))
dialect_options['delimiter'] = ensure_str(opts.pop('delimiter', ','))
if dialect_options['quotechar'] == dialect_options['escapechar']:
dialect_options['doublequote'] = True
del dialect_options['escapechar']
@ -340,7 +344,7 @@ class CopyTask(object):
dialect_options['doublequote'] = False
copy_options = dict()
copy_options['nullval'] = opts.pop('null', '')
copy_options['nullval'] = ensure_str(opts.pop('null', ''))
copy_options['header'] = bool(opts.pop('header', '').lower() == 'true')
copy_options['encoding'] = opts.pop('encoding', 'utf8')
copy_options['maxrequests'] = int(opts.pop('maxrequests', 6))
@ -362,7 +366,7 @@ class CopyTask(object):
copy_options['consistencylevel'] = shell.consistency_level
copy_options['decimalsep'] = opts.pop('decimalsep', '.')
copy_options['thousandssep'] = opts.pop('thousandssep', '')
copy_options['boolstyle'] = [s.strip() for s in opts.pop('boolstyle', 'True, False').split(',')]
copy_options['boolstyle'] = [ensure_str(s.strip()) for s in opts.pop('boolstyle', 'True, False').split(',')]
copy_options['numprocesses'] = int(opts.pop('numprocesses', self.get_num_processes(16)))
copy_options['begintoken'] = opts.pop('begintoken', '')
copy_options['endtoken'] = opts.pop('endtoken', '')
@ -565,7 +569,7 @@ class ExportWriter(object):
if self.header:
writer = csv.writer(self.current_dest.output, **self.options.dialect)
writer.writerow([str(c) for c in self.columns])
writer.writerow([ensure_str(c) for c in self.columns])
return True
@ -1724,7 +1728,7 @@ class ExportProcess(ChildProcess):
return # no rows in this range
try:
output = StringIO()
output = StringIO() if six.PY3 else BytesIO()
writer = csv.writer(output, **self.options.dialect)
for row in rows:
@ -1754,7 +1758,7 @@ class ExportProcess(ChildProcess):
float_precision=cqltype.precision, nullval=self.nullval, quote=False,
decimal_sep=self.decimal_sep, thousands_sep=self.thousands_sep,
boolean_styles=self.boolean_styles)
return formatted
return formatted if six.PY3 else formatted.encode('utf8')
def close(self):
ChildProcess.close(self)
@ -1894,7 +1898,7 @@ class ImportConversion(object):
select_query = 'SELECT * FROM %s.%s WHERE %s' % (protect_name(parent.ks),
protect_name(parent.table),
where_clause)
return parent.session.prepare(select_query)
return parent.session.prepare(ensure_str(select_query))
@staticmethod
def unprotect(v):
@ -1930,20 +1934,20 @@ class ImportConversion(object):
return BlobType(v[2:].decode("hex"))
def convert_text(v, **_):
return str(v)
return ensure_str(v)
def convert_uuid(v, **_):
return UUID(v)
def convert_bool(v, **_):
return True if v.lower() == self.boolean_styles[0].lower() else False
return True if v.lower() == ensure_str(self.boolean_styles[0]).lower() else False
def get_convert_integer_fcn(adapter=int):
"""
Return a slow and a fast integer conversion function depending on self.thousands_sep
"""
if self.thousands_sep:
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ''))
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, ensure_str('')))
else:
return lambda v, ct=cql_type: adapter(v)
@ -1951,8 +1955,8 @@ class ImportConversion(object):
"""
Return a slow and a fast decimal conversion function depending on self.thousands_sep and self.decimal_sep
"""
empty_str = ''
dot_str = '.'
empty_str = ensure_str('')
dot_str = ensure_str('.')
if self.thousands_sep and self.decimal_sep:
return lambda v, ct=cql_type: adapter(v.replace(self.thousands_sep, empty_str).replace(self.decimal_sep, dot_str))
elif self.thousands_sep:
@ -2016,8 +2020,14 @@ class ImportConversion(object):
def convert_datetime(val, **_):
try:
dtval = datetime.datetime.strptime(val, self.date_time_format)
return dtval.timestamp() * 1000
if six.PY2:
# Python 2 implementation
tval = time.strptime(val, self.date_time_format)
return timegm(tval) * 1e3 # scale seconds to millis for the raw value
else:
# Python 3 implementation
dtval = datetime.datetime.strptime(val, self.date_time_format)
return dtval.timestamp() * 1000
except ValueError:
pass # if it's not in the default format we try CQL formats
@ -2068,8 +2078,8 @@ class ImportConversion(object):
"""
See ImmutableDict above for a discussion of why a special object is needed here.
"""
split_format_str = '{%s}'
sep = ':'
split_format_str = ensure_str('{%s}')
sep = ensure_str(':')
return ImmutableDict(frozenset((convert_mandatory(ct.subtypes[0], v[0]), convert(ct.subtypes[1], v[1]))
for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]))
@ -2082,8 +2092,8 @@ class ImportConversion(object):
Also note that it is possible that the subfield names in the csv are in the
wrong order, so we must sort them according to ct.fieldnames, see CASSANDRA-12959.
"""
split_format_str = '{%s}'
sep = ':'
split_format_str = ensure_str('{%s}')
sep = ensure_str(':')
vals = [v for v in [split(split_format_str % vv, sep=sep) for vv in split(val)]]
dict_vals = dict((unprotect(v[0]), v[1]) for v in vals)
sorted_converted_vals = [(n, convert(t, dict_vals[n]) if n in dict_vals else self.get_null_val())
@ -2141,7 +2151,7 @@ class ImportConversion(object):
or "NULL" otherwise. Note that for counters we never use prepared statements, so we
only check is_counter when use_prepared_statements is false.
"""
return None if self.use_prepared_statements else ("0" if self.is_counter else "NULL")
return None if self.use_prepared_statements else (ensure_str("0") if self.is_counter else ensure_str("NULL"))
def convert_row(self, row):
"""
@ -2426,6 +2436,7 @@ class ImportProcess(ChildProcess):
if self.ttl >= 0:
query += 'USING TTL %s' % (self.ttl,)
make_statement = self.wrap_make_statement(self.make_non_prepared_batch_statement)
query = ensure_str(query)
conv = ImportConversion(self, table_meta, prepared_statement)
tm = TokenMap(self.ks, self.hostname, self.local_dc, self.session)
@ -2493,12 +2504,12 @@ class ImportProcess(ChildProcess):
set_clause = []
for i, value in enumerate(row):
if i in conv.primary_key_indexes:
where_clause.append("{}={}".format(self.valid_columns[i], str(value)))
where_clause.append(ensure_text("{}={}").format(self.valid_columns[i], ensure_text(value)))
else:
set_clause.append("{}={}+{}".format(self.valid_columns[i], self.valid_columns[i], str(value)))
set_clause.append(ensure_text("{}={}+{}").format(self.valid_columns[i], self.valid_columns[i], ensure_text(value)))
full_query_text = query % (','.join(set_clause), ' AND '.join(where_clause))
statement.add(full_query_text)
full_query_text = query % (ensure_text(',').join(set_clause), ensure_text(' AND ').join(where_clause))
statement.add(ensure_str(full_query_text))
return statement
def make_prepared_batch_statement(self, query, _, batch, replicas):
@ -2522,7 +2533,7 @@ class ImportProcess(ChildProcess):
statement = BatchStatement(batch_type=BatchType.UNLOGGED, consistency_level=self.consistency_level)
statement.replicas = replicas
statement.keyspace = self.ks
field_sep = ','
field_sep = b',' if six.PY2 else ','
statement._statements_and_parameters = [(False, query % (field_sep.join(r),), ()) for r in batch['rows']]
return statement

2
pylib/cqlshlib/displaying.py
View File

@ -41,7 +41,7 @@ def get_str(val):
return val
class FormattedValue:
class FormattedValue(object):
def __init__(self, strval, coloredval=None, displaywidth=None):
self.strval = strval

17
pylib/cqlshlib/formatting.py
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import binascii
import calendar
import datetime
import math
@ -21,6 +22,8 @@ import os
import re
import sys
from six import ensure_text
from collections import defaultdict
from cassandra.cqltypes import EMPTY
@ -118,7 +121,7 @@ class DateTimeFormat:
self.milliseconds_only = milliseconds_only # the microseconds part, .NNNNNN, wil be rounded to .NNN
class CqlType:
class CqlType(object):
"""
A class for converting a string into a cql type name that can match a formatter
and a list of its sub-types, if any.
@ -200,7 +203,7 @@ class CqlType:
def format_value_default(val, colormap, **_):
val = str(val)
val = ensure_text(str(val))
escapedval = val.replace('\\', '\\\\')
bval = controlchars_re.sub(_show_control_chars, escapedval)
return bval if colormap is NO_COLOR_MAP else color_text(bval, colormap)
@ -232,7 +235,7 @@ def formatter_for(typname):
return registrator
class BlobType:
class BlobType(object):
def __init__(self, val):
self.val = val
@ -242,7 +245,7 @@ class BlobType:
@formatter_for('BlobType')
def format_value_blob(val, colormap, **_):
bval = '0x' + val.hex()
bval = ensure_text('0x') + ensure_text(binascii.hexlify(val))
return colorme(bval, colormap, 'blob')
@ -252,7 +255,7 @@ formatter_for('blob')(format_value_blob)
def format_python_formatted_type(val, colormap, color, quote=False):
bval = str(val)
bval = ensure_text(str(val))
if quote:
bval = "'%s'" % bval
return colorme(bval, colormap, color)
@ -322,7 +325,7 @@ formatter_for('double')(format_floating_point_type)
def format_integer_type(val, colormap, thousands_sep=None, **_):
# base-10 only for now; support others?
bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
bval = str(bval)
bval = ensure_text(bval)
return colorme(bval, colormap, 'int')
@ -357,7 +360,7 @@ def format_value_timestamp(val, colormap, date_time_format, quote=False, **_):
if date_time_format.milliseconds_only:
bval = round_microseconds(bval)
else:
bval = str(val)
bval = ensure_text(str(val))
if quote:
bval = "'%s'" % bval

2
pylib/cqlshlib/helptopics.py
View File

@ -15,7 +15,7 @@
# limitations under the License.
class CQL3HelpTopics:
class CQL3HelpTopics(object):
def get_help_topics(self):
return [t[5:] for t in dir(self) if t.startswith('help_')]

3
pylib/cqlshlib/test/ansi_colors.py
View File

@ -15,6 +15,7 @@
# limitations under the License.
import re
import six
LIGHT = 0o10
@ -105,7 +106,7 @@ class ColoredChar(object):
class ColoredText(object):
def __init__(self, source=''):
if isinstance(source, str):
if isinstance(source, six.text_type):
plain, colors = self.parse_ansi_colors(source)
self.chars = list(map(ColoredChar, plain, colors))
else:

2
pylib/cqlshlib/test/run_cqlsh.py
View File

@ -30,7 +30,7 @@ from os.path import join
import pty
DEFAULT_PREFIX = os.linesep
DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + r'(\S+@)?cqlsh(:\S+)?> '
DEFAULT_CQLSH_PROMPT = DEFAULT_PREFIX + '(\S+@)?cqlsh(:\S+)?> '
DEFAULT_CQLSH_TERM = 'xterm'
try:

4
pylib/cqlshlib/test/winpty.py
View File

@ -15,8 +15,8 @@
# limitations under the License.
from threading import Thread
from io import StringIO
from queue import Queue, Empty
from six import StringIO
from six.moves.queue import Queue, Empty
class WinPty(object):

4
pylib/cqlshlib/tracing.py
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
from datetime import datetime, timedelta
import time
from cassandra.query import QueryTrace, TraceUnavailable
@ -73,6 +73,8 @@ def make_trace_rows(trace):
if trace.duration:
finished_at = (datetime_from_utc_to_local(trace.started_at) + trace.duration)
rows.append(['Request complete', str(finished_at), trace.coordinator, total_micro_seconds(trace.duration), trace.client])
else:
finished_at = trace.duration = "--"
return rows

2
pylib/cqlshlib/util.py
View File

@ -21,7 +21,7 @@ import pstats
from datetime import timedelta, tzinfo
from io import StringIO
from six import StringIO
try:
from line_profiler import LineProfiler

4
pylib/requirements.txt
View File

@ -1,3 +1,7 @@
# See python driver docs: six have to be installed before
# cythonizing the driver, perhaps only on old pips.
# http://datastax.github.io/python-driver/installation.html#cython-based-extensions
six>=1.12.0
-e git+https://github.com/datastax/python-driver.git@cassandra-test#egg=cassandra-driver
# Used ccm version is tracked by cassandra-test branch in ccm repo. Please create a PR there for fixes or upgrades to new releases.
-e git+https://github.com/riptano/ccm.git@cassandra-test#egg=ccm