mirror of https://github.com/apache/cassandra
Match cassandra-loader options in COPY FROM (3.0 version)
patch by Stefania; reviewed by pauloricardomg for CASSANDRA-9303
This commit is contained in:
parent
3dbc56f163
commit
f288387917
|
|
@ -25,6 +25,7 @@ Merged from 2.2:
|
|||
* Disable reloading of GossipingPropertyFileSnitch (CASSANDRA-9474)
|
||||
* Verify tables in pseudo-system keyspaces at startup (CASSANDRA-10761)
|
||||
Merged from 2.1:
|
||||
* Match cassandra-loader options in COPY FROM (CASSANDRA-9303)
|
||||
* Fix binding to any address in CqlBulkRecordWriter (CASSANDRA-9309)
|
||||
* cqlsh fails to decode utf-8 characters for text typed columns (CASSANDRA-10875)
|
||||
* Log error when stream session fails (CASSANDRA-9294)
|
||||
|
|
|
|||
7
NEWS.txt
7
NEWS.txt
|
|
@ -383,6 +383,13 @@ Upgrading
|
|||
to exclude data centers when the global status is enabled, see CASSANDRA-9035 for details.
|
||||
|
||||
|
||||
2.1.13
|
||||
======
|
||||
|
||||
New features
|
||||
------------
|
||||
- New options for cqlsh COPY FROM and COPY TO, see CASSANDRA-9303 for details.
|
||||
|
||||
2.1.10
|
||||
=====
|
||||
|
||||
|
|
|
|||
135
bin/cqlsh.py
135
bin/cqlsh.py
|
|
@ -41,7 +41,6 @@ import optparse
|
|||
import os
|
||||
import platform
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
import warnings
|
||||
import webbrowser
|
||||
|
|
@ -152,7 +151,8 @@ cqlshlibdir = os.path.join(CASSANDRA_PATH, 'pylib')
|
|||
if os.path.isdir(cqlshlibdir):
|
||||
sys.path.insert(0, cqlshlibdir)
|
||||
|
||||
from cqlshlib import cql3handling, cqlhandling, copyutil, pylexotron, sslhandling
|
||||
from cqlshlib import cql3handling, cqlhandling, pylexotron, sslhandling
|
||||
from cqlshlib.copyutil import ExportTask, ImportTask
|
||||
from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
|
||||
RED, FormattedValue, colorme)
|
||||
from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
|
||||
|
|
@ -458,10 +458,12 @@ def complete_copy_column_names(ctxt, cqlsh):
|
|||
return set(colnames[1:]) - set(existcols)
|
||||
|
||||
|
||||
COPY_COMMON_OPTIONS = ['DELIMITER', 'QUOTE', 'ESCAPE', 'HEADER', 'NULL',
|
||||
'MAXATTEMPTS', 'REPORTFREQUENCY']
|
||||
COPY_FROM_OPTIONS = ['CHUNKSIZE', 'INGESTRATE', 'MAXBATCHSIZE', 'MINBATCHSIZE']
|
||||
COPY_TO_OPTIONS = ['ENCODING', 'TIMEFORMAT', 'PAGESIZE', 'PAGETIMEOUT', 'MAXREQUESTS']
|
||||
COPY_COMMON_OPTIONS = ['DELIMITER', 'QUOTE', 'ESCAPE', 'HEADER', 'NULL', 'DATETIMEFORMAT',
|
||||
'MAXATTEMPTS', 'REPORTFREQUENCY', 'DECIMALSEP', 'THOUSANDSSEP', 'BOOLSTYLE',
|
||||
'NUMPROCESSES', 'CONFIGFILE', 'RATEFILE']
|
||||
COPY_FROM_OPTIONS = ['CHUNKSIZE', 'INGESTRATE', 'MAXBATCHSIZE', 'MINBATCHSIZE', 'MAXROWS',
|
||||
'SKIPROWS', 'SKIPCOLS', 'MAXPARSEERRORS', 'MAXINSERTERRORS', 'ERRFILE']
|
||||
COPY_TO_OPTIONS = ['ENCODING', 'PAGESIZE', 'PAGETIMEOUT', 'BEGINTOKEN', 'ENDTOKEN', 'MAXOUTPUTSIZE', 'MAXREQUESTS']
|
||||
|
||||
|
||||
@cqlsh_syntax_completer('copyOption', 'optnames')
|
||||
|
|
@ -581,23 +583,6 @@ warnings.showwarning = show_warning_without_quoting_line
|
|||
warnings.filterwarnings('always', category=cql3handling.UnexpectedTableStructure)
|
||||
|
||||
|
||||
def describe_interval(seconds):
|
||||
desc = []
|
||||
for length, unit in ((86400, 'day'), (3600, 'hour'), (60, 'minute')):
|
||||
num = int(seconds) / length
|
||||
if num > 0:
|
||||
desc.append('%d %s' % (num, unit))
|
||||
if num > 1:
|
||||
desc[-1] += 's'
|
||||
seconds %= length
|
||||
words = '%.03f seconds' % seconds
|
||||
if len(desc) > 1:
|
||||
words = ', '.join(desc) + ', and ' + words
|
||||
elif len(desc) == 1:
|
||||
words = desc[0] + ' and ' + words
|
||||
return words
|
||||
|
||||
|
||||
def insert_driver_hooks():
|
||||
extend_cql_deserialization()
|
||||
auto_format_udts()
|
||||
|
|
@ -664,8 +649,7 @@ class Shell(cmd.Cmd):
|
|||
last_hist = None
|
||||
shunted_query_out = None
|
||||
use_paging = True
|
||||
csv_dialect_defaults = dict(delimiter=',', doublequote=False,
|
||||
escapechar='\\', quotechar='"')
|
||||
|
||||
default_page_size = 100
|
||||
|
||||
def __init__(self, hostname, port, color=False,
|
||||
|
|
@ -1781,32 +1765,67 @@ class Shell(cmd.Cmd):
|
|||
COPY x TO: Exports data from a Cassandra table in CSV format.
|
||||
|
||||
COPY <table_name> [ ( column [, ...] ) ]
|
||||
FROM ( '<filename>' | STDIN )
|
||||
FROM ( '<file_pattern_1, file_pattern_2, ... file_pattern_n>' | STDIN )
|
||||
[ WITH <option>='value' [AND ...] ];
|
||||
|
||||
File patterns are either file names or valid python glob expressions, e.g. *.csv or folder/*.csv.
|
||||
|
||||
COPY <table_name> [ ( column [, ...] ) ]
|
||||
TO ( '<filename>' | STDOUT )
|
||||
[ WITH <option>='value' [AND ...] ];
|
||||
|
||||
Available options and defaults:
|
||||
Available common COPY options and defaults:
|
||||
|
||||
DELIMITER=',' - character that appears between records
|
||||
QUOTE='"' - quoting character to be used to quote fields
|
||||
ESCAPE='\' - character to appear before the QUOTE char when quoted
|
||||
HEADER=false - whether to ignore the first line
|
||||
NULL='' - string that represents a null value
|
||||
ENCODING='utf8' - encoding for CSV output (COPY TO)
|
||||
TIMEFORMAT= - timestamp strftime format (COPY TO)
|
||||
DATETIMEFORMAT= - timestamp strftime format
|
||||
'%Y-%m-%d %H:%M:%S%z' defaults to time_format value in cqlshrc
|
||||
MAXREQUESTS=6 - the maximum number of requests each worker process can work on in parallel (COPY TO)
|
||||
PAGESIZE=1000 - the page size for fetching results (COPY TO)
|
||||
PAGETIMEOUT=10 - the page timeout for fetching results (COPY TO)
|
||||
MAXATTEMPTS=5 - the maximum number of attempts for errors
|
||||
CHUNKSIZE=1000 - the size of chunks passed to worker processes (COPY FROM)
|
||||
INGESTRATE=100000 - an approximate ingest rate in rows per second (COPY FROM)
|
||||
MAXBATCHSIZE=20 - the maximum size of an import batch (COPY FROM)
|
||||
MINBATCHSIZE=2 - the minimum size of an import batch (COPY FROM)
|
||||
MAXATTEMPTS=5 - the maximum number of attempts per batch or range
|
||||
REPORTFREQUENCY=0.25 - the frequency with which we display status updates in seconds
|
||||
DECIMALSEP='.' - the separator for decimal values
|
||||
THOUSANDSSEP='' - the separator for thousands digit groups
|
||||
BOOLSTYLE='True,False' - the representation for booleans, case insensitive, specify true followed by false,
|
||||
for example yes,no or 1,0
|
||||
NUMPROCESSES=n - the number of worker processes, by default the number of cores minus one
|
||||
capped at 16
|
||||
CONFIGFILE='' - a configuration file with the same format as .cqlshrc (see the Python ConfigParser
|
||||
documentation) where you can specify WITH options under the following optional
|
||||
sections: [copy], [copy-to], [copy-from], [copy:ks.table], [copy-to:ks.table],
|
||||
[copy-from:ks.table], where <ks> is your keyspace name and <table> is your table
|
||||
name. Options are read from these sections, in the order specified
|
||||
above, and command line options always override options in configuration files.
|
||||
Depending on the COPY direction, only the relevant copy-from or copy-to sections
|
||||
are used. If no configfile is specified then .cqlshrc is searched instead.
|
||||
RATEFILE='' - an optional file where to print the output statistics
|
||||
|
||||
Available COPY FROM options and defaults:
|
||||
|
||||
CHUNKSIZE=1000 - the size of chunks passed to worker processes
|
||||
INGESTRATE=100000 - an approximate ingest rate in rows per second
|
||||
MINBATCHSIZE=2 - the minimum size of an import batch
|
||||
MAXBATCHSIZE=20 - the maximum size of an import batch
|
||||
MAXROWS=-1 - the maximum number of rows, -1 means no maximum
|
||||
SKIPROWS=0 - the number of rows to skip
|
||||
SKIPCOLS='' - a comma separated list of column names to skip
|
||||
MAXPARSEERRORS=-1 - the maximum global number of parsing errors, -1 means no maximum
|
||||
MAXINSERTERRORS=-1 - the maximum global number of insert errors, -1 means no maximum
|
||||
ERRFILE='' - a file where to store all rows that could not be imported, by default this is
|
||||
import_ks_table.err where <ks> is your keyspace and <table> is your table name.
|
||||
|
||||
Available COPY TO options and defaults:
|
||||
|
||||
ENCODING='utf8' - encoding for CSV output
|
||||
PAGESIZE='1000' - the page size for fetching results
|
||||
PAGETIMEOUT=10 - the page timeout in seconds for fetching results
|
||||
BEGINTOKEN='' - the minimum token string to consider when exporting data
|
||||
ENDTOKEN='' - the maximum token string to consider when exporting data
|
||||
MAXREQUESTS=6 - the maximum number of requests each worker process can work on in parallel
|
||||
MAXOUTPUTSIZE='-1' - the maximum size of the output file measured in number of lines,
|
||||
beyond this maximum the output file will be split into segments,
|
||||
-1 means unlimited.
|
||||
|
||||
When entering CSV data on STDIN, you can use the sequence "\."
|
||||
on a line by itself to end the data input.
|
||||
|
|
@ -1817,55 +1836,31 @@ class Shell(cmd.Cmd):
|
|||
ks = self.current_keyspace
|
||||
if ks is None:
|
||||
raise NoKeyspaceError("Not in any keyspace.")
|
||||
cf = self.cql_unprotect_name(parsed.get_binding('cfname'))
|
||||
table = self.cql_unprotect_name(parsed.get_binding('cfname'))
|
||||
columns = parsed.get_binding('colnames', None)
|
||||
if columns is not None:
|
||||
columns = map(self.cql_unprotect_name, columns)
|
||||
else:
|
||||
# default to all known columns
|
||||
columns = self.get_column_names(ks, cf)
|
||||
columns = self.get_column_names(ks, table)
|
||||
|
||||
fname = parsed.get_binding('fname', None)
|
||||
if fname is not None:
|
||||
fname = os.path.expanduser(self.cql_unprotect_value(fname))
|
||||
fname = self.cql_unprotect_value(fname)
|
||||
|
||||
copyoptnames = map(str.lower, parsed.get_binding('optnames', ()))
|
||||
copyoptvals = map(self.cql_unprotect_value, parsed.get_binding('optvals', ()))
|
||||
cleancopyoptvals = [optval.decode('string-escape') for optval in copyoptvals]
|
||||
opts = dict(zip(copyoptnames, cleancopyoptvals))
|
||||
|
||||
print "\nStarting copy of %s.%s with columns %s." % (ks, cf, columns)
|
||||
|
||||
timestart = time.time()
|
||||
opts = dict(zip(copyoptnames, copyoptvals))
|
||||
|
||||
direction = parsed.get_binding('dir').upper()
|
||||
if direction == 'FROM':
|
||||
rows = self.perform_csv_import(ks, cf, columns, fname, opts)
|
||||
verb = 'imported'
|
||||
task = ImportTask(self, ks, table, columns, fname, opts, DEFAULT_PROTOCOL_VERSION, CONFIG_FILE)
|
||||
elif direction == 'TO':
|
||||
rows = self.perform_csv_export(ks, cf, columns, fname, opts)
|
||||
verb = 'exported'
|
||||
task = ExportTask(self, ks, table, columns, fname, opts, DEFAULT_PROTOCOL_VERSION, CONFIG_FILE)
|
||||
else:
|
||||
raise SyntaxError("Unknown direction %s" % direction)
|
||||
|
||||
timeend = time.time()
|
||||
print "\n%d rows %s in %s." % (rows, verb, describe_interval(timeend - timestart))
|
||||
|
||||
def perform_csv_import(self, ks, cf, columns, fname, opts):
|
||||
csv_options, dialect_options, unrecognized_options = copyutil.parse_options(self, opts)
|
||||
if unrecognized_options:
|
||||
self.printerr('Unrecognized COPY FROM options: %s' % ', '.join(unrecognized_options.keys()))
|
||||
return 0
|
||||
|
||||
return copyutil.ImportTask(self, ks, cf, columns, fname, csv_options, dialect_options,
|
||||
DEFAULT_PROTOCOL_VERSION, CONFIG_FILE).run()
|
||||
|
||||
def perform_csv_export(self, ks, cf, columns, fname, opts):
|
||||
csv_options, dialect_options, unrecognized_options = copyutil.parse_options(self, opts)
|
||||
if unrecognized_options:
|
||||
self.printerr('Unrecognized COPY TO options: %s' % ', '.join(unrecognized_options.keys()))
|
||||
return 0
|
||||
|
||||
return copyutil.ExportTask(self, ks, cf, columns, fname, csv_options, dialect_options,
|
||||
DEFAULT_PROTOCOL_VERSION, CONFIG_FILE).run()
|
||||
task.run()
|
||||
|
||||
def do_show(self, parsed):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ completekey = tab
|
|||
;browser =
|
||||
|
||||
[cql]
|
||||
version = 3.1.5
|
||||
version = 3.2.1
|
||||
|
||||
[connection]
|
||||
hostname = 127.0.0.1
|
||||
|
|
@ -68,3 +68,18 @@ max_trace_wait = 10.0
|
|||
|
||||
|
||||
; vim: set ft=dosini :
|
||||
|
||||
;; optional options for COPY TO and COPY FROM
|
||||
;[copy]
|
||||
;maxattempts=10
|
||||
;numprocesses=4
|
||||
|
||||
;; optional options for COPY FROM
|
||||
;[copy-from]
|
||||
;chunksize=5000
|
||||
;ingestrate=50000
|
||||
|
||||
;; optional options for COPY TO
|
||||
;[copy-to]
|
||||
;pagesize=2000
|
||||
;pagetimeout=20
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -60,7 +60,8 @@ empty_colormap = defaultdict(lambda: '')
|
|||
|
||||
|
||||
def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False,
|
||||
nullval=None, date_time_format=None, float_precision=None):
|
||||
nullval=None, date_time_format=None, float_precision=None,
|
||||
decimal_sep=None, thousands_sep=None, boolean_styles=None):
|
||||
if nullval is None:
|
||||
nullval = default_null_placeholder
|
||||
if val is None:
|
||||
|
|
@ -75,7 +76,8 @@ def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False,
|
|||
float_precision = default_float_precision
|
||||
return format_value(cqltype, val, encoding=encoding, colormap=colormap,
|
||||
date_time_format=date_time_format, float_precision=float_precision,
|
||||
nullval=nullval)
|
||||
nullval=nullval, decimal_sep=decimal_sep, thousands_sep=thousands_sep,
|
||||
boolean_styles=boolean_styles)
|
||||
|
||||
|
||||
def color_text(bval, colormap, displaywidth=None):
|
||||
|
|
@ -155,7 +157,9 @@ def format_python_formatted_type(val, colormap, color, quote=False):
|
|||
|
||||
|
||||
@formatter_for('Decimal')
|
||||
def format_value_decimal(val, colormap, **_):
|
||||
def format_value_decimal(val, float_precision, colormap, decimal_sep=None, thousands_sep=None, **_):
|
||||
if (decimal_sep and decimal_sep != '.') or thousands_sep:
|
||||
return format_floating_point_type(val, colormap, float_precision, decimal_sep, thousands_sep)
|
||||
return format_python_formatted_type(val, colormap, 'decimal')
|
||||
|
||||
|
||||
|
|
@ -170,34 +174,60 @@ def formatter_value_inet(val, colormap, quote=False, **_):
|
|||
|
||||
|
||||
@formatter_for('bool')
|
||||
def format_value_boolean(val, colormap, **_):
|
||||
def format_value_boolean(val, colormap, boolean_styles=None, **_):
|
||||
if boolean_styles:
|
||||
val = boolean_styles[0] if val else boolean_styles[1]
|
||||
return format_python_formatted_type(val, colormap, 'boolean')
|
||||
|
||||
|
||||
def format_floating_point_type(val, colormap, float_precision, **_):
|
||||
def format_floating_point_type(val, colormap, float_precision, decimal_sep=None, thousands_sep=None, **_):
|
||||
if math.isnan(val):
|
||||
bval = 'NaN'
|
||||
elif math.isinf(val):
|
||||
bval = 'Infinity' if val > 0 else '-Infinity'
|
||||
else:
|
||||
exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1
|
||||
if -4 <= exponent < float_precision:
|
||||
# when this is true %g will not use scientific notation,
|
||||
# increasing precision should not change this decision
|
||||
# so we increase the precision to take into account the
|
||||
# digits to the left of the decimal point
|
||||
float_precision = float_precision + exponent + 1
|
||||
bval = '%.*g' % (float_precision, val)
|
||||
if thousands_sep:
|
||||
dpart, ipart = math.modf(val)
|
||||
bval = format_integer_with_thousands_sep(ipart, thousands_sep)
|
||||
dpart_str = ('%.*f' % (float_precision, math.fabs(dpart)))[2:].rstrip('0')
|
||||
if dpart_str:
|
||||
bval += '%s%s' % ('.' if not decimal_sep else decimal_sep, dpart_str)
|
||||
else:
|
||||
exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1
|
||||
if -4 <= exponent < float_precision:
|
||||
# when this is true %g will not use scientific notation,
|
||||
# increasing precision should not change this decision
|
||||
# so we increase the precision to take into account the
|
||||
# digits to the left of the decimal point
|
||||
float_precision = float_precision + exponent + 1
|
||||
bval = '%.*g' % (float_precision, val)
|
||||
if decimal_sep:
|
||||
bval = bval.replace('.', decimal_sep)
|
||||
|
||||
return colorme(bval, colormap, 'float')
|
||||
|
||||
formatter_for('float')(format_floating_point_type)
|
||||
|
||||
|
||||
def format_integer_type(val, colormap, **_):
|
||||
def format_integer_type(val, colormap, thousands_sep=None, **_):
|
||||
# base-10 only for now; support others?
|
||||
bval = str(val)
|
||||
bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
|
||||
return colorme(bval, colormap, 'int')
|
||||
|
||||
# We can get rid of this in cassandra-2.2
|
||||
if sys.version_info >= (2, 7):
|
||||
def format_integer_with_thousands_sep(val, thousands_sep=','):
|
||||
return "{:,.0f}".format(val).replace(',', thousands_sep)
|
||||
else:
|
||||
def format_integer_with_thousands_sep(val, thousands_sep=','):
|
||||
if val < 0:
|
||||
return '-' + format_integer_with_thousands_sep(-val, thousands_sep)
|
||||
result = ''
|
||||
while val >= 1000:
|
||||
val, r = divmod(val, 1000)
|
||||
result = "%s%03d%s" % (thousands_sep, r, result)
|
||||
return "%d%s" % (val, result)
|
||||
|
||||
formatter_for('long')(format_integer_type)
|
||||
formatter_for('int')(format_integer_type)
|
||||
|
||||
|
|
@ -242,10 +272,12 @@ formatter_for('unicode')(format_value_text)
|
|||
|
||||
|
||||
def format_simple_collection(val, lbracket, rbracket, encoding,
|
||||
colormap, date_time_format, float_precision, nullval):
|
||||
colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles):
|
||||
subs = [format_value(type(sval), sval, encoding=encoding, colormap=colormap,
|
||||
date_time_format=date_time_format, float_precision=float_precision,
|
||||
nullval=nullval, quote=True)
|
||||
nullval=nullval, quote=True, decimal_sep=decimal_sep,
|
||||
thousands_sep=thousands_sep, boolean_styles=boolean_styles)
|
||||
for sval in val]
|
||||
bval = lbracket + ', '.join(get_str(sval) for sval in subs) + rbracket
|
||||
if colormap is NO_COLOR_MAP:
|
||||
|
|
@ -259,32 +291,40 @@ def format_simple_collection(val, lbracket, rbracket, encoding,
|
|||
|
||||
|
||||
@formatter_for('list')
|
||||
def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
|
||||
def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles, **_):
|
||||
return format_simple_collection(val, '[', ']', encoding, colormap,
|
||||
date_time_format, float_precision, nullval)
|
||||
date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles)
|
||||
|
||||
|
||||
@formatter_for('tuple')
|
||||
def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
|
||||
def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles, **_):
|
||||
return format_simple_collection(val, '(', ')', encoding, colormap,
|
||||
date_time_format, float_precision, nullval)
|
||||
date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles)
|
||||
|
||||
|
||||
@formatter_for('set')
|
||||
def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
|
||||
def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles, **_):
|
||||
return format_simple_collection(sorted(val), '{', '}', encoding, colormap,
|
||||
date_time_format, float_precision, nullval)
|
||||
date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles)
|
||||
formatter_for('frozenset')(format_value_set)
|
||||
formatter_for('sortedset')(format_value_set)
|
||||
formatter_for('SortedSet')(format_value_set)
|
||||
|
||||
|
||||
@formatter_for('dict')
|
||||
def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
|
||||
def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles, **_):
|
||||
def subformat(v):
|
||||
return format_value(type(v), v, encoding=encoding, colormap=colormap,
|
||||
date_time_format=date_time_format, float_precision=float_precision,
|
||||
nullval=nullval, quote=True)
|
||||
nullval=nullval, quote=True, decimal_sep=decimal_sep,
|
||||
thousands_sep=thousands_sep, boolean_styles=boolean_styles)
|
||||
|
||||
subs = [(subformat(k), subformat(v)) for (k, v) in sorted(val.items())]
|
||||
bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}'
|
||||
|
|
@ -303,13 +343,15 @@ formatter_for('OrderedMap')(format_value_map)
|
|||
formatter_for('OrderedMapSerializedKey')(format_value_map)
|
||||
|
||||
|
||||
def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
|
||||
def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval,
|
||||
decimal_sep, thousands_sep, boolean_styles, **_):
|
||||
def format_field_value(v):
|
||||
if v is None:
|
||||
return colorme(nullval, colormap, 'error')
|
||||
return format_value(type(v), v, encoding=encoding, colormap=colormap,
|
||||
date_time_format=date_time_format, float_precision=float_precision,
|
||||
nullval=nullval, quote=True)
|
||||
nullval=nullval, quote=True, decimal_sep=decimal_sep,
|
||||
thousands_sep=thousands_sep, boolean_styles=boolean_styles)
|
||||
|
||||
def format_field_name(name):
|
||||
return format_value_text(name, encoding=encoding, colormap=colormap, quote=False)
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ import org.apache.cassandra.cql3.*;
|
|||
import org.apache.cassandra.db.*;
|
||||
import org.apache.cassandra.db.partitions.PartitionUpdate;
|
||||
import org.apache.cassandra.db.rows.RowIterator;
|
||||
import org.apache.cassandra.dht.Range;
|
||||
import org.apache.cassandra.dht.Token;
|
||||
import org.apache.cassandra.exceptions.*;
|
||||
import org.apache.cassandra.service.*;
|
||||
import org.apache.cassandra.tracing.Tracing;
|
||||
|
|
@ -48,7 +50,7 @@ import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse
|
|||
*/
|
||||
public class BatchStatement implements CQLStatement
|
||||
{
|
||||
public static enum Type
|
||||
public enum Type
|
||||
{
|
||||
LOGGED, UNLOGGED, COUNTER
|
||||
}
|
||||
|
|
@ -297,12 +299,22 @@ public class BatchStatement implements CQLStatement
|
|||
Set<DecoratedKey> keySet = new HashSet<>();
|
||||
Set<String> tableNames = new HashSet<>();
|
||||
|
||||
Map<String, Collection<Range<Token>>> localTokensByKs = new HashMap<>();
|
||||
boolean localPartitionsOnly = true;
|
||||
for (PartitionUpdate update : updates)
|
||||
{
|
||||
keySet.add(update.partitionKey());
|
||||
tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName));
|
||||
|
||||
if (localPartitionsOnly)
|
||||
localPartitionsOnly &= isPartitionLocal(localTokensByKs, update);
|
||||
}
|
||||
|
||||
// CASSANDRA-9303: If we only have local mutations we do not warn
|
||||
if (localPartitionsOnly)
|
||||
return;
|
||||
|
||||
|
||||
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, UNLOGGED_BATCH_WARNING,
|
||||
keySet.size(), keySet.size() == 1 ? "" : "s",
|
||||
tableNames.size() == 1 ? "" : "s", tableNames);
|
||||
|
|
@ -313,6 +325,18 @@ public class BatchStatement implements CQLStatement
|
|||
}
|
||||
}
|
||||
|
||||
private boolean isPartitionLocal(Map<String, Collection<Range<Token>>> localTokensByKs, PartitionUpdate update)
|
||||
{
|
||||
Collection<Range<Token>> localRanges = localTokensByKs.get(update.metadata().ksName);
|
||||
if (localRanges == null)
|
||||
{
|
||||
localRanges = StorageService.instance.getLocalRanges(update.metadata().ksName);
|
||||
localTokensByKs.put(update.metadata().ksName, localRanges);
|
||||
}
|
||||
|
||||
return Range.isInRanges(update.partitionKey().getToken(), localRanges);
|
||||
}
|
||||
|
||||
public ResultMessage execute(QueryState queryState, QueryOptions options) throws RequestExecutionException, RequestValidationException
|
||||
{
|
||||
return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options));
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import org.apache.cassandra.transport.SimpleClient;
|
|||
import org.apache.cassandra.transport.messages.QueryMessage;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNull;
|
||||
|
||||
public class ClientWarningsTest extends CQLTester
|
||||
{
|
||||
|
|
@ -52,12 +53,11 @@ public class ClientWarningsTest extends CQLTester
|
|||
|
||||
QueryMessage query = new QueryMessage(createBatchStatement2(1), QueryOptions.DEFAULT);
|
||||
Message.Response resp = client.execute(query);
|
||||
assertEquals(1, resp.getWarnings().size());
|
||||
assertNull(resp.getWarnings());
|
||||
|
||||
query = new QueryMessage(createBatchStatement2(DatabaseDescriptor.getBatchSizeWarnThreshold()), QueryOptions.DEFAULT);
|
||||
resp = client.execute(query);
|
||||
assertEquals(2, resp.getWarnings().size());
|
||||
|
||||
assertEquals(1, resp.getWarnings().size());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,4 +19,4 @@ if "%OS%" == "Windows_NT" setlocal
|
|||
pushd "%~dp0"
|
||||
call cassandra.in.bat
|
||||
if NOT DEFINED STRESS_HOME set STRESS_HOME=%CD%\..
|
||||
"%JAVA_HOME%\bin\java" -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*
|
||||
"%JAVA_HOME%\bin\java" %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*
|
||||
|
|
|
|||
Loading…
Reference in New Issue