Match cassandra-loader options in COPY FROM (3.2 version)

patch by Stefania; reviewed by pauloricardomg for CASSANDRA-9303
This commit is contained in:
Stefania Alborghetti 2016-01-06 12:12:12 +01:00 committed by Sylvain Lebresne
parent 08acfe1172
commit 85b8d02aae
9 changed files with 1133 additions and 416 deletions

View File

@ -43,6 +43,7 @@ Merged from 2.2:
* Add new types to Stress (CASSANDRA-9556)
* Add property to allow listening on broadcast interface (CASSANDRA-9748)
Merged from 2.1:
* Match cassandra-loader options in COPY FROM (CASSANDRA-9303)
* Fix binding to any address in CqlBulkRecordWriter (CASSANDRA-9309)
* cqlsh fails to decode utf-8 characters for text typed columns (CASSANDRA-10875)
* Log error when stream session fails (CASSANDRA-9294)

View File

@ -400,6 +400,13 @@ Upgrading
to exclude data centers when the global status is enabled, see CASSANDRA-9035 for details.
2.1.13
======
New features
------------
- New options for cqlsh COPY FROM and COPY TO, see CASSANDRA-9303 for details.
2.1.10
=====

View File

@ -41,7 +41,6 @@ import optparse
import os
import platform
import sys
import time
import traceback
import warnings
import webbrowser
@ -152,7 +151,8 @@ cqlshlibdir = os.path.join(CASSANDRA_PATH, 'pylib')
if os.path.isdir(cqlshlibdir):
sys.path.insert(0, cqlshlibdir)
from cqlshlib import cql3handling, cqlhandling, copyutil, pylexotron, sslhandling
from cqlshlib import cql3handling, cqlhandling, pylexotron, sslhandling
from cqlshlib.copyutil import ExportTask, ImportTask
from cqlshlib.displaying import (ANSI_RESET, BLUE, COLUMN_NAME_COLORS, CYAN,
RED, FormattedValue, colorme)
from cqlshlib.formatting import (DEFAULT_DATE_FORMAT, DEFAULT_NANOTIME_FORMAT,
@ -458,10 +458,12 @@ def complete_copy_column_names(ctxt, cqlsh):
return set(colnames[1:]) - set(existcols)
COPY_COMMON_OPTIONS = ['DELIMITER', 'QUOTE', 'ESCAPE', 'HEADER', 'NULL',
'MAXATTEMPTS', 'REPORTFREQUENCY']
COPY_FROM_OPTIONS = ['CHUNKSIZE', 'INGESTRATE', 'MAXBATCHSIZE', 'MINBATCHSIZE', 'TTL']
COPY_TO_OPTIONS = ['ENCODING', 'TIMEFORMAT', 'PAGESIZE', 'PAGETIMEOUT', 'MAXREQUESTS']
COPY_COMMON_OPTIONS = ['DELIMITER', 'QUOTE', 'ESCAPE', 'HEADER', 'NULL', 'DATETIMEFORMAT',
'MAXATTEMPTS', 'REPORTFREQUENCY', 'DECIMALSEP', 'THOUSANDSSEP', 'BOOLSTYLE',
'NUMPROCESSES', 'CONFIGFILE', 'RATEFILE']
COPY_FROM_OPTIONS = ['CHUNKSIZE', 'INGESTRATE', 'MAXBATCHSIZE', 'MINBATCHSIZE', 'MAXROWS',
'SKIPROWS', 'SKIPCOLS', 'MAXPARSEERRORS', 'MAXINSERTERRORS', 'ERRFILE', 'TTL']
COPY_TO_OPTIONS = ['ENCODING', 'PAGESIZE', 'PAGETIMEOUT', 'BEGINTOKEN', 'ENDTOKEN', 'MAXOUTPUTSIZE', 'MAXREQUESTS']
@cqlsh_syntax_completer('copyOption', 'optnames')
@ -581,23 +583,6 @@ warnings.showwarning = show_warning_without_quoting_line
warnings.filterwarnings('always', category=cql3handling.UnexpectedTableStructure)
def describe_interval(seconds):
desc = []
for length, unit in ((86400, 'day'), (3600, 'hour'), (60, 'minute')):
num = int(seconds) / length
if num > 0:
desc.append('%d %s' % (num, unit))
if num > 1:
desc[-1] += 's'
seconds %= length
words = '%.03f seconds' % seconds
if len(desc) > 1:
words = ', '.join(desc) + ', and ' + words
elif len(desc) == 1:
words = desc[0] + ' and ' + words
return words
def insert_driver_hooks():
extend_cql_deserialization()
auto_format_udts()
@ -664,8 +649,7 @@ class Shell(cmd.Cmd):
last_hist = None
shunted_query_out = None
use_paging = True
csv_dialect_defaults = dict(delimiter=',', doublequote=False,
escapechar='\\', quotechar='"')
default_page_size = 100
def __init__(self, hostname, port, color=False,
@ -1781,33 +1765,68 @@ class Shell(cmd.Cmd):
COPY x TO: Exports data from a Cassandra table in CSV format.
COPY <table_name> [ ( column [, ...] ) ]
FROM ( '<filename>' | STDIN )
FROM ( '<file_pattern_1, file_pattern_2, ... file_pattern_n>' | STDIN )
[ WITH <option>='value' [AND ...] ];
File patterns are either file names or valid python glob expressions, e.g. *.csv or folder/*.csv.
COPY <table_name> [ ( column [, ...] ) ]
TO ( '<filename>' | STDOUT )
[ WITH <option>='value' [AND ...] ];
Available options and defaults:
Available common COPY options and defaults:
DELIMITER=',' - character that appears between records
QUOTE='"' - quoting character to be used to quote fields
ESCAPE='\' - character to appear before the QUOTE char when quoted
HEADER=false - whether to ignore the first line
NULL='' - string that represents a null value
ENCODING='utf8' - encoding for CSV output (COPY TO)
TIMEFORMAT= - timestamp strftime format (COPY TO)
DATETIMEFORMAT= - timestamp strftime format
'%Y-%m-%d %H:%M:%S%z' defaults to time_format value in cqlshrc
MAXREQUESTS=6 - the maximum number of requests each worker process can work on in parallel (COPY TO)
PAGESIZE=1000 - the page size for fetching results (COPY TO)
PAGETIMEOUT=10 - the page timeout for fetching results (COPY TO)
MAXATTEMPTS=5 - the maximum number of attempts for errors
CHUNKSIZE=1000 - the size of chunks passed to worker processes (COPY FROM)
INGESTRATE=100000 - an approximate ingest rate in rows per second (COPY FROM)
MAXBATCHSIZE=20 - the maximum size of an import batch (COPY FROM)
MINBATCHSIZE=2 - the minimum size of an import batch (COPY FROM)
MAXATTEMPTS=5 - the maximum number of attempts per batch or range
REPORTFREQUENCY=0.25 - the frequency with which we display status updates in seconds
TTL=3600 - the time to live in seconds, by default data will not expire (COPY FROM)
DECIMALSEP='.' - the separator for decimal values
THOUSANDSSEP='' - the separator for thousands digit groups
BOOLSTYLE='True,False' - the representation for booleans, case insensitive, specify true followed by false,
for example yes,no or 1,0
NUMPROCESSES=n - the number of worker processes, by default the number of cores minus one
capped at 16
CONFIGFILE='' - a configuration file with the same format as .cqlshrc (see the Python ConfigParser
documentation) where you can specify WITH options under the following optional
sections: [copy], [copy-to], [copy-from], [copy:ks.table], [copy-to:ks.table],
[copy-from:ks.table], where <ks> is your keyspace name and <table> is your table
name. Options are read from these sections, in the order specified
above, and command line options always override options in configuration files.
Depending on the COPY direction, only the relevant copy-from or copy-to sections
are used. If no configfile is specified then .cqlshrc is searched instead.
RATEFILE='' - an optional file where to print the output statistics
Available COPY FROM options and defaults:
CHUNKSIZE=1000 - the size of chunks passed to worker processes
INGESTRATE=100000 - an approximate ingest rate in rows per second
MINBATCHSIZE=2 - the minimum size of an import batch
MAXBATCHSIZE=20 - the maximum size of an import batch
MAXROWS=-1 - the maximum number of rows, -1 means no maximum
SKIPROWS=0 - the number of rows to skip
SKIPCOLS='' - a comma separated list of column names to skip
MAXPARSEERRORS=-1 - the maximum global number of parsing errors, -1 means no maximum
MAXINSERTERRORS=-1 - the maximum global number of insert errors, -1 means no maximum
ERRFILE='' - a file where to store all rows that could not be imported, by default this is
import_ks_table.err where <ks> is your keyspace and <table> is your table name.
TTL=3600 - the time to live in seconds, by default data will not expire
Available COPY TO options and defaults:
ENCODING='utf8' - encoding for CSV output
PAGESIZE='1000' - the page size for fetching results
PAGETIMEOUT=10 - the page timeout in seconds for fetching results
BEGINTOKEN='' - the minimum token string to consider when exporting data
ENDTOKEN='' - the maximum token string to consider when exporting data
MAXREQUESTS=6 - the maximum number of requests each worker process can work on in parallel
MAXOUTPUTSIZE='-1' - the maximum size of the output file measured in number of lines,
beyond this maximum the output file will be split into segments,
-1 means unlimited.
When entering CSV data on STDIN, you can use the sequence "\."
on a line by itself to end the data input.
@ -1818,55 +1837,31 @@ class Shell(cmd.Cmd):
ks = self.current_keyspace
if ks is None:
raise NoKeyspaceError("Not in any keyspace.")
cf = self.cql_unprotect_name(parsed.get_binding('cfname'))
table = self.cql_unprotect_name(parsed.get_binding('cfname'))
columns = parsed.get_binding('colnames', None)
if columns is not None:
columns = map(self.cql_unprotect_name, columns)
else:
# default to all known columns
columns = self.get_column_names(ks, cf)
columns = self.get_column_names(ks, table)
fname = parsed.get_binding('fname', None)
if fname is not None:
fname = os.path.expanduser(self.cql_unprotect_value(fname))
fname = self.cql_unprotect_value(fname)
copyoptnames = map(str.lower, parsed.get_binding('optnames', ()))
copyoptvals = map(self.cql_unprotect_value, parsed.get_binding('optvals', ()))
cleancopyoptvals = [optval.decode('string-escape') for optval in copyoptvals]
opts = dict(zip(copyoptnames, cleancopyoptvals))
print "\nStarting copy of %s.%s with columns %s." % (ks, cf, columns)
timestart = time.time()
opts = dict(zip(copyoptnames, copyoptvals))
direction = parsed.get_binding('dir').upper()
if direction == 'FROM':
rows = self.perform_csv_import(ks, cf, columns, fname, opts)
verb = 'imported'
task = ImportTask(self, ks, table, columns, fname, opts, DEFAULT_PROTOCOL_VERSION, CONFIG_FILE)
elif direction == 'TO':
rows = self.perform_csv_export(ks, cf, columns, fname, opts)
verb = 'exported'
task = ExportTask(self, ks, table, columns, fname, opts, DEFAULT_PROTOCOL_VERSION, CONFIG_FILE)
else:
raise SyntaxError("Unknown direction %s" % direction)
timeend = time.time()
print "\n%d rows %s in %s." % (rows, verb, describe_interval(timeend - timestart))
def perform_csv_import(self, ks, cf, columns, fname, opts):
csv_options, dialect_options, unrecognized_options = copyutil.parse_options(self, opts)
if unrecognized_options:
self.printerr('Unrecognized COPY FROM options: %s' % ', '.join(unrecognized_options.keys()))
return 0
return copyutil.ImportTask(self, ks, cf, columns, fname, csv_options, dialect_options,
DEFAULT_PROTOCOL_VERSION, CONFIG_FILE).run()
def perform_csv_export(self, ks, cf, columns, fname, opts):
csv_options, dialect_options, unrecognized_options = copyutil.parse_options(self, opts)
if unrecognized_options:
self.printerr('Unrecognized COPY TO options: %s' % ', '.join(unrecognized_options.keys()))
return 0
return copyutil.ExportTask(self, ks, cf, columns, fname, csv_options, dialect_options,
DEFAULT_PROTOCOL_VERSION, CONFIG_FILE).run()
task.run()
def do_show(self, parsed):
"""

View File

@ -43,7 +43,7 @@ completekey = tab
;browser =
[cql]
version = 3.1.5
version = 3.2.1
[connection]
hostname = 127.0.0.1
@ -68,3 +68,18 @@ max_trace_wait = 10.0
; vim: set ft=dosini :
;; optional options for COPY TO and COPY FROM
;[copy]
;maxattempts=10
;numprocesses=4
;; optional options for COPY FROM
;[copy-from]
;chunksize=5000
;ingestrate=50000
;; optional options for COPY TO
;[copy-to]
;pagesize=2000
;pagetimeout=20

File diff suppressed because it is too large Load Diff

View File

@ -60,7 +60,8 @@ empty_colormap = defaultdict(lambda: '')
def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False,
nullval=None, date_time_format=None, float_precision=None):
nullval=None, date_time_format=None, float_precision=None,
decimal_sep=None, thousands_sep=None, boolean_styles=None):
if nullval is None:
nullval = default_null_placeholder
if val is None:
@ -75,7 +76,8 @@ def format_by_type(cqltype, val, encoding, colormap=None, addcolor=False,
float_precision = default_float_precision
return format_value(cqltype, val, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
nullval=nullval)
nullval=nullval, decimal_sep=decimal_sep, thousands_sep=thousands_sep,
boolean_styles=boolean_styles)
def color_text(bval, colormap, displaywidth=None):
@ -155,7 +157,9 @@ def format_python_formatted_type(val, colormap, color, quote=False):
@formatter_for('Decimal')
def format_value_decimal(val, colormap, **_):
def format_value_decimal(val, float_precision, colormap, decimal_sep=None, thousands_sep=None, **_):
if (decimal_sep and decimal_sep != '.') or thousands_sep:
return format_floating_point_type(val, colormap, float_precision, decimal_sep, thousands_sep)
return format_python_formatted_type(val, colormap, 'decimal')
@ -170,34 +174,60 @@ def formatter_value_inet(val, colormap, quote=False, **_):
@formatter_for('bool')
def format_value_boolean(val, colormap, **_):
def format_value_boolean(val, colormap, boolean_styles=None, **_):
if boolean_styles:
val = boolean_styles[0] if val else boolean_styles[1]
return format_python_formatted_type(val, colormap, 'boolean')
def format_floating_point_type(val, colormap, float_precision, **_):
def format_floating_point_type(val, colormap, float_precision, decimal_sep=None, thousands_sep=None, **_):
if math.isnan(val):
bval = 'NaN'
elif math.isinf(val):
bval = 'Infinity' if val > 0 else '-Infinity'
else:
exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1
if -4 <= exponent < float_precision:
# when this is true %g will not use scientific notation,
# increasing precision should not change this decision
# so we increase the precision to take into account the
# digits to the left of the decimal point
float_precision = float_precision + exponent + 1
bval = '%.*g' % (float_precision, val)
if thousands_sep:
dpart, ipart = math.modf(val)
bval = format_integer_with_thousands_sep(ipart, thousands_sep)
dpart_str = ('%.*f' % (float_precision, math.fabs(dpart)))[2:].rstrip('0')
if dpart_str:
bval += '%s%s' % ('.' if not decimal_sep else decimal_sep, dpart_str)
else:
exponent = int(math.log10(abs(val))) if abs(val) > sys.float_info.epsilon else -sys.maxsize - 1
if -4 <= exponent < float_precision:
# when this is true %g will not use scientific notation,
# increasing precision should not change this decision
# so we increase the precision to take into account the
# digits to the left of the decimal point
float_precision = float_precision + exponent + 1
bval = '%.*g' % (float_precision, val)
if decimal_sep:
bval = bval.replace('.', decimal_sep)
return colorme(bval, colormap, 'float')
formatter_for('float')(format_floating_point_type)
def format_integer_type(val, colormap, **_):
def format_integer_type(val, colormap, thousands_sep=None, **_):
# base-10 only for now; support others?
bval = str(val)
bval = format_integer_with_thousands_sep(val, thousands_sep) if thousands_sep else str(val)
return colorme(bval, colormap, 'int')
# We can get rid of this in cassandra-2.2
if sys.version_info >= (2, 7):
def format_integer_with_thousands_sep(val, thousands_sep=','):
return "{:,.0f}".format(val).replace(',', thousands_sep)
else:
def format_integer_with_thousands_sep(val, thousands_sep=','):
if val < 0:
return '-' + format_integer_with_thousands_sep(-val, thousands_sep)
result = ''
while val >= 1000:
val, r = divmod(val, 1000)
result = "%s%03d%s" % (thousands_sep, r, result)
return "%d%s" % (val, result)
formatter_for('long')(format_integer_type)
formatter_for('int')(format_integer_type)
@ -242,10 +272,12 @@ formatter_for('unicode')(format_value_text)
def format_simple_collection(val, lbracket, rbracket, encoding,
colormap, date_time_format, float_precision, nullval):
colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles):
subs = [format_value(type(sval), sval, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
nullval=nullval, quote=True)
nullval=nullval, quote=True, decimal_sep=decimal_sep,
thousands_sep=thousands_sep, boolean_styles=boolean_styles)
for sval in val]
bval = lbracket + ', '.join(get_str(sval) for sval in subs) + rbracket
if colormap is NO_COLOR_MAP:
@ -259,32 +291,40 @@ def format_simple_collection(val, lbracket, rbracket, encoding,
@formatter_for('list')
def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
def format_value_list(val, encoding, colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles, **_):
return format_simple_collection(val, '[', ']', encoding, colormap,
date_time_format, float_precision, nullval)
date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles)
@formatter_for('tuple')
def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
def format_value_tuple(val, encoding, colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles, **_):
return format_simple_collection(val, '(', ')', encoding, colormap,
date_time_format, float_precision, nullval)
date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles)
@formatter_for('set')
def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
def format_value_set(val, encoding, colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles, **_):
return format_simple_collection(sorted(val), '{', '}', encoding, colormap,
date_time_format, float_precision, nullval)
date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles)
formatter_for('frozenset')(format_value_set)
formatter_for('sortedset')(format_value_set)
formatter_for('SortedSet')(format_value_set)
@formatter_for('dict')
def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
def format_value_map(val, encoding, colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles, **_):
def subformat(v):
return format_value(type(v), v, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
nullval=nullval, quote=True)
nullval=nullval, quote=True, decimal_sep=decimal_sep,
thousands_sep=thousands_sep, boolean_styles=boolean_styles)
subs = [(subformat(k), subformat(v)) for (k, v) in sorted(val.items())]
bval = '{' + ', '.join(get_str(k) + ': ' + get_str(v) for (k, v) in subs) + '}'
@ -303,13 +343,15 @@ formatter_for('OrderedMap')(format_value_map)
formatter_for('OrderedMapSerializedKey')(format_value_map)
def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval, **_):
def format_value_utype(val, encoding, colormap, date_time_format, float_precision, nullval,
decimal_sep, thousands_sep, boolean_styles, **_):
def format_field_value(v):
if v is None:
return colorme(nullval, colormap, 'error')
return format_value(type(v), v, encoding=encoding, colormap=colormap,
date_time_format=date_time_format, float_precision=float_precision,
nullval=nullval, quote=True)
nullval=nullval, quote=True, decimal_sep=decimal_sep,
thousands_sep=thousands_sep, boolean_styles=boolean_styles)
def format_field_name(name):
return format_value_text(name, encoding=encoding, colormap=colormap, quote=False)

View File

@ -34,6 +34,8 @@ import org.apache.cassandra.cql3.*;
import org.apache.cassandra.db.*;
import org.apache.cassandra.db.partitions.PartitionUpdate;
import org.apache.cassandra.db.rows.RowIterator;
import org.apache.cassandra.dht.Range;
import org.apache.cassandra.dht.Token;
import org.apache.cassandra.exceptions.*;
import org.apache.cassandra.service.*;
import org.apache.cassandra.tracing.Tracing;
@ -297,12 +299,22 @@ public class BatchStatement implements CQLStatement
Set<DecoratedKey> keySet = new HashSet<>();
Set<String> tableNames = new HashSet<>();
Map<String, Collection<Range<Token>>> localTokensByKs = new HashMap<>();
boolean localPartitionsOnly = true;
for (PartitionUpdate update : updates)
{
keySet.add(update.partitionKey());
tableNames.add(String.format("%s.%s", update.metadata().ksName, update.metadata().cfName));
if (localPartitionsOnly)
localPartitionsOnly &= isPartitionLocal(localTokensByKs, update);
}
// CASSANDRA-9303: If we only have local mutations we do not warn
if (localPartitionsOnly)
return;
NoSpamLogger.log(logger, NoSpamLogger.Level.WARN, 1, TimeUnit.MINUTES, UNLOGGED_BATCH_WARNING,
keySet.size(), keySet.size() == 1 ? "" : "s",
tableNames.size() == 1 ? "" : "s", tableNames);
@ -313,6 +325,18 @@ public class BatchStatement implements CQLStatement
}
}
private boolean isPartitionLocal(Map<String, Collection<Range<Token>>> localTokensByKs, PartitionUpdate update)
{
Collection<Range<Token>> localRanges = localTokensByKs.get(update.metadata().ksName);
if (localRanges == null)
{
localRanges = StorageService.instance.getLocalRanges(update.metadata().ksName);
localTokensByKs.put(update.metadata().ksName, localRanges);
}
return Range.isInRanges(update.partitionKey().getToken(), localRanges);
}
public ResultMessage execute(QueryState queryState, QueryOptions options) throws RequestExecutionException, RequestValidationException
{
return execute(queryState, BatchQueryOptions.withoutPerStatementVariables(options));

View File

@ -31,6 +31,7 @@ import org.apache.cassandra.transport.SimpleClient;
import org.apache.cassandra.transport.messages.QueryMessage;
import static junit.framework.Assert.assertEquals;
import static org.junit.Assert.assertNull;
public class ClientWarningsTest extends CQLTester
{
@ -52,12 +53,11 @@ public class ClientWarningsTest extends CQLTester
QueryMessage query = new QueryMessage(createBatchStatement2(1), QueryOptions.DEFAULT);
Message.Response resp = client.execute(query);
assertEquals(1, resp.getWarnings().size());
assertNull(resp.getWarnings());
query = new QueryMessage(createBatchStatement2(DatabaseDescriptor.getBatchSizeWarnThreshold()), QueryOptions.DEFAULT);
resp = client.execute(query);
assertEquals(2, resp.getWarnings().size());
assertEquals(1, resp.getWarnings().size());
}
}

View File

@ -19,4 +19,4 @@ if "%OS%" == "Windows_NT" setlocal
pushd "%~dp0"
call cassandra.in.bat
if NOT DEFINED STRESS_HOME set STRESS_HOME=%CD%\..
"%JAVA_HOME%\bin\java" -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*
"%JAVA_HOME%\bin\java" %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% org.apache.cassandra.stress.Stress %*