Migrate sstableloader code to its own tools directory and artifact

As part of a broader effort to decouple java driver code from the
server code, this moves sstableloader to its own tools directory.

As sstableloader is also used as a library (CASSANDRA-10637), added
a new artifact 'cassandra-sstableloader' that will get deployed to
maven along with 'cassandra-all'.

While I expect this is likely a niche use case, this will allow users
to continue using BulkExport as a library.

Moves sstableloader-specific targets to its own build.xml in
tools/sstableloader/build.xml.

Also updates IDE project files and circleci to utilize new
sstableloader-specific targets.

patch by Andy Tolbert; reviewed by Stefan Miklosovic and Mick Semb Wever for CASSANDRA-20328
This commit is contained in:
Andy Tolbert 2025-05-13 08:23:50 -05:00
parent 7ab1e3827c
commit f80a234d1e
39 changed files with 4456 additions and 199 deletions

View File

@ -100,6 +100,7 @@ Running other types of tests with docker:
.build/docker/run-tests.sh -a test
.build/docker/run-tests.sh -a stress-test
.build/docker/run-tests.sh -a fqltool-test
.build/docker/run-tests.sh -a sstableloader-test
.build/docker/run-tests.sh -a microbench
.build/docker/run-tests.sh -a test-cdc
.build/docker/run-tests.sh -a test-compression

View File

@ -226,9 +226,9 @@
<arg value="-Dsonar.working.directory=${sonar.workdir}/scanner"/>
<arg value="-Dsonar.sourceEncoding=UTF-8"/>
<arg value="-Dsonar.scm.disabled=true"/>
<arg value="-Dsonar.sources=${build.src.java},${fqltool.build.src},${stress.build.src}"/>
<arg value="-Dsonar.sources=${build.src.java},${fqltool.build.src},${sstableloader.build.src},${stress.build.src}"/>
<arg value="-Dsonar.java.libraries=${build.dir.lib}/jars/jsr305-*.jar"/>
<arg value="-Dsonar.java.binaries=${build.classes.main},${fqltool.build.classes},${stress.build.classes}"/>
<arg value="-Dsonar.java.binaries=${build.classes.main},${fqltool.build.classes},${sstableloader.build.classes},${stress.build.classes}"/>
<arg value="-Dsonar.java.source=11"/>
<arg value="-Dsonar.projectVersion=${timestamp}"/>
<arg value="-Dsonar.qualitygate.wait=true"/>

View File

@ -40,7 +40,7 @@ error() {
# legacy argument handling
case ${1} in
"build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
"build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
test_type="-a ${1}"
if [[ -z ${2} ]]; then
test_list=""
@ -182,7 +182,7 @@ docker_flags="-m 5g --memory-swap 5g"
case ${test_target/-repeat/} in
"build_dtest_jars")
;;
"stress-test" | "fqltool-test" )
"stress-test" | "fqltool-test" | "sstableloader-test" )
[[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
;;
# test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway

View File

@ -671,7 +671,7 @@
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
<version>4.1-alpha2-SNAPSHOT</version>
<version>@version@</version>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>

View File

@ -66,7 +66,7 @@ print_help() {
# legacy argument handling
case ${1} in
"build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
"build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
test_type="-a ${1}"
if [[ -z ${2} ]]; then
test_list=""
@ -285,7 +285,7 @@ _main() {
# check split_chunk is compatible with target (if not a regexp)
if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
case ${target} in
"stress-test" | "fqltool-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
"stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
error 1 "Target ${target} does not suport splits."
;;
*)
@ -344,6 +344,11 @@ _main() {
ant fqltool-build-test ${ANT_TEST_OPTS}
ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
;;
"sstableloader-test")
# hard fail on test compilation, but dont fail the test run so unstable test reports are processed
ant sstableloader-build-test ${ANT_TEST_OPTS}
ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
;;
"microbench")
ant $target ${ANT_TEST_OPTS} -Dmaven.test.failure.ignore=true
;;

View File

@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-parent</artifactId>
<version>@version@</version>
<relativePath>@final.name@-parent.pom</relativePath>
</parent>
<artifactId>cassandra-sstableloader</artifactId>
<version>@version@</version>
<name>Apache Cassandra SSTableLoader</name>
<description>Standalone SSTableLoader for Apache Cassandra.</description>
<url>https://cassandra.apache.org</url>
<inceptionYear>2025</inceptionYear>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
<scm>
<connection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</connection>
<developerConnection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</developerConnection>
<url>https://gitbox.apache.org/repos/asf?p=cassandra.git</url>
</scm>
<dependencies>
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
</dependency>
</dependencies>
</project>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -63,6 +63,13 @@ default_env_vars: &default_env_vars
# The number of times that new, modified or manually specified fqltool unit tests should be run.
REPEATED_UTESTS_FQLTOOL_COUNT: 500
# Comma-separated list of tests that should be included in the repeated run for sstableloader unit tests,
# in addition to automatically detected new and modified tests. For example:
# REPEATED_UTESTS_SSTABLELOADER: org.apache.cassandra.tools.LoaderOptionsTest
REPEATED_UTESTS_SSTABLELOADER:
# The number of times that new, modified or manually specified sstableloader unit tests should be run.
REPEATED_UTESTS_SSTABLELOADER_COUNT: 500
# Comma-separated list of tests that should be included in the repeated run for long unit tests,
# in addition to automatically detected new and modified tests. For example:
# REPEATED_UTESTS_LONG: org.apache.cassandra.db.commitlog.CommitLogStressTest
@ -537,6 +544,30 @@ j11_separate_jobs: &j11_separate_jobs
requires:
- start_j17_utests_fqltool_repeat
- j11_build
- start_j11_utests_sstableloader:
type: approval
- j11_utests_sstableloader:
requires:
- start_j11_utests_sstableloader
- j11_build
- start_j17_utests_sstableloader:
type: approval
- j17_utests_sstableloader:
requires:
- start_j17_utests_sstableloader
- j11_build
- start_j11_utests_sstableloader_repeat:
type: approval
- j11_utests_sstableloader_repeat:
requires:
- start_j11_utests_sstableloader_repeat
- j11_build
- start_j17_utests_sstableloader_repeat:
type: approval
- j17_utests_sstableloader_repeat:
requires:
- start_j17_utests_sstableloader_repeat
- j11_build
- start_j11_utests_system_keyspace_directory:
type: approval
- j11_utests_system_keyspace_directory:
@ -974,6 +1005,24 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
requires:
- start_utests_fqltool
- j11_build
- start_utests_sstableloader:
type: approval
- j11_utests_sstableloader:
requires:
- start_utests_sstableloader
- j11_build
- j17_utests_sstableloader:
requires:
- start_utests_sstableloader
- j11_build
- j11_utests_sstableloader_repeat:
requires:
- start_utests_sstableloader
- j11_build
- j17_utests_sstableloader_repeat:
requires:
- start_utests_sstableloader
- j11_build
- start_utests_system_keyspace_directory:
type: approval
- j11_utests_system_keyspace_directory:
@ -1356,6 +1405,18 @@ j17_separate_jobs: &j17_separate_jobs
requires:
- start_j17_utests_fqltool_repeat
- j17_build
- start_j17_utests_sstableloader:
type: approval
- j17_utests_sstableloader:
requires:
- start_j17_utests_sstableloader
- j17_build
- start_j17_utests_sstableloader_repeat:
type: approval
- j17_utests_sstableloader_repeat:
requires:
- start_j17_utests_sstableloader_repeat
- j17_build
- start_j17_utests_system_keyspace_directory:
type: approval
- j17_utests_system_keyspace_directory:
@ -1541,6 +1602,16 @@ j17_pre-commit_jobs: &j17_pre-commit_jobs
requires:
- start_utests_fqltool
- j17_build
- start_utests_sstableloader:
type: approval
- j17_utests_sstableloader:
requires:
- start_utests_sstableloader
- j17_build
- j17_utests_sstableloader_repeat:
requires:
- start_utests_sstableloader
- j17_build
- start_utests_system_keyspace_directory:
type: approval
- j17_utests_system_keyspace_directory:
@ -1875,6 +1946,22 @@ jobs:
- run_junit_tests:
target: fqltool-test
j11_utests_sstableloader:
<<: *j11_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
- run_junit_tests:
target: sstableloader-test
j17_utests_sstableloader:
<<: *j17_seq_executor
steps:
- attach_workspace:
at: /home/cassandra
- run_junit_tests:
target: sstableloader-test
j11_utests_system_keyspace_directory:
<<: *j11_par_executor
steps:
@ -2380,6 +2467,22 @@ jobs:
- log_environment
- run_utests_fqltool_repeat
j11_utests_sstableloader_repeat:
<<: *j11_repeated_utest_executor
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- run_utests_sstableloader_repeat
j17_utests_sstableloader_repeat:
<<: *j17_repeated_utest_executor
steps:
- attach_workspace:
at: /home/cassandra
- log_environment
- run_utests_sstableloader_repeat
j11_utests_long_repeat:
<<: *j11_repeated_utest_executor
steps:
@ -3114,6 +3217,14 @@ commands:
count: ${REPEATED_UTESTS_FQLTOOL_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
run_utests_sstableloader_repeat:
steps:
- run_repeated_utests:
target: sstableloader-test
tests: ${REPEATED_UTESTS_SSTABLELOADER}
count: ${REPEATED_UTESTS_SSTABLELOADER_COUNT}
stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
run_utests_stress_repeat:
steps:
- run_repeated_utests:
@ -3237,6 +3348,7 @@ commands:
$target == "test-oa" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "sstableloader-test" || \
$target == "long-test" || \
$target == "stress-test" || \
$target == "test-simulator-dtest" ]]; then
@ -3365,6 +3477,7 @@ commands:
$target == "test-oa" || \
$target == "test-system-keyspace-directory" || \
$target == "fqltool-test" || \
$target == "sstableloader-test" || \
$target == "long-test" || \
$target == "stress-test" || \
$target == "test-simulator-dtest" ]]; then

View File

@ -51,6 +51,8 @@ print_help()
echo " -e REPEATED_UTESTS_COUNT=500"
echo " -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest"
echo " -e REPEATED_UTESTS_FQLTOOL_COUNT=500"
echo " -e REPEATED_UTESTS_SSTABLELOADER=org.apache.cassandra.tools.LoaderOptionsTest"
echo " -e REPEATED_UTESTS_SSTABLELOADER_COUNT=500"
echo " -e REPEATED_UTESTS_LONG=org.apache.cassandra.db.commitlog.CommitLogStressTest"
echo " -e REPEATED_UTESTS_LONG_COUNT=100"
echo " -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest"
@ -131,6 +133,8 @@ if $has_env_vars && $check_env_vars; then
[ "$key" != "REPEATED_UTESTS_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_FQLTOOL" ] &&
[ "$key" != "REPEATED_UTESTS_FQLTOOL_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_SSTABLELOADER" ] &&
[ "$key" != "REPEATED_UTESTS_SSTABLELOADER_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_LONG" ] &&
[ "$key" != "REPEATED_UTESTS_LONG_COUNT" ] &&
[ "$key" != "REPEATED_UTESTS_STRESS" ] &&
@ -171,7 +175,7 @@ if $free; then
elif $paid; then
($all || $free) && die "Cannot use option -p with options -a or -f"
echo "Generating new config.yml file for paid tier from config_template.yml"
patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
patch --silent -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml
rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
@ -188,7 +192,7 @@ elif $all; then
rm $BASEDIR/config.yml.FREE.tmp
# setup config for paid tier
patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
patch --silent -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml.PAID
rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
@ -241,6 +245,7 @@ if $detect_changed_tests; then
add_diff_tests "REPEATED_UTESTS_LONG" "test/long/" "org.apache.cassandra"
add_diff_tests "REPEATED_UTESTS_STRESS" "tools/stress/test/unit/" "org.apache.cassandra.stress"
add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool"
add_diff_tests "REPEATED_UTESTS_SSTABLELOADER" "tools/sstableloader/test/unit/" "org.apache.cassandra.tools"
add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test"
add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test"
add_diff_tests "REPEATED_JVM_UPGRADE_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.upgrade"
@ -305,6 +310,10 @@ delete_repeated_jobs()
delete_job "$1" "j11_utests_fqltool_repeat"
delete_job "$1" "j17_utests_fqltool_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_SSTABLELOADER=")); then
delete_job "$1" "j11_utests_sstableloader_repeat"
delete_job "$1" "j17_utests_sstableloader_repeat"
fi
if (! (echo "$env_vars" | grep -q "REPEATED_SIMULATOR_DTESTS=")); then
delete_job "$1" "j11_simulator_dtests_repeat"
fi
@ -386,6 +395,7 @@ build_dev_min_jobs()
delete_job "$1" "j11_utests_cdc"
delete_job "$1" "j11_utests_compression"
delete_job "$1" "j11_utests_fqltool"
delete_job "$1" "j11_utests_sstableloader"
delete_job "$1" "j11_utests_long"
delete_job "$1" "j11_utests_stress"
delete_job "$1" "j11_utests_system_keyspace_directory"
@ -394,6 +404,7 @@ build_dev_min_jobs()
delete_job "$1" "j17_utests_cdc"
delete_job "$1" "j17_utests_compression"
delete_job "$1" "j17_utests_fqltool"
delete_job "$1" "j17_utests_sstableloader"
delete_job "$1" "j17_utests_long"
delete_job "$1" "j17_utests_stress"
delete_job "$1" "j11_utests_latest"
@ -403,6 +414,7 @@ build_dev_min_jobs()
delete_job "$1" "start_utests_stress"
delete_job "$1" "start_utests_long"
delete_job "$1" "start_utests_fqltool"
delete_job "$1" "start_utests_sstableloader"
delete_job "$1" "start_utests_compression"
delete_job "$1" "start_utests_cdc"
delete_job "$1" "start_j17_cqlsh-dtests-latest"

View File

@ -127,9 +127,9 @@ def pipelineProfiles() {
return [
'packaging': ['artifacts', 'lint', 'debian', 'redhat'],
'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'],
'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
'custom': []
]
}
@ -164,6 +164,7 @@ def tasks() {
// (some buffer on the heaviest split under the 1h max is required, ref `timeout(…)` in `test(…)`)
'cqlsh-test': [splits: 1],
'fqltool-test': [splits: 1, size: 'small'],
'sstableloader-test': [splits: 1, size: 'small'],
'test-cdc': [splits: 20],
'test': [splits: 20],
'test-latest': [splits: 20],

View File

@ -1,4 +1,5 @@
5.1
* Migrate sstableloader code to its own tools directory and artifact (CASSANDRA-20328)
* Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623)
* Avoid duplicate hardlink error upon forceful taking of ephemeral snapshots during repair (CASSANDRA-20490)
* When a custom disk error handler fails to initiate, fail the startup of a node instead of using the no-op handler (CASSANDRA-20614)

View File

@ -32,18 +32,16 @@ elif [ -r "$CASSANDRA_INCLUDE" ]; then
. "$CASSANDRA_INCLUDE"
fi
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
# SSTableLoader has been moved to tools/bin, this script simply
# invokes the script in the new path.
SSTABLELOADER_PATH="$CASSANDRA_HOME/tools/bin/sstableloader"
if [ ! -f "$SSTABLELOADER_PATH" ]; then
echo "Error: sstableloader has moved to the tools directory. \
Detected that $SSTABLELOADER_PATH does not exist." >&2
exit 2
fi
if [ "x$MAX_HEAP_SIZE" = "x" ]; then
MAX_HEAP_SIZE="256M"
fi
"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
-Dcassandra.storagedir="$cassandra_storagedir" \
-Dlogback.configurationFile=logback-tools.xml \
org.apache.cassandra.tools.BulkLoader "$@"
"$SSTABLELOADER_PATH" "$@"
# vi:ai sw=4 ts=4 tw=0 et

View File

@ -385,26 +385,38 @@
</fileset>
</path>
<macrodef name="create-javadoc">
<attribute name="destdir"/>
<element name="filesets"/>
<sequential>
<javadoc destdir="@{destdir}" author="true" version="true" use="true"
windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
bottom="Copyright &amp;copy; 2009- The Apache Software Foundation"
useexternalfile="yes" encoding="UTF-8" failonerror="false"
maxmemory="256m" additionalparam="${jdk11plus-javac-exports}">
<filesets/>
</javadoc>
<fail message="javadoc failed">
<condition>
<not>
<available file="@{destdir}/index-all.html" />
</not>
</condition>
</fail>
</sequential>
</macrodef>
<macrodef name="create-javadoc">
<attribute name="destdir"/>
<element name="filesets"/>
<sequential>
<javadoc destdir="@{destdir}" author="true" version="true" use="true"
windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
bottom="Copyright &amp;copy; 2009- The Apache Software Foundation"
useexternalfile="yes" encoding="UTF-8" failonerror="false"
maxmemory="256m" additionalparam="${jdk11plus-javac-exports}">
<filesets/>
</javadoc>
<fail message="javadoc failed">
<condition>
<not>
<available file="@{destdir}/index-all.html" />
</not>
</condition>
</fail>
</sequential>
</macrodef>
<!-- filterset to apply to pom templates -->
<filterset id="pom-template">
<filter token="version" value="${version}"/>
<filter token="final.name" value="${final.name}"/>
<filter token="jamm.version" value="${jamm.version}"/>
<filter token="allocation-instrumenter.version" value="${allocation-instrumenter.version}"/>
<filter token="ecj.version" value="${ecj.version}"/>
<filter token="asm.version" value="${asm.version}"/>
<filter token="jacoco.version" value="${jacoco.version}"/>
<filter token="jflex.version" value="${jflex.version}"/>
</filterset>
<!--
Setup the output directories.
@ -746,24 +758,13 @@
</target>
<target name="write-poms" unless="without.maven">
<filterset id="pom-template">
<filter token="version" value="${version}"/>
<filter token="final.name" value="${final.name}"/>
<filter token="jamm.version" value="${jamm.version}"/>
<filter token="allocation-instrumenter.version" value="${allocation-instrumenter.version}"/>
<filter token="ecj.version" value="${ecj.version}"/>
<filter token="asm.version" value="${asm.version}"/>
<filter token="jacoco.version" value="${jacoco.version}"/>
<filter token="jflex.version" value="${jflex.version}"/>
</filterset>
<copy file=".build/cassandra-deps-template.xml" tofile="${build.dir}/${final.name}.pom">
<copy file=".build/cassandra-deps-maven-pom.xml" tofile="${build.dir}/${final.name}.pom">
<filterset refid="pom-template"/>
</copy>
<copy file=".build/parent-pom-template.xml" tofile="${build.dir}/${final.name}-parent.pom">
<copy file=".build/parent-maven-pom.xml" tofile="${build.dir}/${final.name}-parent.pom">
<filterset refid="pom-template"/>
</copy>
<copy file=".build/cassandra-build-deps-template.xml" tofile="${build.dir}/tmp-${final.name}-deps.pom">
<copy file=".build/cassandra-build-maven-pom.xml" tofile="${build.dir}/tmp-${final.name}-deps.pom">
<filterset refid="pom-template"/>
</copy>
</target>
@ -864,7 +865,7 @@
</target>
<target name="jar"
depends="_main-jar,build-test,stress-jar,fqltool-jar,simulator-jars,write-poms"
depends="_main-jar,build-test,stress-jar,fqltool-jar,simulator-jars,sstableloader-jar,write-poms"
description="Assemble Cassandra JAR files">
</target>
@ -949,7 +950,7 @@
</target>
<!-- creates release tarballs -->
<target name="artifacts" depends="_artifacts-init,check,sources-jar"
<target name="artifacts" depends="_artifacts-init,check,sources-jar,sstableloader-sources-jar"
description="Create Cassandra tarball and maven artifacts">
<tar compression="gzip" longfile="gnu"
destfile="${build.dir}/${final.name}-bin.tar.gz">
@ -1082,7 +1083,7 @@
<antcall target="build-test" inheritRefs="true"/>
</target>
<target name="build-test" depends="_main-jar,stress-build-test,fqltool-build,resolver-dist-lib,simulator-jars"
<target name="build-test" depends="_main-jar,stress-build-test,fqltool-build-test,sstableloader-build-test,resolver-dist-lib,simulator-jars"
description="Compile test classes">
<antcall target="_build-test"/>
<antcall target="_check-test-names"/>
@ -1101,6 +1102,7 @@
<classpath>
<path refid="cassandra.classpath.test"/>
<pathelement location="${fqltool.build.classes}"/>
<pathelement location="${sstableloader.build.classes}"/>
</classpath>
<compilerarg value="-XDignore.symbol.file"/>
<!-- needed to compile org.apache.cassandra.utils.memory.BufferPoolTest -->
@ -1129,7 +1131,7 @@
<macrodef name="check-test-names">
<attribute name="annotationName"/>
<attribute name="regex"/>
<attribute name="scanClassPath" default="${test.classes}:${fqltool.test.classes}:${stress.test.classes}"/>
<attribute name="scanClassPath" default="${test.classes}:${fqltool.test.classes}:${stress.test.classes}:${sstableloader.test.classes}"/>
<attribute name="packageName" default="org.apache.cassandra"/>
<attribute name="expand" default="true"/>
<attribute name="normalize" default="true"/>
@ -1141,9 +1143,11 @@
<path refid="cassandra.classpath.test"/>
<pathelement location="${fqltool.build.classes}"/>
<pathelement location="${stress.build.classes}"/>
<pathelement location="${sstableloader.build.classes}"/>
<pathelement location="${test.classes}"/>
<pathelement location="${fqltool.test.classes}"/>
<pathelement location="${stress.test.classes}"/>
<pathelement location="${sstableloader.test.classes}"/>
</classpath>
<jvmarg line="${java-jvmargs}"/>
<jvmarg value="-DscanClassPath=@{scanClassPath}"/>
@ -1270,10 +1274,12 @@
<pathelement path="${java.class.path}"/>
<pathelement location="${stress.build.classes}"/>
<pathelement location="${fqltool.build.classes}"/>
<pathelement location="${sstableloader.build.classes}"/>
<pathelement location="${test.classes}"/>
<path refid="cassandra.classpath.test" />
<pathelement location="${stress.test.classes}"/>
<pathelement location="${fqltool.test.classes}"/>
<pathelement location="${sstableloader.test.classes}"/>
<pathelement location="${test.conf}"/>
<pathelement path="${java.class.path}"/>
<path refid="cassandra.classpath" />
@ -2079,8 +2085,10 @@
<classpathentry kind="src" output="${test.classes}" path="test/resources" />
<classpathentry kind="src" path="tools/stress/src"/>
<classpathentry kind="src" path="tools/fqltool/src"/>
<classpathentry kind="src" path="tools/sstableloader/src"/>
<classpathentry kind="src" output="build/test/stress-classes" path="tools/stress/test/unit" />
<classpathentry kind="src" output="build/test/fqltool-classes" path="tools/fqltool/test/unit" />
<classpathentry kind="src" output="build/test/sstableloader-classes" path="tools/sstableloader/test/unit" />
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="output" path="build/classes/eclipse"/>
<classpathentry kind="lib" path="test/conf"/>
@ -2140,7 +2148,7 @@
<!-- Publish artifacts to remote Maven repository -->
<target name="publish"
depends="mvn-install,artifacts"
depends="mvn-install,sstableloader-mvn-install,artifacts"
description="Publishes the artifacts to the Maven repository">
<!-- Accord -->
<deploy pomFile="${accord.local.repository}/${accord.final.name}.pom"
@ -2170,6 +2178,16 @@
file="${build.dir}/${final.name}-javadoc.jar"
classifier="javadoc"/>
<!-- the cassandra-sstableloader jar -->
<deploy pomFile="${build.dir}/${final.name}-sstableloader.pom"
file="${build.dir}/tools/lib/sstableloader.jar"/>
<deploy pomFile="${build.dir}/${final.name}-sstableloader.pom"
file="${build.dir}/${final.name}-sstableloader-sources.jar"
classifier="sources"/>
<deploy pomFile="${build.dir}/${final.name}-sstableloader.pom"
file="${build.dir}/${final.name}-sstableloader-javadoc.jar"
classifier="javadoc"/>
<!-- the distribution -->
<sign-dist file="${build.dir}/${final.name}-bin.tar.gz" />
<sign-dist file="${build.dir}/${final.name}-src.tar.gz" />
@ -2185,4 +2203,5 @@
<import file="${build.helpers.dir}/build-bench.xml"/>
<import file="${build.helpers.dir}/build-sonar.xml"/>
<import file="${build.helpers.dir}/build-accord.xml" />
<import file="${basedir}/tools/sstableloader/build.xml"/>
</project>

View File

@ -16,7 +16,6 @@ bin/cassandra.in.sh usr/share/cassandra
bin/cassandra usr/sbin
bin/nodetool usr/bin
bin/sstableutil usr/bin
bin/sstableloader usr/bin
bin/cqlsh usr/bin
bin/cqlsh.py usr/bin
bin/sstablescrub usr/bin
@ -28,6 +27,7 @@ tools/bin/auditlogviewer usr/bin
tools/bin/jmxtool usr/bin
tools/bin/hash_password usr/bin
tools/bin/sstablepartitions usr/bin
tools/bin/sstableloader usr/bin
lib/*.jar usr/share/cassandra/lib
lib/*.zip usr/share/cassandra/lib
lib/x86_64/* usr/share/cassandra/lib/x86_64

4
debian/rules vendored
View File

@ -67,6 +67,10 @@ install: build
dh_install $(BUILD_DIR)/tools/lib/fqltool.jar \
usr/share/cassandra
# Copy sstableloader jars
dh_install $(BUILD_DIR)/tools/lib/sstableloader.jar \
usr/share/cassandra
dh_link usr/share/cassandra/apache-cassandra-$(VERSION).jar \
usr/share/cassandra/apache-cassandra.jar

View File

@ -30,6 +30,8 @@
<sourceFolder url="file://$MODULE_DIR$/tools/stress/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/tools/fqltool/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tools/fqltool/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/tools/sstableloader/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tools/sstableloader/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/unit" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/long" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/memory" isTestSource="true" />

View File

@ -360,6 +360,7 @@
<filter targetName="build-project" isVisible="false" />
<filter targetName="stress-build" isVisible="true" />
<filter targetName="fqltool-build" isVisible="true" />
<filter targetName="sstableloader-build" isVisible="true" />
<filter targetName="_write-poms" isVisible="false" />
<filter targetName="write-poms" isVisible="false" />
<filter targetName="jar" isVisible="true" />

View File

@ -5,6 +5,7 @@
<ant antfile="../build.xml" inheritall="false" target="build"/>
<ant antfile="../build.xml" inheritall="false" target="build-test"/>
<ant antfile="../build.xml" inheritall="false" target="fqltool-build-test"/>
<ant antfile="../build.xml" inheritall="false" target="sstableloader-build-test"/>
<ant antfile="../build.xml" inheritall="false" target="stress-build-test"/>
</target>
<target name="clean">

View File

@ -35,6 +35,12 @@
<location>${project.dir}/tools/fqltool/src</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>src/sstableloader</label>
<type>java</type>
<location>${project.dir}/tools/sstableloader/src</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>src/stress</label>
<type>java</type>
@ -53,6 +59,12 @@
<location>${project.dir}/tools/fqltool/test/unit</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>test/sstableloader/</label>
<type>java</type>
<location>${project.dir}/tools/sstableloader/test/unit</location>
<encoding>UTF-8</encoding>
</source-folder>
<source-folder>
<label>test/stress</label>
<type>java</type>
@ -164,6 +176,12 @@
<script>../build.xml</script>
<build-target>fqltool-build</build-target>
</export>
<export>
<type>folder</type>
<location>${project.dir}/build/classes/sstableloader</location>
<script>../build.xml</script>
<build-target>sstableloader-build</build-target>
</export>
<export>
<type>folder</type>
<location>${project.dir}/build/classes/stress</location>
@ -194,6 +212,12 @@
<script>../build.xml</script>
<build-target>fqltool-build-test</build-target>
</export>
<export>
<type>folder</type>
<location>${project.dir}/build/test/sstableloader-classes</location>
<script>../build.xml</script>
<build-target>sstableloader-build-test</build-target>
</export>
<export>
<type>folder</type>
<location>${project.dir}/build/test/stress-classes</location>
@ -214,6 +238,10 @@
<label>src/fqltool</label>
<location>${project.dir}/tools/fqltool/src</location>
</source-folder>
<source-folder style="packages">
<label>src/sstableloader</label>
<location>${project.dir}/tools/sstableloader/src</location>
</source-folder>
<source-folder style="packages">
<label>src/stress</label>
<location>${project.dir}/tools/stress/src</location>
@ -226,6 +254,10 @@
<label>test/fqltool/</label>
<location>${project.dir}/tools/fqltool/test/unit</location>
</source-folder>
<source-folder style="packages">
<label>test/sstableloader/</label>
<location>${project.dir}/tools/sstableloader/test/unit</location>
</source-folder>
<source-folder style="packages">
<label>test/stress</label>
<location>${project.dir}/tools/stress/test/unit</location>
@ -295,6 +327,11 @@
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
<built-to>${project.dir}/build/classes/fqltool</built-to>
</compilation-unit>
<compilation-unit>
<package-root>${project.dir}/tools/sstableloader/src</package-root>
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
<built-to>${project.dir}/build/classes/sstableloader</built-to>
</compilation-unit>
<compilation-unit>
<package-root>${project.dir}/tools/stress/src</package-root>
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
@ -313,7 +350,7 @@
<package-root>${project.dir}/test/simulator/test</package-root>
<package-root>${project.dir}/test/harry/main</package-root>
<unit-tests/>
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/:${project.dir}/build/classes/stress/</classpath>
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/:${project.dir}/build/classes/sstableloader:${project.dir}/build/classes/stress/</classpath>
<built-to>${project.dir}/build/test/classes</built-to>
</compilation-unit>
<compilation-unit>
@ -322,6 +359,12 @@
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/</classpath>
<built-to>${project.dir}/build/test/fqltool-classes</built-to>
</compilation-unit>
<compilation-unit>
<package-root>${project.dir}/tools/sstableloader/test/unit</package-root>
<unit-tests/>
<classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/sstableloader/</classpath>
<built-to>${project.dir}/build/test/sstableloader-classes</built-to>
</compilation-unit>
<compilation-unit>
<package-root>${project.dir}/tools/stress/test/unit</package-root>
<unit-tests/>

View File

@ -120,6 +120,9 @@ cp -p %{_get_dist_dir}/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/
# copy fqltool jar
cp -p %{_get_dist_dir}/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/
# copy sstableloader jar
cp -p %{_get_dist_dir}/tools/lib/sstableloader.jar %{buildroot}/usr/share/%{username}/
# copy binaries
mv bin/cassandra %{buildroot}/usr/sbin/
cp -p bin/* %{buildroot}/usr/bin/

View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
public class CmdLineOptions extends Options
{
/**
* Add option with argument and argument name
*
* @param opt shortcut for option name
* @param longOpt complete option name
* @param argName argument name
* @param description description of the option
* @return updated Options object
*/
public Options addOption(String opt, String longOpt, String argName, String description)
{
Option option = new Option(opt, longOpt, true, description);
option.setArgName(argName);
return addOption(option);
}
/**
* Add option with argument and argument name that accepts being defined multiple times as a list
*
* @param opt shortcut for option name
* @param longOpt complete option name
* @param argName argument name
* @param description description of the option
* @return updated Options object
*/
public Options addOptionList(String opt, String longOpt, String argName, String description)
{
Option option = new Option(opt, longOpt, true, description);
option.setArgName(argName);
option.setArgs(Option.UNLIMITED_VALUES);
return addOption(option);
}
/**
* Add option without argument
*
* @param opt shortcut for option name
* @param longOpt complete option name
* @param description description of the option
* @return updated Options object
*/
public Options addOption(String opt, String longOpt, String description)
{
return addOption(new Option(opt, longOpt, false, description));
}
}

View File

@ -30,7 +30,6 @@ import java.io.IOException;
import java.util.function.BiPredicate;
import org.apache.cassandra.io.util.File;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
public class StandaloneSSTableUtil
{

View File

@ -51,7 +51,6 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Pair;

View File

@ -50,7 +50,7 @@ import org.apache.cassandra.tcm.ClusterMetadataService;
import org.apache.cassandra.utils.JVMStabilityInspector;
import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
public class StandaloneSplitter

View File

@ -46,7 +46,6 @@ import org.apache.cassandra.utils.JVMStabilityInspector;
import org.apache.cassandra.utils.OutputHandler;
import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
public class StandaloneUpgrader
{

View File

@ -53,7 +53,6 @@ import org.apache.cassandra.utils.OutputHandler;
import org.apache.cassandra.utils.Throwables;
import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
public class StandaloneVerifier
{

View File

@ -39,7 +39,7 @@ if [ -d $CASSANDRA_HOME/build ] ; then
if [ "$jars_cnt" = "1" ]; then
cassandra_bin="`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar | grep -v javadoc | grep -v sources`"
cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool"
cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool:$CASSANDRA_HOME/build/classes/sstableloader"
CLASSPATH="$CLASSPATH:$cassandra_bin"
fi
fi

49
tools/bin/sstableloader Executable file
View File

@ -0,0 +1,49 @@
#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
# Locations (in order) to use when searching for an include file.
for include in "`dirname "$0"`/cassandra.in.sh" \
"$HOME/.cassandra.in.sh" \
/usr/share/cassandra/cassandra.in.sh \
/usr/local/share/cassandra/cassandra.in.sh \
/opt/cassandra/cassandra.in.sh; do
if [ -r "$include" ]; then
. "$include"
break
fi
done
elif [ -r "$CASSANDRA_INCLUDE" ]; then
. "$CASSANDRA_INCLUDE"
fi
if [ -z "$CLASSPATH" ]; then
echo "You must set the CLASSPATH var" >&2
exit 1
fi
if [ "x$MAX_HEAP_SIZE" = "x" ]; then
MAX_HEAP_SIZE="256M"
fi
"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
-Dcassandra.storagedir="$cassandra_storagedir" \
-Dlogback.configurationFile=logback-tools.xml \
org.apache.cassandra.tools.BulkLoader "$@"
# vi:ai sw=4 ts=4 tw=0 et

View File

@ -0,0 +1,125 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project basedir="../../" name="apache-cassandra-sstableloader"
xmlns:if="ant:if"
xmlns:unless="ant:unless">
<!-- pull common definitions from parent project -->
<import file="../../build.xml"/>
<property name="sstableloader-javadoc.dir" value="${build.dir}/sstableloader-javadoc"/>
<property name="sstableloader.build.src" value="${basedir}/tools/sstableloader/src" />
<property name="sstableloader.test.src" value="${basedir}/tools/sstableloader/test/unit" />
<property name="sstableloader.build.classes" value="${build.classes}/sstableloader" />
<property name="sstableloader.test.classes" value="${build.dir}/test/sstableloader-classes" />
<property name="sstableloader.manifest" value="${sstableloader.build.classes}/MANIFEST.MF" />
<target name="sstableloader-init">
<mkdir dir="${sstableloader.test.classes}"/>
</target>
<target name="sstableloader-build" depends="sstableloader-init,build" description="build sstableloader">
<antcall target="_sstableloader_build"/>
</target>
<target name="_sstableloader_build">
<mkdir dir="${sstableloader.build.classes}" />
<javac compiler="modern" debug="true" debuglevel="${debuglevel}"
source="${ant.java.version}" target="${ant.java.version}"
encoding="utf-8" destdir="${sstableloader.build.classes}" includeantruntime="true">
<src path="${sstableloader.build.src}" />
<classpath>
<path refid="cassandra.classpath" />
</classpath>
</javac>
</target>
<target name="sstableloader-build-test" depends="sstableloader-build" description="Compile sstableloader tests">
<javac debug="true" debuglevel="${debuglevel}" destdir="${sstableloader.test.classes}"
source="${ant.java.version}" target="${ant.java.version}"
includeantruntime="false" encoding="utf-8">
<classpath>
<path refid="cassandra.classpath.test"/>
<pathelement location="${sstableloader.build.classes}" />
</classpath>
<src path="${sstableloader.test.src}"/>
</javac>
</target>
<target name="sstableloader-test" depends="sstableloader-build-test, maybe-build-test" description="Runs sstableloader tests">
<testmacro inputdir="${sstableloader.test.src}"
timeout="${test.timeout}">
</testmacro>
</target>
<target name="sstableloader-write-pom">
<copy file=".build/sstableloader-deps-maven-pom.xml" tofile="${build.dir}/${final.name}-sstableloader.pom">
<filterset refid="pom-template"/>
</copy>
</target>
<target name="sstableloader-jar"
depends="sstableloader-build,sstableloader-write-pom"
description="Assemble Cassandra sstableloader JAR files">
<!-- sstableloader jar -->
<manifest file="${sstableloader.manifest}">
<attribute name="Built-By" value="Apache Cassandra Project"/>
<attribute name="Main-Class" value="org.apache.cassandra.tools.BulkLoader"/>
</manifest>
<mkdir dir="${sstableloader.build.classes}/META-INF" />
<mkdir dir="${build.dir}/tools/lib/" />
<jar destfile="${build.dir}/tools/lib/sstableloader.jar" manifest="${sstableloader.manifest}">
<fileset dir="${sstableloader.build.classes}"/>
</jar>
</target>
<target name="sstableloader-javadoc" depends="build" description="Create javadoc" unless="no-javadoc">
<create-javadoc destdir="${sstableloader-javadoc.dir}">
<filesets>
<fileset dir="${sstableloader.build.src}" defaultexcludes="yes">
<include name="org/apache/**/*.java"/>
</fileset>
</filesets>
</create-javadoc>
</target>
<target name="sstableloader-javadoc-jar" depends="sstableloader-javadoc" unless="no-javadoc" description="Assemble sstableloader JavaDoc JAR file">
<jar jarfile="${build.dir}/${final.name}-sstableloader-javadoc.jar" basedir="${sstableloader-javadoc.dir}"/>
<delete quiet="true" dir="${sstableloader-javadoc.dir}"/>
</target>
<target name="sstableloader-sources-jar" depends="init" description="Assemble sstableloader Sources JAR file">
<jar jarfile="${build.dir}/${final.name}-sstableloader-sources.jar">
<fileset dir="${sstableloader.build.src}" defaultexcludes="yes">
<include name="org/apache/**/*.java"/>
</fileset>
</jar>
</target>
<target name="sstableloader-mvn-install" depends="sstableloader-sources-jar,sstableloader-javadoc-jar,mvn-install"
description="Installs the sstableloader artifacts in the Maven Local Repository">
<install pomFile="${build.dir}/${final.name}-sstableloader.pom"
file="${build.dir}/tools/lib/sstableloader.jar"/>
<install pomFile="${build.dir}/${final.name}-sstableloader.pom"
file="${build.dir}/${final.name}-sstableloader-sources.jar"
classifier="sources"/>
<install pomFile="${build.dir}/${final.name}-sstableloader.pom"
file="${build.dir}/${final.name}-sstableloader-javadoc.jar"
classifier="javadoc"/>
</target>
</project>

View File

@ -25,8 +25,6 @@ import javax.net.ssl.SSLEngine;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import com.datastax.driver.core.AuthProvider;
import com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions;
@ -313,52 +311,4 @@ public class BulkLoader
return new BulkLoadConnectionFactory(serverEncOptions, storagePort);
}
}
public static class CmdLineOptions extends Options
{
/**
* Add option with argument and argument name
* @param opt shortcut for option name
* @param longOpt complete option name
* @param argName argument name
* @param description description of the option
* @return updated Options object
*/
public Options addOption(String opt, String longOpt, String argName, String description)
{
Option option = new Option(opt, longOpt, true, description);
option.setArgName(argName);
return addOption(option);
}
/**
* Add option with argument and argument name that accepts being defined multiple times as a list
* @param opt shortcut for option name
* @param longOpt complete option name
* @param argName argument name
* @param description description of the option
* @return updated Options object
*/
public Options addOptionList(String opt, String longOpt, String argName, String description)
{
Option option = new Option(opt, longOpt, true, description);
option.setArgName(argName);
option.setArgs(Option.UNLIMITED_VALUES);
return addOption(option);
}
/**
* Add option without argument
* @param opt shortcut for option name
* @param longOpt complete option name
* @param description description of the option
* @return updated Options object
*/
public Options addOption(String opt, String longOpt, String description)
{
return addOption(new Option(opt, longOpt, false, description));
}
}
}

View File

@ -49,7 +49,6 @@ import org.apache.cassandra.config.YamlConfigurationLoader;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.MEBIBYTES_PER_SECOND;
import static org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions.ClientAuth.REQUIRED;
@ -77,9 +76,8 @@ public class LoaderOptions
/**
* Throttle defined in megabits per second. CASSANDRA-10637 introduced a builder and is the preferred way to
* provide options instead of using these constant fields.
* @deprecated Use {@code throttle-mib} instead
* @deprecated Use {@code throttle-mib} instead. See CASSANDRA-17677
*/
/** @deprecated See CASSANDRA-17677 */
@Deprecated(since = "5.0")
public static final String THROTTLE_MBITS = "throttle";
public static final String THROTTLE_MEBIBYTES = "throttle-mib";

View File

@ -25,12 +25,11 @@ import java.nio.file.Paths;
import java.security.Permission;
import com.google.common.net.HostAndPort;
import org.apache.commons.io.FileUtils;
import org.junit.Test;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.transport.TlsTestUtils;
import static org.apache.cassandra.tools.OfflineToolUtils.sstableDirName;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -102,8 +101,8 @@ public class LoaderOptionsTest
sstableDirName("legacy_sstables", "legacy_ma_simple") };
LoaderOptions options = LoaderOptions.builder().parseArgs(args).build();
// Below two lines validating server encryption options is to verify that we are loading config from the yaml
assertEquals(TlsTestUtils.SERVER_KEYSTORE_PATH, options.serverEncOptions.keystore);
assertEquals(TlsTestUtils.SERVER_KEYSTORE_PASSWORD, options.serverEncOptions.keystore_password);
assertEquals("test/conf/cassandra_ssl_test.keystore", options.serverEncOptions.keystore);
assertEquals("cassandra", options.serverEncOptions.keystore_password);
// Below asserts validate the overrides for the client encryption options from the command line
// Since the values are provided by (and local to) this test, they are hardcoded
assertEquals("JKS", options.clientEncOptions.store_type);
@ -245,5 +244,40 @@ public class LoaderOptionsTest
System.setSecurityManager(null);
}
}
// Copied from OfflineToolUtils
public static String sstableDirName(String ks, String cf) throws IOException
{
return sstableDir(ks, cf).absolutePath();
}
public static File sstableDir(String ks, String cf) throws IOException
{
File dataDir = copySSTables();
File ksDir = new File(dataDir, ks);
File[] cfDirs = ksDir.tryList((dir, name) -> cf.equals(name) || name.startsWith(cf + '-'));
return cfDirs[0];
}
public static File copySSTables() throws IOException
{
File dataDir = new File("build/test/cassandra/data");
File srcDir = new File("test/data/legacy-sstables/ma");
FileUtils.copyDirectory(new File(srcDir, "legacy_tables").toJavaIOFile(), new File(dataDir, "legacy_sstables").toJavaIOFile());
return dataDir;
}
// Copied from SystemExitException in unit tests
private static class SystemExitException extends Error
{
public final int status;
public SystemExitException(int status)
{
this.status = status;
}
}
}