Migrate sstableloader code to its own tools directory and artifact

As part of a broader effort to decouple java driver code from the
server code, this moves sstableloader to its own tools directory.

As sstableloader is also used as a library (CASSANDRA-10637), added
a new artifact 'cassandra-sstableloader' that will get deployed to
maven along with 'cassandra-all'.

While I expect this is likely a niche use case, this will allow users
to continue using BulkExport as a library.

Moves sstableloader-specific targets to its own build.xml in
tools/sstableloader/build.xml.

Also updates IDE project files and circleci to utilize new
sstableloader-specific targets.

patch by Andy Tolbert; reviewed by Stefan Miklosovic and Mick Semb Wever for CASSANDRA-20328

.build/README.md

@@ -100,6 +100,7 @@ Running other types of tests with docker:
     .build/docker/run-tests.sh -a test
     .build/docker/run-tests.sh -a stress-test
     .build/docker/run-tests.sh -a fqltool-test
+    .build/docker/run-tests.sh -a sstableloader-test
     .build/docker/run-tests.sh -a microbench
     .build/docker/run-tests.sh -a test-cdc
     .build/docker/run-tests.sh -a test-compression

.build/build-sonar.xml

@@ -226,9 +226,9 @@
             <arg value="-Dsonar.working.directory=${sonar.workdir}/scanner"/>
             <arg value="-Dsonar.sourceEncoding=UTF-8"/>
             <arg value="-Dsonar.scm.disabled=true"/>
-            <arg value="-Dsonar.sources=${build.src.java},${fqltool.build.src},${stress.build.src}"/>
+            <arg value="-Dsonar.sources=${build.src.java},${fqltool.build.src},${sstableloader.build.src},${stress.build.src}"/>
             <arg value="-Dsonar.java.libraries=${build.dir.lib}/jars/jsr305-*.jar"/>
-            <arg value="-Dsonar.java.binaries=${build.classes.main},${fqltool.build.classes},${stress.build.classes}"/>
+            <arg value="-Dsonar.java.binaries=${build.classes.main},${fqltool.build.classes},${sstableloader.build.classes},${stress.build.classes}"/>
             <arg value="-Dsonar.java.source=11"/>
             <arg value="-Dsonar.projectVersion=${timestamp}"/>
             <arg value="-Dsonar.qualitygate.wait=true"/>

.build/docker/run-tests.sh

@@ -40,7 +40,7 @@ error() {
 
 # legacy argument handling
 case ${1} in
-  "build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+  "build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "dtest" | "dtest-novnode" | "dtest-latest" | "dtest-large" | "dtest-large-novnode" | "dtest-upgrade" | "dtest-upgrade-novnode"| "dtest-upgrade-large" | "dtest-upgrade-novnode-large" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
     test_type="-a ${1}"
     if [[ -z ${2} ]]; then
       test_list=""
@@ -182,7 +182,7 @@ docker_flags="-m 5g --memory-swap 5g"
 case ${test_target/-repeat/} in
     "build_dtest_jars")
     ;;
-    "stress-test" | "fqltool-test" )
+    "stress-test" | "fqltool-test" | "sstableloader-test" )
         [[ ${mem} -gt $((1 * 1024 * 1024 * 1024 * ${jenkins_executors})) ]] || { error 1 "${target} require minimum docker memory 1g (per jenkins executor (${jenkins_executors})), found ${mem}"; }
     ;;
     # test-burn doesn't have enough tests in it to split beyond 8, and burn and long we want a bit more resources anyway

.build/parent-maven-pom.xml

@@ -671,7 +671,7 @@
       <dependency>
         <groupId>org.apache.cassandra</groupId>
         <artifactId>cassandra-all</artifactId>
-        <version>4.1-alpha2-SNAPSHOT</version>
+        <version>@version@</version>
       </dependency>
       <dependency>
         <groupId>io.dropwizard.metrics</groupId>

.build/run-tests.sh

@@ -66,7 +66,7 @@ print_help() {
 
 # legacy argument handling
 case ${1} in
-  "build_dtest_jars" | "stress-test" | "fqltool-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
+  "build_dtest_jars" | "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "test-burn" | "long-test" | "cqlsh-test" | "simulator-dtest" | "test" | "test-cdc" | "test-compression" | "test-oa" | "test-system-keyspace-directory" | "test-latest" | "jvm-dtest" | "jvm-dtest-upgrade" | "jvm-dtest-novnode" | "jvm-dtest-upgrade-novnode")
     test_type="-a ${1}"
     if [[ -z ${2} ]]; then
       test_list=""
@@ -285,7 +285,7 @@ _main() {
   # check split_chunk is compatible with target (if not a regexp)
   if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
     case ${target} in
-      "stress-test" | "fqltool-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
+      "stress-test" | "fqltool-test" | "sstableloader-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
           error 1 "Target ${target} does not suport splits."
           ;;
         *)
@@ -344,6 +344,11 @@ _main() {
       ant fqltool-build-test ${ANT_TEST_OPTS}
       ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
       ;;
+    "sstableloader-test")
+      # hard fail on test compilation, but dont fail the test run so unstable test reports are processed
+      ant sstableloader-build-test ${ANT_TEST_OPTS}
+      ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}"
+      ;;
     "microbench")
       ant $target ${ANT_TEST_OPTS} -Dmaven.test.failure.ignore=true
       ;;

.build/sstableloader-deps-maven-pom.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+      http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.cassandra</groupId>
+    <artifactId>cassandra-parent</artifactId>
+    <version>@version@</version>
+    <relativePath>@final.name@-parent.pom</relativePath>
+  </parent>
+  <artifactId>cassandra-sstableloader</artifactId>
+  <version>@version@</version>
+  <name>Apache Cassandra SSTableLoader</name>
+  <description>Standalone SSTableLoader for Apache Cassandra.</description>
+  <url>https://cassandra.apache.org</url>
+  <inceptionYear>2025</inceptionYear>
+  <licenses>
+    <license>
+      <name>The Apache Software License, Version 2.0</name>
+      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+  <scm>
+    <connection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</connection>
+    <developerConnection>scm:https://gitbox.apache.org/repos/asf/cassandra.git</developerConnection>
+    <url>https://gitbox.apache.org/repos/asf?p=cassandra.git</url>
+  </scm>
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.cassandra</groupId>
+      <artifactId>cassandra-all</artifactId>
+    </dependency>
+  </dependencies>
+</project>

.circleci/config_template.yml

@@ -63,6 +63,13 @@ default_env_vars: &default_env_vars
     # The number of times that new, modified or manually specified fqltool unit tests should be run.
     REPEATED_UTESTS_FQLTOOL_COUNT: 500
 
+    # Comma-separated list of tests that should be included in the repeated run for sstableloader unit tests,
+    # in addition to automatically detected new and modified tests. For example:
+    # REPEATED_UTESTS_SSTABLELOADER: org.apache.cassandra.tools.LoaderOptionsTest
+    REPEATED_UTESTS_SSTABLELOADER:
+    # The number of times that new, modified or manually specified sstableloader unit tests should be run.
+    REPEATED_UTESTS_SSTABLELOADER_COUNT: 500
+
     # Comma-separated list of tests that should be included in the repeated run for long unit tests,
     # in addition to automatically detected new and modified tests. For example:
     # REPEATED_UTESTS_LONG: org.apache.cassandra.db.commitlog.CommitLogStressTest
@@ -537,6 +544,30 @@ j11_separate_jobs: &j11_separate_jobs
         requires:
           - start_j17_utests_fqltool_repeat
           - j11_build
+    - start_j11_utests_sstableloader:
+        type: approval
+    - j11_utests_sstableloader:
+        requires:
+          - start_j11_utests_sstableloader
+          - j11_build
+    - start_j17_utests_sstableloader:
+        type: approval
+    - j17_utests_sstableloader:
+        requires:
+          - start_j17_utests_sstableloader
+          - j11_build
+    - start_j11_utests_sstableloader_repeat:
+        type: approval
+    - j11_utests_sstableloader_repeat:
+        requires:
+          - start_j11_utests_sstableloader_repeat
+          - j11_build
+    - start_j17_utests_sstableloader_repeat:
+        type: approval
+    - j17_utests_sstableloader_repeat:
+        requires:
+          - start_j17_utests_sstableloader_repeat
+          - j11_build
     - start_j11_utests_system_keyspace_directory:
         type: approval
     - j11_utests_system_keyspace_directory:
@@ -974,6 +1005,24 @@ j11_pre-commit_jobs: &j11_pre-commit_jobs
         requires:
           - start_utests_fqltool
           - j11_build
+    - start_utests_sstableloader:
+        type: approval
+    - j11_utests_sstableloader:
+        requires:
+          - start_utests_sstableloader
+          - j11_build
+    - j17_utests_sstableloader:
+        requires:
+          - start_utests_sstableloader
+          - j11_build
+    - j11_utests_sstableloader_repeat:
+        requires:
+          - start_utests_sstableloader
+          - j11_build
+    - j17_utests_sstableloader_repeat:
+        requires:
+          - start_utests_sstableloader
+          - j11_build
     - start_utests_system_keyspace_directory:
         type: approval
     - j11_utests_system_keyspace_directory:
@@ -1356,6 +1405,18 @@ j17_separate_jobs: &j17_separate_jobs
         requires:
           - start_j17_utests_fqltool_repeat
           - j17_build
+    - start_j17_utests_sstableloader:
+        type: approval
+    - j17_utests_sstableloader:
+        requires:
+          - start_j17_utests_sstableloader
+          - j17_build
+    - start_j17_utests_sstableloader_repeat:
+        type: approval
+    - j17_utests_sstableloader_repeat:
+        requires:
+          - start_j17_utests_sstableloader_repeat
+          - j17_build
     - start_j17_utests_system_keyspace_directory:
         type: approval
     - j17_utests_system_keyspace_directory:
@@ -1541,6 +1602,16 @@ j17_pre-commit_jobs: &j17_pre-commit_jobs
         requires:
           - start_utests_fqltool
           - j17_build
+    - start_utests_sstableloader:
+        type: approval
+    - j17_utests_sstableloader:
+        requires:
+          - start_utests_sstableloader
+          - j17_build
+    - j17_utests_sstableloader_repeat:
+        requires:
+          - start_utests_sstableloader
+          - j17_build
     - start_utests_system_keyspace_directory:
         type: approval
     - j17_utests_system_keyspace_directory:
@@ -1875,6 +1946,22 @@ jobs:
       - run_junit_tests:
           target: fqltool-test
 
+  j11_utests_sstableloader:
+    <<: *j11_seq_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - run_junit_tests:
+          target: sstableloader-test
+
+  j17_utests_sstableloader:
+    <<: *j17_seq_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - run_junit_tests:
+          target: sstableloader-test
+
   j11_utests_system_keyspace_directory:
     <<: *j11_par_executor
     steps:
@@ -2380,6 +2467,22 @@ jobs:
       - log_environment
       - run_utests_fqltool_repeat
 
+  j11_utests_sstableloader_repeat:
+    <<: *j11_repeated_utest_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - log_environment
+      - run_utests_sstableloader_repeat
+
+  j17_utests_sstableloader_repeat:
+    <<: *j17_repeated_utest_executor
+    steps:
+      - attach_workspace:
+          at: /home/cassandra
+      - log_environment
+      - run_utests_sstableloader_repeat
+
   j11_utests_long_repeat:
     <<: *j11_repeated_utest_executor
     steps:
@@ -3114,6 +3217,14 @@ commands:
           count: ${REPEATED_UTESTS_FQLTOOL_COUNT}
           stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
 
+  run_utests_sstableloader_repeat:
+    steps:
+      - run_repeated_utests:
+          target: sstableloader-test
+          tests: ${REPEATED_UTESTS_SSTABLELOADER}
+          count: ${REPEATED_UTESTS_SSTABLELOADER_COUNT}
+          stop_on_failure: ${REPEATED_TESTS_STOP_ON_FAILURE}
+
   run_utests_stress_repeat:
     steps:
       - run_repeated_utests:
@@ -3237,6 +3348,7 @@ commands:
                       $target == "test-oa" || \
                       $target == "test-system-keyspace-directory" || \
                       $target == "fqltool-test" || \
+                      $target == "sstableloader-test" || \
                       $target == "long-test" || \
                       $target == "stress-test" || \
                       $target == "test-simulator-dtest" ]]; then
@@ -3365,6 +3477,7 @@ commands:
                       $target == "test-oa" || \
                       $target == "test-system-keyspace-directory" || \
                       $target == "fqltool-test" || \
+                      $target == "sstableloader-test" || \
                       $target == "long-test" || \
                       $target == "stress-test" || \
                       $target == "test-simulator-dtest" ]]; then

.circleci/generate.sh

@@ -51,6 +51,8 @@ print_help()
   echo "                   -e REPEATED_UTESTS_COUNT=500"
   echo "                   -e REPEATED_UTESTS_FQLTOOL=org.apache.cassandra.fqltool.FQLCompareTest"
   echo "                   -e REPEATED_UTESTS_FQLTOOL_COUNT=500"
+  echo "                   -e REPEATED_UTESTS_SSTABLELOADER=org.apache.cassandra.tools.LoaderOptionsTest"
+  echo "                   -e REPEATED_UTESTS_SSTABLELOADER_COUNT=500"
   echo "                   -e REPEATED_UTESTS_LONG=org.apache.cassandra.db.commitlog.CommitLogStressTest"
   echo "                   -e REPEATED_UTESTS_LONG_COUNT=100"
   echo "                   -e REPEATED_UTESTS_STRESS=org.apache.cassandra.stress.generate.DistributionGaussianTest"
@@ -131,6 +133,8 @@ if $has_env_vars && $check_env_vars; then
        [ "$key" != "REPEATED_UTESTS_COUNT" ] &&
        [ "$key" != "REPEATED_UTESTS_FQLTOOL" ] &&
        [ "$key" != "REPEATED_UTESTS_FQLTOOL_COUNT" ] &&
+       [ "$key" != "REPEATED_UTESTS_SSTABLELOADER" ] &&
+       [ "$key" != "REPEATED_UTESTS_SSTABLELOADER_COUNT" ] &&
        [ "$key" != "REPEATED_UTESTS_LONG" ] &&
        [ "$key" != "REPEATED_UTESTS_LONG_COUNT" ] &&
        [ "$key" != "REPEATED_UTESTS_STRESS" ] &&
@@ -171,7 +175,7 @@ if $free; then
 elif $paid; then
   ($all || $free) && die "Cannot use option -p with options -a or -f"
   echo "Generating new config.yml file for paid tier from config_template.yml"
-  patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
+  patch --silent -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
   circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp
   cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml
   rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
@@ -188,7 +192,7 @@ elif $all; then
   rm $BASEDIR/config.yml.FREE.tmp
 
   # setup config for paid tier
-  patch -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
+  patch --silent -o $BASEDIR/config_template.yml.PAID $BASEDIR/config_template.yml $BASEDIR/config_template.yml.PAID.patch
   circleci config process $BASEDIR/config_template.yml.PAID > $BASEDIR/config.yml.PAID.tmp
   cat $BASEDIR/license.yml $BASEDIR/config.yml.PAID.tmp > $BASEDIR/config.yml.PAID
   rm $BASEDIR/config_template.yml.PAID $BASEDIR/config.yml.PAID.tmp
@@ -241,6 +245,7 @@ if $detect_changed_tests; then
   add_diff_tests "REPEATED_UTESTS_LONG" "test/long/" "org.apache.cassandra"
   add_diff_tests "REPEATED_UTESTS_STRESS" "tools/stress/test/unit/" "org.apache.cassandra.stress"
   add_diff_tests "REPEATED_UTESTS_FQLTOOL" "tools/fqltool/test/unit/" "org.apache.cassandra.fqltool"
+  add_diff_tests "REPEATED_UTESTS_SSTABLELOADER" "tools/sstableloader/test/unit/" "org.apache.cassandra.tools"
   add_diff_tests "REPEATED_SIMULATOR_DTESTS" "test/simulator/test/" "org.apache.cassandra.simulator.test"
   add_diff_tests "REPEATED_JVM_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.test"
   add_diff_tests "REPEATED_JVM_UPGRADE_DTESTS" "test/distributed/" "org.apache.cassandra.distributed.upgrade"
@@ -305,6 +310,10 @@ delete_repeated_jobs()
     delete_job "$1" "j11_utests_fqltool_repeat"
     delete_job "$1" "j17_utests_fqltool_repeat"
   fi
+  if (! (echo "$env_vars" | grep -q "REPEATED_UTESTS_SSTABLELOADER=")); then
+    delete_job "$1" "j11_utests_sstableloader_repeat"
+    delete_job "$1" "j17_utests_sstableloader_repeat"
+  fi
   if (! (echo "$env_vars" | grep -q "REPEATED_SIMULATOR_DTESTS=")); then
     delete_job "$1" "j11_simulator_dtests_repeat"
   fi
@@ -386,6 +395,7 @@ build_dev_min_jobs()
   delete_job "$1" "j11_utests_cdc"
   delete_job "$1" "j11_utests_compression"
   delete_job "$1" "j11_utests_fqltool"
+  delete_job "$1" "j11_utests_sstableloader"
   delete_job "$1" "j11_utests_long"
   delete_job "$1" "j11_utests_stress"
   delete_job "$1" "j11_utests_system_keyspace_directory"
@@ -394,6 +404,7 @@ build_dev_min_jobs()
   delete_job "$1" "j17_utests_cdc"
   delete_job "$1" "j17_utests_compression"
   delete_job "$1" "j17_utests_fqltool"
+  delete_job "$1" "j17_utests_sstableloader"
   delete_job "$1" "j17_utests_long"
   delete_job "$1" "j17_utests_stress"
   delete_job "$1" "j11_utests_latest"
@@ -403,6 +414,7 @@ build_dev_min_jobs()
   delete_job "$1" "start_utests_stress"
   delete_job "$1" "start_utests_long"
   delete_job "$1" "start_utests_fqltool"
+  delete_job "$1" "start_utests_sstableloader"
   delete_job "$1" "start_utests_compression"
   delete_job "$1" "start_utests_cdc"
   delete_job "$1" "start_j17_cqlsh-dtests-latest"

.jenkins/Jenkinsfile

@@ -127,9 +127,9 @@ def pipelineProfiles() {
   return [
     'packaging': ['artifacts', 'lint', 'debian', 'redhat'],
     'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'],
-    'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
+    'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'],
-    'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
+    'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'],
-    'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
+    'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-novnode-large'],
     'custom': []
   ]
 }
@@ -164,6 +164,7 @@ def tasks() {
      //   (some buffer on the heaviest split under the 1h max is required, ref `timeout(…)` in `test(…)`)
     'cqlsh-test': [splits: 1],
     'fqltool-test': [splits: 1, size: 'small'],
+    'sstableloader-test': [splits: 1, size: 'small'],
     'test-cdc': [splits: 20],
     'test': [splits: 20],
     'test-latest': [splits: 20],

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Migrate sstableloader code to its own tools directory and artifact (CASSANDRA-20328)
  * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623)
  * Avoid duplicate hardlink error upon forceful taking of ephemeral snapshots during repair (CASSANDRA-20490)
  * When a custom disk error handler fails to initiate, fail the startup of a node instead of using the no-op handler (CASSANDRA-20614)

bin/sstableloader

@@ -32,18 +32,16 @@ elif [ -r "$CASSANDRA_INCLUDE" ]; then
     . "$CASSANDRA_INCLUDE"
 fi
 
-if [ -z "$CLASSPATH" ]; then
+# SSTableLoader has been moved to tools/bin, this script simply
-    echo "You must set the CLASSPATH var" >&2
+# invokes the script in the new path.
-    exit 1
+SSTABLELOADER_PATH="$CASSANDRA_HOME/tools/bin/sstableloader"
+
+if [ ! -f "$SSTABLELOADER_PATH" ]; then
+    echo "Error: sstableloader has moved to the tools directory. \
+Detected that $SSTABLELOADER_PATH does not exist." >&2
+    exit 2
 fi
 
-if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+"$SSTABLELOADER_PATH" "$@"
-    MAX_HEAP_SIZE="256M"
-fi
-
-"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
-        -Dcassandra.storagedir="$cassandra_storagedir" \
-        -Dlogback.configurationFile=logback-tools.xml \
-        org.apache.cassandra.tools.BulkLoader "$@"
 
 # vi:ai sw=4 ts=4 tw=0 et

build.xml

@@ -385,26 +385,38 @@
         </fileset>
     </path>
 
-  <macrodef name="create-javadoc">
+    <macrodef name="create-javadoc">
-    <attribute name="destdir"/>
+      <attribute name="destdir"/>
-    <element name="filesets"/>
+      <element name="filesets"/>
-    <sequential>
+      <sequential>
-      <javadoc destdir="@{destdir}" author="true" version="true" use="true"
+        <javadoc destdir="@{destdir}" author="true" version="true" use="true"
-        windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
+          windowtitle="${ant.project.name} API" classpathref="cassandra.classpath"
-        bottom="Copyright &amp;copy; 2009- The Apache Software Foundation"
+          bottom="Copyright &amp;copy; 2009- The Apache Software Foundation"
-        useexternalfile="yes" encoding="UTF-8" failonerror="false"
+          useexternalfile="yes" encoding="UTF-8" failonerror="false"
-        maxmemory="256m" additionalparam="${jdk11plus-javac-exports}">
+          maxmemory="256m" additionalparam="${jdk11plus-javac-exports}">
-        <filesets/>
+          <filesets/>
-      </javadoc>
+        </javadoc>
-      <fail message="javadoc failed">
+        <fail message="javadoc failed">
-        <condition>
+          <condition>
-            <not>
+              <not>
-                <available file="@{destdir}/index-all.html" />
+                  <available file="@{destdir}/index-all.html" />
-            </not>
+              </not>
-        </condition>
+          </condition>
-      </fail>
+        </fail>
-    </sequential>
+      </sequential>
-  </macrodef>
+    </macrodef>
+
+    <!-- filterset to apply to pom templates -->
+    <filterset id="pom-template">
+        <filter token="version" value="${version}"/>
+        <filter token="final.name" value="${final.name}"/>
+        <filter token="jamm.version" value="${jamm.version}"/>
+        <filter token="allocation-instrumenter.version" value="${allocation-instrumenter.version}"/>
+        <filter token="ecj.version" value="${ecj.version}"/>
+        <filter token="asm.version" value="${asm.version}"/>
+        <filter token="jacoco.version" value="${jacoco.version}"/>
+        <filter token="jflex.version" value="${jflex.version}"/>
+    </filterset>
 
     <!--
         Setup the output directories.
@@ -746,24 +758,13 @@
     </target>
 
     <target name="write-poms" unless="without.maven">
-        <filterset id="pom-template">
+        <copy file=".build/cassandra-deps-maven-pom.xml" tofile="${build.dir}/${final.name}.pom">
-            <filter token="version" value="${version}"/>
-            <filter token="final.name" value="${final.name}"/>
-            <filter token="jamm.version" value="${jamm.version}"/>
-            <filter token="allocation-instrumenter.version" value="${allocation-instrumenter.version}"/>
-            <filter token="ecj.version" value="${ecj.version}"/>
-            <filter token="asm.version" value="${asm.version}"/>
-            <filter token="jacoco.version" value="${jacoco.version}"/>
-            <filter token="jflex.version" value="${jflex.version}"/>
-        </filterset>
-
-        <copy file=".build/cassandra-deps-template.xml" tofile="${build.dir}/${final.name}.pom">
             <filterset refid="pom-template"/>
         </copy>
-        <copy file=".build/parent-pom-template.xml" tofile="${build.dir}/${final.name}-parent.pom">
+        <copy file=".build/parent-maven-pom.xml" tofile="${build.dir}/${final.name}-parent.pom">
             <filterset refid="pom-template"/>
         </copy>
-        <copy file=".build/cassandra-build-deps-template.xml" tofile="${build.dir}/tmp-${final.name}-deps.pom">
+        <copy file=".build/cassandra-build-maven-pom.xml" tofile="${build.dir}/tmp-${final.name}-deps.pom">
             <filterset refid="pom-template"/>
         </copy>
     </target>
@@ -864,7 +865,7 @@
     </target>
 
     <target name="jar"
-            depends="_main-jar,build-test,stress-jar,fqltool-jar,simulator-jars,write-poms"
+            depends="_main-jar,build-test,stress-jar,fqltool-jar,simulator-jars,sstableloader-jar,write-poms"
             description="Assemble Cassandra JAR files">
     </target>
 
@@ -949,7 +950,7 @@
     </target>
 
     <!-- creates release tarballs -->
-    <target name="artifacts" depends="_artifacts-init,check,sources-jar"
+    <target name="artifacts" depends="_artifacts-init,check,sources-jar,sstableloader-sources-jar"
             description="Create Cassandra tarball and maven artifacts">
       <tar compression="gzip" longfile="gnu"
         destfile="${build.dir}/${final.name}-bin.tar.gz">
@@ -1082,7 +1083,7 @@
     <antcall target="build-test" inheritRefs="true"/>
   </target>
 
-  <target name="build-test" depends="_main-jar,stress-build-test,fqltool-build,resolver-dist-lib,simulator-jars"
+  <target name="build-test" depends="_main-jar,stress-build-test,fqltool-build-test,sstableloader-build-test,resolver-dist-lib,simulator-jars"
           description="Compile test classes">
     <antcall target="_build-test"/>
     <antcall target="_check-test-names"/>
@@ -1101,6 +1102,7 @@
      <classpath>
         <path refid="cassandra.classpath.test"/>
         <pathelement location="${fqltool.build.classes}"/>
+        <pathelement location="${sstableloader.build.classes}"/>
      </classpath>
      <compilerarg value="-XDignore.symbol.file"/>
      <!-- needed to compile org.apache.cassandra.utils.memory.BufferPoolTest -->
@@ -1129,7 +1131,7 @@
     <macrodef name="check-test-names">
         <attribute name="annotationName"/>
         <attribute name="regex"/>
-        <attribute name="scanClassPath" default="${test.classes}:${fqltool.test.classes}:${stress.test.classes}"/>
+        <attribute name="scanClassPath" default="${test.classes}:${fqltool.test.classes}:${stress.test.classes}:${sstableloader.test.classes}"/>
         <attribute name="packageName" default="org.apache.cassandra"/>
         <attribute name="expand" default="true"/>
         <attribute name="normalize" default="true"/>
@@ -1141,9 +1143,11 @@
                     <path refid="cassandra.classpath.test"/>
                     <pathelement location="${fqltool.build.classes}"/>
                     <pathelement location="${stress.build.classes}"/>
+                    <pathelement location="${sstableloader.build.classes}"/>
                     <pathelement location="${test.classes}"/>
                     <pathelement location="${fqltool.test.classes}"/>
                     <pathelement location="${stress.test.classes}"/>
+                    <pathelement location="${sstableloader.test.classes}"/>
                 </classpath>
                 <jvmarg line="${java-jvmargs}"/>
                 <jvmarg value="-DscanClassPath=@{scanClassPath}"/>
@@ -1270,10 +1274,12 @@
           <pathelement path="${java.class.path}"/>
           <pathelement location="${stress.build.classes}"/>
           <pathelement location="${fqltool.build.classes}"/>
+          <pathelement location="${sstableloader.build.classes}"/>
           <pathelement location="${test.classes}"/>
           <path refid="cassandra.classpath.test" />
           <pathelement location="${stress.test.classes}"/>
           <pathelement location="${fqltool.test.classes}"/>
+          <pathelement location="${sstableloader.test.classes}"/>
           <pathelement location="${test.conf}"/>
           <pathelement path="${java.class.path}"/>
           <path refid="cassandra.classpath" />
@@ -2079,8 +2085,10 @@
   <classpathentry kind="src" output="${test.classes}" path="test/resources" />
   <classpathentry kind="src" path="tools/stress/src"/>
   <classpathentry kind="src" path="tools/fqltool/src"/>
+  <classpathentry kind="src" path="tools/sstableloader/src"/>
   <classpathentry kind="src" output="build/test/stress-classes" path="tools/stress/test/unit" />
   <classpathentry kind="src" output="build/test/fqltool-classes" path="tools/fqltool/test/unit" />
+  <classpathentry kind="src" output="build/test/sstableloader-classes" path="tools/sstableloader/test/unit" />
   <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
   <classpathentry kind="output" path="build/classes/eclipse"/>
   <classpathentry kind="lib" path="test/conf"/>
@@ -2140,7 +2148,7 @@
 
   <!-- Publish artifacts to remote Maven repository -->
   <target name="publish"
-          depends="mvn-install,artifacts"
+          depends="mvn-install,sstableloader-mvn-install,artifacts"
           description="Publishes the artifacts to the Maven repository">
     <!-- Accord -->
     <deploy pomFile="${accord.local.repository}/${accord.final.name}.pom"
@@ -2170,6 +2178,16 @@
             file="${build.dir}/${final.name}-javadoc.jar"
             classifier="javadoc"/>
 
+    <!-- the cassandra-sstableloader jar -->
+    <deploy pomFile="${build.dir}/${final.name}-sstableloader.pom"
+            file="${build.dir}/tools/lib/sstableloader.jar"/>
+    <deploy pomFile="${build.dir}/${final.name}-sstableloader.pom"
+            file="${build.dir}/${final.name}-sstableloader-sources.jar"
+            classifier="sources"/>
+    <deploy pomFile="${build.dir}/${final.name}-sstableloader.pom"
+            file="${build.dir}/${final.name}-sstableloader-javadoc.jar"
+            classifier="javadoc"/>
+
     <!-- the distribution -->
     <sign-dist file="${build.dir}/${final.name}-bin.tar.gz" />
     <sign-dist file="${build.dir}/${final.name}-src.tar.gz" />
@@ -2185,4 +2203,5 @@
   <import file="${build.helpers.dir}/build-bench.xml"/>
   <import file="${build.helpers.dir}/build-sonar.xml"/>
   <import file="${build.helpers.dir}/build-accord.xml" />
+  <import file="${basedir}/tools/sstableloader/build.xml"/>
 </project>

debian/cassandra.install

@@ -16,7 +16,6 @@ bin/cassandra.in.sh usr/share/cassandra
 bin/cassandra usr/sbin
 bin/nodetool usr/bin
 bin/sstableutil usr/bin
-bin/sstableloader usr/bin
 bin/cqlsh usr/bin
 bin/cqlsh.py usr/bin
 bin/sstablescrub usr/bin
@@ -28,6 +27,7 @@ tools/bin/auditlogviewer usr/bin
 tools/bin/jmxtool usr/bin
 tools/bin/hash_password usr/bin
 tools/bin/sstablepartitions usr/bin
+tools/bin/sstableloader usr/bin
 lib/*.jar usr/share/cassandra/lib
 lib/*.zip usr/share/cassandra/lib
 lib/x86_64/* usr/share/cassandra/lib/x86_64

debian/rules

@@ -67,6 +67,10 @@ install: build
 	dh_install $(BUILD_DIR)/tools/lib/fqltool.jar \
 		usr/share/cassandra
 
+	# Copy sstableloader jars
+	dh_install $(BUILD_DIR)/tools/lib/sstableloader.jar \
+		usr/share/cassandra
+
 	dh_link usr/share/cassandra/apache-cassandra-$(VERSION).jar \
 		usr/share/cassandra/apache-cassandra.jar
 

ide/idea-iml-file.xml

@@ -30,6 +30,8 @@
             <sourceFolder url="file://$MODULE_DIR$/tools/stress/test/unit" isTestSource="true" />
             <sourceFolder url="file://$MODULE_DIR$/tools/fqltool/src" isTestSource="false" />
             <sourceFolder url="file://$MODULE_DIR$/tools/fqltool/test/unit" isTestSource="true" />
+            <sourceFolder url="file://$MODULE_DIR$/tools/sstableloader/src" isTestSource="false" />
+            <sourceFolder url="file://$MODULE_DIR$/tools/sstableloader/test/unit" isTestSource="true" />
             <sourceFolder url="file://$MODULE_DIR$/test/unit" isTestSource="true" />
             <sourceFolder url="file://$MODULE_DIR$/test/long" isTestSource="true" />
             <sourceFolder url="file://$MODULE_DIR$/test/memory" isTestSource="true" />

ide/idea/workspace.xml

@@ -360,6 +360,7 @@
         <filter targetName="build-project" isVisible="false" />
         <filter targetName="stress-build" isVisible="true" />
         <filter targetName="fqltool-build" isVisible="true" />
+        <filter targetName="sstableloader-build" isVisible="true" />
         <filter targetName="_write-poms" isVisible="false" />
         <filter targetName="write-poms" isVisible="false" />
         <filter targetName="jar" isVisible="true" />

ide/nbproject/ide-actions.xml

@@ -5,6 +5,7 @@
         <ant antfile="../build.xml" inheritall="false" target="build"/>
         <ant antfile="../build.xml" inheritall="false" target="build-test"/>
         <ant antfile="../build.xml" inheritall="false" target="fqltool-build-test"/>
+        <ant antfile="../build.xml" inheritall="false" target="sstableloader-build-test"/>
         <ant antfile="../build.xml" inheritall="false" target="stress-build-test"/>
     </target>
     <target name="clean">

ide/nbproject/project.xml

@@ -35,6 +35,12 @@
                     <location>${project.dir}/tools/fqltool/src</location>
                     <encoding>UTF-8</encoding>
                 </source-folder>
+                <source-folder>
+                    <label>src/sstableloader</label>
+                    <type>java</type>
+                    <location>${project.dir}/tools/sstableloader/src</location>
+                    <encoding>UTF-8</encoding>
+                </source-folder>
                 <source-folder>
                     <label>src/stress</label>
                     <type>java</type>
@@ -53,6 +59,12 @@
                     <location>${project.dir}/tools/fqltool/test/unit</location>
                     <encoding>UTF-8</encoding>
                 </source-folder>
+                <source-folder>
+                    <label>test/sstableloader/</label>
+                    <type>java</type>
+                    <location>${project.dir}/tools/sstableloader/test/unit</location>
+                    <encoding>UTF-8</encoding>
+                </source-folder>
                 <source-folder>
                     <label>test/stress</label>
                     <type>java</type>
@@ -164,6 +176,12 @@
                 <script>../build.xml</script>
                 <build-target>fqltool-build</build-target>
             </export>
+            <export>
+                <type>folder</type>
+                <location>${project.dir}/build/classes/sstableloader</location>
+                <script>../build.xml</script>
+                <build-target>sstableloader-build</build-target>
+            </export>
             <export>
                 <type>folder</type>
                 <location>${project.dir}/build/classes/stress</location>
@@ -194,6 +212,12 @@
                 <script>../build.xml</script>
                 <build-target>fqltool-build-test</build-target>
             </export>
+            <export>
+                <type>folder</type>
+                <location>${project.dir}/build/test/sstableloader-classes</location>
+                <script>../build.xml</script>
+                <build-target>sstableloader-build-test</build-target>
+            </export>
             <export>
                 <type>folder</type>
                 <location>${project.dir}/build/test/stress-classes</location>
@@ -214,6 +238,10 @@
                         <label>src/fqltool</label>
                         <location>${project.dir}/tools/fqltool/src</location>
                     </source-folder>
+                    <source-folder style="packages">
+                        <label>src/sstableloader</label>
+                        <location>${project.dir}/tools/sstableloader/src</location>
+                    </source-folder>
                     <source-folder style="packages">
                         <label>src/stress</label>
                         <location>${project.dir}/tools/stress/src</location>
@@ -226,6 +254,10 @@
                         <label>test/fqltool/</label>
                         <location>${project.dir}/tools/fqltool/test/unit</location>
                     </source-folder>
+                    <source-folder style="packages">
+                        <label>test/sstableloader/</label>
+                        <location>${project.dir}/tools/sstableloader/test/unit</location>
+                    </source-folder>
                     <source-folder style="packages">
                         <label>test/stress</label>
                         <location>${project.dir}/tools/stress/test/unit</location>
@@ -295,6 +327,11 @@
                 <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
                 <built-to>${project.dir}/build/classes/fqltool</built-to>
             </compilation-unit>
+            <compilation-unit>
+                <package-root>${project.dir}/tools/sstableloader/src</package-root>
+                <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
+                <built-to>${project.dir}/build/classes/sstableloader</built-to>
+            </compilation-unit>
             <compilation-unit>
                 <package-root>${project.dir}/tools/stress/src</package-root>
                 <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main</classpath>
@@ -313,7 +350,7 @@
                 <package-root>${project.dir}/test/simulator/test</package-root>
                 <package-root>${project.dir}/test/harry/main</package-root>
                 <unit-tests/>
-                <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/:${project.dir}/build/classes/stress/</classpath>
+                <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/:${project.dir}/build/classes/sstableloader:${project.dir}/build/classes/stress/</classpath>
                 <built-to>${project.dir}/build/test/classes</built-to>
             </compilation-unit>
             <compilation-unit>
@@ -322,6 +359,12 @@
                 <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/fqltool/</classpath>
                 <built-to>${project.dir}/build/test/fqltool-classes</built-to>
             </compilation-unit>
+            <compilation-unit>
+                <package-root>${project.dir}/tools/sstableloader/test/unit</package-root>
+                <unit-tests/>
+                <classpath mode="compile">${cassandra.classpath.jars}:${project.dir}/build/classes/main:${project.dir}/build/classes/sstableloader/</classpath>
+                <built-to>${project.dir}/build/test/sstableloader-classes</built-to>
+            </compilation-unit>
             <compilation-unit>
                 <package-root>${project.dir}/tools/stress/test/unit</package-root>
                 <unit-tests/>

redhat/cassandra.spec

@@ -120,6 +120,9 @@ cp -p %{_get_dist_dir}/tools/lib/stress.jar %{buildroot}/usr/share/%{username}/
 # copy fqltool jar
 cp -p %{_get_dist_dir}/tools/lib/fqltool.jar %{buildroot}/usr/share/%{username}/
 
+# copy sstableloader jar
+cp -p %{_get_dist_dir}/tools/lib/sstableloader.jar %{buildroot}/usr/share/%{username}/
+
 # copy binaries
 mv bin/cassandra %{buildroot}/usr/sbin/
 cp -p bin/* %{buildroot}/usr/bin/

src/java/org/apache/cassandra/tools/CmdLineOptions.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.tools;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+
+public class CmdLineOptions extends Options
+{
+    /**
+     * Add option with argument and argument name
+     *
+     * @param opt         shortcut for option name
+     * @param longOpt     complete option name
+     * @param argName     argument name
+     * @param description description of the option
+     * @return updated Options object
+     */
+    public Options addOption(String opt, String longOpt, String argName, String description)
+    {
+        Option option = new Option(opt, longOpt, true, description);
+        option.setArgName(argName);
+
+        return addOption(option);
+    }
+
+    /**
+     * Add option with argument and argument name that accepts being defined multiple times as a list
+     *
+     * @param opt         shortcut for option name
+     * @param longOpt     complete option name
+     * @param argName     argument name
+     * @param description description of the option
+     * @return updated Options object
+     */
+    public Options addOptionList(String opt, String longOpt, String argName, String description)
+    {
+        Option option = new Option(opt, longOpt, true, description);
+        option.setArgName(argName);
+        option.setArgs(Option.UNLIMITED_VALUES);
+
+        return addOption(option);
+    }
+
+    /**
+     * Add option without argument
+     *
+     * @param opt         shortcut for option name
+     * @param longOpt     complete option name
+     * @param description description of the option
+     * @return updated Options object
+     */
+    public Options addOption(String opt, String longOpt, String description)
+    {
+        return addOption(new Option(opt, longOpt, false, description));
+    }
+}

src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java

@@ -30,7 +30,6 @@ import java.io.IOException;
 import java.util.function.BiPredicate;
 
 import org.apache.cassandra.io.util.File;
-import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
 
 public class StandaloneSSTableUtil
 {

src/java/org/apache/cassandra/tools/StandaloneScrubber.java

@@ -51,7 +51,6 @@ import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.tcm.ClusterMetadataService;
-import org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Pair;

src/java/org/apache/cassandra/tools/StandaloneSplitter.java

@@ -50,7 +50,7 @@ import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.utils.JVMStabilityInspector;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
-import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
+
 import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
 public class StandaloneSplitter

src/java/org/apache/cassandra/tools/StandaloneUpgrader.java

@@ -46,7 +46,6 @@ import org.apache.cassandra.utils.JVMStabilityInspector;
 import org.apache.cassandra.utils.OutputHandler;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
-import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
 
 public class StandaloneUpgrader
 {

src/java/org/apache/cassandra/tools/StandaloneVerifier.java

@@ -53,7 +53,6 @@ import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Throwables;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
-import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
 
 public class StandaloneVerifier
 {

tools/bin/cassandra.in.sh

@@ -39,7 +39,7 @@ if [ -d $CASSANDRA_HOME/build ] ; then
 
     if [ "$jars_cnt" = "1" ]; then
         cassandra_bin="`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar | grep -v javadoc | grep -v sources`"
-        cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool"
+        cassandra_bin="$cassandra_bin:$CASSANDRA_HOME/build/classes/stress:$CASSANDRA_HOME/build/classes/fqltool:$CASSANDRA_HOME/build/classes/sstableloader"
         CLASSPATH="$CLASSPATH:$cassandra_bin"
     fi
 fi

tools/bin/sstableloader

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+    # Locations (in order) to use when searching for an include file.
+    for include in "`dirname "$0"`/cassandra.in.sh" \
+                   "$HOME/.cassandra.in.sh" \
+                   /usr/share/cassandra/cassandra.in.sh \
+                   /usr/local/share/cassandra/cassandra.in.sh \
+                   /opt/cassandra/cassandra.in.sh; do
+        if [ -r "$include" ]; then
+            . "$include"
+            break
+        fi
+    done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+    . "$CASSANDRA_INCLUDE"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+    echo "You must set the CLASSPATH var" >&2
+    exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+    MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+        -Dcassandra.storagedir="$cassandra_storagedir" \
+        -Dlogback.configurationFile=logback-tools.xml \
+        org.apache.cassandra.tools.BulkLoader "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et

tools/sstableloader/build.xml

@@ -0,0 +1,125 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project basedir="../../" name="apache-cassandra-sstableloader"
+         xmlns:if="ant:if"
+         xmlns:unless="ant:unless">
+
+    <!-- pull common definitions from parent project -->
+    <import file="../../build.xml"/>
+
+    <property name="sstableloader-javadoc.dir" value="${build.dir}/sstableloader-javadoc"/>
+    <property name="sstableloader.build.src" value="${basedir}/tools/sstableloader/src" />
+    <property name="sstableloader.test.src" value="${basedir}/tools/sstableloader/test/unit" />
+    <property name="sstableloader.build.classes" value="${build.classes}/sstableloader" />
+    <property name="sstableloader.test.classes" value="${build.dir}/test/sstableloader-classes" />
+    <property name="sstableloader.manifest" value="${sstableloader.build.classes}/MANIFEST.MF" />
+
+    <target name="sstableloader-init">
+        <mkdir dir="${sstableloader.test.classes}"/>
+    </target>
+
+    <target name="sstableloader-build" depends="sstableloader-init,build" description="build sstableloader">
+        <antcall target="_sstableloader_build"/>
+    </target>
+
+    <target name="_sstableloader_build">
+        <mkdir dir="${sstableloader.build.classes}" />
+        <javac compiler="modern" debug="true" debuglevel="${debuglevel}"
+               source="${ant.java.version}" target="${ant.java.version}"
+               encoding="utf-8" destdir="${sstableloader.build.classes}" includeantruntime="true">
+            <src path="${sstableloader.build.src}" />
+            <classpath>
+                <path refid="cassandra.classpath" />
+            </classpath>
+        </javac>
+    </target>
+
+    <target name="sstableloader-build-test" depends="sstableloader-build" description="Compile sstableloader tests">
+        <javac debug="true" debuglevel="${debuglevel}" destdir="${sstableloader.test.classes}"
+               source="${ant.java.version}" target="${ant.java.version}"
+               includeantruntime="false" encoding="utf-8">
+            <classpath>
+                <path refid="cassandra.classpath.test"/>
+                <pathelement location="${sstableloader.build.classes}" />
+            </classpath>
+            <src path="${sstableloader.test.src}"/>
+        </javac>
+    </target>
+
+    <target name="sstableloader-test" depends="sstableloader-build-test, maybe-build-test" description="Runs sstableloader tests">
+        <testmacro inputdir="${sstableloader.test.src}"
+                       timeout="${test.timeout}">
+        </testmacro>
+    </target>
+
+    <target name="sstableloader-write-pom">
+        <copy file=".build/sstableloader-deps-maven-pom.xml" tofile="${build.dir}/${final.name}-sstableloader.pom">
+            <filterset refid="pom-template"/>
+        </copy>
+    </target>
+
+    <target name="sstableloader-jar"
+            depends="sstableloader-build,sstableloader-write-pom"
+            description="Assemble Cassandra sstableloader JAR files">
+
+        <!-- sstableloader jar -->
+        <manifest file="${sstableloader.manifest}">
+            <attribute name="Built-By" value="Apache Cassandra Project"/>
+            <attribute name="Main-Class" value="org.apache.cassandra.tools.BulkLoader"/>
+        </manifest>
+        <mkdir dir="${sstableloader.build.classes}/META-INF" />
+        <mkdir dir="${build.dir}/tools/lib/" />
+        <jar destfile="${build.dir}/tools/lib/sstableloader.jar" manifest="${sstableloader.manifest}">
+            <fileset dir="${sstableloader.build.classes}"/>
+        </jar>
+    </target>
+
+    <target name="sstableloader-javadoc" depends="build" description="Create javadoc" unless="no-javadoc">
+      <create-javadoc destdir="${sstableloader-javadoc.dir}">
+        <filesets>
+          <fileset dir="${sstableloader.build.src}" defaultexcludes="yes">
+            <include name="org/apache/**/*.java"/>
+          </fileset>
+        </filesets>
+      </create-javadoc>
+    </target>
+
+    <target name="sstableloader-javadoc-jar" depends="sstableloader-javadoc" unless="no-javadoc" description="Assemble sstableloader JavaDoc JAR file">
+      <jar jarfile="${build.dir}/${final.name}-sstableloader-javadoc.jar" basedir="${sstableloader-javadoc.dir}"/>
+      <delete quiet="true" dir="${sstableloader-javadoc.dir}"/>
+    </target>
+
+    <target name="sstableloader-sources-jar" depends="init" description="Assemble sstableloader Sources JAR file">
+      <jar jarfile="${build.dir}/${final.name}-sstableloader-sources.jar">
+        <fileset dir="${sstableloader.build.src}" defaultexcludes="yes">
+          <include name="org/apache/**/*.java"/>
+        </fileset>
+      </jar>
+    </target>
+
+    <target name="sstableloader-mvn-install" depends="sstableloader-sources-jar,sstableloader-javadoc-jar,mvn-install"
+            description="Installs the sstableloader artifacts in the Maven Local Repository">
+      <install pomFile="${build.dir}/${final.name}-sstableloader.pom"
+               file="${build.dir}/tools/lib/sstableloader.jar"/>
+      <install pomFile="${build.dir}/${final.name}-sstableloader.pom"
+               file="${build.dir}/${final.name}-sstableloader-sources.jar"
+               classifier="sources"/>
+      <install pomFile="${build.dir}/${final.name}-sstableloader.pom"
+               file="${build.dir}/${final.name}-sstableloader-javadoc.jar"
+               classifier="javadoc"/>
+    </target>
+</project>

tools/sstableloader/src/org/apache/cassandra/tools/BulkLoader.java

@@ -25,8 +25,6 @@ import javax.net.ssl.SSLEngine;
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
 
 import com.datastax.driver.core.AuthProvider;
 import com.datastax.driver.core.RemoteEndpointAwareJdkSSLOptions;
@@ -313,52 +311,4 @@ public class BulkLoader
             return new BulkLoadConnectionFactory(serverEncOptions, storagePort);
         }
     }
-
-    public static class CmdLineOptions extends Options
-    {
-        /**
-         * Add option with argument and argument name
-         * @param opt shortcut for option name
-         * @param longOpt complete option name
-         * @param argName argument name
-         * @param description description of the option
-         * @return updated Options object
-         */
-        public Options addOption(String opt, String longOpt, String argName, String description)
-        {
-            Option option = new Option(opt, longOpt, true, description);
-            option.setArgName(argName);
-
-            return addOption(option);
-        }
-
-        /**
-         * Add option with argument and argument name that accepts being defined multiple times as a list
-         * @param opt shortcut for option name
-         * @param longOpt complete option name
-         * @param argName argument name
-         * @param description description of the option
-         * @return updated Options object
-         */
-        public Options addOptionList(String opt, String longOpt, String argName, String description)
-        {
-            Option option = new Option(opt, longOpt, true, description);
-            option.setArgName(argName);
-            option.setArgs(Option.UNLIMITED_VALUES);
-
-            return addOption(option);
-        }
-
-        /**
-         * Add option without argument
-         * @param opt shortcut for option name
-         * @param longOpt complete option name
-         * @param description description of the option
-         * @return updated Options object
-         */
-        public Options addOption(String opt, String longOpt, String description)
-        {
-            return addOption(new Option(opt, longOpt, false, description));
-        }
-    }
 }

tools/sstableloader/src/org/apache/cassandra/tools/LoaderOptions.java

@@ -49,7 +49,6 @@ import org.apache.cassandra.config.YamlConfigurationLoader;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.locator.InetAddressAndPort;
-import org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
 
 import static org.apache.cassandra.config.DataRateSpec.DataRateUnit.MEBIBYTES_PER_SECOND;
 import static org.apache.cassandra.config.EncryptionOptions.ClientEncryptionOptions.ClientAuth.REQUIRED;
@@ -77,9 +76,8 @@ public class LoaderOptions
     /**
      * Throttle defined in megabits per second. CASSANDRA-10637 introduced a builder and is the preferred way to
      * provide options instead of using these constant fields.
-     * @deprecated Use {@code throttle-mib} instead
+     * @deprecated Use {@code throttle-mib} instead. See CASSANDRA-17677
      */
-    /** @deprecated See CASSANDRA-17677 */
     @Deprecated(since = "5.0")
     public static final String THROTTLE_MBITS = "throttle";
     public static final String THROTTLE_MEBIBYTES = "throttle-mib";

tools/sstableloader/test/unit/org/apache/cassandra/tools/LoaderOptionsTest.java

@@ -25,12 +25,11 @@ import java.nio.file.Paths;
 import java.security.Permission;
 
 import com.google.common.net.HostAndPort;
+import org.apache.commons.io.FileUtils;
 import org.junit.Test;
 
 import org.apache.cassandra.io.util.File;
-import org.apache.cassandra.transport.TlsTestUtils;
 
-import static org.apache.cassandra.tools.OfflineToolUtils.sstableDirName;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -102,8 +101,8 @@ public class LoaderOptionsTest
                           sstableDirName("legacy_sstables", "legacy_ma_simple") };
         LoaderOptions options = LoaderOptions.builder().parseArgs(args).build();
         // Below two lines validating server encryption options is to verify that we are loading config from the yaml
-        assertEquals(TlsTestUtils.SERVER_KEYSTORE_PATH, options.serverEncOptions.keystore);
+        assertEquals("test/conf/cassandra_ssl_test.keystore", options.serverEncOptions.keystore);
-        assertEquals(TlsTestUtils.SERVER_KEYSTORE_PASSWORD, options.serverEncOptions.keystore_password);
+        assertEquals("cassandra", options.serverEncOptions.keystore_password);
         // Below asserts validate the overrides for the client encryption options from the command line
         // Since the values are provided by (and local to) this test, they are hardcoded
         assertEquals("JKS", options.clientEncOptions.store_type);
@@ -245,5 +244,40 @@ public class LoaderOptionsTest
             System.setSecurityManager(null);
         }
     }
+
+    // Copied from OfflineToolUtils
+
+    public static String sstableDirName(String ks, String cf) throws IOException
+    {
+        return sstableDir(ks, cf).absolutePath();
+    }
+
+    public static File sstableDir(String ks, String cf) throws IOException
+    {
+        File dataDir = copySSTables();
+        File ksDir = new File(dataDir, ks);
+        File[] cfDirs = ksDir.tryList((dir, name) -> cf.equals(name) || name.startsWith(cf + '-'));
+        return cfDirs[0];
+    }
+
+    public static File copySSTables() throws IOException
+    {
+        File dataDir = new File("build/test/cassandra/data");
+        File srcDir = new File("test/data/legacy-sstables/ma");
+        FileUtils.copyDirectory(new File(srcDir, "legacy_tables").toJavaIOFile(), new File(dataDir, "legacy_sstables").toJavaIOFile());
+        return dataDir;
+    }
+
+    // Copied from SystemExitException in unit tests
+
+    private static class SystemExitException extends Error
+    {
+        public final int status;
+
+        public SystemExitException(int status)
+        {
+            this.status = status;
+        }
+    }
 }