Add support to generate a One-Shot heap dump on unhandled exceptions

Patch by Caleb Rackliffe; reviewed by Josh McKenzie, David Capwell, and Jon Meredith for CASSANDRA-17795

Co-authored-by: Caleb Rackliffe <calebrackliffe@gmail.com>
Co-authored-by: Josh McKenzie <jmckenzie@apache.org>

CHANGES.txt

@@ -1,4 +1,5 @@
 4.2
+ * Add support to generate a One-Shot heap dump on unhandled exceptions (CASSANDRA-17795)
  * Rate-limit new client connection auth setup to avoid overwhelming bcrypt (CASSANDRA-17812)
  * DataOutputBuffer#scratchBuffer can use off-heap or on-heap memory as a means to control memory allocations (CASSANDRA-16471)
  * Add ability to read the TTLs and write times of the elements of a collection and/or UDT (CASSANDRA-8877)

build.xml

@@ -1500,6 +1500,7 @@
              more aggressively rather than waiting. See CASSANDRA-14922 for more details.
         -->
         <jvmarg value="-XX:SoftRefLRUPolicyMSPerMB=0" />
+        <jvmarg value="-XX:HeapDumpPath=build/test" />
         <jvmarg value="-Dcassandra.test.driver.connection_timeout_ms=${test.driver.connection_timeout_ms}"/>
         <jvmarg value="-Dcassandra.test.driver.read_timeout_ms=${test.driver.read_timeout_ms}"/>
         <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>

conf/cassandra.yaml

@@ -107,6 +107,16 @@ auto_hints_cleanup_enabled: false
 #     parameters:
 #         -
 
+# Directory where Cassandra should store results of a One-Shot troubleshooting heapdump for uncaught exceptions.
+# Note: this value can be overridden by the -XX:HeapDumpPath JVM env param with a relative local path for testing if
+# so desired.
+# If not set, the default directory is $CASSANDRA_HOME/heapdump
+# heap_dump_path: /var/lib/cassandra/heapdump
+
+# Enable / disable automatic dump of heap on first uncaught exception
+# If not set, the default value is false
+# dump_heap_on_uncaught_exception: true
+
 # Enable / disable persistent hint windows.
 #
 # If set to false, a hint will be stored only in case a respective node

ide/idea/workspace.xml

@@ -143,7 +143,7 @@
     <configuration default="true" type="Application" factoryName="Application">
       <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
       <option name="MAIN_CLASS_NAME" value="" />
-      <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
+      <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true -XX:HeapDumpPath=build/test" />
       <option name="PROGRAM_PARAMETERS" value="" />
       <option name="WORKING_DIRECTORY" value="" />
       <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
@@ -167,7 +167,7 @@
       <option name="MAIN_CLASS_NAME" value="" />
       <option name="METHOD_NAME" value="" />
       <option name="TEST_OBJECT" value="class" />
-      <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMetaspaceSize=384M -XX:SoftRefLRUPolicyMSPerMB=0 -Dcassandra.strict.runtime.checks=true -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dmigration-sstable-root=$PROJECT_DIR$/test/data/migration-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.tolerate_sstable_size=true -Dcassandra.skip_sync=true -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
+      <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMetaspaceSize=384M -XX:SoftRefLRUPolicyMSPerMB=0 -XX:HeapDumpPath=build/test -Dcassandra.strict.runtime.checks=true -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dmigration-sstable-root=$PROJECT_DIR$/test/data/migration-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.tolerate_sstable_size=true -Dcassandra.skip_sync=true -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
       <option name="PARAMETERS" value="" />
       <fork_mode value="class" />
       <option name="WORKING_DIRECTORY" value="" />
@@ -187,7 +187,7 @@
     <configuration default="false" name="Cassandra" type="Application" factoryName="Application">
       <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
       <option name="MAIN_CLASS_NAME" value="org.apache.cassandra.service.CassandraDaemon" />
-      <option name="VM_PARAMETERS" value="-Dcassandra-foreground=yes -Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7199 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -ea -Xmx1G -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
+      <option name="VM_PARAMETERS" value="-Dcassandra-foreground=yes -Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7199 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -ea -Xmx1G -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true -XXHeapDumpPath=build/test" />
       <option name="PROGRAM_PARAMETERS" value="" />
       <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
       <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />

src/java/org/apache/cassandra/config/Config.java

@@ -1179,4 +1179,7 @@ public class Config
 
         logger.info("Node configuration:[{}]", Joiner.on("; ").join(configMap.entrySet()));
     }
+
+    public volatile boolean dump_heap_on_uncaught_exception = false;
+    public String heap_dump_path = "heapdump";
 }

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -18,6 +18,8 @@
 package org.apache.cassandra.config;
 
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
@@ -25,6 +27,8 @@ import java.net.NetworkInterface;
 import java.net.SocketException;
 import java.net.UnknownHostException;
 import java.nio.file.FileStore;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
@@ -32,11 +36,13 @@ import java.util.Enumeration;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
 
 import javax.annotation.Nullable;
 
@@ -906,6 +912,9 @@ public class DatabaseDescriptor
             conf.paxos_state_purging = PaxosStatePurging.legacy;
 
         logInitializationOutcome(logger);
+
+        if (conf.dump_heap_on_uncaught_exception && DatabaseDescriptor.getHeapDumpPath() == null)
+            throw new ConfigurationException(String.format("Invalid configuration. Heap dump is enabled but cannot create heap dump output path: %s.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
     }
 
     @VisibleForTesting
@@ -1582,6 +1591,13 @@ public class DatabaseDescriptor
                     throw new ConfigurationException("cdc_raw_directory must be specified", false);
                 FileUtils.createDirectory(conf.cdc_raw_directory);
             }
+
+            boolean created = maybeCreateHeapDumpPath();
+            if (!created && conf.dump_heap_on_uncaught_exception)
+            {
+                logger.error(String.format("cassandra.yaml:dump_heap_on_uncaught_exception is enabled but unable to create heap dump path %s. Disabling.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
+                conf.dump_heap_on_uncaught_exception = false;
+            }
         }
         catch (ConfigurationException e)
         {
@@ -4397,4 +4413,67 @@ public class DatabaseDescriptor
     {
         conf.min_tracked_partition_tombstone_count = value;
     }
+
+    public static boolean getDumpHeapOnUncaughtException()
+    {
+        return conf.dump_heap_on_uncaught_exception;
+    }
+
+    /**
+     * @return Whether the path exists (be it created now or already prior)
+     */
+    private static boolean maybeCreateHeapDumpPath()
+    {
+        if (!conf.dump_heap_on_uncaught_exception)
+            return false;
+
+        Path heap_dump_path = getHeapDumpPath();
+        if (heap_dump_path == null)
+        {
+            logger.warn("Neither -XX:HeapDumpPath nor cassandra.yaml:heap_dump_path are set; unable to create a directory to hold the output.");
+            return false;
+        }
+        if (PathUtils.exists(Paths.get(conf.heap_dump_path)))
+            return true;
+        return PathUtils.createDirectoryIfNotExists(Paths.get(conf.heap_dump_path));
+    }
+
+    /**
+     * As this is at its heart a debug operation (getting a one-shot heapdump from an uncaught exception), we support
+     * both the more evolved cassandra.yaml approach but also the -XX param to override it on a one-off basis so you don't
+     * have to change the full config of a node or a cluster in order to get a heap dump from a single node that's
+     * misbehaving.
+     * @return the absolute path of the -XX param if provided, else the heap_dump_path in cassandra.yaml
+     */
+    public static Path getHeapDumpPath()
+    {
+        RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
+        Optional<String> pathArg = runtimeMxBean.getInputArguments().stream().filter(s -> s.startsWith("-XX:HeapDumpPath=")).findFirst();
+
+        if (pathArg.isPresent())
+        {
+            Pattern HEAP_DUMP_PATH_SPLITTER = Pattern.compile("HeapDumpPath=");
+            String fullHeapPathString = HEAP_DUMP_PATH_SPLITTER.split(pathArg.get())[1];
+            Path absolutePath = Paths.get(fullHeapPathString).toAbsolutePath();
+            Path basePath = fullHeapPathString.endsWith(".hprof") ? absolutePath.subpath(0, absolutePath.getNameCount() - 1) : absolutePath;
+            return Paths.get("/").resolve(basePath);
+        }
+        if (conf.heap_dump_path == null)
+            throw new ConfigurationException("Attempted to get heap dump path without -XX:HeapDumpPath or cassandra.yaml:heap_dump_path set.");
+        return Paths.get(conf.heap_dump_path);
+    }
+
+    public static void setDumpHeapOnUncaughtException(boolean enabled)
+    {
+        conf.dump_heap_on_uncaught_exception = enabled;
+        boolean pathExists = maybeCreateHeapDumpPath();
+
+        if (enabled && !pathExists)
+        {
+            logger.error("Attempted to enable heap dump but cannot create the requested path. Disabling.");
+            conf.dump_heap_on_uncaught_exception = false;
+        }
+        else
+            logger.info("Setting dump_heap_on_uncaught_exception to {}", enabled);
+    }
 }

src/java/org/apache/cassandra/service/StorageProxy.java

@@ -3167,4 +3167,16 @@ public class StorageProxy implements StorageProxyMBean
     {
         return PaxosState.getDisableCoordinatorLocking();
     }
+
+    @Override
+    public boolean getDumpHeapOnUncaughtException()
+    {
+        return DatabaseDescriptor.getDumpHeapOnUncaughtException();
+    }
+
+    @Override
+    public void setDumpHeapOnUncaughtException(boolean enabled)
+    {
+        DatabaseDescriptor.setDumpHeapOnUncaughtException(enabled);
+    }
 }

src/java/org/apache/cassandra/service/StorageProxyMBean.java

@@ -135,4 +135,7 @@ public interface StorageProxyMBean
 
     void setPaxosCoordinatorLockingDisabled(boolean disabled);
     boolean getPaxosCoordinatorLockingDisabled();
+
+    public boolean getDumpHeapOnUncaughtException();
+    public void setDumpHeapOnUncaughtException(boolean enabled);
 }

src/java/org/apache/cassandra/utils/HeapUtils.java

@@ -22,15 +22,25 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.lang.management.ManagementFactory;
+import java.nio.file.FileStore;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import javax.management.MBeanServer;
 
-import org.apache.cassandra.io.util.File;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.text.StrBuilder;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.sun.management.HotSpotDiagnosticMXBean;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
+
 import static org.apache.cassandra.config.CassandraRelevantEnv.JAVA_HOME;
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
 
 /**
  * Utility to log heap histogram.
@@ -40,6 +50,8 @@ public final class HeapUtils
 {
     private static final Logger logger = LoggerFactory.getLogger(HeapUtils.class);
 
+    private static final Lock DUMP_LOCK = new ReentrantLock();
+
     /**
      * Generates a HEAP histogram in the log file.
      */
@@ -73,6 +85,72 @@ public final class HeapUtils
         }
     }
 
+    /**
+     * @return full path to the created heap dump file
+     */
+    public static String maybeCreateHeapDump()
+    {
+        // Make sure that only one heap dump can be in progress across all threads, and abort for
+        // threads that cannot immediately acquire the lock, allowing them to fail normally.
+        if (DUMP_LOCK.tryLock())
+        {
+            try
+            {
+                if (DatabaseDescriptor.getDumpHeapOnUncaughtException())
+                {
+                    MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+
+                    Path absoluteBasePath = DatabaseDescriptor.getHeapDumpPath();
+                    // We should never reach this point with this value null as we initialize the bool only after confirming
+                    // the -XX param / .yaml conf is present on initial init and the JMX entry point, but still worth checking.
+                    if (absoluteBasePath == null)
+                    {
+                        DatabaseDescriptor.setDumpHeapOnUncaughtException(false);
+                        throw new RuntimeException("Cannot create heap dump unless -XX:HeapDumpPath or cassandra.yaml:heap_dump_path is specified.");
+                    }
+
+                    long maxMemoryBytes = Runtime.getRuntime().maxMemory();
+                    long freeSpaceBytes = PathUtils.tryGetSpace(absoluteBasePath, FileStore::getUnallocatedSpace);
+
+                    // Abort if there isn't enough room on the target disk to dump the entire heap and then copy it.
+                    if (freeSpaceBytes < 2 * maxMemoryBytes)
+                        throw new RuntimeException("Cannot allocated space for a heap dump snapshot. There are only " + freeSpaceBytes + " bytes free at " + absoluteBasePath + '.');
+
+                    HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class);
+                    String filename = String.format("pid%s-epoch%s.hprof", HeapUtils.getProcessId().toString(), currentTimeMillis());
+                    String fullPath = Paths.get(absoluteBasePath.toString(), filename).toString();
+
+                    logger.info("Writing heap dump to {} on partition w/ {} free bytes...", absoluteBasePath, freeSpaceBytes);
+                    mxBean.dumpHeap(fullPath, false);
+                    logger.info("Heap dump written to {}", fullPath);
+
+                    // Disable further heap dump creations until explicitly re-enabled.
+                    DatabaseDescriptor.setDumpHeapOnUncaughtException(false);
+
+                    return fullPath;
+                }
+                else
+                {
+                    logger.debug("Heap dump creation on uncaught exceptions is disabled.");
+                }
+            }
+            catch (Throwable e)
+            {
+                logger.warn("Unable to create heap dump.", e);
+            }
+            finally
+            {
+                DUMP_LOCK.unlock();
+            }
+        }
+        else
+        {
+            logger.debug("Heap dump creation is already in progress. Request aborted.");
+        }
+
+        return null;
+    }
+
     /**
      * Retrieve the path to the JCMD executable.
      * @return the path to the JCMD executable or null if it cannot be found.

src/java/org/apache/cassandra/utils/JVMStabilityInspector.java

@@ -133,6 +133,9 @@ public final class JVMStabilityInspector
             isUnstable = true;
         }
 
+        // Anything other than an OOM, we should try and heap dump to capture what's going on if configured to do so
+        HeapUtils.maybeCreateHeapDump();
+
         if (t instanceof InterruptedException)
             throw new UncheckedInterruptedException((InterruptedException) t);
 

test/conf/cassandra.yaml

@@ -54,6 +54,9 @@ drop_compact_storage_enabled: true
 file_cache_enabled: true
 auto_hints_cleanup_enabled: true
 
+heap_dump_path: build/test
+dump_heap_on_uncaught_exception: true
+
 read_thresholds_enabled: true
 coordinator_read_size_warn_threshold: 1024KiB
 coordinator_read_size_fail_threshold: 4096KiB

test/long/org/apache/cassandra/utils/HeapUtilsTest.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.nio.file.Paths;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class HeapUtilsTest
+{
+    @BeforeClass
+    public static void setup()
+    {
+        DatabaseDescriptor.daemonInitialization();
+    }
+
+    @Test
+    public void shouldDumpHeapWithPathArgSpecified()
+    {
+        DatabaseDescriptor.setDumpHeapOnUncaughtException(true);
+        String path = HeapUtils.maybeCreateHeapDump();
+        assertNotNull(path);
+        assertTrue(Paths.get(path).toFile().exists());
+        assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException());
+
+        // After the first dump, subsequent requests should be no-ops...
+        path = HeapUtils.maybeCreateHeapDump();
+        assertNull(path);
+
+        // ...until creation is manually re-enabled.
+        DatabaseDescriptor.setDumpHeapOnUncaughtException(true);
+        assertTrue(DatabaseDescriptor.getDumpHeapOnUncaughtException());
+        assertNotNull(DatabaseDescriptor.getHeapDumpPath());
+        path = HeapUtils.maybeCreateHeapDump();
+        assertNotNull(path);
+        assertTrue(Paths.get(path).toFile().exists());
+        assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException());
+    }
+}

test/unit/org/apache/cassandra/tools/BulkLoaderTest.java

@@ -64,7 +64,9 @@ public class BulkLoaderTest extends OfflineToolUtils
                                                         "globalEventExecutor-[1-9]-[1-9]",
                                                         "globalEventExecutor-[1-9]-[1-9]",
                                                         "Shutdown-checker",
-                                                        "cluster[0-9]-connection-reaper-[0-9]" });
+                                                        "cluster[0-9]-connection-reaper-[0-9]",
+                                                        "Attach Listener",
+                                                        "process reaper"});
         assertSchemaNotLoaded();
         assertCLSMNotLoaded();
         assertSystemKSNotLoaded();
@@ -92,11 +94,13 @@ public class BulkLoaderTest extends OfflineToolUtils
                                                         "globalEventExecutor-[1-9]-[1-9]",
                                                         "globalEventExecutor-[1-9]-[1-9]",
                                                         "Shutdown-checker",
-                                                        "cluster[0-9]-connection-reaper-[0-9]" });
+                                                        "cluster[0-9]-connection-reaper-[0-9]",
-        assertSchemaNotLoaded();
+                                                        "Attach Listener",
-        assertCLSMNotLoaded();
+                                                        "process reaper"});
-        assertSystemKSNotLoaded();
+    assertSchemaNotLoaded();
-        assertKeyspaceNotLoaded();
+    assertCLSMNotLoaded();
+    assertSystemKSNotLoaded();
+    assertKeyspaceNotLoaded();
         assertServerNotLoaded();
     }
 
@@ -120,10 +124,12 @@ public class BulkLoaderTest extends OfflineToolUtils
                                                         "globalEventExecutor-[1-9]-[1-9]",
                                                         "globalEventExecutor-[1-9]-[1-9]",
                                                         "Shutdown-checker",
-                                                        "cluster[0-9]-connection-reaper-[0-9]" });
+                                                        "cluster[0-9]-connection-reaper-[0-9]",
-        assertSchemaNotLoaded();
+                                                        "Attach Listener",
-        assertCLSMNotLoaded();
+                                                        "process reaper"});
-        assertSystemKSNotLoaded();
+    assertSchemaNotLoaded();
+    assertCLSMNotLoaded();
+    assertSystemKSNotLoaded();
         assertKeyspaceNotLoaded();
         assertServerNotLoaded();
     }