natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support to generate a One-Shot heap dump on unhandled exceptions 

Patch by Caleb Rackliffe; reviewed by Josh McKenzie, David Capwell, and Jon Meredith for CASSANDRA-17795

Co-authored-by: Caleb Rackliffe <calebrackliffe@gmail.com>
Co-authored-by: Josh McKenzie <jmckenzie@apache.org>
This commit is contained in:
Josh McKenzie 2022-08-02 16:02:03 -04:00
parent 09b282d1fd
commit 3e0b94565a
13 changed files with 277 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.2
* Add support to generate a One-Shot heap dump on unhandled exceptions (CASSANDRA-17795)
* Rate-limit new client connection auth setup to avoid overwhelming bcrypt (CASSANDRA-17812)
* DataOutputBuffer#scratchBuffer can use off-heap or on-heap memory as a means to control memory allocations (CASSANDRA-16471)
* Add ability to read the TTLs and write times of the elements of a collection and/or UDT (CASSANDRA-8877)

1
build.xml
View File

@ -1500,6 +1500,7 @@
more aggressively rather than waiting. See CASSANDRA-14922 for more details.
-->
<jvmarg value="-XX:SoftRefLRUPolicyMSPerMB=0" />
<jvmarg value="-XX:HeapDumpPath=build/test" />
<jvmarg value="-Dcassandra.test.driver.connection_timeout_ms=${test.driver.connection_timeout_ms}"/>
<jvmarg value="-Dcassandra.test.driver.read_timeout_ms=${test.driver.read_timeout_ms}"/>
<jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/>

10
conf/cassandra.yaml
View File

@ -107,6 +107,16 @@ auto_hints_cleanup_enabled: false
# parameters:
# -
# Directory where Cassandra should store results of a One-Shot troubleshooting heapdump for uncaught exceptions.
# Note: this value can be overridden by the -XX:HeapDumpPath JVM env param with a relative local path for testing if
# so desired.
# If not set, the default directory is $CASSANDRA_HOME/heapdump
# heap_dump_path: /var/lib/cassandra/heapdump
# Enable / disable automatic dump of heap on first uncaught exception
# If not set, the default value is false
# dump_heap_on_uncaught_exception: true
# Enable / disable persistent hint windows.
#
# If set to false, a hint will be stored only in case a respective node

6
ide/idea/workspace.xml
View File

@ -143,7 +143,7 @@
<configuration default="true" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="" />
<option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
<option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true -XX:HeapDumpPath=build/test" />
<option name="PROGRAM_PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
@ -167,7 +167,7 @@
<option name="MAIN_CLASS_NAME" value="" />
<option name="METHOD_NAME" value="" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMetaspaceSize=384M -XX:SoftRefLRUPolicyMSPerMB=0 -Dcassandra.strict.runtime.checks=true -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dmigration-sstable-root=$PROJECT_DIR$/test/data/migration-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.tolerate_sstable_size=true -Dcassandra.skip_sync=true -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
<option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMetaspaceSize=384M -XX:SoftRefLRUPolicyMSPerMB=0 -XX:HeapDumpPath=build/test -Dcassandra.strict.runtime.checks=true -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dmigration-sstable-root=$PROJECT_DIR$/test/data/migration-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.tolerate_sstable_size=true -Dcassandra.skip_sync=true -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
<option name="PARAMETERS" value="" />
<fork_mode value="class" />
<option name="WORKING_DIRECTORY" value="" />
@ -187,7 +187,7 @@
<configuration default="false" name="Cassandra" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="org.apache.cassandra.service.CassandraDaemon" />
<option name="VM_PARAMETERS" value="-Dcassandra-foreground=yes -Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7199 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -ea -Xmx1G -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" />
<option name="VM_PARAMETERS" value="-Dcassandra-foreground=yes -Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7199 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -ea -Xmx1G -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true -XXHeapDumpPath=build/test" />
<option name="PROGRAM_PARAMETERS" value="" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />

3
src/java/org/apache/cassandra/config/Config.java
View File

@ -1179,4 +1179,7 @@ public class Config
logger.info("Node configuration:[{}]", Joiner.on("; ").join(configMap.entrySet()));
}
public volatile boolean dump_heap_on_uncaught_exception = false;
public String heap_dump_path = "heapdump";
}

79
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -18,6 +18,8 @@
package org.apache.cassandra.config;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.net.Inet4Address;
import java.net.Inet6Address;
import java.net.InetAddress;
@ -25,6 +27,8 @@ import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.nio.file.FileStore;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@ -32,11 +36,13 @@ import java.util.Enumeration;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import javax.annotation.Nullable;
@ -906,6 +912,9 @@ public class DatabaseDescriptor
conf.paxos_state_purging = PaxosStatePurging.legacy;
logInitializationOutcome(logger);
if (conf.dump_heap_on_uncaught_exception && DatabaseDescriptor.getHeapDumpPath() == null)
throw new ConfigurationException(String.format("Invalid configuration. Heap dump is enabled but cannot create heap dump output path: %s.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
}
@VisibleForTesting
@ -1582,6 +1591,13 @@ public class DatabaseDescriptor
throw new ConfigurationException("cdc_raw_directory must be specified", false);
FileUtils.createDirectory(conf.cdc_raw_directory);
}
boolean created = maybeCreateHeapDumpPath();
if (!created && conf.dump_heap_on_uncaught_exception)
{
logger.error(String.format("cassandra.yaml:dump_heap_on_uncaught_exception is enabled but unable to create heap dump path %s. Disabling.", conf.heap_dump_path != null ? conf.heap_dump_path : "null"));
conf.dump_heap_on_uncaught_exception = false;
}
}
catch (ConfigurationException e)
{
@ -4397,4 +4413,67 @@ public class DatabaseDescriptor
{
conf.min_tracked_partition_tombstone_count = value;
}
public static boolean getDumpHeapOnUncaughtException()
{
return conf.dump_heap_on_uncaught_exception;
}
/**
* @return Whether the path exists (be it created now or already prior)
*/
private static boolean maybeCreateHeapDumpPath()
{
if (!conf.dump_heap_on_uncaught_exception)
return false;
Path heap_dump_path = getHeapDumpPath();
if (heap_dump_path == null)
{
logger.warn("Neither -XX:HeapDumpPath nor cassandra.yaml:heap_dump_path are set; unable to create a directory to hold the output.");
return false;
}
if (PathUtils.exists(Paths.get(conf.heap_dump_path)))
return true;
return PathUtils.createDirectoryIfNotExists(Paths.get(conf.heap_dump_path));
}
/**
* As this is at its heart a debug operation (getting a one-shot heapdump from an uncaught exception), we support
* both the more evolved cassandra.yaml approach but also the -XX param to override it on a one-off basis so you don't
* have to change the full config of a node or a cluster in order to get a heap dump from a single node that's
* misbehaving.
* @return the absolute path of the -XX param if provided, else the heap_dump_path in cassandra.yaml
*/
public static Path getHeapDumpPath()
{
RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
Optional<String> pathArg = runtimeMxBean.getInputArguments().stream().filter(s -> s.startsWith("-XX:HeapDumpPath=")).findFirst();
if (pathArg.isPresent())
{
Pattern HEAP_DUMP_PATH_SPLITTER = Pattern.compile("HeapDumpPath=");
String fullHeapPathString = HEAP_DUMP_PATH_SPLITTER.split(pathArg.get())[1];
Path absolutePath = Paths.get(fullHeapPathString).toAbsolutePath();
Path basePath = fullHeapPathString.endsWith(".hprof") ? absolutePath.subpath(0, absolutePath.getNameCount() - 1) : absolutePath;
return Paths.get("/").resolve(basePath);
}
if (conf.heap_dump_path == null)
throw new ConfigurationException("Attempted to get heap dump path without -XX:HeapDumpPath or cassandra.yaml:heap_dump_path set.");
return Paths.get(conf.heap_dump_path);
}
public static void setDumpHeapOnUncaughtException(boolean enabled)
{
conf.dump_heap_on_uncaught_exception = enabled;
boolean pathExists = maybeCreateHeapDumpPath();
if (enabled && !pathExists)
{
logger.error("Attempted to enable heap dump but cannot create the requested path. Disabling.");
conf.dump_heap_on_uncaught_exception = false;
}
else
logger.info("Setting dump_heap_on_uncaught_exception to {}", enabled);
}
}

12
src/java/org/apache/cassandra/service/StorageProxy.java
View File

@ -3167,4 +3167,16 @@ public class StorageProxy implements StorageProxyMBean
{
return PaxosState.getDisableCoordinatorLocking();
}
@Override
public boolean getDumpHeapOnUncaughtException()
{
return DatabaseDescriptor.getDumpHeapOnUncaughtException();
}
@Override
public void setDumpHeapOnUncaughtException(boolean enabled)
{
DatabaseDescriptor.setDumpHeapOnUncaughtException(enabled);
}
}

3
src/java/org/apache/cassandra/service/StorageProxyMBean.java
View File

@ -135,4 +135,7 @@ public interface StorageProxyMBean
void setPaxosCoordinatorLockingDisabled(boolean disabled);
boolean getPaxosCoordinatorLockingDisabled();
public boolean getDumpHeapOnUncaughtException();
public void setDumpHeapOnUncaughtException(boolean enabled);
}

82
src/java/org/apache/cassandra/utils/HeapUtils.java
View File

@ -22,15 +22,25 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.management.ManagementFactory;
import java.nio.file.FileStore;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.management.MBeanServer;
import org.apache.cassandra.io.util.File;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.text.StrBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.sun.management.HotSpotDiagnosticMXBean;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.PathUtils;
import static org.apache.cassandra.config.CassandraRelevantEnv.JAVA_HOME;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
/**
* Utility to log heap histogram.
@ -40,6 +50,8 @@ public final class HeapUtils
{
private static final Logger logger = LoggerFactory.getLogger(HeapUtils.class);
private static final Lock DUMP_LOCK = new ReentrantLock();
/**
* Generates a HEAP histogram in the log file.
*/
@ -73,6 +85,72 @@ public final class HeapUtils
}
}
/**
* @return full path to the created heap dump file
*/
public static String maybeCreateHeapDump()
{
// Make sure that only one heap dump can be in progress across all threads, and abort for
// threads that cannot immediately acquire the lock, allowing them to fail normally.
if (DUMP_LOCK.tryLock())
{
try
{
if (DatabaseDescriptor.getDumpHeapOnUncaughtException())
{
MBeanServer server = ManagementFactory.getPlatformMBeanServer();
Path absoluteBasePath = DatabaseDescriptor.getHeapDumpPath();
// We should never reach this point with this value null as we initialize the bool only after confirming
// the -XX param / .yaml conf is present on initial init and the JMX entry point, but still worth checking.
if (absoluteBasePath == null)
{
DatabaseDescriptor.setDumpHeapOnUncaughtException(false);
throw new RuntimeException("Cannot create heap dump unless -XX:HeapDumpPath or cassandra.yaml:heap_dump_path is specified.");
}
long maxMemoryBytes = Runtime.getRuntime().maxMemory();
long freeSpaceBytes = PathUtils.tryGetSpace(absoluteBasePath, FileStore::getUnallocatedSpace);
// Abort if there isn't enough room on the target disk to dump the entire heap and then copy it.
if (freeSpaceBytes < 2 * maxMemoryBytes)
throw new RuntimeException("Cannot allocated space for a heap dump snapshot. There are only " + freeSpaceBytes + " bytes free at " + absoluteBasePath + '.');
HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class);
String filename = String.format("pid%s-epoch%s.hprof", HeapUtils.getProcessId().toString(), currentTimeMillis());
String fullPath = Paths.get(absoluteBasePath.toString(), filename).toString();
logger.info("Writing heap dump to {} on partition w/ {} free bytes...", absoluteBasePath, freeSpaceBytes);
mxBean.dumpHeap(fullPath, false);
logger.info("Heap dump written to {}", fullPath);
// Disable further heap dump creations until explicitly re-enabled.
DatabaseDescriptor.setDumpHeapOnUncaughtException(false);
return fullPath;
}
else
{
logger.debug("Heap dump creation on uncaught exceptions is disabled.");
}
}
catch (Throwable e)
{
logger.warn("Unable to create heap dump.", e);
}
finally
{
DUMP_LOCK.unlock();
}
}
else
{
logger.debug("Heap dump creation is already in progress. Request aborted.");
}
return null;
}
/**
* Retrieve the path to the JCMD executable.
* @return the path to the JCMD executable or null if it cannot be found.

3
src/java/org/apache/cassandra/utils/JVMStabilityInspector.java
View File

@ -133,6 +133,9 @@ public final class JVMStabilityInspector
isUnstable = true;
}
// Anything other than an OOM, we should try and heap dump to capture what's going on if configured to do so
HeapUtils.maybeCreateHeapDump();
if (t instanceof InterruptedException)
throw new UncheckedInterruptedException((InterruptedException) t);

3
test/conf/cassandra.yaml
View File

@ -54,6 +54,9 @@ drop_compact_storage_enabled: true
file_cache_enabled: true
auto_hints_cleanup_enabled: true
heap_dump_path: build/test
dump_heap_on_uncaught_exception: true
read_thresholds_enabled: true
coordinator_read_size_warn_threshold: 1024KiB
coordinator_read_size_fail_threshold: 4096KiB

63
test/long/org/apache/cassandra/utils/HeapUtilsTest.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.utils;
import java.nio.file.Paths;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.config.DatabaseDescriptor;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class HeapUtilsTest
{
@BeforeClass
public static void setup()
{
DatabaseDescriptor.daemonInitialization();
}
@Test
public void shouldDumpHeapWithPathArgSpecified()
{
DatabaseDescriptor.setDumpHeapOnUncaughtException(true);
String path = HeapUtils.maybeCreateHeapDump();
assertNotNull(path);
assertTrue(Paths.get(path).toFile().exists());
assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException());
// After the first dump, subsequent requests should be no-ops...
path = HeapUtils.maybeCreateHeapDump();
assertNull(path);
// ...until creation is manually re-enabled.
DatabaseDescriptor.setDumpHeapOnUncaughtException(true);
assertTrue(DatabaseDescriptor.getDumpHeapOnUncaughtException());
assertNotNull(DatabaseDescriptor.getHeapDumpPath());
path = HeapUtils.maybeCreateHeapDump();
assertNotNull(path);
assertTrue(Paths.get(path).toFile().exists());
assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException());
}
}

26
test/unit/org/apache/cassandra/tools/BulkLoaderTest.java
View File

@ -64,7 +64,9 @@ public class BulkLoaderTest extends OfflineToolUtils
"globalEventExecutor-[1-9]-[1-9]",
"globalEventExecutor-[1-9]-[1-9]",
"Shutdown-checker",
"cluster[0-9]-connection-reaper-[0-9]" });
"cluster[0-9]-connection-reaper-[0-9]",
"Attach Listener",
"process reaper"});
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
@ -92,11 +94,13 @@ public class BulkLoaderTest extends OfflineToolUtils
"globalEventExecutor-[1-9]-[1-9]",
"globalEventExecutor-[1-9]-[1-9]",
"Shutdown-checker",
"cluster[0-9]-connection-reaper-[0-9]" });
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
assertKeyspaceNotLoaded();
"cluster[0-9]-connection-reaper-[0-9]",
"Attach Listener",
"process reaper"});
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
assertKeyspaceNotLoaded();
assertServerNotLoaded();
}
@ -120,10 +124,12 @@ public class BulkLoaderTest extends OfflineToolUtils
"globalEventExecutor-[1-9]-[1-9]",
"globalEventExecutor-[1-9]-[1-9]",
"Shutdown-checker",
"cluster[0-9]-connection-reaper-[0-9]" });
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
"cluster[0-9]-connection-reaper-[0-9]",
"Attach Listener",
"process reaper"});
assertSchemaNotLoaded();
assertCLSMNotLoaded();
assertSystemKSNotLoaded();
assertKeyspaceNotLoaded();
assertServerNotLoaded();
}