Limit the number of held heap dumps to not consume disk space excessively

patch by Igor Kamyshnikov; reviewed by Dmitry Konstantinov, Stefan Miklosovic for CASSANDRA-20457

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Limit the number of held heap dumps to not consume disk space excessively (CASSANDRA-20457)
  * Accord: BEGIN TRANSACTIONs IF condition logic does not properly support meaningless emptiness and null values (CASSANDRA-20667)
  * Accord: startup race condition where accord journal tries to access the 2i index before its ready (CASSANDRA-20686)
  * Adopt Unsafe::invokeCleaner for Direct ByteBuffer cleaning (CASSANDRA-20677)

bin/cassandra

@@ -174,6 +174,23 @@ case "`uname -s`" in
     ;;
 esac
 
+clean_heap_dump_files()
+{
+    if [ "$CASSANDRA_HEAPDUMP_KEEP_NEWEST_N_FILES" != "" ] && \
+        [ "$CASSANDRA_HEAPDUMP_KEEP_NEWEST_N_FILES" -ge 0 ] && \
+        [ -d "$CASSANDRA_HEAPDUMP_DIR" ]; then
+        # find all files under CASSANDRA_HEAPDUMP_DIR,
+        # sort by last modification date descending,
+        # print those, that need to be removed
+        ls -pt1 "$CASSANDRA_HEAPDUMP_DIR" | grep -v / | \
+        grep "^cassandra-.*-pid.*[.]hprof$" | \
+        awk "{ if (NR > $CASSANDRA_HEAPDUMP_KEEP_NEWEST_N_FILES) print \$0}" | \
+        while IFS= read -r file; do
+          rm -f "$CASSANDRA_HEAPDUMP_DIR/$file"
+        done
+    fi
+}
+
 launch_service()
 {
     pidpath="$1"
@@ -188,6 +205,8 @@ launch_service()
         cassandra_parms="$cassandra_parms -Dcassandra-pidfile=$pidpath"
     fi
 
+    clean_heap_dump_files
+
     # The cassandra-foreground option will tell CassandraDaemon not
     # to close stdout/stderr, but it's up to us not to background.
     if [ "x$foreground" != "x" ]; then
@@ -247,6 +266,8 @@ while true; do
         ;;
         -H)
             properties="$properties -XX:HeapDumpPath=$2"
+            # disable automatic heap dump files management as HeapDumpPath was overridden
+            CASSANDRA_HEAPDUMP_KEEP_NEWEST_N_FILES=-1
             shift 2
         ;;
         -E)

conf/cassandra-env.sh

@@ -199,6 +199,14 @@ if [ "x$CASSANDRA_HEAPDUMP_DIR" = "x" ]; then
 fi
 JVM_OPTS="$JVM_OPTS -XX:HeapDumpPath=$CASSANDRA_HEAPDUMP_DIR/cassandra-`date +%s`-pid$$.hprof"
 
+# Cassandra heap dump files management options:
+#  N >= 0 - keep N newest files
+# -1 - disable clean up
+# defaults to 2 if not set
+if [ "$CASSANDRA_HEAPDUMP_KEEP_NEWEST_N_FILES" = "" ]; then
+    CASSANDRA_HEAPDUMP_KEEP_NEWEST_N_FILES=2
+fi
+
 # stop the jvm on OutOfMemoryError as it can result in some data corruption
 # uncomment the preferred option
 # ExitOnOutOfMemoryError and CrashOnOutOfMemoryError require a JRE greater or equals to 1.7 update 101 or 1.8 update 92