Extend nodetool tablestats for dictionary memory usage

patch by Stefan Miklosovic; reviewed by Yifan Cai for CASSANDRA-20940

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Extend nodetool tablestats for dictionary memory usage (CASSANDRA-20940)
  * Introduce separate GCInspector thresholds for concurrent GC events (CASSANDRA-20980)
  * Reduce contention in MemtableAllocator.allocate (CASSANDRA-20226)
  * Add export, list, import sub-commands for nodetool compressiondictionary (CASSANDRA-20941)

src/java/org/apache/cassandra/db/compression/CompressionDictionary.java

@@ -56,6 +56,15 @@ public interface CompressionDictionary extends AutoCloseable
      */
     int checksum();
 
+    /**
+     * Get memory occupied of this dictionary as a whole.
+     * Use for metrics exposing used memory in total. The value
+     * return by this method is a best-effort estimation.
+     *
+     * @return memory occuppied by this compression dictionary, in bytes
+     */
+    int estimatedOccupiedMemoryBytes();
+
     /**
      * Get the kind of the compression algorithm
      *

src/java/org/apache/cassandra/db/compression/CompressionDictionaryCache.java

@@ -19,6 +19,7 @@
 package org.apache.cassandra.db.compression;
 
 import java.time.Duration;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
 
@@ -112,6 +113,14 @@ public class CompressionDictionaryCache implements ICompressionDictionaryCache
         }
     }
 
+    @Override
+    public long cachedDictionariesMemoryUsed()
+    {
+        AtomicInteger value = new AtomicInteger();
+        cache.asMap().forEach((key, dict) -> value.addAndGet(dict.estimatedOccupiedMemoryBytes()));
+        return value.get();
+    }
+
     @Override
     public synchronized void close()
     {

src/java/org/apache/cassandra/db/compression/CompressionDictionaryManager.java

@@ -192,6 +192,12 @@ public class CompressionDictionaryManager implements CompressionDictionaryManage
         cache.add(compressionDictionary);
     }
 
+    @Override
+    public long cachedDictionariesMemoryUsed()
+    {
+        return cache.cachedDictionariesMemoryUsed();
+    }
+
     @Override
     public void onNewDictionaryTrained(CompressionDictionary.DictId dictionaryId)
     {

src/java/org/apache/cassandra/db/compression/ICompressionDictionaryCache.java

@@ -53,4 +53,11 @@ public interface ICompressionDictionaryCache extends AutoCloseable
      * @param compressionDictionary the compression dictionary to cache, may be null
      */
     void add(@Nullable CompressionDictionary compressionDictionary);
+
+    /**
+     * Gives number of bytes cached compression dictionaries occupy in this cache.
+     *
+     * @return number of bytes cached dictionaries occupy
+     */
+    long cachedDictionariesMemoryUsed();
 }

src/java/org/apache/cassandra/db/compression/ZstdCompressionDictionary.java

@@ -86,6 +86,16 @@ public class ZstdCompressionDictionary implements CompressionDictionary, SelfRef
         return checksum;
     }
 
+    @Override
+    public int estimatedOccupiedMemoryBytes()
+    {
+        int occupied = rawDictionary.length;
+        occupied += dictDecompress != null ? rawDictionary.length : 0;
+        occupied += zstdDictCompressPerLevel.size() * rawDictionary.length;
+
+        return occupied;
+    }
+
     @Override
     public boolean equals(Object o)
     {

src/java/org/apache/cassandra/metrics/TableMetrics.java

@@ -45,6 +45,7 @@ import com.codahale.metrics.Timer;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compression.CompressionDictionaryManager;
 import org.apache.cassandra.db.lifecycle.SSTableSet;
 import org.apache.cassandra.db.lifecycle.View;
 import org.apache.cassandra.db.memtable.Memtable;
@@ -154,6 +155,8 @@ public class TableMetrics
     public final Gauge<Long> maxPartitionSize;
     /** Size of the smallest compacted partition */
     public final Gauge<Long> meanPartitionSize;
+    /** Memory usage of cached compression dictionaries */
+    public final Gauge<Long> compressionDictionariesMemoryUsed;
     /** Off heap memory used by compression meta data*/
     public final Gauge<Long> compressionMetadataOffHeapMemoryUsed;
     /** Tombstones scanned in queries on this CF */
@@ -771,6 +774,19 @@ public class TableMetrics
                 return count > 0 ? sum / count : 0;
             }
         });
+        compressionDictionariesMemoryUsed = createTableGauge("CompressionDictionariesMemoryUsed", new Gauge<Long>()
+        {
+            @Override
+            public Long getValue()
+            {
+                CompressionDictionaryManager compressionDictionaryManager = cfs.compressionDictionaryManager();
+                if (compressionDictionaryManager != null)
+                    return compressionDictionaryManager.cachedDictionariesMemoryUsed();
+                else
+                    return 0L;
+            }
+        });
+
         compressionMetadataOffHeapMemoryUsed = createTableGauge("CompressionMetadataOffHeapMemoryUsed", new Gauge<Long>()
         {
             public Long getValue()

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -2088,6 +2088,7 @@ public class NodeProbe implements AutoCloseable
                 case "BloomFilterFalseRatio":
                 case "BloomFilterOffHeapMemoryUsed":
                 case "IndexSummaryOffHeapMemoryUsed":
+                case "CompressionDictionariesMemoryUsed":
                 case "CompressionMetadataOffHeapMemoryUsed":
                 case "CompressionRatio":
                 case "EstimatedColumnCountHistogram":

src/java/org/apache/cassandra/tools/nodetool/TableStats.java

@@ -58,7 +58,7 @@ public class TableStats extends AbstractCommand
                         + "average_tombstones_per_slice_last_five_minutes, bloom_filter_false_positives, "
                         + "bloom_filter_false_ratio, bloom_filter_off_heap_memory_used, bloom_filter_space_used, "
                         + "compacted_partition_maximum_bytes, compacted_partition_mean_bytes, "
-                        + "compacted_partition_minimum_bytes, compression_metadata_off_heap_memory_used, "
+                        + "compacted_partition_minimum_bytes, compression_dictionaries_memory_used, compression_metadata_off_heap_memory_used, "
                         + "full_name, index_summary_off_heap_memory_used, local_read_count, "
                         + "local_read_latency_ms, local_write_latency_ms, "
                         + "maximum_live_cells_per_slice_last_five_minutes, "

src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java

@@ -57,6 +57,8 @@ public class StatsTable
     public String bloomFilterOffHeapMemoryUsed;
     public boolean indexSummaryOffHeapUsed = false;
     public String indexSummaryOffHeapMemoryUsed;
+    public boolean compressionDictionariesUsed = false;
+    public String compressionDictionariesMemoryUsed;
     public boolean compressionMetadataOffHeapUsed = false;
     public String compressionMetadataOffHeapMemoryUsed;
     public long compactedPartitionMinimumBytes;

src/java/org/apache/cassandra/tools/nodetool/stats/StatsTableComparator.java

@@ -53,6 +53,7 @@ public class StatsTableComparator implements Comparator<StatsTable>
                                                        "compacted_partition_maximum_bytes",
                                                        "compacted_partition_mean_bytes",
                                                        "compacted_partition_minimum_bytes",
+                                                       "compression_dictionaries_memory_used",
                                                        "compression_metadata_off_heap_memory_used", "dropped_mutations",
                                                        "full_name", "index_summary_off_heap_memory_used",
                                                        "local_read_count", "local_read_latency_ms",
@@ -179,6 +180,11 @@ public class StatsTableComparator implements Comparator<StatsTable>
             result = sign * Long.valueOf(stx.compactedPartitionMinimumBytes)
                             .compareTo(Long.valueOf(sty.compactedPartitionMinimumBytes));
         }
+        else if (sortKey.equals("compression_dictionaries_memory_used"))
+        {
+            result = compareFileSizes(stx.compressionDictionariesMemoryUsed,
+                                      sty.compressionDictionariesMemoryUsed);
+        }
         else if (sortKey.equals("compression_metadata_off_heap_memory_used"))
         {
             if (stx.compressionMetadataOffHeapUsed && !sty.compressionMetadataOffHeapUsed)

src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java

@@ -156,6 +156,8 @@ public class TableStatsHolder implements StatsHolder
             mpTable.put("bloom_filter_off_heap_memory_used", table.bloomFilterOffHeapMemoryUsed);
         if (table.indexSummaryOffHeapUsed)
             mpTable.put("index_summary_off_heap_memory_used", table.indexSummaryOffHeapMemoryUsed);
+        if (table.compressionDictionariesUsed)
+            mpTable.put("compression_dictionaries_memory_used", table.compressionDictionariesMemoryUsed);
         if (table.compressionMetadataOffHeapUsed)
             mpTable.put("compression_metadata_off_heap_memory_used",
                         table.compressionMetadataOffHeapMemoryUsed);
@@ -282,6 +284,7 @@ public class TableStatsHolder implements StatsHolder
                 Long bloomFilterOffHeapSize = null;
                 Long indexSummaryOffHeapSize = null;
                 Long compressionMetadataOffHeapSize = null;
+                Long compressionDictionariesMemoryUsed = null;
                 Long offHeapSize = null;
                 Double percentRepaired = null;
                 Long bytesRepaired = null;
@@ -295,6 +298,7 @@ public class TableStatsHolder implements StatsHolder
                     memtableOffHeapSize = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "MemtableOffHeapSize");
                     bloomFilterOffHeapSize = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "BloomFilterOffHeapMemoryUsed");
                     indexSummaryOffHeapSize = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "IndexSummaryOffHeapMemoryUsed");
+                    compressionDictionariesMemoryUsed = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "CompressionDictionariesMemoryUsed");
                     compressionMetadataOffHeapSize = (Long) probe.getColumnFamilyMetric(keyspaceName, tableName, "CompressionMetadataOffHeapMemoryUsed");
                     offHeapSize = memtableOffHeapSize + bloomFilterOffHeapSize + indexSummaryOffHeapSize + compressionMetadataOffHeapSize;
                     percentRepaired = (Double) probe.getColumnFamilyMetric(keyspaceName, tableName, "PercentRepaired");
@@ -380,6 +384,11 @@ public class TableStatsHolder implements StatsHolder
                     statsTable.indexSummaryOffHeapUsed = true;
                     statsTable.indexSummaryOffHeapMemoryUsed = FileUtils.stringifyFileSize(indexSummaryOffHeapSize, humanReadable);
                 }
+                if (compressionDictionariesMemoryUsed != null)
+                {
+                    statsTable.compressionDictionariesUsed = true;
+                    statsTable.compressionDictionariesMemoryUsed = FileUtils.stringifyFileSize(compressionDictionariesMemoryUsed, humanReadable);
+                }
                 if (compressionMetadataOffHeapSize != null)
                 {
                     statsTable.compressionMetadataOffHeapUsed = true;

src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java

@@ -137,6 +137,9 @@ public class TableStatsPrinter<T extends StatsHolder>
                 out.println(indent + "Bloom filter off heap memory used: " + table.bloomFilterOffHeapMemoryUsed);
             if (table.indexSummaryOffHeapUsed)
                 out.println(indent + "Index summary off heap memory used: " + table.indexSummaryOffHeapMemoryUsed);
+            if (table.compressionDictionariesUsed)
+                out.println(indent + "Compression dictionaries memory used: " + table.compressionDictionariesMemoryUsed);
+
             if (table.compressionMetadataOffHeapUsed)
                 out.println(indent + "Compression metadata off heap memory used: " + table.compressionMetadataOffHeapMemoryUsed);
 

test/resources/nodetool/help/tablestats

@@ -46,6 +46,7 @@ OPTIONS
             bloom_filter_off_heap_memory_used, bloom_filter_space_used,
             compacted_partition_maximum_bytes, compacted_partition_mean_bytes,
             compacted_partition_minimum_bytes,
+            compression_dictionaries_memory_used,
             compression_metadata_off_heap_memory_used, full_name,
             index_summary_off_heap_memory_used, local_read_count,
             local_read_latency_ms, local_write_latency_ms,

test/unit/org/apache/cassandra/tools/nodetool/stats/StatsTableComparatorTest.java

@@ -157,6 +157,11 @@ public class StatsTableComparatorTest extends TableStatsTestBase
                        "table1 > table3 > table5 > table2 > table4 > table6",
                        humanReadable,
                        ascending);
+        runCompareTest(humanReadableTables,
+                       "compression_dictionaries_memory_used",
+                       "table1 > table3 > table5 > table2 > table4 > table6",
+                       humanReadable,
+                       ascending);
     }
 
     @Test

test/unit/org/apache/cassandra/tools/nodetool/stats/TableStatsTestBase.java

@@ -92,6 +92,7 @@ public class TableStatsTestBase
         template.bloomFilterFalseRatio = 0D;
         template.bloomFilterSpaceUsed = "0";
         template.indexSummaryOffHeapMemoryUsed = "0";
+        template.compressionDictionariesMemoryUsed = "0";
         template.compressionMetadataOffHeapMemoryUsed = "0";
         template.compactedPartitionMinimumBytes = 0L;
         template.compactedPartitionMaximumBytes = 0L;
@@ -415,6 +416,13 @@ public class TableStatsTestBase
         humanReadableTable4.memtableDataSize = "999 bytes";
         humanReadableTable5.memtableDataSize = "3.14 MiB";
         humanReadableTable6.memtableDataSize = "0 bytes";
+        // human readable compression dictionaries cached memory used: 1 > 3 > 5 > 2 > 4 > 6
+        humanReadableTable1.compressionDictionariesMemoryUsed = "1.21 TiB";
+        humanReadableTable2.compressionDictionariesMemoryUsed = "42 KiB";
+        humanReadableTable3.compressionDictionariesMemoryUsed = "2.71 GiB";
+        humanReadableTable4.compressionDictionariesMemoryUsed = "999 bytes";
+        humanReadableTable5.compressionDictionariesMemoryUsed = "3.14 MiB";
+        humanReadableTable6.compressionDictionariesMemoryUsed = "0 bytes";
 
         // cretae human-readable SAI disk space used size:
         humanReadableTable5.saiDiskUsedBytes = "40 bytes";