From 8974fdb821db6b72eac5a34d96fe1f78ead3c11f Mon Sep 17 00:00:00 2001 From: Alan Wang Date: Fri, 14 Nov 2025 11:02:53 -0800 Subject: [PATCH] Include Level information for UnifiedCompactionStrategy in nodetool tablestats output Patch by Alan Wang, reviewed by Jyothsna Konisa and marcuse for CASSANDRA-20820 --- CHANGES.txt | 1 + .../cassandra/db/ColumnFamilyStore.java | 36 ++++ .../cassandra/db/ColumnFamilyStoreMBean.java | 42 +++++ .../compaction/CompactionStrategyManager.java | 157 ++++++++++++++++++ .../compaction/UnifiedCompactionStrategy.java | 28 +++- .../tools/nodetool/stats/StatsTable.java | 7 + .../nodetool/stats/TableStatsHolder.java | 33 ++++ .../nodetool/stats/TableStatsPrinter.java | 16 ++ .../test/ColumnFamilyStoreMBeansTest.java | 133 +++++++++++++++ .../CompactionStrategyManagerTest.java | 56 +++++++ 10 files changed, 508 insertions(+), 1 deletion(-) create mode 100644 test/distributed/org/apache/cassandra/distributed/test/ColumnFamilyStoreMBeansTest.java diff --git a/CHANGES.txt b/CHANGES.txt index a78c0876ad..c748e17c1f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * Include Level information for UnifiedCompactionStrategy in nodetool tablestats output (CASSANDRA-20820) * Support low-overhead async profiling (CASSANDRA-20854) * Minor perf optimizations around memtable put logic (CASSANDRA-21088) * When level compaction validates its table properties, it used the wrong default value for sstable_size_in_mb which allowed properties that would later be rejected at runtime (CASSANDRA-20570) diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index d93cf1b2c7..a60d4a01a7 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -3021,6 +3021,42 @@ public T withAllSSTables(final OperationType operationType, Function { + data.sum[data.levelIndex] += data.sstable.tokenSpaceCoverage(); + data.count[data.levelIndex]++; + }, + CompactionStrategyManager::averageArrayFinalizer + ); + } + + public double[] getPerLevelMaxDensityThreshold() + { + return computeUCSMetric( + data -> { + data.max[data.levelIndex] = Math.max(data.max[data.levelIndex], data.level.max); + }, + CompactionStrategyManager::maxArrayFinalizer + ); + } + + public double[] getPerLevelAvgSize() + { + return computeUCSMetric( + data -> { + data.sum[data.levelIndex] += data.sstable.onDiskLength(); + data.count[data.levelIndex]++; + }, + CompactionStrategyManager::averageArrayFinalizer + ); + } + + public double[] getPerLevelAvgDensity() + { + return computeUCSMetric( + data -> { + data.sum[data.levelIndex] += data.strategy.getDensity(data.sstable); + data.count[data.levelIndex]++; + }, + CompactionStrategyManager::averageArrayFinalizer + ); + } + + public double[] getPerLevelAvgDensityMaxDensityThresholdRatio() + { + double[] avgDensity = getPerLevelAvgDensity(); + if (avgDensity == null) + return null; + + double[] maxThreshold = getPerLevelMaxDensityThreshold(); + double[] res = new double[avgDensity.length]; + for (int i = 0; i < avgDensity.length; i++) + res[i] = avgDensity[i] / maxThreshold[i]; + return res; + } + + public double[] getPerLevelMaxDensityMaxDensityThresholdRatio() + { + return computeUCSMetric( + data -> { + data.sum[data.levelIndex] = Math.max(data.sum[data.levelIndex], data.strategy.getDensity(data.sstable)); + data.max[data.levelIndex] = Math.max(data.max[data.levelIndex], data.level.max); + }, + CompactionStrategyManager::ratioArrayFinalizer + ); + } + public boolean isLeveledCompaction() { readLock.lock(); @@ -703,6 +773,93 @@ public class CompactionStrategyManager implements INotificationConsumer } } + /** + * Data class for accumulating UCS metrics computation state. + * Holds intermediate values during metric calculation across all strategies and levels. + */ + @VisibleForTesting + static class CompactionStatsMetricsData + { + final double[] sum = new double[UnifiedCompactionStrategy.MAX_LEVELS]; + final int[] count = new int[UnifiedCompactionStrategy.MAX_LEVELS]; + final double[] max = new double[UnifiedCompactionStrategy.MAX_LEVELS]; + int numberOfLevels = 0; + + int levelIndex; + Level level; + SSTableReader sstable; + UnifiedCompactionStrategy strategy; + } + + /** + * Generic helper to compute UCS metrics across all strategies and levels. + * Reduces code duplication for per-level metric calculations. + * + * @param accumulator processes each sstable and updates the metrics data state + * @param finalizer computes the final result array from the accumulated metrics data + * @return computed metric array, one value per level, or null if not using UCS + */ + private double[] computeUCSMetric(Consumer accumulator, Function finalizer) + { + readLock.lock(); + try + { + if (repaired.first() instanceof UnifiedCompactionStrategy) + { + CompactionStatsMetricsData data = new CompactionStatsMetricsData(); + + for (AbstractCompactionStrategy strategy : getAllStrategies()) + { + UnifiedCompactionStrategy ucsStrategy = (UnifiedCompactionStrategy) strategy; + List levels = ucsStrategy.getLevelsSnapshot(); + + data.numberOfLevels = Math.max(data.numberOfLevels, levels.size()); + data.strategy = ucsStrategy; + + for (int i = 0; i < levels.size(); i++) + { + data.levelIndex = i; + data.level = levels.get(i); + for (SSTableReader sstable : levels.get(i).getSSTables()) + { + data.sstable = sstable; + accumulator.accept(data); + } + } + } + + return finalizer.apply(data); + } + return null; + } + finally { + readLock.unlock(); + } + } + + @VisibleForTesting + static double[] averageArrayFinalizer(CompactionStatsMetricsData data) + { + double[] res = new double[data.numberOfLevels]; + for (int i = 0; i < data.numberOfLevels; i++) + res[i] = data.count[i] == 0 ? 0 : data.sum[i] / data.count[i]; + return res; + } + + @VisibleForTesting + static double[] maxArrayFinalizer(CompactionStatsMetricsData data) + { + return Arrays.copyOf(data.max, data.numberOfLevels); + } + + @VisibleForTesting + static double[] ratioArrayFinalizer(CompactionStatsMetricsData data) { + double[] res = new double[data.numberOfLevels]; + for (int i = 0; i < data.numberOfLevels; i++) + res[i] = data.sum[i] / data.max[i]; + return res; + } + public int[] getSSTableCountPerTWCSBucket() { readLock.lock(); diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java index 61717b5e28..f6a3fbaeb9 100644 --- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java @@ -530,6 +530,24 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy return getLevels(getSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction); } + /** + * @return a list of the levels in the compaction hierarchy, that also includes SSTables that + * are currently undergoing compaction. This is used only for table stats so we can have a consistent + * snapshot of the levels. + */ + List getLevelsSnapshot() + { + Set sstables = getSSTables(); + List suitable = new ArrayList<>(sstables.size()); + for (SSTableReader rdr : sstables) + { + if (isSuitableForCompaction(rdr)) + suitable.add(rdr); + } + + return formLevels(suitable); + } + /** * Groups the sstables passed in into levels. This is used by the strategy to determine * new compactions, and by external tools to analyze the strategy decisions. @@ -547,6 +565,14 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy return formLevels(suitable); } + /** + * Returns the density of the SSTable + */ + public double getDensity(SSTableReader sstable) + { + return shardManager.density(sstable); + } + private List formLevels(List suitable) { maybeUpdateShardManager(); @@ -558,7 +584,7 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy Level level = new Level(controller, index, 0, maxDensity); for (SSTableReader candidate : suitable) { - final double density = shardManager.density(candidate); + final double density = getDensity(candidate); if (density < level.max) { level.add(candidate); diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java index 7bb816ed1d..b809ca199f 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java @@ -29,6 +29,7 @@ public class StatsTable public String tableName; public boolean isIndex; public boolean isLeveledSstable = false; + public boolean isUCSSstable = false; public Object sstableCount; public Object oldSSTableCount; public Long maxSSTableSize; @@ -74,6 +75,12 @@ public class StatsTable public long maximumTombstonesPerSliceLastFiveMinutes; public List sstablesInEachLevel = new ArrayList<>(); public List sstableBytesInEachLevel = new ArrayList<>(); + public List sstableAvgTokenSpaceInEachLevel = new ArrayList<>(); + public List sstableMaxDensityThresholdInEachLevel = new ArrayList<>(); + public List sstableAvgSizeInEachLevel = new ArrayList<>(); + public List sstableAvgDensityInEachLevel = new ArrayList<>(); + public List sstableAvgDensityMaxDensityThresholdRatioInEachLevel = new ArrayList<>(); + public List sstableMaxDensityMaxDensityThresholdRatioInEachLevel = new ArrayList<>(); public int[] sstableCountPerTWCSBucket = null; public Boolean isInCorrectLocation = null; // null: option not active public double droppableTombstoneRatio; diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java index 6e05ac5e43..ab9fa5b1fa 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java @@ -133,6 +133,15 @@ public class TableStatsHolder implements StatsHolder mpTable.put("old_sstable_count", table.oldSSTableCount); mpTable.put("sstables_in_each_level", table.sstablesInEachLevel); mpTable.put("sstable_bytes_in_each_level", table.sstableBytesInEachLevel); + if (table.isUCSSstable) + { + mpTable.put("sstable_avg_token_space_in_each_level", table.sstableAvgTokenSpaceInEachLevel); + mpTable.put("sstable_max_density_threshold_in_each_level", table.sstableMaxDensityThresholdInEachLevel); + mpTable.put("sstable_avg_size_in_each_level", table.sstableAvgSizeInEachLevel); + mpTable.put("sstable_avg_density_in_each_level", table.sstableAvgDensityInEachLevel); + mpTable.put("sstable_avg_density_max_density_threshold_ratio_in_each_level", table.sstableAvgDensityMaxDensityThresholdRatioInEachLevel); + mpTable.put("sstable_max_density_max_density_threshold_ratio_in_each_level", table.sstableMaxDensityMaxDensityThresholdRatioInEachLevel); + } mpTable.put("max_sstable_size", table.maxSSTableSize); mpTable.put("twcs", table.twcs); mpTable.put("space_used_live", table.spaceUsedLive); @@ -286,6 +295,18 @@ public class TableStatsHolder implements StatsHolder } } + addUCSMetric(statsTable, statsTable.sstableAvgTokenSpaceInEachLevel, table.getPerLevelAvgTokenSpace()); + + addUCSMetric(statsTable, statsTable.sstableMaxDensityThresholdInEachLevel, table.getPerLevelMaxDensityThreshold()); + + addUCSMetric(statsTable, statsTable.sstableAvgSizeInEachLevel, table.getPerLevelAvgSize()); + + addUCSMetric(statsTable, statsTable.sstableAvgDensityInEachLevel, table.getPerLevelAvgDensity()); + + addUCSMetric(statsTable, statsTable.sstableAvgDensityMaxDensityThresholdRatioInEachLevel, table.getPerLevelAvgDensityMaxDensityThresholdRatio()); + + addUCSMetric(statsTable, statsTable.sstableMaxDensityMaxDensityThresholdRatioInEachLevel, table.getPerLevelMaxDensityMaxDensityThresholdRatio()); + if (locationCheck) statsTable.isInCorrectLocation = !table.hasMisplacedSSTables(); @@ -465,6 +486,18 @@ public class TableStatsHolder implements StatsHolder } } + private void addUCSMetric(StatsTable statsTable, List acc, double[] values) + { + if (values != null) + { + statsTable.isUCSSstable = true; + for (int level = 0; level < values.length; level++) + { + acc.add(String.format("%.03f", values[level])); + } + } + } + private double getMetricMean(Object metricObject) { if (metricObject instanceof CassandraMetricsRegistry.JmxTimerMBean) { return ((CassandraMetricsRegistry.JmxTimerMBean) metricObject).getMean() / 1000; diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java index b500326999..9d1735dbe2 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java @@ -100,6 +100,22 @@ public class TableStatsPrinter table.sstableBytesInEachLevel) + "]"); } + if (table.isUCSSstable) + { + out.println(indent + "Average token space for SSTables in each level: [" + String.join(", ", + table.sstableAvgTokenSpaceInEachLevel) + "]"); + out.println(indent + "Maximum density threshold for SSTables in each level: [" + String.join(", ", + table.sstableMaxDensityThresholdInEachLevel) + "]"); + out.println(indent + "Average SSTable size in each level: [" + String.join(", ", + table.sstableAvgSizeInEachLevel) + "]"); + out.println(indent + "Average SSTable density in each level: [" + String.join(", ", + table.sstableAvgDensityInEachLevel) + "]"); + out.println(indent + "Average SSTable density to max threshold ratio in each level: [" + String.join(", ", + table.sstableAvgDensityMaxDensityThresholdRatioInEachLevel) + "]"); + out.println(indent + "Maximum SSTable density to max threshold ratio in each level: [" + String.join(", ", + table.sstableMaxDensityMaxDensityThresholdRatioInEachLevel) + "]"); + } + out.println(indent + "Space used (live): " + table.spaceUsedLive); out.println(indent + "Space used (total): " + table.spaceUsedTotal); out.println(indent + "Space used by snapshots (total): " + table.spaceUsedBySnapshotsTotal); diff --git a/test/distributed/org/apache/cassandra/distributed/test/ColumnFamilyStoreMBeansTest.java b/test/distributed/org/apache/cassandra/distributed/test/ColumnFamilyStoreMBeansTest.java new file mode 100644 index 0000000000..8bbdb77a6d --- /dev/null +++ b/test/distributed/org/apache/cassandra/distributed/test/ColumnFamilyStoreMBeansTest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.distributed.test; + +import java.io.IOException; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.distributed.Cluster; + +import static org.junit.Assert.assertTrue; + +public class ColumnFamilyStoreMBeansTest extends TestBaseImpl +{ + private static Cluster CLUSTER; + + @BeforeClass + public static void setup() throws IOException + { + CLUSTER = init(Cluster.build(1) + .start()); + + CLUSTER.schemaChange(withKeyspace("DROP KEYSPACE %s")); + CLUSTER.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}")); + CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.cf (k text, c1 text, c2 text, PRIMARY KEY (k)) WITH compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'L10'}")); + + for (int i = 0; i < 10000; i++) + CLUSTER.get(1).executeInternal(withKeyspace("INSERT INTO %s.cf (k, c1, c2) VALUES (?, 'value1', 'value2');"), Integer.toString(i)); + + CLUSTER.get(1).nodetool("flush"); + } + + @AfterClass + public static void teardownCluster() throws Exception + { + if (CLUSTER != null) + CLUSTER.close(); + } + + @Test + public void testPerLevelAverageTokenSpace() throws Throwable + { + CLUSTER.get(1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf"); + double[] perLevelAvgTokenSpace = cfs.getPerLevelAvgTokenSpace(); + assertTrue(perLevelAvgTokenSpace.length > 0); + for (int i = 0; i < perLevelAvgTokenSpace.length; i++) + assertTrue(perLevelAvgTokenSpace[i] > 0); + }); + } + + @Test + public void testGetPerLevelMaxDensityThreshold() throws Throwable + { + CLUSTER.get(1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf"); + assertTrue(cfs.getPerLevelMaxDensityThreshold().length > 0); + }); + } + + @Test + public void testGetPerLevelAvgSize() throws Throwable + { + CLUSTER.get(1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf"); + double[] perLevelAvgSize = cfs.getPerLevelAvgSize(); + assertTrue(perLevelAvgSize.length > 0); + for (int i = 0; i < perLevelAvgSize.length; i++) + assertTrue(perLevelAvgSize[i] > 0); + }); + } + + @Test + public void testGetPerLevelAvgDensity() throws Throwable + { + CLUSTER.get(1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf"); + double[] perLevelAvgDensity = cfs.getPerLevelAvgDensity(); + assertTrue(perLevelAvgDensity.length > 0); + for (int i = 0; i < perLevelAvgDensity.length; i++) + assertTrue(perLevelAvgDensity[i] > 0); + }); + } + + @Test + public void testGetPerLevelAvgDensityMaxDensityThresholdRatio() throws Throwable + { + CLUSTER.get(1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf"); + double[] perLevelAvgDensityMaxDensityThresholdRatio = cfs.getPerLevelAvgDensityMaxDensityThresholdRatio(); + assertTrue(perLevelAvgDensityMaxDensityThresholdRatio.length > 0); + for (int i = 0; i < perLevelAvgDensityMaxDensityThresholdRatio.length; i++) + { + assertTrue(0 <= perLevelAvgDensityMaxDensityThresholdRatio[i]); + assertTrue(perLevelAvgDensityMaxDensityThresholdRatio[i] < 1); + } + }); + } + + @Test + public void testGetPerLevelMaxDensityMaxDensityThresholdRatio() throws Throwable + { + CLUSTER.get(1).runOnInstance(() -> { + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf"); + double[] perLevelMaxDensityMaxDensityThresholdRatio = cfs.getPerLevelMaxDensityMaxDensityThresholdRatio(); + assertTrue(perLevelMaxDensityMaxDensityThresholdRatio.length > 0); + for (int i = 0; i < perLevelMaxDensityMaxDensityThresholdRatio.length; i++) { + assertTrue(0 <= perLevelMaxDensityMaxDensityThresholdRatio[i]); + assertTrue(perLevelMaxDensityMaxDensityThresholdRatio[i] < 1); + } + }); + } +} diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java index 7907b1dd30..b52b550187 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionStrategyManagerTest.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; @@ -434,6 +435,61 @@ public class CompactionStrategyManagerTest extends CassandraTestBase Collections.emptyMap()), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX)); } + @Test + public void testAverageArrayFinalizer() + { + CompactionStrategyManager.CompactionStatsMetricsData data = new CompactionStrategyManager.CompactionStatsMetricsData(); + Random random = new Random(); + data.numberOfLevels = random.nextInt(32); + + for (int i = 0; i < data.numberOfLevels; i++) { + data.sum[i] = random.nextInt(250); + data.count[i] = random.nextInt(10); + } + + double[] res = CompactionStrategyManager.averageArrayFinalizer(data); + assertEquals(res.length, data.numberOfLevels); + for (int i = 0; i < data.numberOfLevels; i++) + if (data.count[i] == 0) + assertEquals(0, res[i], 0.0); + else + assertEquals(data.sum[i] / data.count[i], res[i], 0.1); + } + + @Test + public void testMaxArrayFinalizer() + { + CompactionStrategyManager.CompactionStatsMetricsData data = new CompactionStrategyManager.CompactionStatsMetricsData(); + Random random = new Random(); + data.numberOfLevels = random.nextInt(32); + + for (int i = 0; i < data.numberOfLevels; i++) + data.max[i] = random.nextInt(250); + + double[] res = CompactionStrategyManager.maxArrayFinalizer(data); + assertEquals(res.length, data.numberOfLevels); + for (int i = 0; i < data.numberOfLevels; i++) + assertEquals(data.max[i], res[i], 0.0); + } + + @Test + public void testRatioArrayFinalizer() + { + CompactionStrategyManager.CompactionStatsMetricsData data = new CompactionStrategyManager.CompactionStatsMetricsData(); + Random random = new Random(); + data.numberOfLevels = random.nextInt(32); + + for (int i = 0; i < data.numberOfLevels; i++) { + data.sum[i] = random.nextInt(250); + data.max[i] = 250 + random.nextInt(500); + } + + double[] res = CompactionStrategyManager.ratioArrayFinalizer(data); + assertEquals(res.length, data.numberOfLevels); + for (int i = 0; i < data.numberOfLevels; i++) + assertEquals(data.sum[i] / data.max[i], res[i], 0.1); + } + private MockCFS createJBODMockCFS(int disks) { // Create #disks data directories to simulate JBOD