Include Level information for UnifiedCompactionStrategy in nodetool tablestats output

Patch by Alan Wang, reviewed by Jyothsna Konisa and marcuse for CASSANDRA-20820
This commit is contained in:
Alan Wang 2025-11-14 11:02:53 -08:00 committed by Marcus Eriksson
parent a3c40f1a67
commit 8974fdb821
10 changed files with 508 additions and 1 deletions

View File

@ -1,4 +1,5 @@
5.1
* Include Level information for UnifiedCompactionStrategy in nodetool tablestats output (CASSANDRA-20820)
* Support low-overhead async profiling (CASSANDRA-20854)
* Minor perf optimizations around memtable put logic (CASSANDRA-21088)
* When level compaction validates its table properties, it used the wrong default value for sstable_size_in_mb which allowed properties that would later be rejected at runtime (CASSANDRA-20570)

View File

@ -3021,6 +3021,42 @@ public <T> T withAllSSTables(final OperationType operationType, Function<Lifecyc
return compactionStrategyManager.getPerLevelSizeBytes();
}
@Override
public double[] getPerLevelAvgTokenSpace()
{
return compactionStrategyManager.getPerLevelAvgTokenSpace();
}
@Override
public double[] getPerLevelMaxDensityThreshold()
{
return compactionStrategyManager.getPerLevelMaxDensityThreshold();
}
@Override
public double[] getPerLevelAvgSize()
{
return compactionStrategyManager.getPerLevelAvgSize();
}
@Override
public double[] getPerLevelAvgDensity()
{
return compactionStrategyManager.getPerLevelAvgDensity();
}
@Override
public double[] getPerLevelAvgDensityMaxDensityThresholdRatio()
{
return compactionStrategyManager.getPerLevelAvgDensityMaxDensityThresholdRatio();
}
@Override
public double[] getPerLevelMaxDensityMaxDensityThresholdRatio()
{
return compactionStrategyManager.getPerLevelMaxDensityMaxDensityThresholdRatio();
}
@Override
public boolean isLeveledCompaction()
{

View File

@ -277,6 +277,48 @@ public interface ColumnFamilyStoreMBean
*/
public long[] getPerLevelSizeBytes();
/**
* @return average of sstable covered token spaces in each level.
* null unless unified compaction strategy is used.
* array index corresponds to level(int[0] is for level 0, ...).
*/
public double[] getPerLevelAvgTokenSpace();
/**
* @return the maximum density each level is allowed to hold.
* null unless unified compaction strategy is used.
* array index corresponds to level(int[0] is for level 0, ...).
*/
public double[] getPerLevelMaxDensityThreshold();
/**
* @return the average size of sstables in each level.
* null unless unified compaction strategy is used.
* array index corresponds to level(int[0] is for level 0, ...).
*/
public double[] getPerLevelAvgSize();
/**
* @return the average density of sstables in each level.
* null unless unified compaction strategy is used.
* array index corresponds to level(int[0] is for level 0, ...).
*/
public double[] getPerLevelAvgDensity();
/**
* @return the ratio of avg density to the maximum density threshold of that level
* in each level. null unless unified compaction strategy is used.
* array index corresponds to level(int[0] is for level 0, ...).
*/
public double[] getPerLevelAvgDensityMaxDensityThresholdRatio();
/**
* @return the ratio of maximum density to the maximum density threshold of that level
* in each level. null unless unified compaction strategy is used.
* array index corresponds to level(int[0] is for level 0, ...).
*/
public double[] getPerLevelMaxDensityMaxDensityThresholdRatio();
/**
* @return true if the table is using LeveledCompactionStrategy. false otherwise.
*/

View File

@ -32,6 +32,8 @@ import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -55,6 +57,7 @@ import org.apache.cassandra.db.commitlog.CommitLogPosition;
import org.apache.cassandra.db.commitlog.IntervalSet;
import org.apache.cassandra.db.compaction.AbstractStrategyHolder.TaskSupplier;
import org.apache.cassandra.db.compaction.PendingRepairManager.CleanupTask;
import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy.Level;
import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.lifecycle.SSTableSet;
@ -691,6 +694,73 @@ public class CompactionStrategyManager implements INotificationConsumer
}
}
public double[] getPerLevelAvgTokenSpace()
{
return computeUCSMetric(
data -> {
data.sum[data.levelIndex] += data.sstable.tokenSpaceCoverage();
data.count[data.levelIndex]++;
},
CompactionStrategyManager::averageArrayFinalizer
);
}
public double[] getPerLevelMaxDensityThreshold()
{
return computeUCSMetric(
data -> {
data.max[data.levelIndex] = Math.max(data.max[data.levelIndex], data.level.max);
},
CompactionStrategyManager::maxArrayFinalizer
);
}
public double[] getPerLevelAvgSize()
{
return computeUCSMetric(
data -> {
data.sum[data.levelIndex] += data.sstable.onDiskLength();
data.count[data.levelIndex]++;
},
CompactionStrategyManager::averageArrayFinalizer
);
}
public double[] getPerLevelAvgDensity()
{
return computeUCSMetric(
data -> {
data.sum[data.levelIndex] += data.strategy.getDensity(data.sstable);
data.count[data.levelIndex]++;
},
CompactionStrategyManager::averageArrayFinalizer
);
}
public double[] getPerLevelAvgDensityMaxDensityThresholdRatio()
{
double[] avgDensity = getPerLevelAvgDensity();
if (avgDensity == null)
return null;
double[] maxThreshold = getPerLevelMaxDensityThreshold();
double[] res = new double[avgDensity.length];
for (int i = 0; i < avgDensity.length; i++)
res[i] = avgDensity[i] / maxThreshold[i];
return res;
}
public double[] getPerLevelMaxDensityMaxDensityThresholdRatio()
{
return computeUCSMetric(
data -> {
data.sum[data.levelIndex] = Math.max(data.sum[data.levelIndex], data.strategy.getDensity(data.sstable));
data.max[data.levelIndex] = Math.max(data.max[data.levelIndex], data.level.max);
},
CompactionStrategyManager::ratioArrayFinalizer
);
}
public boolean isLeveledCompaction()
{
readLock.lock();
@ -703,6 +773,93 @@ public class CompactionStrategyManager implements INotificationConsumer
}
}
/**
* Data class for accumulating UCS metrics computation state.
* Holds intermediate values during metric calculation across all strategies and levels.
*/
@VisibleForTesting
static class CompactionStatsMetricsData
{
final double[] sum = new double[UnifiedCompactionStrategy.MAX_LEVELS];
final int[] count = new int[UnifiedCompactionStrategy.MAX_LEVELS];
final double[] max = new double[UnifiedCompactionStrategy.MAX_LEVELS];
int numberOfLevels = 0;
int levelIndex;
Level level;
SSTableReader sstable;
UnifiedCompactionStrategy strategy;
}
/**
* Generic helper to compute UCS metrics across all strategies and levels.
* Reduces code duplication for per-level metric calculations.
*
* @param accumulator processes each sstable and updates the metrics data state
* @param finalizer computes the final result array from the accumulated metrics data
* @return computed metric array, one value per level, or null if not using UCS
*/
private double[] computeUCSMetric(Consumer<CompactionStatsMetricsData> accumulator, Function<CompactionStatsMetricsData, double[]> finalizer)
{
readLock.lock();
try
{
if (repaired.first() instanceof UnifiedCompactionStrategy)
{
CompactionStatsMetricsData data = new CompactionStatsMetricsData();
for (AbstractCompactionStrategy strategy : getAllStrategies())
{
UnifiedCompactionStrategy ucsStrategy = (UnifiedCompactionStrategy) strategy;
List<Level> levels = ucsStrategy.getLevelsSnapshot();
data.numberOfLevels = Math.max(data.numberOfLevels, levels.size());
data.strategy = ucsStrategy;
for (int i = 0; i < levels.size(); i++)
{
data.levelIndex = i;
data.level = levels.get(i);
for (SSTableReader sstable : levels.get(i).getSSTables())
{
data.sstable = sstable;
accumulator.accept(data);
}
}
}
return finalizer.apply(data);
}
return null;
}
finally {
readLock.unlock();
}
}
@VisibleForTesting
static double[] averageArrayFinalizer(CompactionStatsMetricsData data)
{
double[] res = new double[data.numberOfLevels];
for (int i = 0; i < data.numberOfLevels; i++)
res[i] = data.count[i] == 0 ? 0 : data.sum[i] / data.count[i];
return res;
}
@VisibleForTesting
static double[] maxArrayFinalizer(CompactionStatsMetricsData data)
{
return Arrays.copyOf(data.max, data.numberOfLevels);
}
@VisibleForTesting
static double[] ratioArrayFinalizer(CompactionStatsMetricsData data) {
double[] res = new double[data.numberOfLevels];
for (int i = 0; i < data.numberOfLevels; i++)
res[i] = data.sum[i] / data.max[i];
return res;
}
public int[] getSSTableCountPerTWCSBucket()
{
readLock.lock();

View File

@ -530,6 +530,24 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
return getLevels(getSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction);
}
/**
* @return a list of the levels in the compaction hierarchy, that also includes SSTables that
* are currently undergoing compaction. This is used only for table stats so we can have a consistent
* snapshot of the levels.
*/
List<Level> getLevelsSnapshot()
{
Set<SSTableReader> sstables = getSSTables();
List<SSTableReader> suitable = new ArrayList<>(sstables.size());
for (SSTableReader rdr : sstables)
{
if (isSuitableForCompaction(rdr))
suitable.add(rdr);
}
return formLevels(suitable);
}
/**
* Groups the sstables passed in into levels. This is used by the strategy to determine
* new compactions, and by external tools to analyze the strategy decisions.
@ -547,6 +565,14 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
return formLevels(suitable);
}
/**
* Returns the density of the SSTable
*/
public double getDensity(SSTableReader sstable)
{
return shardManager.density(sstable);
}
private List<Level> formLevels(List<SSTableReader> suitable)
{
maybeUpdateShardManager();
@ -558,7 +584,7 @@ public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
Level level = new Level(controller, index, 0, maxDensity);
for (SSTableReader candidate : suitable)
{
final double density = shardManager.density(candidate);
final double density = getDensity(candidate);
if (density < level.max)
{
level.add(candidate);

View File

@ -29,6 +29,7 @@ public class StatsTable
public String tableName;
public boolean isIndex;
public boolean isLeveledSstable = false;
public boolean isUCSSstable = false;
public Object sstableCount;
public Object oldSSTableCount;
public Long maxSSTableSize;
@ -74,6 +75,12 @@ public class StatsTable
public long maximumTombstonesPerSliceLastFiveMinutes;
public List<String> sstablesInEachLevel = new ArrayList<>();
public List<String> sstableBytesInEachLevel = new ArrayList<>();
public List<String> sstableAvgTokenSpaceInEachLevel = new ArrayList<>();
public List<String> sstableMaxDensityThresholdInEachLevel = new ArrayList<>();
public List<String> sstableAvgSizeInEachLevel = new ArrayList<>();
public List<String> sstableAvgDensityInEachLevel = new ArrayList<>();
public List<String> sstableAvgDensityMaxDensityThresholdRatioInEachLevel = new ArrayList<>();
public List<String> sstableMaxDensityMaxDensityThresholdRatioInEachLevel = new ArrayList<>();
public int[] sstableCountPerTWCSBucket = null;
public Boolean isInCorrectLocation = null; // null: option not active
public double droppableTombstoneRatio;

View File

@ -133,6 +133,15 @@ public class TableStatsHolder implements StatsHolder
mpTable.put("old_sstable_count", table.oldSSTableCount);
mpTable.put("sstables_in_each_level", table.sstablesInEachLevel);
mpTable.put("sstable_bytes_in_each_level", table.sstableBytesInEachLevel);
if (table.isUCSSstable)
{
mpTable.put("sstable_avg_token_space_in_each_level", table.sstableAvgTokenSpaceInEachLevel);
mpTable.put("sstable_max_density_threshold_in_each_level", table.sstableMaxDensityThresholdInEachLevel);
mpTable.put("sstable_avg_size_in_each_level", table.sstableAvgSizeInEachLevel);
mpTable.put("sstable_avg_density_in_each_level", table.sstableAvgDensityInEachLevel);
mpTable.put("sstable_avg_density_max_density_threshold_ratio_in_each_level", table.sstableAvgDensityMaxDensityThresholdRatioInEachLevel);
mpTable.put("sstable_max_density_max_density_threshold_ratio_in_each_level", table.sstableMaxDensityMaxDensityThresholdRatioInEachLevel);
}
mpTable.put("max_sstable_size", table.maxSSTableSize);
mpTable.put("twcs", table.twcs);
mpTable.put("space_used_live", table.spaceUsedLive);
@ -286,6 +295,18 @@ public class TableStatsHolder implements StatsHolder
}
}
addUCSMetric(statsTable, statsTable.sstableAvgTokenSpaceInEachLevel, table.getPerLevelAvgTokenSpace());
addUCSMetric(statsTable, statsTable.sstableMaxDensityThresholdInEachLevel, table.getPerLevelMaxDensityThreshold());
addUCSMetric(statsTable, statsTable.sstableAvgSizeInEachLevel, table.getPerLevelAvgSize());
addUCSMetric(statsTable, statsTable.sstableAvgDensityInEachLevel, table.getPerLevelAvgDensity());
addUCSMetric(statsTable, statsTable.sstableAvgDensityMaxDensityThresholdRatioInEachLevel, table.getPerLevelAvgDensityMaxDensityThresholdRatio());
addUCSMetric(statsTable, statsTable.sstableMaxDensityMaxDensityThresholdRatioInEachLevel, table.getPerLevelMaxDensityMaxDensityThresholdRatio());
if (locationCheck)
statsTable.isInCorrectLocation = !table.hasMisplacedSSTables();
@ -465,6 +486,18 @@ public class TableStatsHolder implements StatsHolder
}
}
private void addUCSMetric(StatsTable statsTable, List<String> acc, double[] values)
{
if (values != null)
{
statsTable.isUCSSstable = true;
for (int level = 0; level < values.length; level++)
{
acc.add(String.format("%.03f", values[level]));
}
}
}
private double getMetricMean(Object metricObject) {
if (metricObject instanceof CassandraMetricsRegistry.JmxTimerMBean) {
return ((CassandraMetricsRegistry.JmxTimerMBean) metricObject).getMean() / 1000;

View File

@ -100,6 +100,22 @@ public class TableStatsPrinter<T extends StatsHolder>
table.sstableBytesInEachLevel) + "]");
}
if (table.isUCSSstable)
{
out.println(indent + "Average token space for SSTables in each level: [" + String.join(", ",
table.sstableAvgTokenSpaceInEachLevel) + "]");
out.println(indent + "Maximum density threshold for SSTables in each level: [" + String.join(", ",
table.sstableMaxDensityThresholdInEachLevel) + "]");
out.println(indent + "Average SSTable size in each level: [" + String.join(", ",
table.sstableAvgSizeInEachLevel) + "]");
out.println(indent + "Average SSTable density in each level: [" + String.join(", ",
table.sstableAvgDensityInEachLevel) + "]");
out.println(indent + "Average SSTable density to max threshold ratio in each level: [" + String.join(", ",
table.sstableAvgDensityMaxDensityThresholdRatioInEachLevel) + "]");
out.println(indent + "Maximum SSTable density to max threshold ratio in each level: [" + String.join(", ",
table.sstableMaxDensityMaxDensityThresholdRatioInEachLevel) + "]");
}
out.println(indent + "Space used (live): " + table.spaceUsedLive);
out.println(indent + "Space used (total): " + table.spaceUsedTotal);
out.println(indent + "Space used by snapshots (total): " + table.spaceUsedBySnapshotsTotal);

View File

@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.distributed.test;
import java.io.IOException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.distributed.Cluster;
import static org.junit.Assert.assertTrue;
public class ColumnFamilyStoreMBeansTest extends TestBaseImpl
{
private static Cluster CLUSTER;
@BeforeClass
public static void setup() throws IOException
{
CLUSTER = init(Cluster.build(1)
.start());
CLUSTER.schemaChange(withKeyspace("DROP KEYSPACE %s"));
CLUSTER.schemaChange(withKeyspace("CREATE KEYSPACE %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}"));
CLUSTER.schemaChange(withKeyspace("CREATE TABLE %s.cf (k text, c1 text, c2 text, PRIMARY KEY (k)) WITH compaction = {'class': 'UnifiedCompactionStrategy', 'scaling_parameters': 'L10'}"));
for (int i = 0; i < 10000; i++)
CLUSTER.get(1).executeInternal(withKeyspace("INSERT INTO %s.cf (k, c1, c2) VALUES (?, 'value1', 'value2');"), Integer.toString(i));
CLUSTER.get(1).nodetool("flush");
}
@AfterClass
public static void teardownCluster() throws Exception
{
if (CLUSTER != null)
CLUSTER.close();
}
@Test
public void testPerLevelAverageTokenSpace() throws Throwable
{
CLUSTER.get(1).runOnInstance(() -> {
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf");
double[] perLevelAvgTokenSpace = cfs.getPerLevelAvgTokenSpace();
assertTrue(perLevelAvgTokenSpace.length > 0);
for (int i = 0; i < perLevelAvgTokenSpace.length; i++)
assertTrue(perLevelAvgTokenSpace[i] > 0);
});
}
@Test
public void testGetPerLevelMaxDensityThreshold() throws Throwable
{
CLUSTER.get(1).runOnInstance(() -> {
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf");
assertTrue(cfs.getPerLevelMaxDensityThreshold().length > 0);
});
}
@Test
public void testGetPerLevelAvgSize() throws Throwable
{
CLUSTER.get(1).runOnInstance(() -> {
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf");
double[] perLevelAvgSize = cfs.getPerLevelAvgSize();
assertTrue(perLevelAvgSize.length > 0);
for (int i = 0; i < perLevelAvgSize.length; i++)
assertTrue(perLevelAvgSize[i] > 0);
});
}
@Test
public void testGetPerLevelAvgDensity() throws Throwable
{
CLUSTER.get(1).runOnInstance(() -> {
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf");
double[] perLevelAvgDensity = cfs.getPerLevelAvgDensity();
assertTrue(perLevelAvgDensity.length > 0);
for (int i = 0; i < perLevelAvgDensity.length; i++)
assertTrue(perLevelAvgDensity[i] > 0);
});
}
@Test
public void testGetPerLevelAvgDensityMaxDensityThresholdRatio() throws Throwable
{
CLUSTER.get(1).runOnInstance(() -> {
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf");
double[] perLevelAvgDensityMaxDensityThresholdRatio = cfs.getPerLevelAvgDensityMaxDensityThresholdRatio();
assertTrue(perLevelAvgDensityMaxDensityThresholdRatio.length > 0);
for (int i = 0; i < perLevelAvgDensityMaxDensityThresholdRatio.length; i++)
{
assertTrue(0 <= perLevelAvgDensityMaxDensityThresholdRatio[i]);
assertTrue(perLevelAvgDensityMaxDensityThresholdRatio[i] < 1);
}
});
}
@Test
public void testGetPerLevelMaxDensityMaxDensityThresholdRatio() throws Throwable
{
CLUSTER.get(1).runOnInstance(() -> {
ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore("cf");
double[] perLevelMaxDensityMaxDensityThresholdRatio = cfs.getPerLevelMaxDensityMaxDensityThresholdRatio();
assertTrue(perLevelMaxDensityMaxDensityThresholdRatio.length > 0);
for (int i = 0; i < perLevelMaxDensityMaxDensityThresholdRatio.length; i++) {
assertTrue(0 <= perLevelMaxDensityMaxDensityThresholdRatio[i]);
assertTrue(perLevelMaxDensityMaxDensityThresholdRatio[i] < 1);
}
});
}
}

View File

@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicInteger;
@ -434,6 +435,61 @@ public class CompactionStrategyManagerTest extends CassandraTestBase
Collections.emptyMap()), CompactionStrategyManager.TWCS_BUCKET_COUNT_MAX));
}
@Test
public void testAverageArrayFinalizer()
{
CompactionStrategyManager.CompactionStatsMetricsData data = new CompactionStrategyManager.CompactionStatsMetricsData();
Random random = new Random();
data.numberOfLevels = random.nextInt(32);
for (int i = 0; i < data.numberOfLevels; i++) {
data.sum[i] = random.nextInt(250);
data.count[i] = random.nextInt(10);
}
double[] res = CompactionStrategyManager.averageArrayFinalizer(data);
assertEquals(res.length, data.numberOfLevels);
for (int i = 0; i < data.numberOfLevels; i++)
if (data.count[i] == 0)
assertEquals(0, res[i], 0.0);
else
assertEquals(data.sum[i] / data.count[i], res[i], 0.1);
}
@Test
public void testMaxArrayFinalizer()
{
CompactionStrategyManager.CompactionStatsMetricsData data = new CompactionStrategyManager.CompactionStatsMetricsData();
Random random = new Random();
data.numberOfLevels = random.nextInt(32);
for (int i = 0; i < data.numberOfLevels; i++)
data.max[i] = random.nextInt(250);
double[] res = CompactionStrategyManager.maxArrayFinalizer(data);
assertEquals(res.length, data.numberOfLevels);
for (int i = 0; i < data.numberOfLevels; i++)
assertEquals(data.max[i], res[i], 0.0);
}
@Test
public void testRatioArrayFinalizer()
{
CompactionStrategyManager.CompactionStatsMetricsData data = new CompactionStrategyManager.CompactionStatsMetricsData();
Random random = new Random();
data.numberOfLevels = random.nextInt(32);
for (int i = 0; i < data.numberOfLevels; i++) {
data.sum[i] = random.nextInt(250);
data.max[i] = 250 + random.nextInt(500);
}
double[] res = CompactionStrategyManager.ratioArrayFinalizer(data);
assertEquals(res.length, data.numberOfLevels);
for (int i = 0; i < data.numberOfLevels; i++)
assertEquals(data.sum[i] / data.max[i], res[i], 0.1);
}
private MockCFS createJBODMockCFS(int disks)
{
// Create #disks data directories to simulate JBOD