natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enhance nodetool compactionhistory to report compaction type and strategy 

patch by Arvind Kandpal; reviewed by Maxwell Guo and Jyothsna Konisa for CASSANDRA-20081
This commit is contained in:
Maxwell Guo 2026-01-26 18:38:00 +08:00
parent afa55123c8
commit 6d60102422
6 changed files with 97 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/java/org/apache/cassandra/db/compaction/CompactionHistoryTabularData.java
View File

@ -42,7 +42,8 @@ public class CompactionHistoryTabularData
private static final String[] ITEM_DESCS = new String[]{ "time uuid", "keyspace name",
"column family name", "compaction finished at",
"total bytes in", "total bytes out", "total rows merged", "compaction properties" };
"total bytes in", "total bytes out", "total rows merged",
"compaction properties" };
private static final String TYPE_NAME = "CompactionHistory";
@ -53,10 +54,10 @@ public class CompactionHistoryTabularData
private static final CompositeType COMPOSITE_TYPE;
private static final TabularType TABULAR_TYPE;
public static final String COMPACTION_TYPE_PROPERTY = "compaction_type";
static
static
{
try
{

13
src/java/org/apache/cassandra/db/compaction/CompactionTask.java
View File

@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.RateLimiter;
@ -64,7 +63,6 @@ import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import org.apache.cassandra.utils.concurrent.Refs;
import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.cassandra.utils.Clock.Global.nanoTime;
@ -325,8 +323,15 @@ public class CompactionTask extends AbstractCompactionTask
for (int i = 0; i < mergedRowCounts.length; i++)
totalSourceRows += mergedRowCounts[i] * (i + 1);
String mergeSummary = updateCompactionHistory(taskId, cfs.getKeyspaceName(), cfs.getTableName(), mergedRowCounts, startsize, endsize,
ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType.type));
Map<String, String> props = new HashMap<>();
props.put("strategy", cfs.getCompactionStrategyManager().getCompactionParams().klass().getSimpleName());
props.put(CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY, compactionType.type);
if (getLevel() > 0)
props.put("level", Integer.toString(getLevel()));
String mergeSummary = updateCompactionHistory(taskId, cfs.getKeyspaceName(), cfs.getTableName(), mergedRowCounts, startsize, endsize, props);
logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d. %s to %s (~%d%% of original) in %,dms. Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s. %,d total partitions merged to %,d. Partition merge counts were {%s}. Time spent writing keys = %,dms",
transaction.opIdString(),

9
test/distributed/org/apache/cassandra/distributed/upgrade/CompactionHistorySystemTableUpgradeTest.java
View File

@ -66,14 +66,17 @@ public class CompactionHistorySystemTableUpgradeTest extends UpgradeTestBase
cluster.stream().forEach(node -> node.nodetool("disableautocompaction"));
ToolRunner.ToolResult toolHistory = invokeNodetoolJvmDtest(cluster.get(1), "compactionhistory");
toolHistory.assertOnCleanExit();
// upgraded system.compaction_history data verify
assertCompactionHistoryOutPut(toolHistory, KEYSPACE, "tb", ImmutableMap.of());
assertCompactionHistoryOutPut(toolHistory, KEYSPACE, "tb", ImmutableMap.of(), "UNKNOWN");
// force compact
cluster.stream().forEach(node -> node.nodetool("compact"));
toolHistory = invokeNodetoolJvmDtest(cluster.get(1), "compactionhistory");
toolHistory.assertOnCleanExit();
assertCompactionHistoryOutPut(toolHistory, KEYSPACE, "tb", ImmutableMap.of(COMPACTION_TYPE_PROPERTY, OperationType.MAJOR_COMPACTION.type));
assertCompactionHistoryOutPut(toolHistory, KEYSPACE, "tb",
ImmutableMap.of(COMPACTION_TYPE_PROPERTY, OperationType.MAJOR_COMPACTION.type),
OperationType.MAJOR_COMPACTION.type);
})
.run();
}

35
test/unit/org/apache/cassandra/db/SystemKeyspaceTest.java
View File

@ -22,8 +22,10 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.BeforeClass;
@ -45,6 +47,7 @@ import org.apache.cassandra.transport.ProtocolVersion;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.CassandraVersion;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.TimeUUID;
import static java.lang.String.format;
import static org.apache.cassandra.cql3.QueryProcessor.executeInternal;
@ -163,6 +166,38 @@ public class SystemKeyspaceTest
assertEquals(DatabaseDescriptor.getStoragePort(), row.getInt("listen_port"));
}
@Test
public void testCompactionHistory()
{
String ks = "test_ks";
String cf = "test_cf";
long now = System.currentTimeMillis();
Map<Integer, Long> rowsMerged = Collections.singletonMap(1, 100L);
Map<String, String> props = Collections.singletonMap("strategy", "STCS");
String compactionType = "TestMajor";
Map<String, String> propertiesWithType = new HashMap<>(props);
propertiesWithType.put("compaction_type", compactionType);
SystemKeyspace.updateCompactionHistory(
TimeUUID.Generator.nextTimeUUID(),
ks,
cf,
now,
1000,
500,
rowsMerged,
propertiesWithType
);
UntypedResultSet result = executeInternal("SELECT compaction_properties FROM system.compaction_history WHERE keyspace_name=? AND columnfamily_name=? ALLOW FILTERING", ks, cf);
assertNotNull(result);
Map<String, String> resProps = result.one().getMap("compaction_properties", org.apache.cassandra.db.marshal.UTF8Type.instance, org.apache.cassandra.db.marshal.UTF8Type.instance);
assertEquals(compactionType, resProps.get("compaction_type"));
}
private String getOlderVersionString()
{
String version = FBUtilities.getReleaseVersionString();

8
test/unit/org/apache/cassandra/db/compaction/CompactionTaskTest.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.Assert;
@ -37,6 +38,7 @@ import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.SystemKeyspace;
import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.Schema;
@ -92,7 +94,7 @@ public class CompactionTaskTest
task.execute(CompactionManager.instance.active);
}
UntypedResultSet rows = QueryProcessor.executeInternal(format("SELECT id FROM system.%s where id = %s",
UntypedResultSet rows = QueryProcessor.executeInternal(format("SELECT id, compaction_properties FROM system.%s where id = %s",
SystemKeyspace.COMPACTION_HISTORY,
id.toString()));
@ -103,6 +105,10 @@ public class CompactionTaskTest
TimeUUID persistedId = one.getTimeUUID("id");
Assert.assertEquals(id, persistedId);
Map<String, String> properties = one.getMap("compaction_properties", UTF8Type.instance, UTF8Type.instance);
Assert.assertTrue("Strategy missing in properties", properties.containsKey("strategy"));
Assert.assertEquals("Compaction", properties.get("compaction_type"));
}
@Test

46
test/unit/org/apache/cassandra/tools/nodetool/CompactionHistoryTest.java
View File

@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@ -43,7 +44,6 @@ import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.db.compaction.CompactionHistoryTabularData.COMPACTION_TYPE_PROPERTY;
import static org.apache.cassandra.tools.ToolRunner.invokeNodetool;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertTrue;
@ -102,35 +102,59 @@ public class CompactionHistoryTest extends CQLTester
ImmutableList.Builder<String> builder = ImmutableList.builder();
List<String> cmds = builder.addAll(cmd).add(keyspace()).add(currentTable()).build();
compactionHistoryResultVerify(keyspace(), currentTable(), ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType), cmds);
String cql = "select keyspace_name,columnfamily_name,compaction_properties from system." + SystemKeyspace.COMPACTION_HISTORY +
Map<String, String> baseProperties = ImmutableMap.of("strategy", "SizeTieredCompactionStrategy");
Map<String, String> expectedProperties = new HashMap<>(baseProperties);
expectedProperties.put("compaction_type", compactionType);
compactionHistoryResultVerify(keyspace(), currentTable(), expectedProperties, compactionType, cmds);
String cql = "select keyspace_name,columnfamily_name,compaction_properties from system." + SystemKeyspace.COMPACTION_HISTORY +
" where keyspace_name = '" + keyspace() + "' AND columnfamily_name = '" + currentTable() + "' ALLOW FILTERING";
Object[][] objects = new Object[systemTableRecord][];
for (int i = 0; i != systemTableRecord; ++i)
{
objects[i] = row(keyspace(), currentTable(), ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType));
objects[i] = row(keyspace(), currentTable(), expectedProperties);
}
assertRows(execute(cql), objects);
}
private void compactionHistoryResultVerify(String keyspace, String table, Map<String, String> properties, List<String> cmds)
private void compactionHistoryResultVerify(String keyspace, String table, Map<String, String> properties, String compType, List<String> cmds)
{
ToolResult toolCompact = invokeNodetool(cmds);
toolCompact.assertOnCleanExit();
ToolResult toolHistory = invokeNodetool("compactionhistory");
toolHistory.assertOnCleanExit();
assertCompactionHistoryOutPut(toolHistory, keyspace, table, properties);
assertCompactionHistoryOutPut(toolHistory, keyspace, table, properties, compType);
}
public static void assertCompactionHistoryOutPut(ToolResult toolHistory, String keyspace, String table, Map<String, String> properties)
public static void assertCompactionHistoryOutPut(ToolResult toolHistory, String keyspace, String table, Map<String, String> properties, String compType)
{
String stdout = toolHistory.getStdout();
String[] resultArray = stdout.split(System.lineSeparator());
assertTrue(Arrays.stream(resultArray)
.anyMatch(result -> result.contains('{' + FBUtilities.toString(properties) + '}')
&& result.contains(keyspace)
&& result.contains(table)));
boolean matchFound = Arrays.stream(resultArray).anyMatch(result -> {
if (!result.contains(keyspace) || !result.contains(table) || !result.contains(compType))
{
return false;
}
for (Map.Entry<String, String> entry : properties.entrySet())
{
if (!result.contains(entry.getKey()) || !result.contains(entry.getValue()))
{
return false;
}
}
return true;
});
assertTrue("Output did not contain expected data.\nExpected KS: " + keyspace +
"\nTable: " + table +
"\nType: " + compType +
"\nProps: " + properties +
"\nActual Output:\n" + stdout, matchFound);
}
}