natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'cassandra-6.0' into trunk 

* cassandra-6.0:
  Uncompressed size is being used for compressed tables in maintenance operations
This commit is contained in:
Dmitry Konstantinov 2026-05-13 13:32:42 +01:00
parent 0625a4d525 51a58ef22e
commit b6f175bfc2
29 changed files with 164 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -6,6 +6,7 @@ Merged from 6.0:
* Differentiate between legitimate cases where the first entry is the same as the last entry and empty bounds in SSTableCursorWriter#addIndexBlock() (CASSANDRA-21255)
* Introduce minimum_threshold for data resurrection startup check (CASSANDRA-21293)
Merged from 5.0:
* Use estimated compressed size for tables to check if there is enough free space for a compaction (CASSANDRA-21245)
* Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348)

4
src/java/org/apache/cassandra/cache/AutoSavingCache.java vendored
View File

@ -330,6 +330,7 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
type,
0,
keysEstimate,
keysEstimate,
Unit.KEYS,
nextTimeUUID(),
getCacheDataPath(CURRENT_VERSION).toPath().toString());
@ -344,7 +345,8 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
{
// keyset can change in size, thus total can too
// TODO need to check for this one... was: info.forProgress(keysWritten, Math.max(keysWritten, keys.size()));
return info.forProgress(keysWritten, Math.max(keysWritten, keysEstimate));
long totalKeys = Math.max(keysWritten, keysEstimate);
return info.forProgress(keysWritten, totalKeys, totalKeys);
}
public void saveCache()

5
src/java/org/apache/cassandra/db/compaction/ActiveCompactions.java
View File

@ -49,13 +49,14 @@ public class ActiveCompactions implements ActiveCompactionsTracker
{
compactions.remove(ci);
CompactionManager.instance.getMetrics().bytesCompacted.inc(ci.getCompactionInfo().getTotal());
CompactionManager.instance.getMetrics().compressedBytesCompacted.inc(ci.getCompactionInfo().getTotalCompressed());
CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark();
}
/**
* Get the estimated number of bytes remaining to write per sstable directory
*/
public Map<File, Long> estimatedRemainingWriteBytes()
public Map<File, Long> estimatedRemainingWriteToDiskBytes()
{
synchronized (compactions)
{
@ -66,7 +67,7 @@ public class ActiveCompactions implements ActiveCompactionsTracker
List<File> directories = compactionInfo.getTargetDirectories();
if (directories == null || directories.isEmpty())
continue;
long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteBytes() / directories.size();
long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteToDiskBytes() / directories.size();
for (File directory : directories)
writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum);
}

41
src/java/org/apache/cassandra/db/compaction/CompactionInfo.java
View File

@ -42,6 +42,7 @@ public final class CompactionInfo
public static final String COLUMNFAMILY = "columnfamily";
public static final String COMPLETED = "completed";
public static final String TOTAL = "total";
public static final String TOTAL_COMPRESSED = "totalCompressed";
public static final String TASK_TYPE = "taskType";
public static final String UNIT = "unit";
public static final String COMPACTION_ID = "compactionId";
@ -52,16 +53,18 @@ public final class CompactionInfo
private final OperationType tasktype;
private final long completed;
private final long total;
private final long totalCompressed;
private final Unit unit;
private final TimeUUID compactionId;
private final ImmutableSet<SSTableReader> sstables;
private final String targetDirectory;
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection<? extends SSTableReader> sstables, String targetDirectory)
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId, Collection<? extends SSTableReader> sstables, String targetDirectory)
{
this.tasktype = tasktype;
this.completed = completed;
this.total = total;
this.totalCompressed = totalCompressed;
this.metadata = metadata;
this.unit = unit;
this.compactionId = compactionId;
@ -69,38 +72,38 @@ public final class CompactionInfo
this.targetDirectory = targetDirectory;
}
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<SSTableReader> sstables, String targetDirectory)
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, TimeUUID compactionId, Collection<SSTableReader> sstables, String targetDirectory)
{
this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, targetDirectory);
this(metadata, tasktype, completed, total, totalCompressed, Unit.BYTES, compactionId, sstables, targetDirectory);
}
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<? extends SSTableReader> sstables)
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, TimeUUID compactionId, Collection<? extends SSTableReader> sstables)
{
this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, null);
this(metadata, tasktype, completed, total, totalCompressed, Unit.BYTES, compactionId, sstables, null);
}
/**
* Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask where we don't know
* the sstables at construction
*/
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId)
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId)
{
return withoutSSTables(metadata, tasktype, completed, total, unit, compactionId, null);
return withoutSSTables(metadata, tasktype, completed, total, totalCompressed, unit, compactionId, null);
}
/**
* Special compaction info where we always need to cancel the compaction - for example AutoSavingCache where we don't know
* the sstables at construction
*/
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, String targetDirectory)
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId, String targetDirectory)
{
return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of(), targetDirectory);
return new CompactionInfo(metadata, tasktype, completed, total, totalCompressed, unit, compactionId, ImmutableSet.of(), targetDirectory);
}
/** @return A copy of this CompactionInfo with updated progress. */
public CompactionInfo forProgress(long complete, long total)
public CompactionInfo forProgress(long complete, long total, long totalCompressed)
{
return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables, targetDirectory);
return new CompactionInfo(metadata, tasktype, complete, total, totalCompressed, unit, compactionId, sstables, targetDirectory);
}
public Optional<String> getKeyspace()
@ -128,6 +131,11 @@ public final class CompactionInfo
return total;
}
public long getTotalCompressed()
{
return totalCompressed;
}
public OperationType getTaskType()
{
return tasktype;
@ -183,12 +191,16 @@ public final class CompactionInfo
/**
* Note that this estimate is based on the amount of data we have left to read - it assumes input
* size == output size for a compaction, which is not really true, but should most often provide a worst case
* remaining write size.
* remaining write size. We also scale by the effective compression ratio since total/completed are for the uncompressed size.
*/
public long estimatedRemainingWriteBytes()
public long estimatedRemainingWriteToDiskBytes()
{
if (unit == Unit.BYTES && tasktype.writesData)
return getTotal() - getCompleted();
{
final long total = getTotal();
double compressionRatio = total == 0 ? 1 : ((double) totalCompressed / (double)total);
return (long)(compressionRatio * (total - getCompleted()));
}
return 0;
}
@ -216,6 +228,7 @@ public final class CompactionInfo
ret.put(COLUMNFAMILY, getTable().orElse(null));
ret.put(COMPLETED, Long.toString(completed));
ret.put(TOTAL, Long.toString(total));
ret.put(TOTAL_COMPRESSED, Long.toString(totalCompressed));
ret.put(TASK_TYPE, tasktype.toString());
ret.put(UNIT, unit.toString());
ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString());

7
src/java/org/apache/cassandra/db/compaction/CompactionIterator.java
View File

@ -158,6 +158,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
private final long nowInSec;
private final TimeUUID compactionId;
private final long totalBytes;
private final long totalCompressedBytes;
private long bytesRead;
private long totalSourceCQLRows;
@ -231,9 +232,14 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
this.bytesRead = 0;
long bytes = 0;
long compressedBytes = 0;
for (ISSTableScanner scanner : scanners)
{
bytes += scanner.getLengthInBytes();
compressedBytes += scanner.getCompressedLengthInBytes();
}
this.totalBytes = bytes;
this.totalCompressedBytes = compressedBytes;
this.mergeCounters = new long[scanners.size()];
// note that we leak `this` from the constructor when calling beginCompaction below, this means we have to get the sstables before
// calling that to avoid a NPE.
@ -281,6 +287,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
type,
bytesRead,
totalBytes,
totalCompressedBytes,
compactionId,
sstables,
targetDirectory);

2
src/java/org/apache/cassandra/db/compaction/CompactionTask.java
View File

@ -489,7 +489,7 @@ public class CompactionTask extends AbstractCompactionTask
for (File directory : newCompactionDatadirs)
expectedNewWriteSize.put(directory, writeSizePerOutputDatadir);
Map<File, Long> expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteBytes();
Map<File, Long> expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes();
// todo: abort streams if they block compactions
if (cfs.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize))

9
src/java/org/apache/cassandra/db/compaction/CursorCompactor.java
View File

@ -210,6 +210,7 @@ public class CursorCompactor extends CompactionInfo.Holder
private final long nowInSec;
private final TimeUUID compactionId;
private final long totalInputBytes;
private final long totalCompressedInputBytes;
private final StatefulCursor[] sstableCursors;
private final boolean[] sstableCursorsEqualsNext;
private final boolean hasStaticColumns;
@ -268,9 +269,14 @@ public class CursorCompactor extends CompactionInfo.Holder
this.compactionId = compactionId;
long inputBytes = 0;
long compressedInputBytes = 0;
for (ISSTableScanner scanner : scanners)
{
inputBytes += scanner.getLengthInBytes();
compressedInputBytes += scanner.getCompressedLengthInBytes();
}
this.totalInputBytes = inputBytes;
this.totalCompressedInputBytes = compressedInputBytes;
this.partitionMergeCounters = new long[scanners.size()];
this.staticRowMergeCounters = new long[partitionMergeCounters.length];
this.rowMergeCounters = new long[partitionMergeCounters.length];
@ -1461,6 +1467,7 @@ public class CursorCompactor extends CompactionInfo.Holder
type,
getBytesRead(),
totalInputBytes,
totalCompressedInputBytes,
compactionId,
sstables,
targetDirectory);
@ -1666,4 +1673,4 @@ public class CursorCompactor extends CompactionInfo.Holder
}
preSortedArray[insertInto] = newElement;
}
}
}

4
src/java/org/apache/cassandra/db/view/ViewBuilderTask.java
View File

@ -205,13 +205,13 @@ public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<L
if (range.left.getPartitioner().splitter().isPresent())
{
long progress = prevToken == null ? 0 : Math.round(prevToken.getPartitioner().splitter().get().positionInRange(prevToken, range) * 1000);
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, Unit.RANGES, compactionId);
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, 1000, Unit.RANGES, compactionId);
}
// When there is no splitter, estimate based on number of total keys but
// take the max with keysBuilt + 1 to avoid having more completed than total
long keysTotal = Math.max(keysBuilt + 1, baseCfs.estimatedKeysForRange(range));
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, Unit.KEYS, compactionId);
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, keysTotal, Unit.KEYS, compactionId);
}
@Override

4
src/java/org/apache/cassandra/db/virtual/SSTableTasksTable.java
View File

@ -39,6 +39,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
private final static String PROGRESS = "progress";
private final static String SSTABLES = "sstables";
private final static String TOTAL = "total";
private final static String TOTAL_COMPRESSED = "total_compressed";
private final static String UNIT = "unit";
private final static String TARGET_DIRECTORY = "target_directory";
@ -56,6 +57,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
.addRegularColumn(PROGRESS, LongType.instance)
.addRegularColumn(SSTABLES, Int32Type.instance)
.addRegularColumn(TOTAL, LongType.instance)
.addRegularColumn(TOTAL_COMPRESSED, LongType.instance)
.addRegularColumn(UNIT, UTF8Type.instance)
.addRegularColumn(TARGET_DIRECTORY, UTF8Type.instance)
.build());
@ -69,6 +71,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
{
long completed = task.getCompleted();
long total = task.getTotal();
long totalCompressed = task.getTotalCompressed();
double completionRatio = total == 0L ? 1.0 : (((double) completed) / total);
@ -80,6 +83,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
.column(PROGRESS, completed)
.column(SSTABLES, task.getSSTables().size())
.column(TOTAL, total)
.column(TOTAL_COMPRESSED, totalCompressed)
.column(UNIT, toLowerCaseLocalized(task.getUnit().toString()))
.column(TARGET_DIRECTORY, task.targetDirectory());
}

12
src/java/org/apache/cassandra/index/accord/RouteSecondaryIndexBuilder.java
View File

@ -57,6 +57,7 @@ public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
private final boolean isFullRebuild;
private final boolean isInitialBuild;
private final long totalSizeInBytes;
private final long totalCompressedSizeInBytes;
private long bytesProcessed = 0;
public RouteSecondaryIndexBuilder(RouteJournalIndex index,
@ -72,7 +73,15 @@ public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
this.sstables = sstables;
this.isFullRebuild = isFullRebuild;
this.isInitialBuild = isInitialBuild;
this.totalSizeInBytes = sstables.stream().mapToLong(SSTableReader::uncompressedLength).sum();
long uncompressedSum = 0L;
long compressedSum = 0L;
for (SSTableReader sstable : sstables)
{
uncompressedSum += sstable.uncompressedLength();
compressedSum += sstable.onDiskLength();
}
this.totalSizeInBytes = uncompressedSum;
this.totalCompressedSizeInBytes = compressedSum;
}
@Override
@ -82,6 +91,7 @@ public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
OperationType.INDEX_BUILD,
bytesProcessed,
totalSizeInBytes,
totalCompressedSizeInBytes,
compactionId,
sstables);
}

4
src/java/org/apache/cassandra/index/internal/CollatedViewIndexBuilder.java
View File

@ -61,8 +61,10 @@ public class CollatedViewIndexBuilder extends SecondaryIndexBuilder
OperationType.INDEX_BUILD,
iter.getBytesRead(),
iter.getTotalBytes(),
iter.getTotalBytes(),
compactionId,
sstables);
sstables
);
}
public void build()

1
src/java/org/apache/cassandra/index/sai/StorageAttachedIndexBuilder.java
View File

@ -250,6 +250,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder
OperationType.INDEX_BUILD,
bytesProcessed,
totalSizeInBytes,
totalSizeInBytes,
compactionId,
sstables.keySet());
}

1
src/java/org/apache/cassandra/index/sasi/SASIIndexBuilder.java
View File

@ -134,6 +134,7 @@ class SASIIndexBuilder extends SecondaryIndexBuilder
OperationType.INDEX_BUILD,
bytesProcessed,
totalBytesToProcess,
totalBytesToProcess,
compactionId,
sstables.keySet(),
targetDirectory);

1
src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java
View File

@ -384,6 +384,7 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
OperationType.SCRUB,
dataFile.getFilePointer(),
dataFile.length(),
sstable.onDiskLength(),
scrubCompactionId,
ImmutableSet.of(sstable),
File.getPath(sstable.getFilename()).getParent().toString());

1
src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
View File

@ -499,6 +499,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
OperationType.VERIFY,
dataFile.getFilePointer(),
dataFile.length(),
sstable.onDiskLength(),
verificationCompactionId,
ImmutableSet.of(sstable));
}

2
src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryRedistribution.java
View File

@ -363,7 +363,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
public CompactionInfo getCompactionInfo()
{
return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, memoryPoolBytes, Unit.BYTES, compactionId);
}
public boolean isGlobal()

3
src/java/org/apache/cassandra/metrics/CompactionMetrics.java
View File

@ -54,6 +54,8 @@ public class CompactionMetrics
public final Meter totalCompactionsCompleted;
/** Total number of bytes compacted since server [re]start */
public final Counter bytesCompacted;
/** Estimated compressed bytes compacted since server [re]start, computed by scaling uncompressed bytes by the compression ratio */
public final Counter compressedBytesCompacted;
/** Recent/current throughput of compactions take */
public final Meter bytesCompactedThroughput;
/** Time spent redistributing index summaries */
@ -150,6 +152,7 @@ public class CompactionMetrics
});
totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted"));
bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted"));
compressedBytesCompacted = Metrics.counter(factory.createMetricName("CompressedBytesCompacted"));
bytesCompactedThroughput = Metrics.meter(factory.createMetricName("BytesCompactedThroughput"));
// compaction failure metrics

2
src/java/org/apache/cassandra/streaming/StreamSession.java
View File

@ -969,7 +969,7 @@ public class StreamSession
for (FileStore fs : allWriteableFileStores)
newStreamBytesToWritePerFileStore.merge(fs, totalBytesInPerFileStore, Long::sum);
}
Map<FileStore, Long> totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteBytes(),
Map<FileStore, Long> totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes(),
fileStoreMapper);
long totalStreamRemaining = StreamManager.instance.getTotalRemainingOngoingBytes();
long totalBytesStreamRemainingPerFileStore = totalStreamRemaining / Math.max(1, allFileStores.size());

1
src/java/org/apache/cassandra/tools/NodeProbe.java
View File

@ -2225,6 +2225,7 @@ public class NodeProbe implements AutoCloseable
switch(metricName)
{
case "BytesCompacted":
case "CompressedBytesCompacted":
case "CompactionsAborted":
case "CompactionsReduced":
case "SSTablesDroppedFromCompaction":

25
src/java/org/apache/cassandra/tools/nodetool/CompactionStats.java
View File

@ -89,11 +89,17 @@ public class CompactionStats extends AbstractCommand
tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedMetrics.getCount()));
CassandraMetricsRegistry.JmxCounterMBean bytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("BytesCompacted");
CassandraMetricsRegistry.JmxCounterMBean compressedBytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompressedBytesCompacted");
if (humanReadable)
{
tableBuilder.add("data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(bytesCompacted.getCount()))));
tableBuilder.add("compressed data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(compressedBytesCompacted.getCount()))));
}
else
{
tableBuilder.add("data compacted", Long.toString(bytesCompacted.getCount()));
tableBuilder.add("compressed data compacted", Long.toString(compressedBytesCompacted.getCount()));
}
CassandraMetricsRegistry.JmxCounterMBean compactionsAborted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompactionsAborted");
tableBuilder.add("compactions aborted", Long.toString(compactionsAborted.getCount()));
@ -132,13 +138,14 @@ public class CompactionStats extends AbstractCommand
long remainingBytes = 0;
if (vtableOutput)
table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit", "target directory");
table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "total compressed", "unit", "target directory");
else
table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress");
for (Map<String, String> c : compactions)
{
long total = Long.parseLong(c.get(CompactionInfo.TOTAL));
String totalCompressedValue = c.get(CompactionInfo.TOTAL_COMPRESSED);
long completed = Long.parseLong(c.get(CompactionInfo.COMPLETED));
String taskType = c.get(CompactionInfo.TASK_TYPE);
String keyspace = c.get(CompactionInfo.KEYSPACE);
@ -148,12 +155,22 @@ public class CompactionStats extends AbstractCommand
String[] tables = c.get(CompactionInfo.SSTABLES).split(",");
String progressStr = toFileSize ? FileUtils.stringifyFileSize(completed) : Long.toString(completed);
String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total);
String totalCompressedStr;
if (totalCompressedValue != null)
{
long totalCompressed = Long.parseLong(totalCompressedValue);
totalCompressedStr = toFileSize ? FileUtils.stringifyFileSize(totalCompressed) : Long.toString(totalCompressed);
}
else
{
totalCompressedStr = "n/a";
}
String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + '%';
String id = c.get(CompactionInfo.COMPACTION_ID);
if (vtableOutput)
{
String targetDirectory = c.get(CompactionInfo.TARGET_DIRECTORY);
table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit, targetDirectory);
table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, totalCompressedStr, unit, targetDirectory);
}
else
table.add(id, taskType, keyspace, columnFamily, progressStr, totalStr, unit, percentComplete);
@ -172,4 +189,4 @@ public class CompactionStats extends AbstractCommand
table.printTo(out);
}
}
}

4
test/distributed/org/apache/cassandra/distributed/test/CompactionDiskSpaceTest.java
View File

@ -115,7 +115,7 @@ public class CompactionDiskSpaceTest extends TestBaseImpl
public static void install(ClassLoader cl, Integer node)
{
new ByteBuddy().rebase(ActiveCompactions.class)
.method(named("estimatedRemainingWriteBytes"))
.method(named("estimatedRemainingWriteToDiskBytes"))
.intercept(MethodDelegation.to(BB.class))
.make()
.load(cl, ClassLoadingStrategy.Default.INJECTION);
@ -127,7 +127,7 @@ public class CompactionDiskSpaceTest extends TestBaseImpl
.load(cl, ClassLoadingStrategy.Default.INJECTION);
}
public static Map<File, Long> estimatedRemainingWriteBytes()
public static Map<File, Long> estimatedRemainingWriteToDiskBytes()
{
if (sstableDir != null)
return ImmutableMap.of(sstableDir, estimatedRemaining.get());

6
test/distributed/org/apache/cassandra/distributed/test/SecondaryIndexCompactionTest.java
View File

@ -60,7 +60,7 @@ public class SecondaryIndexCompactionTest extends TestBaseImpl
// emulate ongoing index compaction:
CompactionInfo.Holder h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables);
CompactionManager.instance.active.beginCompaction(h);
CompactionManager.instance.active.estimatedRemainingWriteBytes();
CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes();
CompactionManager.instance.active.finishCompaction(h);
});
}
@ -79,7 +79,7 @@ public class SecondaryIndexCompactionTest extends TestBaseImpl
@Override
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, nextTimeUUID(), sstables);
return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, 300, nextTimeUUID(), sstables);
}
@Override
@ -88,4 +88,4 @@ public class SecondaryIndexCompactionTest extends TestBaseImpl
return false;
}
}
}
}

4
test/distributed/org/apache/cassandra/distributed/test/StreamsDiskSpaceTest.java
View File

@ -73,7 +73,7 @@ public class StreamsDiskSpaceTest extends TestBaseImpl
.withConfig(config -> config.set("hinted_handoff_enabled", false)
.with(GOSSIP)
.with(NETWORK))
.withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteBytes"))
.withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteToDiskBytes"))
.start()))
{
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
@ -139,7 +139,7 @@ public class StreamsDiskSpaceTest extends TestBaseImpl
return ongoing.get();
}
public static Map<File, Long> estimatedRemainingWriteBytes()
public static Map<File, Long> estimatedRemainingWriteToDiskBytes()
{
Map<File, Long> ret = new HashMap<>();
if (datadir != null)

4
test/unit/org/apache/cassandra/db/compaction/CompactionInfoTest.java
View File

@ -39,7 +39,7 @@ public class CompactionInfoTest extends AbstractPendingAntiCompactionTest
{
ColumnFamilyStore cfs = MockSchema.newCFS();
TimeUUID expectedTaskId = nextTimeUUID();
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, expectedTaskId, new ArrayList<>());
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, 1000, expectedTaskId, new ArrayList<>());
Assertions.assertThat(compactionInfo.toString())
.contains(expectedTaskId.toString());
}
@ -50,7 +50,7 @@ public class CompactionInfoTest extends AbstractPendingAntiCompactionTest
UUID tableId = UUID.randomUUID();
TimeUUID taskId = nextTimeUUID();
ColumnFamilyStore cfs = MockSchema.newCFS(builder -> builder.id(TableId.fromUUID(tableId)));
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, taskId, new ArrayList<>());
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, 300, taskId, new ArrayList<>());
Assertions.assertThat(compactionInfo.toString())
.isEqualTo("Compaction(%s, 0 / 1000 bytes)@%s(mockks, mockcf1)", taskId, tableId);
}

31
test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
View File

@ -85,7 +85,7 @@ public class CompactionsCQLTest extends CQLTester
public void before() throws IOException
{
strategy = DatabaseDescriptor.getCorruptedTombstoneStrategy();
CommitLog.instance.resetUnsafe(true);
}
@ -877,7 +877,9 @@ public class CompactionsCQLTest extends CQLTester
execute("insert into %s (id, i) values (?,?)", i, i);
getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
}
CompactionInfo.Holder holder = holder(OperationType.COMPACTION);
// When we have an existing compaction with sstables of total size more than double the available space,
// we should not be able to then run a major compaction
CompactionInfo.Holder holder = holder(OperationType.COMPACTION, 2);
CompactionManager.instance.active.beginCompaction(holder);
try
{
@ -893,7 +895,19 @@ public class CompactionsCQLTest extends CQLTester
CompactionManager.instance.active.finishCompaction(holder);
}
// don't block compactions if there is a huge validation
holder = holder(OperationType.VALIDATION);
holder = holder(OperationType.VALIDATION, 2);
CompactionManager.instance.active.beginCompaction(holder);
try
{
getCurrentColumnFamilyStore().forceMajorCompaction();
}
finally
{
CompactionManager.instance.active.finishCompaction(holder);
}
// Should be able to run when the sstables in question are 90% of the total available space
holder = holder(OperationType.COMPACTION, 0.9);
CompactionManager.instance.active.beginCompaction(holder);
try
{
@ -905,7 +919,7 @@ public class CompactionsCQLTest extends CQLTester
}
}
private CompactionInfo.Holder holder(OperationType opType)
private CompactionInfo.Holder holder(OperationType opType, double availableSpaceMultiplier)
{
CompactionInfo.Holder holder = new CompactionInfo.Holder()
{
@ -915,12 +929,17 @@ public class CompactionsCQLTest extends CQLTester
for (File f : getCurrentColumnFamilyStore().getDirectories().getCFDirectories())
availableSpace += PathUtils.tryGetSpace(f.toPath(), FileStore::getUsableSpace);
Set<SSTableReader> liveSSTables = getCurrentColumnFamilyStore().getLiveSSTables();
long totalDiskUsage = (long)(availableSpace * availableSpaceMultiplier);
// Arbitrary compression ratio of 3.4
long totalUncompressedSize = (long) ((double) totalDiskUsage * 3.4);
return new CompactionInfo(getCurrentColumnFamilyStore().metadata(),
opType,
+0,
+availableSpace * 2,
totalUncompressedSize,
totalDiskUsage,
nextTimeUUID(),
getCurrentColumnFamilyStore().getLiveSSTables());
liveSSTables);
}
public boolean isGlobal()

6
test/unit/org/apache/cassandra/db/repair/PendingAntiCompactionTest.java
View File

@ -609,7 +609,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, nextTimeUUID(), compacting);
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, 1000, nextTimeUUID(), compacting);
}
public boolean isGlobal()
@ -650,7 +650,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
}
public boolean isGlobal()
@ -703,7 +703,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
}
public boolean isGlobal()

5
test/unit/org/apache/cassandra/db/virtual/SSTableTasksTableTest.java
View File

@ -70,6 +70,7 @@ public class SSTableTasksTableTest extends CQLTester
long bytesCompacted = 123;
long bytesTotal = 123456;
long totalCompressedBytes = 112233;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
@ -81,7 +82,7 @@ public class SSTableTasksTableTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, directory);
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, directory);
}
public boolean isGlobal()
@ -94,7 +95,7 @@ public class SSTableTasksTableTest extends CQLTester
UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks");
assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal,
toLowerCaseLocalized(OperationType.COMPACTION.toString()), bytesCompacted, sstables.size(),
directory, bytesTotal, CompactionInfo.Unit.BYTES.toString()));
directory, bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES.toString()));
CompactionManager.instance.active.finishCompaction(compactionHolder);
result = execute("SELECT * FROM vts.sstable_tasks");

2
test/unit/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryManagerTest.java
View File

@ -655,7 +655,7 @@ public class IndexSummaryManagerTest<R extends SSTableReader & IndexSummarySuppo
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting);
return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, 0, nextTimeUUID(), compacting);
}
public boolean isGlobal()

39
test/unit/org/apache/cassandra/tools/nodetool/CompactionStatsTest.java
View File

@ -57,6 +57,7 @@ public class CompactionStatsTest extends CQLTester
long bytesCompacted = 123;
long bytesTotal = 123456;
long totalCompressedBytes = 112233;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
@ -65,7 +66,7 @@ public class CompactionStatsTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
}
public boolean isGlobal()
@ -87,6 +88,7 @@ public class CompactionStatsTest extends CQLTester
assertThat(stdout).containsPattern("pending tasks\\s+[0-9]*");
assertThat(stdout).containsPattern("compactions completed\\s+[0-9]*");
assertThat(stdout).containsPattern("data compacted\\s+[0-9]*");
assertThat(stdout).containsPattern("compressed data compacted\\s+[0-9]*");
assertThat(stdout).containsPattern("compactions aborted\\s+[0-9]*");
assertThat(stdout).containsPattern("compactions reduced\\s+[0-9]*");
assertThat(stdout).containsPattern("sstables dropped from compaction\\s+[0-9]*");
@ -109,6 +111,7 @@ public class CompactionStatsTest extends CQLTester
long bytesCompacted = 123;
long bytesTotal = 123456;
long totalCompressedBytes = 112233;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
@ -118,7 +121,7 @@ public class CompactionStatsTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, targetDirectory);
}
public boolean isGlobal()
@ -131,7 +134,7 @@ public class CompactionStatsTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
}
public boolean isGlobal()
@ -143,16 +146,16 @@ public class CompactionStatsTest extends CQLTester
CompactionManager.instance.active.beginCompaction(compactionHolder);
CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V");
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+total compressed\\s+unit\\s+target directory");
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES,
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES,
targetDirectory);
assertThat(stdout).containsPattern(expectedStatsPattern);
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES);
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES);
assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
CompactionManager.instance.active.finishCompaction(compactionHolder);
@ -168,6 +171,7 @@ public class CompactionStatsTest extends CQLTester
long bytesCompacted = 123;
long bytesTotal = 123456;
long totalCompressedBytes = 112233;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
@ -176,7 +180,7 @@ public class CompactionStatsTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
}
public boolean isGlobal()
@ -205,6 +209,7 @@ public class CompactionStatsTest extends CQLTester
long bytesCompacted = 123;
long bytesTotal = 123456;
long totalCompressedBytes = 112233;
TimeUUID compactionId = nextTimeUUID();
List<SSTableReader> sstables = IntStream.range(0, 10)
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
@ -214,7 +219,7 @@ public class CompactionStatsTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, targetDirectory);
}
public boolean isGlobal()
@ -227,7 +232,7 @@ public class CompactionStatsTest extends CQLTester
{
public CompactionInfo getCompactionInfo()
{
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
}
public boolean isGlobal()
@ -239,15 +244,15 @@ public class CompactionStatsTest extends CQLTester
CompactionManager.instance.active.beginCompaction(compactionHolder);
CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable");
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+total compressed\\s+unit\\s+target directory");
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES,
OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", "109.6 KiB", CompactionInfo.Unit.BYTES,
targetDirectory);
assertThat(stdout).containsPattern(expectedStatsPattern);
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES);
OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", "109.6 KiB", CompactionInfo.Unit.BYTES);
assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
CompactionManager.instance.active.finishCompaction(compactionHolder);
@ -277,4 +282,4 @@ public class CompactionStatsTest extends CQLTester
return stdout.get();
}
}
}