mirror of https://github.com/apache/cassandra
Merge branch 'cassandra-6.0' into trunk
* cassandra-6.0: Uncompressed size is being used for compressed tables in maintenance operations
This commit is contained in:
commit
b6f175bfc2
|
|
@ -6,6 +6,7 @@ Merged from 6.0:
|
||||||
* Differentiate between legitimate cases where the first entry is the same as the last entry and empty bounds in SSTableCursorWriter#addIndexBlock() (CASSANDRA-21255)
|
* Differentiate between legitimate cases where the first entry is the same as the last entry and empty bounds in SSTableCursorWriter#addIndexBlock() (CASSANDRA-21255)
|
||||||
* Introduce minimum_threshold for data resurrection startup check (CASSANDRA-21293)
|
* Introduce minimum_threshold for data resurrection startup check (CASSANDRA-21293)
|
||||||
Merged from 5.0:
|
Merged from 5.0:
|
||||||
|
* Use estimated compressed size for tables to check if there is enough free space for a compaction (CASSANDRA-21245)
|
||||||
* Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348)
|
* Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -330,6 +330,7 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
|
||||||
type,
|
type,
|
||||||
0,
|
0,
|
||||||
keysEstimate,
|
keysEstimate,
|
||||||
|
keysEstimate,
|
||||||
Unit.KEYS,
|
Unit.KEYS,
|
||||||
nextTimeUUID(),
|
nextTimeUUID(),
|
||||||
getCacheDataPath(CURRENT_VERSION).toPath().toString());
|
getCacheDataPath(CURRENT_VERSION).toPath().toString());
|
||||||
|
|
@ -344,7 +345,8 @@ public class AutoSavingCache<K extends CacheKey, V> extends InstrumentingCache<K
|
||||||
{
|
{
|
||||||
// keyset can change in size, thus total can too
|
// keyset can change in size, thus total can too
|
||||||
// TODO need to check for this one... was: info.forProgress(keysWritten, Math.max(keysWritten, keys.size()));
|
// TODO need to check for this one... was: info.forProgress(keysWritten, Math.max(keysWritten, keys.size()));
|
||||||
return info.forProgress(keysWritten, Math.max(keysWritten, keysEstimate));
|
long totalKeys = Math.max(keysWritten, keysEstimate);
|
||||||
|
return info.forProgress(keysWritten, totalKeys, totalKeys);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void saveCache()
|
public void saveCache()
|
||||||
|
|
|
||||||
|
|
@ -49,13 +49,14 @@ public class ActiveCompactions implements ActiveCompactionsTracker
|
||||||
{
|
{
|
||||||
compactions.remove(ci);
|
compactions.remove(ci);
|
||||||
CompactionManager.instance.getMetrics().bytesCompacted.inc(ci.getCompactionInfo().getTotal());
|
CompactionManager.instance.getMetrics().bytesCompacted.inc(ci.getCompactionInfo().getTotal());
|
||||||
|
CompactionManager.instance.getMetrics().compressedBytesCompacted.inc(ci.getCompactionInfo().getTotalCompressed());
|
||||||
CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark();
|
CompactionManager.instance.getMetrics().totalCompactionsCompleted.mark();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the estimated number of bytes remaining to write per sstable directory
|
* Get the estimated number of bytes remaining to write per sstable directory
|
||||||
*/
|
*/
|
||||||
public Map<File, Long> estimatedRemainingWriteBytes()
|
public Map<File, Long> estimatedRemainingWriteToDiskBytes()
|
||||||
{
|
{
|
||||||
synchronized (compactions)
|
synchronized (compactions)
|
||||||
{
|
{
|
||||||
|
|
@ -66,7 +67,7 @@ public class ActiveCompactions implements ActiveCompactionsTracker
|
||||||
List<File> directories = compactionInfo.getTargetDirectories();
|
List<File> directories = compactionInfo.getTargetDirectories();
|
||||||
if (directories == null || directories.isEmpty())
|
if (directories == null || directories.isEmpty())
|
||||||
continue;
|
continue;
|
||||||
long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteBytes() / directories.size();
|
long remainingWriteBytesPerDataDir = compactionInfo.estimatedRemainingWriteToDiskBytes() / directories.size();
|
||||||
for (File directory : directories)
|
for (File directory : directories)
|
||||||
writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum);
|
writeBytesPerSSTableDir.merge(directory, remainingWriteBytesPerDataDir, Long::sum);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,7 @@ public final class CompactionInfo
|
||||||
public static final String COLUMNFAMILY = "columnfamily";
|
public static final String COLUMNFAMILY = "columnfamily";
|
||||||
public static final String COMPLETED = "completed";
|
public static final String COMPLETED = "completed";
|
||||||
public static final String TOTAL = "total";
|
public static final String TOTAL = "total";
|
||||||
|
public static final String TOTAL_COMPRESSED = "totalCompressed";
|
||||||
public static final String TASK_TYPE = "taskType";
|
public static final String TASK_TYPE = "taskType";
|
||||||
public static final String UNIT = "unit";
|
public static final String UNIT = "unit";
|
||||||
public static final String COMPACTION_ID = "compactionId";
|
public static final String COMPACTION_ID = "compactionId";
|
||||||
|
|
@ -52,16 +53,18 @@ public final class CompactionInfo
|
||||||
private final OperationType tasktype;
|
private final OperationType tasktype;
|
||||||
private final long completed;
|
private final long completed;
|
||||||
private final long total;
|
private final long total;
|
||||||
|
private final long totalCompressed;
|
||||||
private final Unit unit;
|
private final Unit unit;
|
||||||
private final TimeUUID compactionId;
|
private final TimeUUID compactionId;
|
||||||
private final ImmutableSet<SSTableReader> sstables;
|
private final ImmutableSet<SSTableReader> sstables;
|
||||||
private final String targetDirectory;
|
private final String targetDirectory;
|
||||||
|
|
||||||
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, Collection<? extends SSTableReader> sstables, String targetDirectory)
|
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId, Collection<? extends SSTableReader> sstables, String targetDirectory)
|
||||||
{
|
{
|
||||||
this.tasktype = tasktype;
|
this.tasktype = tasktype;
|
||||||
this.completed = completed;
|
this.completed = completed;
|
||||||
this.total = total;
|
this.total = total;
|
||||||
|
this.totalCompressed = totalCompressed;
|
||||||
this.metadata = metadata;
|
this.metadata = metadata;
|
||||||
this.unit = unit;
|
this.unit = unit;
|
||||||
this.compactionId = compactionId;
|
this.compactionId = compactionId;
|
||||||
|
|
@ -69,38 +72,38 @@ public final class CompactionInfo
|
||||||
this.targetDirectory = targetDirectory;
|
this.targetDirectory = targetDirectory;
|
||||||
}
|
}
|
||||||
|
|
||||||
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<SSTableReader> sstables, String targetDirectory)
|
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, TimeUUID compactionId, Collection<SSTableReader> sstables, String targetDirectory)
|
||||||
{
|
{
|
||||||
this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, targetDirectory);
|
this(metadata, tasktype, completed, total, totalCompressed, Unit.BYTES, compactionId, sstables, targetDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, TimeUUID compactionId, Collection<? extends SSTableReader> sstables)
|
public CompactionInfo(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, TimeUUID compactionId, Collection<? extends SSTableReader> sstables)
|
||||||
{
|
{
|
||||||
this(metadata, tasktype, completed, total, Unit.BYTES, compactionId, sstables, null);
|
this(metadata, tasktype, completed, total, totalCompressed, Unit.BYTES, compactionId, sstables, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask where we don't know
|
* Special compaction info where we always need to cancel the compaction - for example ViewBuilderTask where we don't know
|
||||||
* the sstables at construction
|
* the sstables at construction
|
||||||
*/
|
*/
|
||||||
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId)
|
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId)
|
||||||
{
|
{
|
||||||
return withoutSSTables(metadata, tasktype, completed, total, unit, compactionId, null);
|
return withoutSSTables(metadata, tasktype, completed, total, totalCompressed, unit, compactionId, null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Special compaction info where we always need to cancel the compaction - for example AutoSavingCache where we don't know
|
* Special compaction info where we always need to cancel the compaction - for example AutoSavingCache where we don't know
|
||||||
* the sstables at construction
|
* the sstables at construction
|
||||||
*/
|
*/
|
||||||
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, Unit unit, TimeUUID compactionId, String targetDirectory)
|
public static CompactionInfo withoutSSTables(TableMetadata metadata, OperationType tasktype, long completed, long total, long totalCompressed, Unit unit, TimeUUID compactionId, String targetDirectory)
|
||||||
{
|
{
|
||||||
return new CompactionInfo(metadata, tasktype, completed, total, unit, compactionId, ImmutableSet.of(), targetDirectory);
|
return new CompactionInfo(metadata, tasktype, completed, total, totalCompressed, unit, compactionId, ImmutableSet.of(), targetDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return A copy of this CompactionInfo with updated progress. */
|
/** @return A copy of this CompactionInfo with updated progress. */
|
||||||
public CompactionInfo forProgress(long complete, long total)
|
public CompactionInfo forProgress(long complete, long total, long totalCompressed)
|
||||||
{
|
{
|
||||||
return new CompactionInfo(metadata, tasktype, complete, total, unit, compactionId, sstables, targetDirectory);
|
return new CompactionInfo(metadata, tasktype, complete, total, totalCompressed, unit, compactionId, sstables, targetDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Optional<String> getKeyspace()
|
public Optional<String> getKeyspace()
|
||||||
|
|
@ -128,6 +131,11 @@ public final class CompactionInfo
|
||||||
return total;
|
return total;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public long getTotalCompressed()
|
||||||
|
{
|
||||||
|
return totalCompressed;
|
||||||
|
}
|
||||||
|
|
||||||
public OperationType getTaskType()
|
public OperationType getTaskType()
|
||||||
{
|
{
|
||||||
return tasktype;
|
return tasktype;
|
||||||
|
|
@ -183,12 +191,16 @@ public final class CompactionInfo
|
||||||
/**
|
/**
|
||||||
* Note that this estimate is based on the amount of data we have left to read - it assumes input
|
* Note that this estimate is based on the amount of data we have left to read - it assumes input
|
||||||
* size == output size for a compaction, which is not really true, but should most often provide a worst case
|
* size == output size for a compaction, which is not really true, but should most often provide a worst case
|
||||||
* remaining write size.
|
* remaining write size. We also scale by the effective compression ratio since total/completed are for the uncompressed size.
|
||||||
*/
|
*/
|
||||||
public long estimatedRemainingWriteBytes()
|
public long estimatedRemainingWriteToDiskBytes()
|
||||||
{
|
{
|
||||||
if (unit == Unit.BYTES && tasktype.writesData)
|
if (unit == Unit.BYTES && tasktype.writesData)
|
||||||
return getTotal() - getCompleted();
|
{
|
||||||
|
final long total = getTotal();
|
||||||
|
double compressionRatio = total == 0 ? 1 : ((double) totalCompressed / (double)total);
|
||||||
|
return (long)(compressionRatio * (total - getCompleted()));
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -216,6 +228,7 @@ public final class CompactionInfo
|
||||||
ret.put(COLUMNFAMILY, getTable().orElse(null));
|
ret.put(COLUMNFAMILY, getTable().orElse(null));
|
||||||
ret.put(COMPLETED, Long.toString(completed));
|
ret.put(COMPLETED, Long.toString(completed));
|
||||||
ret.put(TOTAL, Long.toString(total));
|
ret.put(TOTAL, Long.toString(total));
|
||||||
|
ret.put(TOTAL_COMPRESSED, Long.toString(totalCompressed));
|
||||||
ret.put(TASK_TYPE, tasktype.toString());
|
ret.put(TASK_TYPE, tasktype.toString());
|
||||||
ret.put(UNIT, unit.toString());
|
ret.put(UNIT, unit.toString());
|
||||||
ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString());
|
ret.put(COMPACTION_ID, compactionId == null ? "" : compactionId.toString());
|
||||||
|
|
|
||||||
|
|
@ -158,6 +158,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
|
||||||
private final long nowInSec;
|
private final long nowInSec;
|
||||||
private final TimeUUID compactionId;
|
private final TimeUUID compactionId;
|
||||||
private final long totalBytes;
|
private final long totalBytes;
|
||||||
|
private final long totalCompressedBytes;
|
||||||
private long bytesRead;
|
private long bytesRead;
|
||||||
private long totalSourceCQLRows;
|
private long totalSourceCQLRows;
|
||||||
|
|
||||||
|
|
@ -231,9 +232,14 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
|
||||||
this.bytesRead = 0;
|
this.bytesRead = 0;
|
||||||
|
|
||||||
long bytes = 0;
|
long bytes = 0;
|
||||||
|
long compressedBytes = 0;
|
||||||
for (ISSTableScanner scanner : scanners)
|
for (ISSTableScanner scanner : scanners)
|
||||||
|
{
|
||||||
bytes += scanner.getLengthInBytes();
|
bytes += scanner.getLengthInBytes();
|
||||||
|
compressedBytes += scanner.getCompressedLengthInBytes();
|
||||||
|
}
|
||||||
this.totalBytes = bytes;
|
this.totalBytes = bytes;
|
||||||
|
this.totalCompressedBytes = compressedBytes;
|
||||||
this.mergeCounters = new long[scanners.size()];
|
this.mergeCounters = new long[scanners.size()];
|
||||||
// note that we leak `this` from the constructor when calling beginCompaction below, this means we have to get the sstables before
|
// note that we leak `this` from the constructor when calling beginCompaction below, this means we have to get the sstables before
|
||||||
// calling that to avoid a NPE.
|
// calling that to avoid a NPE.
|
||||||
|
|
@ -281,6 +287,7 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
|
||||||
type,
|
type,
|
||||||
bytesRead,
|
bytesRead,
|
||||||
totalBytes,
|
totalBytes,
|
||||||
|
totalCompressedBytes,
|
||||||
compactionId,
|
compactionId,
|
||||||
sstables,
|
sstables,
|
||||||
targetDirectory);
|
targetDirectory);
|
||||||
|
|
|
||||||
|
|
@ -489,7 +489,7 @@ public class CompactionTask extends AbstractCompactionTask
|
||||||
for (File directory : newCompactionDatadirs)
|
for (File directory : newCompactionDatadirs)
|
||||||
expectedNewWriteSize.put(directory, writeSizePerOutputDatadir);
|
expectedNewWriteSize.put(directory, writeSizePerOutputDatadir);
|
||||||
|
|
||||||
Map<File, Long> expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteBytes();
|
Map<File, Long> expectedWriteSize = CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes();
|
||||||
|
|
||||||
// todo: abort streams if they block compactions
|
// todo: abort streams if they block compactions
|
||||||
if (cfs.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize))
|
if (cfs.getDirectories().hasDiskSpaceForCompactionsAndStreams(expectedNewWriteSize, expectedWriteSize))
|
||||||
|
|
|
||||||
|
|
@ -210,6 +210,7 @@ public class CursorCompactor extends CompactionInfo.Holder
|
||||||
private final long nowInSec;
|
private final long nowInSec;
|
||||||
private final TimeUUID compactionId;
|
private final TimeUUID compactionId;
|
||||||
private final long totalInputBytes;
|
private final long totalInputBytes;
|
||||||
|
private final long totalCompressedInputBytes;
|
||||||
private final StatefulCursor[] sstableCursors;
|
private final StatefulCursor[] sstableCursors;
|
||||||
private final boolean[] sstableCursorsEqualsNext;
|
private final boolean[] sstableCursorsEqualsNext;
|
||||||
private final boolean hasStaticColumns;
|
private final boolean hasStaticColumns;
|
||||||
|
|
@ -268,9 +269,14 @@ public class CursorCompactor extends CompactionInfo.Holder
|
||||||
this.compactionId = compactionId;
|
this.compactionId = compactionId;
|
||||||
|
|
||||||
long inputBytes = 0;
|
long inputBytes = 0;
|
||||||
|
long compressedInputBytes = 0;
|
||||||
for (ISSTableScanner scanner : scanners)
|
for (ISSTableScanner scanner : scanners)
|
||||||
|
{
|
||||||
inputBytes += scanner.getLengthInBytes();
|
inputBytes += scanner.getLengthInBytes();
|
||||||
|
compressedInputBytes += scanner.getCompressedLengthInBytes();
|
||||||
|
}
|
||||||
this.totalInputBytes = inputBytes;
|
this.totalInputBytes = inputBytes;
|
||||||
|
this.totalCompressedInputBytes = compressedInputBytes;
|
||||||
this.partitionMergeCounters = new long[scanners.size()];
|
this.partitionMergeCounters = new long[scanners.size()];
|
||||||
this.staticRowMergeCounters = new long[partitionMergeCounters.length];
|
this.staticRowMergeCounters = new long[partitionMergeCounters.length];
|
||||||
this.rowMergeCounters = new long[partitionMergeCounters.length];
|
this.rowMergeCounters = new long[partitionMergeCounters.length];
|
||||||
|
|
@ -1461,6 +1467,7 @@ public class CursorCompactor extends CompactionInfo.Holder
|
||||||
type,
|
type,
|
||||||
getBytesRead(),
|
getBytesRead(),
|
||||||
totalInputBytes,
|
totalInputBytes,
|
||||||
|
totalCompressedInputBytes,
|
||||||
compactionId,
|
compactionId,
|
||||||
sstables,
|
sstables,
|
||||||
targetDirectory);
|
targetDirectory);
|
||||||
|
|
@ -1666,4 +1673,4 @@ public class CursorCompactor extends CompactionInfo.Holder
|
||||||
}
|
}
|
||||||
preSortedArray[insertInto] = newElement;
|
preSortedArray[insertInto] = newElement;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -205,13 +205,13 @@ public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<L
|
||||||
if (range.left.getPartitioner().splitter().isPresent())
|
if (range.left.getPartitioner().splitter().isPresent())
|
||||||
{
|
{
|
||||||
long progress = prevToken == null ? 0 : Math.round(prevToken.getPartitioner().splitter().get().positionInRange(prevToken, range) * 1000);
|
long progress = prevToken == null ? 0 : Math.round(prevToken.getPartitioner().splitter().get().positionInRange(prevToken, range) * 1000);
|
||||||
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, Unit.RANGES, compactionId);
|
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, progress, 1000, 1000, Unit.RANGES, compactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
// When there is no splitter, estimate based on number of total keys but
|
// When there is no splitter, estimate based on number of total keys but
|
||||||
// take the max with keysBuilt + 1 to avoid having more completed than total
|
// take the max with keysBuilt + 1 to avoid having more completed than total
|
||||||
long keysTotal = Math.max(keysBuilt + 1, baseCfs.estimatedKeysForRange(range));
|
long keysTotal = Math.max(keysBuilt + 1, baseCfs.estimatedKeysForRange(range));
|
||||||
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, Unit.KEYS, compactionId);
|
return CompactionInfo.withoutSSTables(baseCfs.metadata(), OperationType.VIEW_BUILD, keysBuilt, keysTotal, keysTotal, Unit.KEYS, compactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
||||||
|
|
@ -39,6 +39,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
|
||||||
private final static String PROGRESS = "progress";
|
private final static String PROGRESS = "progress";
|
||||||
private final static String SSTABLES = "sstables";
|
private final static String SSTABLES = "sstables";
|
||||||
private final static String TOTAL = "total";
|
private final static String TOTAL = "total";
|
||||||
|
private final static String TOTAL_COMPRESSED = "total_compressed";
|
||||||
private final static String UNIT = "unit";
|
private final static String UNIT = "unit";
|
||||||
private final static String TARGET_DIRECTORY = "target_directory";
|
private final static String TARGET_DIRECTORY = "target_directory";
|
||||||
|
|
||||||
|
|
@ -56,6 +57,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
|
||||||
.addRegularColumn(PROGRESS, LongType.instance)
|
.addRegularColumn(PROGRESS, LongType.instance)
|
||||||
.addRegularColumn(SSTABLES, Int32Type.instance)
|
.addRegularColumn(SSTABLES, Int32Type.instance)
|
||||||
.addRegularColumn(TOTAL, LongType.instance)
|
.addRegularColumn(TOTAL, LongType.instance)
|
||||||
|
.addRegularColumn(TOTAL_COMPRESSED, LongType.instance)
|
||||||
.addRegularColumn(UNIT, UTF8Type.instance)
|
.addRegularColumn(UNIT, UTF8Type.instance)
|
||||||
.addRegularColumn(TARGET_DIRECTORY, UTF8Type.instance)
|
.addRegularColumn(TARGET_DIRECTORY, UTF8Type.instance)
|
||||||
.build());
|
.build());
|
||||||
|
|
@ -69,6 +71,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
|
||||||
{
|
{
|
||||||
long completed = task.getCompleted();
|
long completed = task.getCompleted();
|
||||||
long total = task.getTotal();
|
long total = task.getTotal();
|
||||||
|
long totalCompressed = task.getTotalCompressed();
|
||||||
|
|
||||||
double completionRatio = total == 0L ? 1.0 : (((double) completed) / total);
|
double completionRatio = total == 0L ? 1.0 : (((double) completed) / total);
|
||||||
|
|
||||||
|
|
@ -80,6 +83,7 @@ final class SSTableTasksTable extends AbstractVirtualTable
|
||||||
.column(PROGRESS, completed)
|
.column(PROGRESS, completed)
|
||||||
.column(SSTABLES, task.getSSTables().size())
|
.column(SSTABLES, task.getSSTables().size())
|
||||||
.column(TOTAL, total)
|
.column(TOTAL, total)
|
||||||
|
.column(TOTAL_COMPRESSED, totalCompressed)
|
||||||
.column(UNIT, toLowerCaseLocalized(task.getUnit().toString()))
|
.column(UNIT, toLowerCaseLocalized(task.getUnit().toString()))
|
||||||
.column(TARGET_DIRECTORY, task.targetDirectory());
|
.column(TARGET_DIRECTORY, task.targetDirectory());
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,7 @@ public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
|
||||||
private final boolean isFullRebuild;
|
private final boolean isFullRebuild;
|
||||||
private final boolean isInitialBuild;
|
private final boolean isInitialBuild;
|
||||||
private final long totalSizeInBytes;
|
private final long totalSizeInBytes;
|
||||||
|
private final long totalCompressedSizeInBytes;
|
||||||
private long bytesProcessed = 0;
|
private long bytesProcessed = 0;
|
||||||
|
|
||||||
public RouteSecondaryIndexBuilder(RouteJournalIndex index,
|
public RouteSecondaryIndexBuilder(RouteJournalIndex index,
|
||||||
|
|
@ -72,7 +73,15 @@ public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
|
||||||
this.sstables = sstables;
|
this.sstables = sstables;
|
||||||
this.isFullRebuild = isFullRebuild;
|
this.isFullRebuild = isFullRebuild;
|
||||||
this.isInitialBuild = isInitialBuild;
|
this.isInitialBuild = isInitialBuild;
|
||||||
this.totalSizeInBytes = sstables.stream().mapToLong(SSTableReader::uncompressedLength).sum();
|
long uncompressedSum = 0L;
|
||||||
|
long compressedSum = 0L;
|
||||||
|
for (SSTableReader sstable : sstables)
|
||||||
|
{
|
||||||
|
uncompressedSum += sstable.uncompressedLength();
|
||||||
|
compressedSum += sstable.onDiskLength();
|
||||||
|
}
|
||||||
|
this.totalSizeInBytes = uncompressedSum;
|
||||||
|
this.totalCompressedSizeInBytes = compressedSum;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -82,6 +91,7 @@ public class RouteSecondaryIndexBuilder extends SecondaryIndexBuilder
|
||||||
OperationType.INDEX_BUILD,
|
OperationType.INDEX_BUILD,
|
||||||
bytesProcessed,
|
bytesProcessed,
|
||||||
totalSizeInBytes,
|
totalSizeInBytes,
|
||||||
|
totalCompressedSizeInBytes,
|
||||||
compactionId,
|
compactionId,
|
||||||
sstables);
|
sstables);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -61,8 +61,10 @@ public class CollatedViewIndexBuilder extends SecondaryIndexBuilder
|
||||||
OperationType.INDEX_BUILD,
|
OperationType.INDEX_BUILD,
|
||||||
iter.getBytesRead(),
|
iter.getBytesRead(),
|
||||||
iter.getTotalBytes(),
|
iter.getTotalBytes(),
|
||||||
|
iter.getTotalBytes(),
|
||||||
compactionId,
|
compactionId,
|
||||||
sstables);
|
sstables
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void build()
|
public void build()
|
||||||
|
|
|
||||||
|
|
@ -250,6 +250,7 @@ public class StorageAttachedIndexBuilder extends SecondaryIndexBuilder
|
||||||
OperationType.INDEX_BUILD,
|
OperationType.INDEX_BUILD,
|
||||||
bytesProcessed,
|
bytesProcessed,
|
||||||
totalSizeInBytes,
|
totalSizeInBytes,
|
||||||
|
totalSizeInBytes,
|
||||||
compactionId,
|
compactionId,
|
||||||
sstables.keySet());
|
sstables.keySet());
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -134,6 +134,7 @@ class SASIIndexBuilder extends SecondaryIndexBuilder
|
||||||
OperationType.INDEX_BUILD,
|
OperationType.INDEX_BUILD,
|
||||||
bytesProcessed,
|
bytesProcessed,
|
||||||
totalBytesToProcess,
|
totalBytesToProcess,
|
||||||
|
totalBytesToProcess,
|
||||||
compactionId,
|
compactionId,
|
||||||
sstables.keySet(),
|
sstables.keySet(),
|
||||||
targetDirectory);
|
targetDirectory);
|
||||||
|
|
|
||||||
|
|
@ -384,6 +384,7 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
|
||||||
OperationType.SCRUB,
|
OperationType.SCRUB,
|
||||||
dataFile.getFilePointer(),
|
dataFile.getFilePointer(),
|
||||||
dataFile.length(),
|
dataFile.length(),
|
||||||
|
sstable.onDiskLength(),
|
||||||
scrubCompactionId,
|
scrubCompactionId,
|
||||||
ImmutableSet.of(sstable),
|
ImmutableSet.of(sstable),
|
||||||
File.getPath(sstable.getFilename()).getParent().toString());
|
File.getPath(sstable.getFilename()).getParent().toString());
|
||||||
|
|
|
||||||
|
|
@ -499,6 +499,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
|
||||||
OperationType.VERIFY,
|
OperationType.VERIFY,
|
||||||
dataFile.getFilePointer(),
|
dataFile.getFilePointer(),
|
||||||
dataFile.length(),
|
dataFile.length(),
|
||||||
|
sstable.onDiskLength(),
|
||||||
verificationCompactionId,
|
verificationCompactionId,
|
||||||
ImmutableSet.of(sstable));
|
ImmutableSet.of(sstable));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -363,7 +363,7 @@ public class IndexSummaryRedistribution extends CompactionInfo.Holder
|
||||||
|
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, Unit.BYTES, compactionId);
|
return CompactionInfo.withoutSSTables(null, OperationType.INDEX_SUMMARY, (memoryPoolBytes - remainingSpace), memoryPoolBytes, memoryPoolBytes, Unit.BYTES, compactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
|
||||||
|
|
@ -54,6 +54,8 @@ public class CompactionMetrics
|
||||||
public final Meter totalCompactionsCompleted;
|
public final Meter totalCompactionsCompleted;
|
||||||
/** Total number of bytes compacted since server [re]start */
|
/** Total number of bytes compacted since server [re]start */
|
||||||
public final Counter bytesCompacted;
|
public final Counter bytesCompacted;
|
||||||
|
/** Estimated compressed bytes compacted since server [re]start, computed by scaling uncompressed bytes by the compression ratio */
|
||||||
|
public final Counter compressedBytesCompacted;
|
||||||
/** Recent/current throughput of compactions take */
|
/** Recent/current throughput of compactions take */
|
||||||
public final Meter bytesCompactedThroughput;
|
public final Meter bytesCompactedThroughput;
|
||||||
/** Time spent redistributing index summaries */
|
/** Time spent redistributing index summaries */
|
||||||
|
|
@ -150,6 +152,7 @@ public class CompactionMetrics
|
||||||
});
|
});
|
||||||
totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted"));
|
totalCompactionsCompleted = Metrics.meter(factory.createMetricName("TotalCompactionsCompleted"));
|
||||||
bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted"));
|
bytesCompacted = Metrics.counter(factory.createMetricName("BytesCompacted"));
|
||||||
|
compressedBytesCompacted = Metrics.counter(factory.createMetricName("CompressedBytesCompacted"));
|
||||||
bytesCompactedThroughput = Metrics.meter(factory.createMetricName("BytesCompactedThroughput"));
|
bytesCompactedThroughput = Metrics.meter(factory.createMetricName("BytesCompactedThroughput"));
|
||||||
|
|
||||||
// compaction failure metrics
|
// compaction failure metrics
|
||||||
|
|
|
||||||
|
|
@ -969,7 +969,7 @@ public class StreamSession
|
||||||
for (FileStore fs : allWriteableFileStores)
|
for (FileStore fs : allWriteableFileStores)
|
||||||
newStreamBytesToWritePerFileStore.merge(fs, totalBytesInPerFileStore, Long::sum);
|
newStreamBytesToWritePerFileStore.merge(fs, totalBytesInPerFileStore, Long::sum);
|
||||||
}
|
}
|
||||||
Map<FileStore, Long> totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteBytes(),
|
Map<FileStore, Long> totalCompactionWriteRemaining = Directories.perFileStore(CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes(),
|
||||||
fileStoreMapper);
|
fileStoreMapper);
|
||||||
long totalStreamRemaining = StreamManager.instance.getTotalRemainingOngoingBytes();
|
long totalStreamRemaining = StreamManager.instance.getTotalRemainingOngoingBytes();
|
||||||
long totalBytesStreamRemainingPerFileStore = totalStreamRemaining / Math.max(1, allFileStores.size());
|
long totalBytesStreamRemainingPerFileStore = totalStreamRemaining / Math.max(1, allFileStores.size());
|
||||||
|
|
|
||||||
|
|
@ -2225,6 +2225,7 @@ public class NodeProbe implements AutoCloseable
|
||||||
switch(metricName)
|
switch(metricName)
|
||||||
{
|
{
|
||||||
case "BytesCompacted":
|
case "BytesCompacted":
|
||||||
|
case "CompressedBytesCompacted":
|
||||||
case "CompactionsAborted":
|
case "CompactionsAborted":
|
||||||
case "CompactionsReduced":
|
case "CompactionsReduced":
|
||||||
case "SSTablesDroppedFromCompaction":
|
case "SSTablesDroppedFromCompaction":
|
||||||
|
|
|
||||||
|
|
@ -89,11 +89,17 @@ public class CompactionStats extends AbstractCommand
|
||||||
tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedMetrics.getCount()));
|
tableBuilder.add("compactions completed", String.valueOf(totalCompactionsCompletedMetrics.getCount()));
|
||||||
|
|
||||||
CassandraMetricsRegistry.JmxCounterMBean bytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("BytesCompacted");
|
CassandraMetricsRegistry.JmxCounterMBean bytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("BytesCompacted");
|
||||||
|
CassandraMetricsRegistry.JmxCounterMBean compressedBytesCompacted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompressedBytesCompacted");
|
||||||
if (humanReadable)
|
if (humanReadable)
|
||||||
|
{
|
||||||
tableBuilder.add("data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(bytesCompacted.getCount()))));
|
tableBuilder.add("data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(bytesCompacted.getCount()))));
|
||||||
|
tableBuilder.add("compressed data compacted", FileUtils.stringifyFileSize(Double.parseDouble(Long.toString(compressedBytesCompacted.getCount()))));
|
||||||
|
}
|
||||||
else
|
else
|
||||||
|
{
|
||||||
tableBuilder.add("data compacted", Long.toString(bytesCompacted.getCount()));
|
tableBuilder.add("data compacted", Long.toString(bytesCompacted.getCount()));
|
||||||
|
tableBuilder.add("compressed data compacted", Long.toString(compressedBytesCompacted.getCount()));
|
||||||
|
}
|
||||||
CassandraMetricsRegistry.JmxCounterMBean compactionsAborted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompactionsAborted");
|
CassandraMetricsRegistry.JmxCounterMBean compactionsAborted = (CassandraMetricsRegistry.JmxCounterMBean) probe.getCompactionMetric("CompactionsAborted");
|
||||||
tableBuilder.add("compactions aborted", Long.toString(compactionsAborted.getCount()));
|
tableBuilder.add("compactions aborted", Long.toString(compactionsAborted.getCount()));
|
||||||
|
|
||||||
|
|
@ -132,13 +138,14 @@ public class CompactionStats extends AbstractCommand
|
||||||
long remainingBytes = 0;
|
long remainingBytes = 0;
|
||||||
|
|
||||||
if (vtableOutput)
|
if (vtableOutput)
|
||||||
table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "unit", "target directory");
|
table.add("keyspace", "table", "task id", "completion ratio", "kind", "progress", "sstables", "total", "total compressed", "unit", "target directory");
|
||||||
else
|
else
|
||||||
table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress");
|
table.add("id", "compaction type", "keyspace", "table", "completed", "total", "unit", "progress");
|
||||||
|
|
||||||
for (Map<String, String> c : compactions)
|
for (Map<String, String> c : compactions)
|
||||||
{
|
{
|
||||||
long total = Long.parseLong(c.get(CompactionInfo.TOTAL));
|
long total = Long.parseLong(c.get(CompactionInfo.TOTAL));
|
||||||
|
String totalCompressedValue = c.get(CompactionInfo.TOTAL_COMPRESSED);
|
||||||
long completed = Long.parseLong(c.get(CompactionInfo.COMPLETED));
|
long completed = Long.parseLong(c.get(CompactionInfo.COMPLETED));
|
||||||
String taskType = c.get(CompactionInfo.TASK_TYPE);
|
String taskType = c.get(CompactionInfo.TASK_TYPE);
|
||||||
String keyspace = c.get(CompactionInfo.KEYSPACE);
|
String keyspace = c.get(CompactionInfo.KEYSPACE);
|
||||||
|
|
@ -148,12 +155,22 @@ public class CompactionStats extends AbstractCommand
|
||||||
String[] tables = c.get(CompactionInfo.SSTABLES).split(",");
|
String[] tables = c.get(CompactionInfo.SSTABLES).split(",");
|
||||||
String progressStr = toFileSize ? FileUtils.stringifyFileSize(completed) : Long.toString(completed);
|
String progressStr = toFileSize ? FileUtils.stringifyFileSize(completed) : Long.toString(completed);
|
||||||
String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total);
|
String totalStr = toFileSize ? FileUtils.stringifyFileSize(total) : Long.toString(total);
|
||||||
|
String totalCompressedStr;
|
||||||
|
if (totalCompressedValue != null)
|
||||||
|
{
|
||||||
|
long totalCompressed = Long.parseLong(totalCompressedValue);
|
||||||
|
totalCompressedStr = toFileSize ? FileUtils.stringifyFileSize(totalCompressed) : Long.toString(totalCompressed);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
totalCompressedStr = "n/a";
|
||||||
|
}
|
||||||
String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + '%';
|
String percentComplete = total == 0 ? "n/a" : new DecimalFormat("0.00").format((double) completed / total * 100) + '%';
|
||||||
String id = c.get(CompactionInfo.COMPACTION_ID);
|
String id = c.get(CompactionInfo.COMPACTION_ID);
|
||||||
if (vtableOutput)
|
if (vtableOutput)
|
||||||
{
|
{
|
||||||
String targetDirectory = c.get(CompactionInfo.TARGET_DIRECTORY);
|
String targetDirectory = c.get(CompactionInfo.TARGET_DIRECTORY);
|
||||||
table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, unit, targetDirectory);
|
table.add(keyspace, columnFamily, id, percentComplete, taskType, progressStr, String.valueOf(tables.length), totalStr, totalCompressedStr, unit, targetDirectory);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
table.add(id, taskType, keyspace, columnFamily, progressStr, totalStr, unit, percentComplete);
|
table.add(id, taskType, keyspace, columnFamily, progressStr, totalStr, unit, percentComplete);
|
||||||
|
|
@ -172,4 +189,4 @@ public class CompactionStats extends AbstractCommand
|
||||||
table.printTo(out);
|
table.printTo(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -115,7 +115,7 @@ public class CompactionDiskSpaceTest extends TestBaseImpl
|
||||||
public static void install(ClassLoader cl, Integer node)
|
public static void install(ClassLoader cl, Integer node)
|
||||||
{
|
{
|
||||||
new ByteBuddy().rebase(ActiveCompactions.class)
|
new ByteBuddy().rebase(ActiveCompactions.class)
|
||||||
.method(named("estimatedRemainingWriteBytes"))
|
.method(named("estimatedRemainingWriteToDiskBytes"))
|
||||||
.intercept(MethodDelegation.to(BB.class))
|
.intercept(MethodDelegation.to(BB.class))
|
||||||
.make()
|
.make()
|
||||||
.load(cl, ClassLoadingStrategy.Default.INJECTION);
|
.load(cl, ClassLoadingStrategy.Default.INJECTION);
|
||||||
|
|
@ -127,7 +127,7 @@ public class CompactionDiskSpaceTest extends TestBaseImpl
|
||||||
.load(cl, ClassLoadingStrategy.Default.INJECTION);
|
.load(cl, ClassLoadingStrategy.Default.INJECTION);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Map<File, Long> estimatedRemainingWriteBytes()
|
public static Map<File, Long> estimatedRemainingWriteToDiskBytes()
|
||||||
{
|
{
|
||||||
if (sstableDir != null)
|
if (sstableDir != null)
|
||||||
return ImmutableMap.of(sstableDir, estimatedRemaining.get());
|
return ImmutableMap.of(sstableDir, estimatedRemaining.get());
|
||||||
|
|
|
||||||
|
|
@ -60,7 +60,7 @@ public class SecondaryIndexCompactionTest extends TestBaseImpl
|
||||||
// emulate ongoing index compaction:
|
// emulate ongoing index compaction:
|
||||||
CompactionInfo.Holder h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables);
|
CompactionInfo.Holder h = new MockHolder(i.getIndexCfs().metadata(), idxSSTables);
|
||||||
CompactionManager.instance.active.beginCompaction(h);
|
CompactionManager.instance.active.beginCompaction(h);
|
||||||
CompactionManager.instance.active.estimatedRemainingWriteBytes();
|
CompactionManager.instance.active.estimatedRemainingWriteToDiskBytes();
|
||||||
CompactionManager.instance.active.finishCompaction(h);
|
CompactionManager.instance.active.finishCompaction(h);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -79,7 +79,7 @@ public class SecondaryIndexCompactionTest extends TestBaseImpl
|
||||||
@Override
|
@Override
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, nextTimeUUID(), sstables);
|
return new CompactionInfo(metadata, OperationType.COMPACTION, 0, 1000, 300, nextTimeUUID(), sstables);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -88,4 +88,4 @@ public class SecondaryIndexCompactionTest extends TestBaseImpl
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,7 @@ public class StreamsDiskSpaceTest extends TestBaseImpl
|
||||||
.withConfig(config -> config.set("hinted_handoff_enabled", false)
|
.withConfig(config -> config.set("hinted_handoff_enabled", false)
|
||||||
.with(GOSSIP)
|
.with(GOSSIP)
|
||||||
.with(NETWORK))
|
.with(NETWORK))
|
||||||
.withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteBytes"))
|
.withInstanceInitializer((cl, id) -> BB.doInstall(cl, id, ActiveCompactions.class, "estimatedRemainingWriteToDiskBytes"))
|
||||||
.start()))
|
.start()))
|
||||||
{
|
{
|
||||||
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
|
cluster.schemaChange("create table " + KEYSPACE + ".tbl (id int primary key, t int) with compaction={'class': 'SizeTieredCompactionStrategy'}");
|
||||||
|
|
@ -139,7 +139,7 @@ public class StreamsDiskSpaceTest extends TestBaseImpl
|
||||||
return ongoing.get();
|
return ongoing.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Map<File, Long> estimatedRemainingWriteBytes()
|
public static Map<File, Long> estimatedRemainingWriteToDiskBytes()
|
||||||
{
|
{
|
||||||
Map<File, Long> ret = new HashMap<>();
|
Map<File, Long> ret = new HashMap<>();
|
||||||
if (datadir != null)
|
if (datadir != null)
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ public class CompactionInfoTest extends AbstractPendingAntiCompactionTest
|
||||||
{
|
{
|
||||||
ColumnFamilyStore cfs = MockSchema.newCFS();
|
ColumnFamilyStore cfs = MockSchema.newCFS();
|
||||||
TimeUUID expectedTaskId = nextTimeUUID();
|
TimeUUID expectedTaskId = nextTimeUUID();
|
||||||
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, expectedTaskId, new ArrayList<>());
|
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, 1000, expectedTaskId, new ArrayList<>());
|
||||||
Assertions.assertThat(compactionInfo.toString())
|
Assertions.assertThat(compactionInfo.toString())
|
||||||
.contains(expectedTaskId.toString());
|
.contains(expectedTaskId.toString());
|
||||||
}
|
}
|
||||||
|
|
@ -50,7 +50,7 @@ public class CompactionInfoTest extends AbstractPendingAntiCompactionTest
|
||||||
UUID tableId = UUID.randomUUID();
|
UUID tableId = UUID.randomUUID();
|
||||||
TimeUUID taskId = nextTimeUUID();
|
TimeUUID taskId = nextTimeUUID();
|
||||||
ColumnFamilyStore cfs = MockSchema.newCFS(builder -> builder.id(TableId.fromUUID(tableId)));
|
ColumnFamilyStore cfs = MockSchema.newCFS(builder -> builder.id(TableId.fromUUID(tableId)));
|
||||||
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, taskId, new ArrayList<>());
|
CompactionInfo compactionInfo = new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, 0, 1000, 300, taskId, new ArrayList<>());
|
||||||
Assertions.assertThat(compactionInfo.toString())
|
Assertions.assertThat(compactionInfo.toString())
|
||||||
.isEqualTo("Compaction(%s, 0 / 1000 bytes)@%s(mockks, mockcf1)", taskId, tableId);
|
.isEqualTo("Compaction(%s, 0 / 1000 bytes)@%s(mockks, mockcf1)", taskId, tableId);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -85,7 +85,7 @@ public class CompactionsCQLTest extends CQLTester
|
||||||
public void before() throws IOException
|
public void before() throws IOException
|
||||||
{
|
{
|
||||||
strategy = DatabaseDescriptor.getCorruptedTombstoneStrategy();
|
strategy = DatabaseDescriptor.getCorruptedTombstoneStrategy();
|
||||||
|
|
||||||
CommitLog.instance.resetUnsafe(true);
|
CommitLog.instance.resetUnsafe(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -877,7 +877,9 @@ public class CompactionsCQLTest extends CQLTester
|
||||||
execute("insert into %s (id, i) values (?,?)", i, i);
|
execute("insert into %s (id, i) values (?,?)", i, i);
|
||||||
getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
|
getCurrentColumnFamilyStore().forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS);
|
||||||
}
|
}
|
||||||
CompactionInfo.Holder holder = holder(OperationType.COMPACTION);
|
// When we have an existing compaction with sstables of total size more than double the available space,
|
||||||
|
// we should not be able to then run a major compaction
|
||||||
|
CompactionInfo.Holder holder = holder(OperationType.COMPACTION, 2);
|
||||||
CompactionManager.instance.active.beginCompaction(holder);
|
CompactionManager.instance.active.beginCompaction(holder);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
|
@ -893,7 +895,19 @@ public class CompactionsCQLTest extends CQLTester
|
||||||
CompactionManager.instance.active.finishCompaction(holder);
|
CompactionManager.instance.active.finishCompaction(holder);
|
||||||
}
|
}
|
||||||
// don't block compactions if there is a huge validation
|
// don't block compactions if there is a huge validation
|
||||||
holder = holder(OperationType.VALIDATION);
|
holder = holder(OperationType.VALIDATION, 2);
|
||||||
|
CompactionManager.instance.active.beginCompaction(holder);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
getCurrentColumnFamilyStore().forceMajorCompaction();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CompactionManager.instance.active.finishCompaction(holder);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should be able to run when the sstables in question are 90% of the total available space
|
||||||
|
holder = holder(OperationType.COMPACTION, 0.9);
|
||||||
CompactionManager.instance.active.beginCompaction(holder);
|
CompactionManager.instance.active.beginCompaction(holder);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
|
@ -905,7 +919,7 @@ public class CompactionsCQLTest extends CQLTester
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private CompactionInfo.Holder holder(OperationType opType)
|
private CompactionInfo.Holder holder(OperationType opType, double availableSpaceMultiplier)
|
||||||
{
|
{
|
||||||
CompactionInfo.Holder holder = new CompactionInfo.Holder()
|
CompactionInfo.Holder holder = new CompactionInfo.Holder()
|
||||||
{
|
{
|
||||||
|
|
@ -915,12 +929,17 @@ public class CompactionsCQLTest extends CQLTester
|
||||||
for (File f : getCurrentColumnFamilyStore().getDirectories().getCFDirectories())
|
for (File f : getCurrentColumnFamilyStore().getDirectories().getCFDirectories())
|
||||||
availableSpace += PathUtils.tryGetSpace(f.toPath(), FileStore::getUsableSpace);
|
availableSpace += PathUtils.tryGetSpace(f.toPath(), FileStore::getUsableSpace);
|
||||||
|
|
||||||
|
Set<SSTableReader> liveSSTables = getCurrentColumnFamilyStore().getLiveSSTables();
|
||||||
|
long totalDiskUsage = (long)(availableSpace * availableSpaceMultiplier);
|
||||||
|
// Arbitrary compression ratio of 3.4
|
||||||
|
long totalUncompressedSize = (long) ((double) totalDiskUsage * 3.4);
|
||||||
return new CompactionInfo(getCurrentColumnFamilyStore().metadata(),
|
return new CompactionInfo(getCurrentColumnFamilyStore().metadata(),
|
||||||
opType,
|
opType,
|
||||||
+0,
|
+0,
|
||||||
+availableSpace * 2,
|
totalUncompressedSize,
|
||||||
|
totalDiskUsage,
|
||||||
nextTimeUUID(),
|
nextTimeUUID(),
|
||||||
getCurrentColumnFamilyStore().getLiveSSTables());
|
liveSSTables);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
|
||||||
|
|
@ -609,7 +609,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, nextTimeUUID(), compacting);
|
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 1000, 1000, nextTimeUUID(), compacting);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -650,7 +650,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
|
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -703,7 +703,7 @@ public class PendingAntiCompactionTest extends AbstractPendingAntiCompactionTest
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
|
return new CompactionInfo(cfs.metadata(), OperationType.ANTICOMPACTION, 0, 0, 0, nextTimeUUID(), cfs.getLiveSSTables());
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,7 @@ public class SSTableTasksTableTest extends CQLTester
|
||||||
|
|
||||||
long bytesCompacted = 123;
|
long bytesCompacted = 123;
|
||||||
long bytesTotal = 123456;
|
long bytesTotal = 123456;
|
||||||
|
long totalCompressedBytes = 112233;
|
||||||
TimeUUID compactionId = nextTimeUUID();
|
TimeUUID compactionId = nextTimeUUID();
|
||||||
List<SSTableReader> sstables = IntStream.range(0, 10)
|
List<SSTableReader> sstables = IntStream.range(0, 10)
|
||||||
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
||||||
|
|
@ -81,7 +82,7 @@ public class SSTableTasksTableTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, directory);
|
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, directory);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -94,7 +95,7 @@ public class SSTableTasksTableTest extends CQLTester
|
||||||
UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks");
|
UntypedResultSet result = execute("SELECT * FROM vts.sstable_tasks");
|
||||||
assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal,
|
assertRows(result, row(CQLTester.KEYSPACE, currentTable(), compactionId, 1.0 * bytesCompacted / bytesTotal,
|
||||||
toLowerCaseLocalized(OperationType.COMPACTION.toString()), bytesCompacted, sstables.size(),
|
toLowerCaseLocalized(OperationType.COMPACTION.toString()), bytesCompacted, sstables.size(),
|
||||||
directory, bytesTotal, CompactionInfo.Unit.BYTES.toString()));
|
directory, bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES.toString()));
|
||||||
|
|
||||||
CompactionManager.instance.active.finishCompaction(compactionHolder);
|
CompactionManager.instance.active.finishCompaction(compactionHolder);
|
||||||
result = execute("SELECT * FROM vts.sstable_tasks");
|
result = execute("SELECT * FROM vts.sstable_tasks");
|
||||||
|
|
|
||||||
|
|
@ -655,7 +655,7 @@ public class IndexSummaryManagerTest<R extends SSTableReader & IndexSummarySuppo
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, nextTimeUUID(), compacting);
|
return new CompactionInfo(cfs.metadata(), OperationType.UNKNOWN, 0, 0, 0, nextTimeUUID(), compacting);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
|
||||||
|
|
@ -57,6 +57,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
|
|
||||||
long bytesCompacted = 123;
|
long bytesCompacted = 123;
|
||||||
long bytesTotal = 123456;
|
long bytesTotal = 123456;
|
||||||
|
long totalCompressedBytes = 112233;
|
||||||
TimeUUID compactionId = nextTimeUUID();
|
TimeUUID compactionId = nextTimeUUID();
|
||||||
List<SSTableReader> sstables = IntStream.range(0, 10)
|
List<SSTableReader> sstables = IntStream.range(0, 10)
|
||||||
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
||||||
|
|
@ -65,7 +66,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
|
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -87,6 +88,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
assertThat(stdout).containsPattern("pending tasks\\s+[0-9]*");
|
assertThat(stdout).containsPattern("pending tasks\\s+[0-9]*");
|
||||||
assertThat(stdout).containsPattern("compactions completed\\s+[0-9]*");
|
assertThat(stdout).containsPattern("compactions completed\\s+[0-9]*");
|
||||||
assertThat(stdout).containsPattern("data compacted\\s+[0-9]*");
|
assertThat(stdout).containsPattern("data compacted\\s+[0-9]*");
|
||||||
|
assertThat(stdout).containsPattern("compressed data compacted\\s+[0-9]*");
|
||||||
assertThat(stdout).containsPattern("compactions aborted\\s+[0-9]*");
|
assertThat(stdout).containsPattern("compactions aborted\\s+[0-9]*");
|
||||||
assertThat(stdout).containsPattern("compactions reduced\\s+[0-9]*");
|
assertThat(stdout).containsPattern("compactions reduced\\s+[0-9]*");
|
||||||
assertThat(stdout).containsPattern("sstables dropped from compaction\\s+[0-9]*");
|
assertThat(stdout).containsPattern("sstables dropped from compaction\\s+[0-9]*");
|
||||||
|
|
@ -109,6 +111,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
|
|
||||||
long bytesCompacted = 123;
|
long bytesCompacted = 123;
|
||||||
long bytesTotal = 123456;
|
long bytesTotal = 123456;
|
||||||
|
long totalCompressedBytes = 112233;
|
||||||
TimeUUID compactionId = nextTimeUUID();
|
TimeUUID compactionId = nextTimeUUID();
|
||||||
List<SSTableReader> sstables = IntStream.range(0, 10)
|
List<SSTableReader> sstables = IntStream.range(0, 10)
|
||||||
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
||||||
|
|
@ -118,7 +121,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
|
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, targetDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -131,7 +134,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
|
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -143,16 +146,16 @@ public class CompactionStatsTest extends CQLTester
|
||||||
CompactionManager.instance.active.beginCompaction(compactionHolder);
|
CompactionManager.instance.active.beginCompaction(compactionHolder);
|
||||||
CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
|
CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
|
||||||
String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V");
|
String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "-V");
|
||||||
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
|
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+total compressed\\s+unit\\s+target directory");
|
||||||
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
||||||
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
||||||
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES,
|
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES,
|
||||||
targetDirectory);
|
targetDirectory);
|
||||||
assertThat(stdout).containsPattern(expectedStatsPattern);
|
assertThat(stdout).containsPattern(expectedStatsPattern);
|
||||||
|
|
||||||
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
||||||
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
||||||
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, CompactionInfo.Unit.BYTES);
|
OperationType.COMPACTION, bytesCompacted, sstables.size(), bytesTotal, totalCompressedBytes, CompactionInfo.Unit.BYTES);
|
||||||
assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
|
assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
|
||||||
|
|
||||||
CompactionManager.instance.active.finishCompaction(compactionHolder);
|
CompactionManager.instance.active.finishCompaction(compactionHolder);
|
||||||
|
|
@ -168,6 +171,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
|
|
||||||
long bytesCompacted = 123;
|
long bytesCompacted = 123;
|
||||||
long bytesTotal = 123456;
|
long bytesTotal = 123456;
|
||||||
|
long totalCompressedBytes = 112233;
|
||||||
TimeUUID compactionId = nextTimeUUID();
|
TimeUUID compactionId = nextTimeUUID();
|
||||||
List<SSTableReader> sstables = IntStream.range(0, 10)
|
List<SSTableReader> sstables = IntStream.range(0, 10)
|
||||||
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
||||||
|
|
@ -176,7 +180,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables);
|
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -205,6 +209,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
|
|
||||||
long bytesCompacted = 123;
|
long bytesCompacted = 123;
|
||||||
long bytesTotal = 123456;
|
long bytesTotal = 123456;
|
||||||
|
long totalCompressedBytes = 112233;
|
||||||
TimeUUID compactionId = nextTimeUUID();
|
TimeUUID compactionId = nextTimeUUID();
|
||||||
List<SSTableReader> sstables = IntStream.range(0, 10)
|
List<SSTableReader> sstables = IntStream.range(0, 10)
|
||||||
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
.mapToObj(i -> MockSchema.sstable(i, i * 10L, i * 10L + 9, cfs))
|
||||||
|
|
@ -214,7 +219,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, compactionId, sstables, targetDirectory);
|
return new CompactionInfo(cfs.metadata(), OperationType.COMPACTION, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables, targetDirectory);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -227,7 +232,7 @@ public class CompactionStatsTest extends CQLTester
|
||||||
{
|
{
|
||||||
public CompactionInfo getCompactionInfo()
|
public CompactionInfo getCompactionInfo()
|
||||||
{
|
{
|
||||||
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, compactionId, sstables);
|
return new CompactionInfo(cfs.metadata(), OperationType.CLEANUP, bytesCompacted, bytesTotal, totalCompressedBytes, compactionId, sstables);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isGlobal()
|
public boolean isGlobal()
|
||||||
|
|
@ -239,15 +244,15 @@ public class CompactionStatsTest extends CQLTester
|
||||||
CompactionManager.instance.active.beginCompaction(compactionHolder);
|
CompactionManager.instance.active.beginCompaction(compactionHolder);
|
||||||
CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
|
CompactionManager.instance.active.beginCompaction(nonCompactionHolder);
|
||||||
String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable");
|
String stdout = waitForNumberOfPendingTasks(2, "compactionstats", "--vtable", "--human-readable");
|
||||||
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+unit\\s+target directory");
|
assertThat(stdout).containsPattern("keyspace\\s+table\\s+task id\\s+completion ratio\\s+kind\\s+progress\\s+sstables\\s+total\\s+total compressed\\s+unit\\s+target directory");
|
||||||
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
String expectedStatsPattern = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
||||||
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
||||||
OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES,
|
OperationType.COMPACTION, "123 bytes", sstables.size(), "120.56 KiB", "109.6 KiB", CompactionInfo.Unit.BYTES,
|
||||||
targetDirectory);
|
targetDirectory);
|
||||||
assertThat(stdout).containsPattern(expectedStatsPattern);
|
assertThat(stdout).containsPattern(expectedStatsPattern);
|
||||||
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
String expectedStatsPatternForNonCompaction = String.format("%s\\s+%s\\s+%s\\s+%.2f%%\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s\\s+%s",
|
||||||
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
CQLTester.KEYSPACE, currentTable(), compactionId, (double) bytesCompacted / bytesTotal * 100,
|
||||||
OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", CompactionInfo.Unit.BYTES);
|
OperationType.CLEANUP, "123 bytes", sstables.size(), "120.56 KiB", "109.6 KiB", CompactionInfo.Unit.BYTES);
|
||||||
assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
|
assertThat(stdout).containsPattern(expectedStatsPatternForNonCompaction);
|
||||||
|
|
||||||
CompactionManager.instance.active.finishCompaction(compactionHolder);
|
CompactionManager.instance.active.finishCompaction(compactionHolder);
|
||||||
|
|
@ -277,4 +282,4 @@ public class CompactionStatsTest extends CQLTester
|
||||||
|
|
||||||
return stdout.get();
|
return stdout.get();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue