natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add duration and count of partition keys to sstablemetadata 

patch by Timothy Tu; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-18639
This commit is contained in:
timothytu12 2023-07-05 13:00:33 -04:00 committed by Stefan Miklosovic
parent 8a955e7040
commit 006ec71f63
No known key found for this signature in database
GPG Key ID: 32F35CB2F546D93E
3 changed files with 74 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.0
* Add duration and count of partition keys to sstablemetadata (CASSANDRA-18639)
* Remove deprecated compaction_tombstone_warning_threshold and compaction_large_partition_warning_threshold from yaml (CASSANDRA-18626)
* Enhance nodetool compactionstats with additional metrics (CASSANDRA-18305)
* Added support for type VECTOR<type, dimension> (CASSANDRA-18504)

46
doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc
View File

@ -16,7 +16,11 @@ sstablemetadata <options> <sstable filename(s)>
[cols=",",]
|===
|--colors |Use ANSI color sequences
|--gc_grace_seconds <arg> |The gc_grace_seconds to use when calculating
|--scan |Full sstable scan for additional details. Only available in 3.0+ sstables. Defaults: false
|--timestamp_unit <arg> |Time unit that cell timestamps are written with
|--unicode |Use unicode to draw histograms and progress bars
droppable tombstones
|===
@ -28,29 +32,34 @@ necessary, find the__Data.db file(s) using sstableutil.
Example:
....
sstableutil keyspace1 standard1 | grep Data
$ sstableutil keyspace1 standard1 | grep Data
/var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db
sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db
$ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db
SSTable: /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big
Partitioner: org.apache.cassandra.dht.Murmur3Partitioner
Bloom Filter FP chance: 0.010000
Minimum timestamp: 1535025576141000
Maximum timestamp: 1535025604309000
SSTable min local deletion time: 2147483647
SSTable max local deletion time: 2147483647
Minimum timestamp: 06/28/2023 15:15:04 (1687979704767589)
Maximum timestamp: 06/28/2023 15:15:09 (1687979709601420)
Duration: Days: 0 Hours: 0 Minutes: 0 Seconds: 4
SSTable min local deletion time: 08/17/292278994 02:12:55 (9223372036854775807)
SSTable max local deletion time: 08/17/292278994 02:12:55 (9223372036854775807)
Compressor: org.apache.cassandra.io.compress.LZ4Compressor
Compression ratio: 0.553999337227618
TTL min: 86400
TTL max: 86400
First token: -9223004712949498654 (key=39373333373831303130)
Last token: 9222554117157811897 (key=4f3438394e39374d3730)
Covered clusterings: ALL
Estimated droppable tombstones: 0.9188263888888889
SSTable Level: 0
Repaired at: 0
Originating host id: badfcb83-a283-402b-818e-78ede00aa3c4
Pending repair: --
Replay positions covered: {CommitLogPosition(segmentId=1535025390651, position=226400)=CommitLogPosition(segmentId=1535025390651, position=6849139)}
totalColumnsSet: 100000
totalRows: 20000
Total Column Cells: 72918
Total Rows: 24306
Estimated tombstone drop times:
1535039100: 80390
1535039160: 5645
@ -214,6 +223,7 @@ KeyType: org.apache.cassandra.db.marshal.BytesType
ClusteringTypes: [org.apache.cassandra.db.marshal.UTF8Type]
StaticColumns: {C3:org.apache.cassandra.db.marshal.BytesType, C4:org.apache.cassandra.db.marshal.BytesType, C0:org.apache.cassandra.db.marshal.BytesType, C1:org.apache.cassandra.db.marshal.BytesType, C2:org.apache.cassandra.db.marshal.BytesType}
RegularColumns: {}
IsTransient: false
....
== Specify gc grace seconds
@ -231,25 +241,25 @@ ref: https://issues.apache.org/jira/browse/CASSANDRA-12208
Example:
....
sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated tombstone drop times" -A4
$ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated tombstone drop times" -A4
Estimated tombstone drop times:
1536599100: 1
1536599640: 1
1536599700: 2
echo $(date +%s)
$ echo $(date +%s)
1536602005
# if gc_grace_seconds was configured at 100, all of the tombstones would be currently droppable
sstablemetadata --gc_grace_seconds 100 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
$ sstablemetadata --gc_grace_seconds 100 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
Estimated droppable tombstones: 4.0E-5
# if gc_grace_seconds was configured at 4700, some of the tombstones would be currently droppable
sstablemetadata --gc_grace_seconds 4700 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
$ sstablemetadata --gc_grace_seconds 4700 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
Estimated droppable tombstones: 9.61111111111111E-6
# if gc_grace_seconds was configured at 100, none of the tombstones would be currently droppable
sstablemetadata --gc_grace_seconds 5000 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
$ sstablemetadata --gc_grace_seconds 5000 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones"
Estimated droppable tombstones: 0.0
....
@ -268,7 +278,9 @@ in reads; defined in the table definition
timestamp of any entry in this sstable, in epoch microseconds
|Maximum
timestamp |maximum timestamp of any entry in this sstable, in epoch
microseconds
microseconds
|Duration |Difference between Maximum timestamp
and Minimum timestamp
|SSTable min local deletion time |minimum timestamp of
deletion date, based on TTL, in epoch seconds
|SSTable max local deletion
@ -291,9 +303,9 @@ this sstable was marked as repaired via sstablerepairedset, in epoch
milliseconds
|Replay positions covered |the interval of time and commitlog
positions related to this sstable
|totalColumnsSet |number of cells in the
table
|totalRows |number of rows in the table
|Total Columns Cells |number of cells in the table
|Total Rows |number of rows in the table
|Total Partitions |total number of partitions
|Estimated tombstone drop
times |approximate number of rows that will expire, ordered by epoch
seconds

58
src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
View File

@ -71,6 +71,8 @@ import static org.apache.cassandra.tools.Util.WHITE;
import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationWords;
import org.apache.cassandra.io.sstable.KeyIterator;
/**
* Shows the contents of sstable metadata
*/
@ -83,7 +85,7 @@ public class SSTableMetadataViewer
private static final String GCGS_KEY = "g";
private static final String TIMESTAMP_UNIT = "t";
private static final String SCAN = "s";
private static Comparator<ValuedByteBuffer> VCOMP = Comparator.comparingLong(ValuedByteBuffer::getValue).reversed();
private static final Comparator<ValuedByteBuffer> VCOMP = Comparator.comparingLong(ValuedByteBuffer::getValue).reversed();
static
{
@ -321,6 +323,20 @@ public class SSTableMetadataViewer
CompactionMetadata compaction = statsComponent.compactionMetadata();
SerializationHeader.Component header = statsComponent.serializationHeader();
Class<? extends ICompressor> compressorClass = null;
TableMetadata metadata = Util.metadataFromSSTable(descriptor);
SSTableReader sstable = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(metadata));
int count = 0;
try (KeyIterator iter = sstable.keyIterator())
{
while (iter.hasNext())
{
iter.next();
count += 1;
}
}
try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor))
{
compressorClass = compression != null ? compression.compressor().getClass() : null;
@ -338,10 +354,12 @@ public class SSTableMetadataViewer
}
if (stats != null)
{
field("Minimum timestamp", stats.minTimestamp, toDateString(stats.minTimestamp, tsUnit));
field("Maximum timestamp", stats.maxTimestamp, toDateString(stats.maxTimestamp, tsUnit));
field("SSTable min local deletion time", stats.minLocalDeletionTime, deletion(stats.minLocalDeletionTime));
field("SSTable max local deletion time", stats.maxLocalDeletionTime, deletion(stats.maxLocalDeletionTime));
TimeUnit tsUnit = TimeUnit.MICROSECONDS;
field("Minimum timestamp", toDateString(stats.minTimestamp, tsUnit), Long.toString(stats.minTimestamp));
field("Maximum timestamp", toDateString(stats.maxTimestamp, tsUnit), Long.toString(stats.maxTimestamp));
field("Duration", durationString(stats.maxTimestamp - stats.minTimestamp));
field("SSTable min local deletion time", deletion(stats.minLocalDeletionTime), Long.toString(stats.minLocalDeletionTime));
field("SSTable max local deletion time", deletion(stats.maxLocalDeletionTime), Long.toString(stats.maxLocalDeletionTime));
field("Compressor", compressorClass != null ? compressorClass.getName() : "-");
if (compressorClass != null)
field("Compression ratio", stats.compressionRatio);
@ -363,8 +381,8 @@ public class SSTableMetadataViewer
field("Originating host id", stats.originatingHostId);
field("Pending repair", stats.pendingRepair);
field("Replay positions covered", stats.commitLogIntervals);
field("totalColumnsSet", stats.totalColumnsSet);
field("totalRows", stats.totalRows);
field("Total Column Cells", stats.totalColumnsSet);
field("Total Rows", stats.totalRows);
field("Estimated tombstone drop times", "");
TermHistogram estDropped = new TermHistogram(stats.estimatedTombstoneDropTime,
@ -376,6 +394,7 @@ public class SSTableMetadataViewer
String::valueOf);
estDropped.printHistogram(out, color, unicode);
field("Partition Size", "");
field("Total Partitions", count);
TermHistogram rowSize = new TermHistogram(stats.estimatedPartitionSize,
"Size (bytes)",
offset -> String.format("%d %s",
@ -410,15 +429,16 @@ public class SSTableMetadataViewer
field("EncodingStats minTTL", encodingStats.minTTL,
toDurationString(encodingStats.minTTL, TimeUnit.SECONDS));
field("EncodingStats minLocalDeletionTime", encodingStats.minLocalDeletionTime,
toDateString(encodingStats.minLocalDeletionTime, TimeUnit.SECONDS));
field("EncodingStats minTimestamp", encodingStats.minTimestamp,
toDateString(encodingStats.minTimestamp, tsUnit));
field("EncodingStats minLocalDeletionTime", toDateString(encodingStats.minLocalDeletionTime,
TimeUnit.SECONDS), Long.toString(encodingStats.minLocalDeletionTime));
field("EncodingStats minTimestamp", toDateString(encodingStats.minTimestamp, tsUnit),
Long.toString(encodingStats.minTimestamp));
field("KeyType", keyType.toString());
field("ClusteringTypes", clusteringTypes.toString());
field("StaticColumns", FBUtilities.toString(statics));
field("RegularColumns", FBUtilities.toString(regulars));
field("IsTransient", stats.isTransient);
if (stats != null)
field("IsTransient", stats.isTransient);
}
}
@ -442,10 +462,20 @@ public class SSTableMetadataViewer
if (color) sb.append(WHITE);
sb.append(" (");
sb.append(comment);
sb.append(")");
sb.append(')');
if (color) sb.append(RESET);
}
this.out.println(sb.toString());
this.out.println(sb);
}
public static String durationString(Long value)
{
long seconds = TimeUnit.MICROSECONDS.toSeconds(value);
long day = TimeUnit.SECONDS.toDays(seconds);
long hours = TimeUnit.SECONDS.toHours(seconds) - (day * 24);
long minute = TimeUnit.SECONDS.toMinutes(seconds) - (TimeUnit.SECONDS.toHours(seconds) * 60);
long second = TimeUnit.SECONDS.toSeconds(seconds) - (TimeUnit.SECONDS.toMinutes(seconds) * 60);
return "Days: " + day + " Hours: " + hours + " Minutes: " + minute + " Seconds: " + second;
}
private static void printUsage()