From e72ec4e82835cef80cf9e9d8c58b25801d4af8c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20de=20la=20Pe=C3=B1a?= Date: Fri, 21 Apr 2023 18:31:32 +0100 Subject: [PATCH] Add sstablepartitions offline tool to find large partitions in sstables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit patch by Andrés de la Peña and Robert Stupp; reviewed by Brandon Williams and Stefan Miklosovic for CASSANDRA-8720 Co-authored-by: Andrés de la Peña Co-authored-by: Robert Stupp --- CHANGES.txt | 1 + NEWS.txt | 1 + debian/cassandra.install | 1 + .../pages/managing/tools/sstable/index.adoc | 1 + .../tools/sstable/sstablepartitions.adoc | 171 ++++ redhat/cassandra.spec | 1 + .../cassandra/io/sstable/Descriptor.java | 43 +- .../org/apache/cassandra/schema/TableId.java | 25 + .../cassandra/tools/SSTablePartitions.java | 889 ++++++++++++++++++ .../tools/SSTablePartitionsTest.java | 654 +++++++++++++ tools/bin/sstablepartitions | 49 + 11 files changed, 1823 insertions(+), 13 deletions(-) create mode 100644 doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc create mode 100644 src/java/org/apache/cassandra/tools/SSTablePartitions.java create mode 100644 test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java create mode 100755 tools/bin/sstablepartitions diff --git a/CHANGES.txt b/CHANGES.txt index 862755286a..970058c239 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * Add sstablepartitions offline tool to find large partitions in sstables (CASSANDRA-8720) * Replace usages of json-simple dependency by Jackson (CASSANDRA-16855) * When decommissioning should set Severity to limit traffic (CASSANDRA-18430) * For Java11 and Java17 remove -XX:-UseBiasedLocking as it is the default already (CASSANDRA-17869) diff --git a/NEWS.txt b/NEWS.txt index b18a5e3a48..4e0792aee6 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -140,6 +140,7 @@ New features - Added new SELECT_MASKED permission. It allows to run SELECT queries selecting the clear values of masked columns. Superusers have it by default, whereas regular users don't have it by default. - Added support for using UDFs as masking functions attached to table columns on the schema. + - Added `sstablepartitions` offline tool to find large partitions in sstables. Upgrading --------- diff --git a/debian/cassandra.install b/debian/cassandra.install index f54d1ad835..dced5a29e2 100644 --- a/debian/cassandra.install +++ b/debian/cassandra.install @@ -25,6 +25,7 @@ tools/bin/fqltool usr/bin tools/bin/auditlogviewer usr/bin tools/bin/jmxtool usr/bin tools/bin/hash_password usr/bin +tools/bin/sstablepartitions usr/bin lib/*.jar usr/share/cassandra/lib lib/*.zip usr/share/cassandra/lib lib/sigar-bin/* usr/share/cassandra/lib/sigar-bin diff --git a/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc index cb787ece21..cc7637c8dd 100644 --- a/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc +++ b/doc/modules/cassandra/pages/managing/tools/sstable/index.adoc @@ -12,6 +12,7 @@ stopped. * xref:tools/sstable/sstableloader.adoc[sstableloader] * xref:tools/sstable/sstablemetadata.adoc[sstablemetadata] * xref:tools/sstable/sstableofflinerelevel.adoc[sstableofflinerelevel] +* xref:tools/sstable/sstablepartitions.adoc[sstablepartitions] * xref:tools/sstable/sstablerepairedset.adoc[sstablerepairdset] * xref:tools/sstable/sstablescrub.adoc[sstablescrub] * xref:tools/sstable/sstablesplit.adoc[sstablesplit] diff --git a/doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc new file mode 100644 index 0000000000..6684946b47 --- /dev/null +++ b/doc/modules/cassandra/pages/managing/tools/sstable/sstablepartitions.adoc @@ -0,0 +1,171 @@ += sstablepartitions + +Identifies large partitions of SSTables and outputs the partition size in bytes, row count, cell count, and tombstone count. + +You can supply any number of sstables file paths, or directories containing sstables. Each sstable will be analyzed separately. + +If a metrics threshold such as `--min-size`, `--min-rows`, `--min-cells` or `--min-tombstones` is provided, +then the partition keys exceeding of the threshold will be printed in the output. +It also prints a summary of metrics for the table. The percentiles in the metrics are estimates, +while the min/max/count metrics are accurate. + +The default output of this tool is meant to be read by human eyes. +Future versions might include small formatting changes or present new data that can fool scripts reading it. +Scripts or other automatic tools should use the `--csv` flag to produce machine-readable output. +Future versions will not change the format of the CSV output except for maybe adding new columns, +so a proper CSV parser consuming the output should keep working. + +Cassandra doesn't need to be running before this tool is executed. + +== Usage + +sstablepartitions + +[cols=",",] +|=== +|-t, --min-size |Partition size threshold, expressed as either the number of bytes or a size with unit of the form 10KiB, 20MiB, 30GiB, etc. +|-w, --min-rows |Partition row count threshold. +|-c, --min-cells |Partition cell count threshold +|-o, --min-tombstones |Partition tombstone count threshold. +|-k, --key |Partition keys to include, instead of scanning all partitions. +|-x, --exclude-key |Partition keys to exclude. +|-r, --recursive |Scan for sstables recursively +|-b, --backups |Include backups present in data directories when scanning directories +|-s, --snaphsots |Include snapshots present in data directories when scanning directories +|-u, --current-timestamp |Timestamp (seconds since epoch, unit time) for TTL expired calculation. +|-y, --partitions-only |Only brief partition information. Exclude per-partition detailed row/cell/tombstone information from process and output. +|-m, --csv |Produced CSV output (machine readable) +|=== + +== Examples + +=== Analyze partition statistics for a single SSTable + +Use the path to the SSTable file as the only argument. + +Example: + +.... +sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db + +Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk) + Partition size Row count Cell count Tombstone count + ~p50 767.519 KiB 770 1916 0 + ~p75 2.238 MiB 2299 5722 0 + ~p90 3.867 MiB 3311 9887 50 + ~p95 16.629 MiB 14237 42510 446 + ~p99 148.267 MiB 126934 379022 1331 + ~p999 368.936 MiB 315852 943127 2759 + min 56.854 KiB 100 150 0 + max 356.067 MiB 310706 932118 2450 + count 210 +.... + +=== Analyze partition statistics for all SSTables in a directory + +Use the path to the SSTables directory as the only argument. + +Example: + +.... +sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb + +Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk) + Partition size Row count Cell count Tombstone count + ~p50 767.519 KiB 770 1916 0 + ~p75 2.238 MiB 2299 5722 0 + ~p90 3.867 MiB 3311 9887 50 + ~p95 16.629 MiB 14237 42510 446 + ~p99 148.267 MiB 126934 379022 1331 + ~p999 368.936 MiB 315852 943127 2759 + min 56.854 KiB 100 150 0 + max 356.067 MiB 310706 932118 2450 + count 210 + +Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #9 (big-nc) (457.540 MiB uncompressed, 174.880 MiB on disk) + Partition size Row count Cell count Tombstone count + ~p50 1.865 MiB 1597 4768 0 + ~p75 13.858 MiB 14237 42510 0 + ~p90 28.735 MiB 29521 73457 50 + ~p95 34.482 MiB 29521 88148 8239 + ~p99 49.654 MiB 42510 126934 14237 + ~p999 49.654 MiB 42510 126934 14237 + min 47.272 KiB 100 150 0 + max 45.133 MiB 39429 118287 13030 + count 57 +.... + +=== Output only partitions over 100MiB in size + +Use the `--min-size` option to specify the minimum size a partition must have to be included in the output. + +Example: + +.... +sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db --min-size 100MiB + +Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk) + Partition: '13' (0000000d) live, size: 105.056 MiB, rows: 91490, cells: 274470, tombstones: 50 (row:50, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) + Partition: '1' (00000001) live, size: 127.241 MiB, rows: 111065, cells: 333195, tombstones: 50 (row:50, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) + Partition: '8' (00000008) live, size: 356.067 MiB, rows: 310706, cells: 932118, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) + Partition: '2' (00000002) live, size: 213.341 MiB, rows: 186582, cells: 559125, tombstones: 978 (row:978, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) +Summary of k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc): + File: /Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db + 4 partitions match + Keys: 13 1 8 2 + Partition size Row count Cell count Tombstone count + ~p50 767.519 KiB 770 1916 0 + ~p75 2.238 MiB 2299 5722 0 + ~p90 3.867 MiB 3311 9887 50 + ~p95 16.629 MiB 14237 42510 446 + ~p99 148.267 MiB 126934 379022 1331 + ~p999 368.936 MiB 315852 943127 2759 + min 56.854 KiB 100 150 0 + max 356.067 MiB 310706 932118 2450 + count 210 +.... + +=== Output only partitions with more than 1000 tombstones + +Use the `--min-tombstones` option to specify the minimum number of tombstones a partition must have to be included in the output. + +Example: + +.... +sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db --min-tombstones 1000 + +Processing k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc) (1.368 GiB uncompressed, 534.979 MiB on disk) + Partition: '55' (00000037) live, size: 1.290 MiB, rows: 2317, cells: 3474, tombstones: 1159 (row:1159, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) + Partition: '28' (0000001c) live, size: 1.198 MiB, rows: 2099, cells: 3147, tombstones: 1050 (row:1050, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) + Partition: '89' (00000059) live, size: 1.346 MiB, rows: 2226, cells: 3339, tombstones: 1113 (row:1113, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) + Partition: '21' (00000015) live, size: 3.853 MiB, rows: 4900, cells: 9927, tombstones: 2450 (row:2450, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0) +Summary of k.t-d7be5e90e90111ed8b54efe3c39cb0bb #8 (big-nc): + File: /Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db + 4 partitions match + Keys: 55 28 89 21 + Partition size Row count Cell count Tombstone count + ~p50 767.519 KiB 770 1916 0 + ~p75 2.238 MiB 2299 5722 0 + ~p90 3.867 MiB 3311 9887 50 + ~p95 16.629 MiB 14237 42510 446 + ~p99 148.267 MiB 126934 379022 1331 + ~p999 368.936 MiB 315852 943127 2759 + min 56.854 KiB 100 150 0 + max 356.067 MiB 310706 932118 2450 + count 210 +.... + +=== Output CSV machine-readable output + +Use the `--csv` option to output a CSV machine-readable output, combined with any threshold value. + +Example: + +.... +sstablepartitions data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db --min-size 100MiB --csv +key,keyBinary,live,offset,size,rowCount,cellCount,tombstoneCount,rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount,cellTombstoneCount,rowTtlExpired,cellTtlExpired,directory,keyspace,table,index,snapshot,backup,generation,format,version +"13",0000000d,true,186403543,110158965,91490,274470,50,50,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc +"1",00000001,true,325141542,133422183,111065,333195,50,50,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc +"8",00000008,true,477133752,373362819,310706,932118,0,0,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc +"2",00000002,true,851841363,223704192,186582,559125,978,978,0,0,0,0,0,/Users/adelapena/src/cassandra/trunk/data/data/k/t-d7be5e90e90111ed8b54efe3c39cb0bb/nc-8-big-Data.db,k,t,,,,8,big,nc +.... \ No newline at end of file diff --git a/redhat/cassandra.spec b/redhat/cassandra.spec index 5004eed585..ae5ad4c308 100644 --- a/redhat/cassandra.spec +++ b/redhat/cassandra.spec @@ -198,6 +198,7 @@ This package contains extra tools for working with Cassandra clusters. %attr(755,root,root) %{_bindir}/sstableofflinerelevel %attr(755,root,root) %{_bindir}/sstablerepairedset %attr(755,root,root) %{_bindir}/sstablesplit +%attr(755,root,root) %{_bindir}/sstablepartitions %attr(755,root,root) %{_bindir}/auditlogviewer %attr(755,root,root) %{_bindir}/jmxtool %attr(755,root,root) %{_bindir}/fqltool diff --git a/src/java/org/apache/cassandra/io/sstable/Descriptor.java b/src/java/org/apache/cassandra/io/sstable/Descriptor.java index b7dcdcd0d1..527a7140e3 100644 --- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java +++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java @@ -262,6 +262,27 @@ public class Descriptor return fromFileWithComponent(file).left; } + public static Component componentFromFile(File file) + { + String name = file.name(); + List tokens = filenameTokens(name); + + return Component.parse(tokens.get(3), formatFromName(name, tokens)); + } + + private static SSTableFormat.Type formatFromName(String fileName, List tokens) + { + String format = tokens.get(2); + try + { + return SSTableFormat.Type.getByName(format); + } + catch (RuntimeException e) + { + throw invalidSSTable(fileName, "unknown 'format' part (%s)", format); + } + } + /** * Parse a sstable filename, extracting both the {@code Descriptor} and {@code Component} part. * The keyspace/table name will be extracted from the directory path. @@ -342,9 +363,8 @@ public class Descriptor return Pair.create(new Descriptor(info.version, parentOf(file.name(), file), keyspace, table, info.id, info.format), info.component); } - private static SSTableInfo validateAndExtractInfo(File file) + private static List filenameTokens(String name) { - String name = file.name(); List tokens = filenameSplitter.splitToList(name); int size = tokens.size(); @@ -359,6 +379,13 @@ public class Descriptor throw new IllegalArgumentException(String.format("%s is of version %s which is now unsupported and cannot be read.", name, tokens.get(size - 3))); throw new IllegalArgumentException(String.format("Invalid sstable file %s: the name doesn't look like a supported sstable file name", name)); } + return tokens; + } + + private static SSTableInfo validateAndExtractInfo(File file) + { + String name = file.name(); + List tokens = filenameTokens(name); String versionString = tokens.get(0); if (!Version.validate(versionString)) @@ -374,17 +401,7 @@ public class Descriptor throw invalidSSTable(name, "the 'id' part (%s) of the name doesn't parse as a valid unique identifier", tokens.get(1)); } - String formatString = tokens.get(2); - SSTableFormat.Type format; - try - { - format = SSTableFormat.Type.getByName(formatString); - } - catch (RuntimeException e) - { - throw invalidSSTable(name, "unknown 'format' part (%s)", formatString); - } - + SSTableFormat.Type format = formatFromName(name, tokens); Component component = Component.parse(tokens.get(3), format); Version version = format.info.getVersion(versionString); diff --git a/src/java/org/apache/cassandra/schema/TableId.java b/src/java/org/apache/cassandra/schema/TableId.java index fd47a47e58..ceeeec315a 100644 --- a/src/java/org/apache/cassandra/schema/TableId.java +++ b/src/java/org/apache/cassandra/schema/TableId.java @@ -20,11 +20,15 @@ package org.apache.cassandra.schema; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.UUID; +import javax.annotation.Nullable; + import org.apache.commons.lang3.ArrayUtils; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Pair; import static org.apache.cassandra.utils.TimeUUID.Generator.nextTimeUUID; @@ -62,6 +66,27 @@ public class TableId return new TableId(UUID.fromString(idString)); } + @Nullable + public static Pair tableNameAndIdFromFilename(String filename) + { + int dash = filename.lastIndexOf('-'); + if (dash <= 0 || dash != filename.length() - 32 - 1) + return null; + + TableId id = fromHexString(filename.substring(dash + 1)); + String tableName = filename.substring(0, dash); + + return Pair.create(tableName, id); + } + + private static TableId fromHexString(String nonDashUUID) + { + ByteBuffer bytes = ByteBufferUtil.hexToBytes(nonDashUUID); + long msb = bytes.getLong(0); + long lsb = bytes.getLong(8); + return fromUUID(new UUID(msb, lsb)); + } + /** * Creates the UUID of a system table. * diff --git a/src/java/org/apache/cassandra/tools/SSTablePartitions.java b/src/java/org/apache/cassandra/tools/SSTablePartitions.java new file mode 100644 index 0000000000..fec5ef94be --- /dev/null +++ b/src/java/org/apache/cassandra/tools/SSTablePartitions.java @@ -0,0 +1,889 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.ToLongFunction; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; + +import org.apache.cassandra.config.DataStorageSpec; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.PartitionPosition; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.ColumnData; +import org.apache.cassandra.db.rows.ComplexColumnData; +import org.apache.cassandra.db.rows.RangeTombstoneMarker; +import org.apache.cassandra.db.rows.Row; +import org.apache.cassandra.db.rows.Unfiltered; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.dht.AbstractBounds; +import org.apache.cassandra.dht.Bounds; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.schema.TableId; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.schema.TableMetadataRef; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.Clock; +import org.apache.cassandra.utils.EstimatedHistogram; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; + +public class SSTablePartitions +{ + private static final String KEY_OPTION = "k"; + private static final String EXCLUDE_KEY_OPTION = "x"; + private static final String RECURSIVE_OPTION = "r"; + private static final String SNAPSHOTS_OPTION = "s"; + private static final String BACKUPS_OPTION = "b"; + private static final String PARTITIONS_ONLY_OPTION = "y"; + private static final String SIZE_THRESHOLD_OPTION = "t"; + private static final String TOMBSTONE_THRESHOLD_OPTION = "o"; + private static final String CELL_THRESHOLD_OPTION = "c"; + private static final String ROW_THRESHOLD_OPTION = "w"; + private static final String CSV_OPTION = "m"; + private static final String CURRENT_TIMESTAMP_OPTION = "u"; + + private static final Options options = new Options(); + + private static final TableId EMPTY_TABLE_ID = TableId.fromUUID(new UUID(0L, 0L)); + + static + { + DatabaseDescriptor.clientInitialization(); + + Option optKey = new Option(KEY_OPTION, "key", true, "Partition keys to include"); + // Number of times -k can be passed on the command line. + optKey.setArgs(Option.UNLIMITED_VALUES); + options.addOption(optKey); + + Option excludeKey = new Option(EXCLUDE_KEY_OPTION, "exclude-key", true, + "Excluded partition key(s) from partition detailed row/cell/tombstone " + + "information (irrelevant, if --partitions-only is given)"); + excludeKey.setArgs(Option.UNLIMITED_VALUES); // Number of times -x can be passed on the command line. + options.addOption(excludeKey); + + Option thresholdKey = new Option(SIZE_THRESHOLD_OPTION, "min-size", true, + "partition size threshold, expressed as either the number of bytes or a " + + "size with unit of the form 10KiB, 20MiB, 30GiB, etc."); + options.addOption(thresholdKey); + + Option tombstoneKey = new Option(TOMBSTONE_THRESHOLD_OPTION, "min-tombstones", true, + "partition tombstone count threshold"); + options.addOption(tombstoneKey); + + Option cellKey = new Option(CELL_THRESHOLD_OPTION, "min-cells", true, "partition cell count threshold"); + options.addOption(cellKey); + + Option rowKey = new Option(ROW_THRESHOLD_OPTION, "min-rows", true, "partition row count threshold"); + options.addOption(rowKey); + + Option currentTimestampKey = new Option(CURRENT_TIMESTAMP_OPTION, "current-timestamp", true, + "timestamp (seconds since epoch, unit time) for TTL expired calculation"); + options.addOption(currentTimestampKey); + + Option recursiveKey = new Option(RECURSIVE_OPTION, "recursive", false, "scan for sstables recursively"); + options.addOption(recursiveKey); + + Option snapshotsKey = new Option(SNAPSHOTS_OPTION, "snapshots", false, + "include snapshots present in data directories (recursive scans)"); + options.addOption(snapshotsKey); + + Option backupsKey = new Option(BACKUPS_OPTION, "backups", false, + "include backups present in data directories (recursive scans)"); + options.addOption(backupsKey); + + Option partitionsOnlyKey = new Option(PARTITIONS_ONLY_OPTION, "partitions-only", false, + "Do not process per-partition detailed row/cell/tombstone information, " + + "only brief information"); + options.addOption(partitionsOnlyKey); + + Option csvKey = new Option(CSV_OPTION, "csv", false, "CSV output (machine readable)"); + options.addOption(csvKey); + } + + /** + * Given arguments specifying a list of SSTables or directories, print information about SSTable partitions. + * + * @param args command lines arguments + * @throws ConfigurationException on configuration failure (wrong params given) + */ + public static void main(String[] args) throws ConfigurationException, IOException + { + CommandLineParser parser = new PosixParser(); + CommandLine cmd; + try + { + cmd = parser.parse(options, args); + } + catch (ParseException e) + { + System.err.println(e.getMessage()); + printUsage(); + System.exit(1); + return; + } + + if (cmd.getArgs().length == 0) + { + System.err.println("You must supply at least one sstable or directory"); + printUsage(); + System.exit(1); + } + + int ec = processArguments(cmd); + + System.exit(ec); + } + + private static void printUsage() + { + String usage = String.format("sstablepartitions %n"); + String header = "Print partition statistics of one or more sstables."; + new HelpFormatter().printHelp(usage, header, options, ""); + } + + private static int processArguments(CommandLine cmd) throws IOException + { + String[] keys = cmd.getOptionValues(KEY_OPTION); + Set excludes = cmd.getOptionValues(EXCLUDE_KEY_OPTION) == null + ? Collections.emptySet() + : ImmutableSet.copyOf(cmd.getOptionValues(EXCLUDE_KEY_OPTION)); + + boolean scanRecursive = cmd.hasOption(RECURSIVE_OPTION); + boolean withSnapshots = cmd.hasOption(SNAPSHOTS_OPTION); + boolean withBackups = cmd.hasOption(BACKUPS_OPTION); + boolean csv = cmd.hasOption(CSV_OPTION); + boolean partitionsOnly = cmd.hasOption(PARTITIONS_ONLY_OPTION); + + long sizeThreshold = Long.MAX_VALUE; + int cellCountThreshold = Integer.MAX_VALUE; + int rowCountThreshold = Integer.MAX_VALUE; + int tombstoneCountThreshold = Integer.MAX_VALUE; + long currentTime = Clock.Global.currentTimeMillis() / 1000L; + + try + { + if (cmd.hasOption(SIZE_THRESHOLD_OPTION)) + { + String threshold = cmd.getOptionValue(SIZE_THRESHOLD_OPTION); + sizeThreshold = NumberUtils.isParsable(threshold) + ? Long.parseLong(threshold) + : new DataStorageSpec.LongBytesBound(threshold).toBytes(); + } + if (cmd.hasOption(CELL_THRESHOLD_OPTION)) + cellCountThreshold = Integer.parseInt(cmd.getOptionValue(CELL_THRESHOLD_OPTION)); + if (cmd.hasOption(ROW_THRESHOLD_OPTION)) + rowCountThreshold = Integer.parseInt(cmd.getOptionValue(ROW_THRESHOLD_OPTION)); + if (cmd.hasOption(TOMBSTONE_THRESHOLD_OPTION)) + tombstoneCountThreshold = Integer.parseInt(cmd.getOptionValue(TOMBSTONE_THRESHOLD_OPTION)); + if (cmd.hasOption(CURRENT_TIMESTAMP_OPTION)) + currentTime = Integer.parseInt(cmd.getOptionValue(CURRENT_TIMESTAMP_OPTION)); + } + catch (NumberFormatException e) + { + System.err.printf("Invalid threshold argument: %s%n", e.getMessage()); + return 1; + } + + if (sizeThreshold < 0 || cellCountThreshold < 0 || tombstoneCountThreshold < 0 || currentTime < 0) + { + System.err.println("Negative values are not allowed"); + return 1; + } + + List directories = new ArrayList<>(); + List descriptors = new ArrayList<>(); + + if (!argumentsToFiles(cmd.getArgs(), descriptors, directories)) + return 1; + + for (File directory : directories) + { + processDirectory(scanRecursive, withSnapshots, withBackups, directory, descriptors); + } + + if (csv) + System.out.println("key,keyBinary,live,offset,size,rowCount,cellCount," + + "tombstoneCount,rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount," + + "cellTombstoneCount,rowTtlExpired,cellTtlExpired," + + "directory,keyspace,table,index," + + "snapshot,backup,generation,format,version"); + + Collections.sort(descriptors); + + for (ExtendedDescriptor desc : descriptors) + { + processSSTable(keys, excludes, desc, + sizeThreshold, cellCountThreshold, rowCountThreshold, tombstoneCountThreshold, partitionsOnly, + csv, currentTime); + } + + return 0; + } + + private static void processDirectory(boolean scanRecursive, + boolean withSnapshots, + boolean withBackups, + File dir, + List descriptors) + { + File[] files = dir.tryList(); + if (files == null) + return; + + for (File file : files) + { + if (file.isFile()) + { + try + { + if (Descriptor.componentFromFile(file) != BigFormat.Components.DATA) + continue; + + ExtendedDescriptor desc = ExtendedDescriptor.guessFromFile(file); + if (desc.snapshot != null && !withSnapshots) + continue; + if (desc.backup != null && !withBackups) + continue; + + descriptors.add(desc); + } + catch (IllegalArgumentException e) + { + // ignore that error when scanning directories + } + } + if (scanRecursive && file.isDirectory()) + { + processDirectory(true, + withSnapshots, withBackups, + file, + descriptors); + } + } + } + + private static boolean argumentsToFiles(String[] args, List descriptors, List directories) + { + boolean err = false; + for (String arg : args) + { + File file = new File(arg); + if (!file.exists()) + { + System.err.printf("Argument '%s' does not resolve to a file or directory%n", arg); + err = true; + } + + if (!file.isReadable()) + { + System.err.printf("Argument '%s' is not a readable file or directory (check permissions)%n", arg); + err = true; + continue; + } + + if (file.isFile()) + { + try + { + descriptors.add(ExtendedDescriptor.guessFromFile(file)); + } + catch (IllegalArgumentException e) + { + System.err.printf("Argument '%s' is not an sstable%n", arg); + err = true; + } + } + if (file.isDirectory()) + directories.add(file); + } + return !err; + } + + private static void processSSTable(String[] keys, + Set excludedKeys, + ExtendedDescriptor desc, + long sizeThreshold, + int cellCountThreshold, + int rowCountThreshold, + int tombstoneCountThreshold, + boolean partitionsOnly, + boolean csv, + long currentTime) throws IOException + { + TableMetadata metadata = Util.metadataFromSSTable(desc.descriptor); + SSTableReader sstable = SSTableReader.openNoValidation(null, desc.descriptor, TableMetadataRef.forOfflineTools(metadata)); + + if (!csv) + System.out.printf("%nProcessing %s (%s uncompressed, %s on disk)%n", + desc, + prettyPrintMemory(sstable.uncompressedLength()), + prettyPrintMemory(sstable.onDiskLength())); + + List matches = new ArrayList<>(); + SSTableStats sstableStats = new SSTableStats(); + + try (ISSTableScanner scanner = buildScanner(sstable, metadata, keys, excludedKeys)) + { + while (scanner.hasNext()) + { + try (UnfilteredRowIterator partition = scanner.next()) + { + ByteBuffer key = partition.partitionKey().getKey(); + boolean isExcluded = excludedKeys.contains(metadata.partitionKeyType.getString(key)); + + PartitionStats partitionStats = new PartitionStats(key, + scanner.getCurrentPosition(), + partition.partitionLevelDeletion().isLive()); + + // Consume the partition to populate the stats. + while (partition.hasNext()) + { + Unfiltered unfiltered = partition.next(); + + // We don't need any details if we are only interested on its size or if it's excluded. + if (!partitionsOnly && !isExcluded) + partitionStats.addUnfiltered(desc, currentTime, unfiltered); + } + + // record the partiton size + partitionStats.endOfPartition(scanner.getCurrentPosition()); + + if (isExcluded) + continue; + + sstableStats.addPartition(partitionStats); + + if (partitionStats.size < sizeThreshold && + partitionStats.rowCount < rowCountThreshold && + partitionStats.cellCount < cellCountThreshold && + partitionStats.tombstoneCount() < tombstoneCountThreshold) + continue; + + matches.add(partitionStats); + if (csv) + partitionStats.printPartitionInfoCSV(metadata, desc); + else + partitionStats.printPartitionInfo(metadata, partitionsOnly); + } + } + } + catch (RuntimeException e) + { + System.err.printf("Failure processing sstable %s: %s%n", desc.descriptor, e); + } + finally + { + sstable.selfRef().release(); + } + + if (!csv) + { + printSummary(metadata, desc, sstableStats, matches, partitionsOnly); + } + } + + private static String prettyPrintMemory(long bytes) + { + return FBUtilities.prettyPrintMemory(bytes, true); + } + + private static ISSTableScanner buildScanner(SSTableReader sstable, + TableMetadata metadata, + String[] keys, + Set excludedKeys) + { + if (keys != null && keys.length > 0) + { + try + { + return sstable.getScanner(Arrays.stream(keys) + .filter(key -> !excludedKeys.contains(key)) + .map(metadata.partitionKeyType::fromString) + .map(k -> sstable.getPartitioner().decorateKey(k)) + .sorted() + .map(DecoratedKey::getToken) + .map(token -> new Bounds<>(token.minKeyBound(), token.maxKeyBound())) + .collect(Collectors.>toList()) + .iterator()); + } + catch (RuntimeException e) + { + System.err.printf("Cannot use one or more partition keys in %s for the partition key type ('%s') " + + "of the underlying table: %s%n", + Arrays.toString(keys), + metadata.partitionKeyType.asCQL3Type(), e); + } + } + return sstable.getScanner(); + } + + private static void printSummary(TableMetadata metadata, + ExtendedDescriptor desc, + SSTableStats stats, + List matches, + boolean partitionsOnly) + { + // Print header + if (!matches.isEmpty()) + { + System.out.printf("Summary of %s:%n" + + " File: %s%n" + + " %d partitions match%n" + + " Keys:", desc, desc.descriptor.fileFor(BigFormat.Components.DATA), matches.size()); + + for (PartitionStats match : matches) + System.out.print(" " + maybeEscapeKeyForSummary(metadata, match.key)); + + System.out.println(); + } + + // Print stats table columns + String format; + if (partitionsOnly) + { + System.out.printf(" %20s%n", "Partition size"); + format = " %-5s %20s%n"; + } + else + { + System.out.printf(" %20s %20s %20s %20s%n", "Partition size", "Row count", "Cell count", "Tombstone count"); + format = " %-5s %20s %20d %20d %20d%n"; + } + + // Print approximate percentiles from the histograms + printPercentile(partitionsOnly, stats, format, "~p50", h -> h.percentile(.5d)); + printPercentile(partitionsOnly, stats, format, "~p75", h -> h.percentile(.75d)); + printPercentile(partitionsOnly, stats, format, "~p90", h -> h.percentile(.90d)); + printPercentile(partitionsOnly, stats, format, "~p95", h -> h.percentile(.95d)); + printPercentile(partitionsOnly, stats, format, "~p99", h -> h.percentile(.99d)); + printPercentile(partitionsOnly, stats, format, "~p999", h -> h.percentile(.999d)); + + // Print accurate metrics (min/max/count) + if (partitionsOnly) + { + System.out.printf(format, "min", prettyPrintMemory(stats.minSize)); + System.out.printf(format, "max", prettyPrintMemory(stats.maxSize)); + } + else + { + System.out.printf(format, + "min", + prettyPrintMemory(stats.minSize), + stats.minRowCount, + stats.minCellCount, + stats.minTombstoneCount); + System.out.printf(format, + "max", + prettyPrintMemory(stats.maxSize), + stats.maxRowCount, + stats.maxCellCount, + stats.maxTombstoneCount); + } + System.out.printf(" count %20d%n", stats.partitionSizeHistogram.count()); + } + + private static void printPercentile(boolean partitionsOnly, + SSTableStats stats, + String format, + String header, + ToLongFunction value) + { + if (partitionsOnly) + { + System.out.printf(format, + header, + prettyPrintMemory(value.applyAsLong(stats.partitionSizeHistogram))); + } + else + { + System.out.printf(format, + header, + prettyPrintMemory(value.applyAsLong(stats.partitionSizeHistogram)), + value.applyAsLong(stats.rowCountHistogram), + value.applyAsLong(stats.cellCountHistogram), + value.applyAsLong(stats.tombstoneCountHistogram)); + } + } + + private static String maybeEscapeKeyForSummary(TableMetadata metadata, ByteBuffer key) + { + String s = metadata.partitionKeyType.getString(key); + if (s.indexOf(' ') == -1) + return s; + return "\"" + StringUtils.replace(s, "\"", "\"\"") + "\""; + } + + static final class SSTableStats + { + // EH of 155 can track a max value of 3520571548412 i.e. 3.5TB + EstimatedHistogram partitionSizeHistogram = new EstimatedHistogram(155, true); + + // EH of 118 can track a max value of 4139110981, i.e., > 4B rows, cells or tombstones + EstimatedHistogram rowCountHistogram = new EstimatedHistogram(118, true); + EstimatedHistogram cellCountHistogram = new EstimatedHistogram(118, true); + EstimatedHistogram tombstoneCountHistogram = new EstimatedHistogram(118, true); + + long minSize = 0; + long maxSize = 0; + + int minRowCount = 0; + int maxRowCount = 0; + + int minCellCount = 0; + int maxCellCount = 0; + + int minTombstoneCount = 0; + int maxTombstoneCount = 0; + + void addPartition(PartitionStats stats) + { + partitionSizeHistogram.add(stats.size); + rowCountHistogram.add(stats.rowCount); + cellCountHistogram.add(stats.cellCount); + tombstoneCountHistogram.add(stats.tombstoneCount()); + + if (minSize == 0 || stats.size < minSize) + minSize = stats.size; + if (stats.size > maxSize) + maxSize = stats.size; + + if (minRowCount == 0 || stats.rowCount < minRowCount) + minRowCount = stats.rowCount; + if (stats.rowCount > maxRowCount) + maxRowCount = stats.rowCount; + + if (minCellCount == 0 || stats.cellCount < minCellCount) + minCellCount = stats.cellCount; + if (stats.cellCount > maxCellCount) + maxCellCount = stats.cellCount; + + if (minTombstoneCount == 0 || stats.tombstoneCount() < minTombstoneCount) + minTombstoneCount = stats.tombstoneCount(); + if (stats.tombstoneCount() > maxTombstoneCount) + maxTombstoneCount = stats.tombstoneCount(); + } + } + + static final class PartitionStats + { + final ByteBuffer key; + final long offset; + final boolean live; + + long size = -1; + int rowCount = 0; + int cellCount = 0; + int rowTombstoneCount = 0; + int rangeTombstoneCount = 0; + int complexTombstoneCount = 0; + int cellTombstoneCount = 0; + int rowTtlExpired = 0; + int cellTtlExpired = 0; + + PartitionStats(ByteBuffer key, long offset, boolean live) + { + this.key = key; + this.offset = offset; + this.live = live; + } + + void endOfPartition(long position) + { + size = position - offset; + } + + int tombstoneCount() + { + return rowTombstoneCount + rangeTombstoneCount + complexTombstoneCount + cellTombstoneCount + rowTtlExpired + cellTtlExpired; + } + + void addUnfiltered(ExtendedDescriptor desc, long currentTime, Unfiltered unfiltered) + { + if (unfiltered instanceof Row) + { + Row row = (Row) unfiltered; + rowCount++; + + if (!row.deletion().isLive()) + rowTombstoneCount++; + + LivenessInfo liveInfo = row.primaryKeyLivenessInfo(); + if (!liveInfo.isEmpty() && liveInfo.isExpiring() && liveInfo.localExpirationTime() < currentTime) + rowTtlExpired++; + + for (ColumnData cd : row) + { + + if (cd.column().isSimple()) + { + addCell((int) currentTime, liveInfo, (Cell) cd); + } + else + { + ComplexColumnData complexData = (ComplexColumnData) cd; + if (!complexData.complexDeletion().isLive()) + complexTombstoneCount++; + + for (Cell cell : complexData) + addCell((int) currentTime, liveInfo, cell); + } + } + } + else if (unfiltered instanceof RangeTombstoneMarker) + { + rangeTombstoneCount++; + } + else + { + throw new UnsupportedOperationException("Unknown kind " + unfiltered.kind() + " in sstable " + desc.descriptor); + } + } + + private void addCell(int currentTime, LivenessInfo liveInfo, Cell cell) + { + cellCount++; + if (cell.isTombstone()) + cellTombstoneCount++; + if (cell.isExpiring() && (liveInfo.isEmpty() || cell.ttl() != liveInfo.ttl()) && !cell.isLive(currentTime)) + cellTtlExpired++; + } + + void printPartitionInfo(TableMetadata metadata, boolean partitionsOnly) + { + String key = metadata.partitionKeyType.getString(this.key); + if (partitionsOnly) + System.out.printf(" Partition: '%s' (%s) %s, size: %s%n", + key, + ByteBufferUtil.bytesToHex(this.key), live ? "live" : "not live", + prettyPrintMemory(size)); + else + System.out.printf(" Partition: '%s' (%s) %s, size: %s, rows: %d, cells: %d, " + + "tombstones: %d (row:%d, range:%d, complex:%d, cell:%d, row-TTLd:%d, cell-TTLd:%d)%n", + key, + ByteBufferUtil.bytesToHex(this.key), + live ? "live" : "not live", + prettyPrintMemory(size), + rowCount, + cellCount, + tombstoneCount(), + rowTombstoneCount, + rangeTombstoneCount, + complexTombstoneCount, + cellTombstoneCount, + rowTtlExpired, + cellTtlExpired); + } + + void printPartitionInfoCSV(TableMetadata metadata, ExtendedDescriptor desc) + { + System.out.printf("\"%s\",%s,%s,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%s,%s,%s,%s,%s,%s,%s,%s,%s%n", + maybeEscapeKeyForSummary(metadata, key), + ByteBufferUtil.bytesToHex(key), + live ? "true" : "false", + offset, size, + rowCount, cellCount, tombstoneCount(), + rowTombstoneCount, rangeTombstoneCount, complexTombstoneCount, cellTombstoneCount, + rowTtlExpired, cellTtlExpired, + desc.descriptor.fileFor(BigFormat.Components.DATA), + notNull(desc.keyspace), + notNull(desc.table), + notNull(desc.index), + notNull(desc.snapshot), + notNull(desc.backup), + desc.descriptor.id, + desc.descriptor.formatType.name, + desc.descriptor.version.getVersion()); + } + } + + static final class ExtendedDescriptor implements Comparable + { + final String keyspace; + final String table; + final String index; + final String snapshot; + final String backup; + final TableId tableId; + final Descriptor descriptor; + + ExtendedDescriptor(String keyspace, String table, TableId tableId, String index, String snapshot, String backup, Descriptor descriptor) + { + this.keyspace = keyspace; + this.table = table; + this.tableId = tableId; + this.index = index; + this.snapshot = snapshot; + this.backup = backup; + this.descriptor = descriptor; + } + + @Override + public String toString() + { + StringBuilder sb = new StringBuilder(); + if (backup != null) + sb.append("Backup:").append(backup).append(' '); + if (snapshot != null) + sb.append("Snapshot:").append(snapshot).append(' '); + if (keyspace != null) + sb.append(keyspace).append('.'); + if (table != null) + sb.append(table); + if (index != null) + sb.append('.').append(index); + if (tableId != null) + sb.append('-').append(tableId.toHexString()); + return sb.append(" #") + .append(descriptor.id) + .append(" (") + .append(descriptor.formatType.name) + .append('-') + .append(descriptor.version.getVersion()) + .append(')') + .toString(); + } + + static ExtendedDescriptor guessFromFile(File fArg) + { + Descriptor desc = Descriptor.fromFile(fArg); + + String snapshot = null; + String backup = null; + String index = null; + + File parent = fArg.parent(); + File grandparent = parent.parent(); + + if (parent.name().length() > 1 && parent.name().startsWith(".") && parent.name().charAt(1) != '.') + { + index = parent.name().substring(1); + parent = parent.parent(); + grandparent = parent.parent(); + } + + if (parent.name().equals(Directories.BACKUPS_SUBDIR)) + { + backup = parent.name(); + parent = parent.parent(); + grandparent = parent.parent(); + } + + if (grandparent.name().equals(Directories.SNAPSHOT_SUBDIR)) + { + snapshot = parent.name(); + parent = grandparent.parent(); + grandparent = parent.parent(); + } + + try + { + Pair tableNameAndId = TableId.tableNameAndIdFromFilename(parent.name()); + if (tableNameAndId != null) + { + return new ExtendedDescriptor(grandparent.name(), + tableNameAndId.left, + tableNameAndId.right, + index, + snapshot, + backup, + desc); + } + } + catch (NumberFormatException e) + { + // ignore non-parseable table-IDs + } + + return new ExtendedDescriptor(null, + null, + null, + index, + snapshot, + backup, + desc); + } + + @Override + public int compareTo(ExtendedDescriptor o) + { + int c = descriptor.directory.toString().compareTo(o.descriptor.directory.toString()); + if (c != 0) + return c; + c = notNull(keyspace).compareTo(notNull(o.keyspace)); + if (c != 0) + return c; + c = notNull(table).compareTo(notNull(o.table)); + if (c != 0) + return c; + c = notNull(tableId).toString().compareTo(notNull(o.tableId).toString()); + if (c != 0) + return c; + c = notNull(index).compareTo(notNull(o.index)); + if (c != 0) + return c; + c = notNull(snapshot).compareTo(notNull(o.snapshot)); + if (c != 0) + return c; + c = notNull(backup).compareTo(notNull(o.backup)); + if (c != 0) + return c; + c = notNull(descriptor.id.toString()).compareTo(notNull(o.descriptor.id.toString())); + if (c != 0) + return c; + return Integer.compare(System.identityHashCode(this), System.identityHashCode(o)); + } + } + + private static String notNull(String s) + { + return s != null ? s : ""; + } + + private static TableId notNull(TableId s) + { + return s != null ? s : EMPTY_TABLE_ID; + } +} diff --git a/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java b/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java new file mode 100644 index 0000000000..247fa8a1e6 --- /dev/null +++ b/test/unit/org/apache/cassandra/tools/SSTablePartitionsTest.java @@ -0,0 +1,654 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.tools; + +import java.io.IOException; + +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.tools.ToolRunner.ToolResult; +import org.apache.cassandra.utils.FBUtilities; +import org.assertj.core.api.AbstractStringAssert; +import org.assertj.core.api.Assertions; + +import static java.lang.String.format; + +/** + * Tests for {@link SSTablePartitions}. + */ +public class SSTablePartitionsTest extends OfflineToolUtils +{ + private static final String SSTABLE_1 = sstable("legacy_ma_simple"); + private static final String SSTABLE_2 = sstable("legacy_ma_clust"); + private static final String HEADER_1 = "\nProcessing #1 (big-ma) (0.169 KiB uncompressed, 0.086 KiB on disk)\n"; + private static final String HEADER_2 = "\nProcessing #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n"; + private static final String BACKUPS_HEADER_1 = "\nProcessing Backup:backups #1 (big-ma) (0.169 KiB uncompressed, 0.086 KiB on disk)\n"; + private static final String BACKUPS_HEADER_2 = "\nProcessing Backup:backups #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n"; + private static final String SNAPSHOTS_HEADER_1 = "\nProcessing Snapshot:snapshot-1 #1 (big-ma) (0.169 KiB uncompressed, 0.086 KiB on disk)\n"; + private static final String SNAPSHOTS_HEADER_2 = "\nProcessing Snapshot:snapshot-1 #1 (big-ma) (328.145 KiB uncompressed, 5.096 KiB on disk)\n"; + private static final String SUMMARY_1 = " Partition size Row count Cell count Tombstone count\n" + + " ~p50 0.034 KiB 1 1 0\n" + + " ~p75 0.034 KiB 1 1 0\n" + + " ~p90 0.034 KiB 1 1 0\n" + + " ~p95 0.034 KiB 1 1 0\n" + + " ~p99 0.034 KiB 1 1 0\n" + + " ~p999 0.034 KiB 1 1 0\n" + + " min 0.032 KiB 1 1 0\n" + + " max 0.034 KiB 1 1 0\n" + + " count 5\n"; + private static final String SUMMARY_2 = " Partition size Row count Cell count Tombstone count\n" + + " ~p50 71.735 KiB 50 50 0\n" + + " ~p75 71.735 KiB 50 50 0\n" + + " ~p90 71.735 KiB 50 50 0\n" + + " ~p95 71.735 KiB 50 50 0\n" + + " ~p99 71.735 KiB 50 50 0\n" + + " ~p999 71.735 KiB 50 50 0\n" + + " min 65.625 KiB 50 50 0\n" + + " max 65.630 KiB 50 50 0\n" + + " count 5\n"; + + @BeforeClass + public static void prepareDirectories() + { + createBackupsAndSnapshots(SSTABLE_1); + createBackupsAndSnapshots(SSTABLE_2); + } + + private static void createBackupsAndSnapshots(String sstable) + { + File parentDir = new File(sstable).parent(); + + File backupsDir = new File(parentDir, Directories.BACKUPS_SUBDIR); + backupsDir.tryCreateDirectory(); + + File snapshotsDir = new File(parentDir, Directories.SNAPSHOT_SUBDIR); + snapshotsDir.tryCreateDirectory(); + + File snapshotDir = new File(snapshotsDir, "snapshot-1"); + snapshotDir.tryCreateDirectory(); + + for (File f : parentDir.tryList(File::isFile)) + { + FileUtils.copyWithOutConfirm(f, new File(backupsDir, f.name())); + FileUtils.copyWithOutConfirm(f, new File(snapshotDir, f.name())); + } + } + + /** + * Runs post-test assertions about loaded classed and started threads. + */ + @After + public void assertPostTestEnv() + { + assertNoUnexpectedThreadsStarted(OPTIONAL_THREADS_WITH_SCHEMA, false); + assertCLSMNotLoaded(); + assertSystemKSNotLoaded(); + assertKeyspaceNotLoaded(); + assertServerNotLoaded(); + } + + /** + * Verify that the tool prints help when no arguments are provided. + */ + @Test + public void testNoArgsPrintsHelp() + { + ToolResult tool = ToolRunner.invokeClass(SSTablePartitions.class); + Assertions.assertThat(tool.getExitCode()).isOne(); + Assertions.assertThat(tool.getCleanedStderr()).contains("You must supply at least one sstable or directory"); + Assertions.assertThat(tool.getStdout()).contains("usage"); + } + + /** + * Verify that the tool prints the right help contents. + * If you added, modified options or help, please update docs if necessary. + */ + @Test + public void testMaybeChangeDocs() + { + ToolResult tool = ToolRunner.invokeClass(SSTablePartitions.class); + Assertions.assertThat(tool.getStdout()) + .isEqualTo("usage: sstablepartitions \n" + + " \n" + + "Print partition statistics of one or more sstables.\n" + + " -b,--backups include backups present in data\n" + + " directories (recursive scans)\n" + + " -c,--min-cells partition cell count threshold\n" + + " -k,--key Partition keys to include\n" + + " -m,--csv CSV output (machine readable)\n" + + " -o,--min-tombstones partition tombstone count threshold\n" + + " -r,--recursive scan for sstables recursively\n" + + " -s,--snapshots include snapshots present in data\n" + + " directories (recursive scans)\n" + + " -t,--min-size partition size threshold, expressed as\n" + + " either the number of bytes or a size with unit of the form 10KiB, 20MiB,\n" + + " 30GiB, etc.\n" + + " -u,--current-timestamp timestamp (seconds since epoch, unit time)\n" + + " for TTL expired calculation\n" + + " -w,--min-rows partition row count threshold\n" + + " -x,--exclude-key Excluded partition key(s) from partition\n" + + " detailed row/cell/tombstone information (irrelevant, if --partitions-only\n" + + " is given)\n" + + " -y,--partitions-only Do not process per-partition detailed\n" + + " row/cell/tombstone information, only brief information\n"); + } + + /** + * Verify that the tool can select single sstable file. + */ + @Test + public void testSingleSSTable() + { + assertThatToolSucceds(SSTABLE_1).isEqualTo(HEADER_1 + SUMMARY_1); + assertThatToolSucceds(SSTABLE_2).isEqualTo(HEADER_2 + SUMMARY_2); + } + + /** + * Verify that the tool can select multiple sstable files. + */ + @Test + public void testMultipleSSTables() + { + assertThatToolSucceds(SSTABLE_1, SSTABLE_2) + .isEqualTo(HEADER_2 + SUMMARY_2 + HEADER_1 + SUMMARY_1); + } + + /** + * Verify that the tool can select all the sstable files in a directory. + */ + @Test + public void testDirectory() + { + assertThatToolSucceds(new File(SSTABLE_1).parentPath()) + .isEqualTo(HEADER_1 + SUMMARY_1); + + assertThatToolSucceds("-r", new File(SSTABLE_1).parent().parentPath()) + .contains(HEADER_1 + SUMMARY_1) + .contains(HEADER_2 + SUMMARY_2); + + assertThatToolSucceds("--recursive", new File(SSTABLE_2).parent().parentPath()) + .contains(HEADER_1 + SUMMARY_1) + .contains(HEADER_2 + SUMMARY_2); + } + + /** + * Test the flag for collecting and printing sstable partition sizes only. + */ + @Test + public void testPartitionsOnly() + { + testPartitionsOnly("-y"); + testPartitionsOnly("--partitions-only"); + } + + private static void testPartitionsOnly(String option) + { + assertThatToolSucceds(SSTABLE_1, option) + .isEqualTo(HEADER_1 + + " Partition size\n" + + " ~p50 0.034 KiB\n" + + " ~p75 0.034 KiB\n" + + " ~p90 0.034 KiB\n" + + " ~p95 0.034 KiB\n" + + " ~p99 0.034 KiB\n" + + " ~p999 0.034 KiB\n" + + " min 0.032 KiB\n" + + " max 0.034 KiB\n" + + " count 5\n"); + + assertThatToolSucceds(SSTABLE_2, "--partitions-only") + .isEqualTo(HEADER_2 + + " Partition size\n" + + " ~p50 71.735 KiB\n" + + " ~p75 71.735 KiB\n" + + " ~p90 71.735 KiB\n" + + " ~p95 71.735 KiB\n" + + " ~p99 71.735 KiB\n" + + " ~p999 71.735 KiB\n" + + " min 65.625 KiB\n" + + " max 65.630 KiB\n" + + " count 5\n"); + } + + /** + * Test the flag for detecting partitions over a certain size threshold. + */ + @Test + public void testMinSize() + { + testMinSize("-t"); + testMinSize("--min-size"); + } + + private static void testMinSize(String option) + { + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "35") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + + HEADER_1 + + " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 4 partitions match\n" + + " Keys: 1 2 3 4\n" + + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "36") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "67201") + .isEqualTo(HEADER_2 + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 4 partitions match\n" + + " Keys: 1 2 3 4\n" + + SUMMARY_2 + HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "67201B") + .isEqualTo(HEADER_2 + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 4 partitions match\n" + + " Keys: 1 2 3 4\n" + + SUMMARY_2 + HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-size", "65KiB") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + HEADER_1 + SUMMARY_1); + } + + /** + * Test the flag for detecting partitions with more cells than a certain threshold. + */ + @Test + public void testMinCells() + { + testMinCells("-c"); + testMinCells("--min-cells"); + } + + private static void testMinCells(String option) + { + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + + HEADER_1 + + " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "2") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "51") + .isEqualTo(HEADER_2 + SUMMARY_2 + + HEADER_1 + SUMMARY_1); + } + + /** + * Test the flag for detecting partitions with more rows than a certain threshold. + */ + @Test + public void testMinRows() + { + testMinRows("-w"); + testMinRows("--min-rows"); + } + + private static void testMinRows(String option) + { + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + + HEADER_1 + + " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "50") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "51") + .isEqualTo(HEADER_2 + SUMMARY_2 + + HEADER_1 + SUMMARY_1); + } + + /** + * Test the flag for detecting partitions with more tombstones than a certain threshold. + */ + @Test + public void testMinTombstones() + { + testMinTombstones("-o"); + testMinTombstones("--min-tombstones"); + } + + private static void testMinTombstones(String option) + { + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, option, "0") + .isEqualTo(HEADER_2 + + " Partition: '0' (30) live, size: 65.625 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 65.630 KiB, rows: 50, cells: 50, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_2 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_2 + + HEADER_1 + + " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 5 partitions match\n" + + " Keys: 0 1 2 3 4\n" + + SUMMARY_1); + + assertThatToolSucceds(SSTABLE_1, SSTABLE_2, "--min-tombstones", "1") + .isEqualTo(HEADER_2 + SUMMARY_2 + HEADER_1 + SUMMARY_1); + } + + /** + * Test the flag for providing a current time. + */ + @Test + public void testCurrentTimestamp() + { + testCurrentTimestamp("-u"); + testCurrentTimestamp("--current-timestamp"); + } + + private static void testCurrentTimestamp(String option) + { + String now = String.valueOf(FBUtilities.nowInSeconds()); + assertThatToolSucceds(SSTABLE_1, option, now).isEqualTo(HEADER_1 + SUMMARY_1); + } + + /** + * Test the flag for including backup sstables. + */ + @Test + public void testBackups() + { + testBackups("-b"); + testBackups("--backups"); + } + + private static void testBackups(String option) + { + assertThatToolSucceds(new File(SSTABLE_1).parentPath(), "-r", option) + .isEqualTo(HEADER_1 + SUMMARY_1 + BACKUPS_HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(new File(SSTABLE_2).parentPath(), "-r", option) + .isEqualTo(HEADER_2 + SUMMARY_2 + BACKUPS_HEADER_2 + SUMMARY_2); + } + + /** + * Test the flag for including snapshot sstables. + */ + @Test + public void testSnapshots() + { + testSnapshots("-s"); + testSnapshots("--snapshots"); + } + + private static void testSnapshots(String option) + { + assertThatToolSucceds(new File(SSTABLE_1).parentPath(), "-r", option) + .isEqualTo(HEADER_1 + SUMMARY_1 + SNAPSHOTS_HEADER_1 + SUMMARY_1); + + assertThatToolSucceds(new File(SSTABLE_2).parentPath(), "-r", option) + .isEqualTo(HEADER_2 + SUMMARY_2 + SNAPSHOTS_HEADER_2 + SUMMARY_2); + } + + /** + * Test the flag for specifying partiton keys to be considered. + */ + @Test + public void testIncludedKeys() + { + testIncludedKeys("-k"); + testIncludedKeys("--key"); + } + + private static void testIncludedKeys(String option) + { + assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "1", option, "3") + .contains(HEADER_1 + + " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 2 partitions match\n" + + " Keys: 1 3\n") + .contains("count 2\n"); + + assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "0", option, "2", option, "4") + .contains(HEADER_1 + + " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 3 partitions match\n" + + " Keys: 0 2 4\n") + .contains("count 3\n"); + + assertThatToolSucceds(SSTABLE_1, "-y","--min-size", "0", option, "0", option, "2", option, "4") + .contains(HEADER_1 + + " Partition: '0' (30) live, size: 0.032 KiB\n" + + " Partition: '2' (32) live, size: 0.034 KiB\n" + + " Partition: '4' (34) live, size: 0.034 KiB\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 3 partitions match\n" + + " Keys: 0 2 4\n") + .contains("count 3\n"); + } + + /** + * Test the flag for specifying partiton keys to be excluded. + */ + @Test + public void testExcludedKeys() + { + testExcludedKeys("-x"); + testExcludedKeys("--exclude-key"); + } + + private static void testExcludedKeys(String option) + { + assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "1", option, "3") + .contains(HEADER_1 + + " Partition: '0' (30) live, size: 0.032 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '2' (32) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '4' (34) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 3 partitions match\n" + + " Keys: 0 2 4\n") + .contains("count 3\n"); + + assertThatToolSucceds(SSTABLE_1, "--min-size", "0", option, "0", option, "2", option, "4") + .contains(HEADER_1 + + " Partition: '1' (31) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + " Partition: '3' (33) live, size: 0.034 KiB, rows: 1, cells: 1, tombstones: 0 (row:0, range:0, complex:0, cell:0, row-TTLd:0, cell-TTLd:0)\n" + + "Summary of #1 (big-ma):\n" + + " File: " + SSTABLE_1 + "\n" + + " 2 partitions match\n" + + " Keys: 1 3\n") + .contains("count 2\n"); + } + + /** + * Test the flag for producing machine-readable CSV output. + */ + @Test + public void testCSV() + { + testCSV("-m"); + testCSV("--csv"); + } + + private static void testCSV(String option) + { + assertThatToolSucceds(option, "--min-size", "35", SSTABLE_1, SSTABLE_2) + .isEqualTo(format("key,keyBinary,live,offset,size,rowCount,cellCount,tombstoneCount," + + "rowTombstoneCount,rangeTombstoneCount,complexTombstoneCount,cellTombstoneCount," + + "rowTtlExpired,cellTtlExpired,directory,keyspace,table,index,snapshot,backup," + + "generation,format,version\n" + + "\"0\",30,true,0,67200,50,50,0,0,0,0,0,0,0,%s,,,,,,1,big,ma\n" + + "\"1\",31,true,67200,67205,50,50,0,0,0,0,0,0,0,% assertThatToolSucceds(String... args) + { + ToolResult tool = invokeTool(args); + Assertions.assertThat(tool.getExitCode()).isZero(); + tool.assertOnCleanExit(); + return Assertions.assertThat(tool.getStdout()); + } + + private static String sstable(String table) + { + try + { + return findOneSSTable("legacy_sstables", table); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + private static ToolResult invokeTool(String... args) + { + return ToolRunner.invokeClass(SSTablePartitions.class, args); + } +} diff --git a/tools/bin/sstablepartitions b/tools/bin/sstablepartitions new file mode 100755 index 0000000000..a5f32481db --- /dev/null +++ b/tools/bin/sstablepartitions @@ -0,0 +1,49 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$CASSANDRA_INCLUDE" = "x" ]; then + # Locations (in order) to use when searching for an include file. + for include in "`dirname "$0"`/cassandra.in.sh" \ + "$HOME/.cassandra.in.sh" \ + /usr/share/cassandra/cassandra.in.sh \ + /usr/local/share/cassandra/cassandra.in.sh \ + /opt/cassandra/cassandra.in.sh; do + if [ -r "$include" ]; then + . "$include" + break + fi + done +elif [ -r "$CASSANDRA_INCLUDE" ]; then + . "$CASSANDRA_INCLUDE" +fi + +if [ -z "$CLASSPATH" ]; then + echo "You must set the CLASSPATH var" >&2 + exit 1 +fi + +if [ "x$MAX_HEAP_SIZE" = "x" ]; then + MAX_HEAP_SIZE="256M" +fi + +"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \ + -Dcassandra.storagedir="$cassandra_storagedir" \ + -Dlogback.configurationFile=logback-tools.xml \ + org.apache.cassandra.tools.SSTablePartitions "$@" + +# vi:ai sw=4 ts=4 tw=0 et