From 24ebd24c79175f7426f4c489dc5a006e75f09dfb Mon Sep 17 00:00:00 2001 From: Jacek Lewandowski Date: Wed, 4 Jan 2023 16:22:52 +0100 Subject: [PATCH] More accurate skipping of sstables in read path This patch improves the following things: 1. SSTable metadata will store a covered slice instead of min/max clusterings. The difference is that for slices there is available the type of a bound rather than just a clustering. In particular it will provide the information whether the lower and upper bound of an sstable is opened or closed. 2. SSTable metadata will store a flag whether the SSTable contains any partition level deletions or not 3. The above two changes required to introduce a new major format for SSTables - oa 4. Single partition read command makes use of the above changes. In particular an sstable can be skipped when it does not intersect with the column filter, does not have partition level deletions and does not have statics; In case there are partition level deletions, but the other conditions are satisfied, only the partition header needs to be accessed (tests attached) 5. Skipping SSTables assuming those three conditions are satisfied has been implemented also for partition range queries (tests attached). Also added minor separate statistics to record the number of accessed sstables in partition reads because now not all of them need to be accessed. 6. Artificial lower bound marker is now an object on its own and is not implemented as a special case of range tombstone bound. 7. Extended the lower bound optimization usage due the 1 and 2 8. Do not initialize iterator just to get a cached partition and associated columns index. The purpose of using lower bound optimization was to avoid opening an iterator of an sstable if possible. 9. Add key range to stats metadata [f369595b1c] Add fields to sstable version and placeholders in stats serializer [f5c3f772e2] Add hasKeyRange and hasLegacyMinMax [3cde51f4e1] Add partition level deletion presence marker to sstable stats [67b2ee2152] Extract AbstractTypeSerializer [c77b475d6c] Refactor slices intersection checking [ceb5af3a38] Store min and max clustering as a slice in stats metadata as and improved min/max [d1f8973929] Implement MetadataCollectorBench [335369da84] Apply partition level deletion presence marker optimizations to single partition read command [2497a009b9] Lower bound optimization - add slices and isReverseOrder fields to UnfilteredRowIteratorWithLowerBound [e32ee31177] Lower bound optimization - Replace usage of RangeTombstoneMarker as a lower bound with ArtificialBoundMarker [e213e712c4] Lower bound optimization - improve usage of lower bound optimization [c4f93006b1] Apply read path improvements to partition range queries [5fa462266c] Add key range to StatsMetadata [79a7339ed4] Use key range from stats if possible [266ed2749b] Added new sstables for LegacySSTableTest patch by Jacek Lewandowski; reviewed by Branimir Lambov and C. Scott Andreas for CASSANDRA-18134 Co-authored-by: Branimir Lambov Co-authored-by: Sylvain Lebresne Co-authored-by: Jacek Lewandowski Co-authored-by: Jakub Zytka --- CHANGES.txt | 1 + NEWS.txt | 2 + .../cassandra/db/AbstractReadQuery.java | 2 +- .../org/apache/cassandra/db/Clustering.java | 14 +- .../apache/cassandra/db/ClusteringBound.java | 71 ++--- .../cassandra/db/ClusteringBoundary.java | 14 +- .../cassandra/db/ClusteringComparator.java | 7 +- .../apache/cassandra/db/ClusteringPrefix.java | 45 ++- .../db/PartitionRangeReadCommand.java | 38 ++- .../org/apache/cassandra/db/ReadCommand.java | 15 +- .../cassandra/db/SerializationHeader.java | 61 ++-- .../db/SinglePartitionReadCommand.java | 133 +++++---- src/java/org/apache/cassandra/db/Slice.java | 89 +++--- src/java/org/apache/cassandra/db/Slices.java | 97 +++---- .../org/apache/cassandra/db/StorageHook.java | 15 +- .../db/filter/ClusteringIndexFilter.java | 12 +- .../db/filter/ClusteringIndexNamesFilter.java | 13 +- .../db/filter/ClusteringIndexSliceFilter.java | 19 +- .../db/marshal/ByteArrayObjectFactory.java | 28 +- .../db/marshal/ByteBufferObjectFactory.java | 28 +- .../PartitionStatisticsCollector.java | 13 +- .../UnfilteredPartitionIterators.java | 2 +- .../db/rows/ArtificialBoundMarker.java | 59 ++++ .../cassandra/db/rows/EncodingStats.java | 8 +- .../UnfilteredRowIteratorWithLowerBound.java | 180 ++++++------ .../sstable/format/SSTableReaderBuilder.java | 15 +- .../io/sstable/format/SSTableWriter.java | 4 +- .../cassandra/io/sstable/format/Version.java | 42 ++- .../io/sstable/format/big/BigFormat.java | 92 ++++-- .../io/sstable/format/big/BigTableWriter.java | 6 +- .../sstable/metadata/MetadataCollector.java | 135 ++++++--- .../io/sstable/metadata/StatsMetadata.java | 239 ++++++++++++---- .../cassandra/metrics/KeyspaceMetrics.java | 7 +- .../cassandra/metrics/TableMetrics.java | 12 +- .../serializers/AbstractTypeSerializer.java | 75 +++++ .../tools/SSTableMetadataViewer.java | 81 +++--- .../apache/cassandra/utils/Comparables.java | 58 ++++ .../apache/cassandra/utils/MergeIterator.java | 5 +- .../utils/bytecomparable/ByteSource.java | 6 +- .../nc-1-big-CompressionInfo.db | Bin 0 -> 207 bytes .../legacy_nc_clust/nc-1-big-Data.db | Bin 0 -> 8713 bytes .../legacy_nc_clust/nc-1-big-Digest.crc32 | 1 + .../legacy_nc_clust/nc-1-big-Filter.db | Bin 0 -> 24 bytes .../legacy_nc_clust/nc-1-big-Index.db | Bin 0 -> 157553 bytes .../legacy_nc_clust/nc-1-big-Statistics.db | Bin 0 -> 9718 bytes .../legacy_nc_clust/nc-1-big-Summary.db | Bin 0 -> 47 bytes .../legacy_nc_clust/nc-1-big-TOC.txt | 8 + .../nc-1-big-CompressionInfo.db | Bin 0 -> 199 bytes .../legacy_nc_clust_counter/nc-1-big-Data.db | Bin 0 -> 7529 bytes .../nc-1-big-Digest.crc32 | 1 + .../nc-1-big-Filter.db | Bin 0 -> 24 bytes .../legacy_nc_clust_counter/nc-1-big-Index.db | Bin 0 -> 157553 bytes .../nc-1-big-Statistics.db | Bin 0 -> 9727 bytes .../nc-1-big-Summary.db | Bin 0 -> 47 bytes .../legacy_nc_clust_counter/nc-1-big-TOC.txt | 8 + .../nc-1-big-CompressionInfo.db | Bin 0 -> 47 bytes .../legacy_nc_simple/nc-1-big-Data.db | Bin 0 -> 87 bytes .../legacy_nc_simple/nc-1-big-Digest.crc32 | 1 + .../legacy_nc_simple/nc-1-big-Filter.db | Bin 0 -> 24 bytes .../legacy_nc_simple/nc-1-big-Index.db | Bin 0 -> 26 bytes .../legacy_nc_simple/nc-1-big-Statistics.db | Bin 0 -> 4822 bytes .../legacy_nc_simple/nc-1-big-Summary.db | Bin 0 -> 47 bytes .../legacy_nc_simple/nc-1-big-TOC.txt | 8 + .../nc-1-big-CompressionInfo.db | Bin 0 -> 47 bytes .../legacy_nc_simple_counter/nc-1-big-Data.db | Bin 0 -> 138 bytes .../nc-1-big-Digest.crc32 | 1 + .../nc-1-big-Filter.db | Bin 0 -> 24 bytes .../nc-1-big-Index.db | Bin 0 -> 27 bytes .../nc-1-big-Statistics.db | Bin 0 -> 4831 bytes .../nc-1-big-Summary.db | Bin 0 -> 47 bytes .../legacy_nc_simple_counter/nc-1-big-TOC.txt | 8 + .../microbench/MetadataCollectorBench.java | 155 +++++++++++ .../miscellaneous/SSTablesIteratedTest.java | 215 +++++++++++++- .../db/compaction/CompactionsTest.java | 6 +- .../apache/cassandra/db/filter/SliceTest.java | 180 ++++++------ .../db/lifecycle/LogTransactionTest.java | 6 +- .../apache/cassandra/db/rows/RowsTest.java | 8 +- ...filteredRowIteratorWithLowerBoundTest.java | 263 ++++++++++++++++++ .../io/sstable/LegacySSTableTest.java | 2 +- .../io/sstable/SSTableMetadataTest.java | 81 ++---- .../metadata/MetadataSerializerTest.java | 33 ++- .../apache/cassandra/schema/MockSchema.java | 11 +- .../ByteSourceComparisonTest.java | 9 +- .../ByteSourceConversionTest.java | 8 +- 84 files changed, 2002 insertions(+), 754 deletions(-) create mode 100644 src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java create mode 100644 src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java create mode 100644 src/java/org/apache/cassandra/utils/Comparables.java create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Data.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Filter.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Index.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Statistics.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Summary.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-TOC.txt create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Data.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Filter.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Index.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Statistics.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Summary.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-TOC.txt create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Data.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Filter.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Index.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Statistics.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Summary.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-TOC.txt create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-CompressionInfo.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Data.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32 create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Filter.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Index.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Statistics.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Summary.db create mode 100644 test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-TOC.txt create mode 100644 test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java create mode 100644 test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java diff --git a/CHANGES.txt b/CHANGES.txt index deb3a64a23..238417174c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.2 + * More accurate skipping of sstables in read path (CASSANDRA-18134) * Prepare for JDK17 experimental support (CASSANDRA-18179) * Remove Scripted UDFs internals; hooks to be added later in CASSANDRA-17281 (CASSANDRA-18252) * Update JNA to 5.13.0 (CASSANDRA-18050) diff --git a/NEWS.txt b/NEWS.txt index be30bf0252..72cabc5402 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -124,6 +124,8 @@ New features - `mask_hash` replaces the data by its hash, according to the specified algorithm. - On virtual tables, it is not strictly necessary to specify `ALLOW FILTERING` for select statements which would normally require it, except `system_views.system_logs`. + - More accurate skipping of sstables in read path due to better handling of min/max clustering and lower bound; + SSTable format has been bumped to 'nc' because there are new fields in stats metadata Upgrading --------- diff --git a/src/java/org/apache/cassandra/db/AbstractReadQuery.java b/src/java/org/apache/cassandra/db/AbstractReadQuery.java index 374d2b2588..c2e4258440 100644 --- a/src/java/org/apache/cassandra/db/AbstractReadQuery.java +++ b/src/java/org/apache/cassandra/db/AbstractReadQuery.java @@ -118,4 +118,4 @@ abstract class AbstractReadQuery extends MonitorableImpl implements ReadQuery } protected abstract void appendCQLWhereClause(StringBuilder sb); -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/Clustering.java b/src/java/org/apache/cassandra/db/Clustering.java index dd91fea2e7..426d3279f9 100644 --- a/src/java/org/apache/cassandra/db/Clustering.java +++ b/src/java/org/apache/cassandra/db/Clustering.java @@ -56,6 +56,18 @@ public interface Clustering extends ClusteringPrefix, IMeasurableMemory return new BufferClustering(newValues); } + @Override + default ClusteringBound asStartBound() + { + return ClusteringBound.inclusiveStartOf(this); + } + + @Override + default ClusteringBound asEndBound() + { + return ClusteringBound.inclusiveEndOf(this); + } + public default String toString(TableMetadata metadata) { StringBuilder sb = new StringBuilder(); @@ -178,4 +190,4 @@ public interface Clustering extends ClusteringPrefix, IMeasurableMemory } } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/ClusteringBound.java b/src/java/org/apache/cassandra/db/ClusteringBound.java index 60a1c6c6be..4afdfe6285 100644 --- a/src/java/org/apache/cassandra/db/ClusteringBound.java +++ b/src/java/org/apache/cassandra/db/ClusteringBound.java @@ -21,22 +21,27 @@ package org.apache.cassandra.db; import java.nio.ByteBuffer; -import java.util.List; -import org.apache.cassandra.db.marshal.ByteBufferAccessor; import org.apache.cassandra.utils.memory.ByteBufferCloner; +import static org.apache.cassandra.db.AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY; + /** * The start or end of a range of clusterings, either inclusive or exclusive. */ public interface ClusteringBound extends ClusteringBoundOrBoundary { /** The smallest start bound, i.e. the one that starts before any row. */ - public static final ClusteringBound BOTTOM = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND, BufferClusteringBound.EMPTY_VALUES_ARRAY); + ClusteringBound BOTTOM = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_START_BOUND, EMPTY_VALUES_ARRAY); /** The biggest end bound, i.e. the one that ends after any row. */ - public static final ClusteringBound TOP = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, BufferClusteringBound.EMPTY_VALUES_ARRAY); + ClusteringBound TOP = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, EMPTY_VALUES_ARRAY); - public static ClusteringPrefix.Kind boundKind(boolean isStart, boolean isInclusive) + /** The biggest start bound, i.e. the one that starts after any row. */ + ClusteringBound MAX_START = new BufferClusteringBound(Kind.EXCL_START_BOUND, EMPTY_VALUES_ARRAY); + /** The smallest end bound, i.e. the one that end before any row. */ + ClusteringBound MIN_END = new BufferClusteringBound(Kind.EXCL_END_BOUND, EMPTY_VALUES_ARRAY); + + static ClusteringPrefix.Kind boundKind(boolean isStart, boolean isInclusive) { return isStart ? (isInclusive ? ClusteringPrefix.Kind.INCL_START_BOUND : ClusteringPrefix.Kind.EXCL_START_BOUND) @@ -69,32 +74,14 @@ public interface ClusteringBound extends ClusteringBoundOrBoundary return kind() == Kind.EXCL_START_BOUND || kind() == Kind.EXCL_END_BOUND; } - // For use by intersects, it's called with the sstable bound opposite to the slice bound - // (so if the slice bound is a start, it's call with the max sstable bound) - default int compareTo(ClusteringComparator comparator, List sstableBound) + default boolean isArtificial() { - for (int i = 0; i < sstableBound.size(); i++) - { - // Say the slice bound is a start. It means we're in the case where the max - // sstable bound is say (1:5) while the slice start is (1). So the start - // does start before the sstable end bound (and intersect it). It's the exact - // inverse with a end slice bound. - if (i >= size()) - return isStart() ? -1 : 1; + return kind() == Kind.SSTABLE_LOWER_BOUND || kind() == Kind.SSTABLE_UPPER_BOUND; + } - int cmp = comparator.compareComponent(i, get(i), accessor(), sstableBound.get(i), ByteBufferAccessor.instance); - if (cmp != 0) - return cmp; - } - - // Say the slice bound is a start. I means we're in the case where the max - // sstable bound is say (1), while the slice start is (1:5). This again means - // that the slice start before the end bound. - if (size() > sstableBound.size()) - return isStart() ? -1 : 1; - - // The slice bound is equal to the sstable bound. Results depends on whether the slice is inclusive or not - return isInclusive() ? 0 : (isStart() ? 1 : -1); + default ClusteringBound artificialLowerBound(boolean isReversed) + { + return create(!isReversed ? Kind.SSTABLE_LOWER_BOUND : Kind.SSTABLE_UPPER_BOUND, this); } static ClusteringBound create(ClusteringPrefix.Kind kind, ClusteringPrefix from) @@ -102,27 +89,27 @@ public interface ClusteringBound extends ClusteringBoundOrBoundary return from.accessor().factory().bound(kind, from.getRawValues()); } - public static ClusteringBound inclusiveStartOf(ClusteringPrefix from) + static ClusteringBound inclusiveStartOf(ClusteringPrefix from) { return create(ClusteringPrefix.Kind.INCL_START_BOUND, from); } - public static ClusteringBound inclusiveEndOf(ClusteringPrefix from) + static ClusteringBound inclusiveEndOf(ClusteringPrefix from) { return create(ClusteringPrefix.Kind.INCL_END_BOUND, from); } - public static ClusteringBound exclusiveStartOf(ClusteringPrefix from) + static ClusteringBound exclusiveStartOf(ClusteringPrefix from) { return create(ClusteringPrefix.Kind.EXCL_START_BOUND, from); } - public static ClusteringBound exclusiveEndOf(ClusteringPrefix from) + static ClusteringBound exclusiveEndOf(ClusteringPrefix from) { return create(ClusteringPrefix.Kind.EXCL_END_BOUND, from); } - public static ClusteringBound create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values) + static ClusteringBound create(ClusteringComparator comparator, boolean isStart, boolean isInclusive, Object... values) { CBuilder builder = CBuilder.create(comparator); for (Object val : values) @@ -134,4 +121,18 @@ public interface ClusteringBound extends ClusteringBoundOrBoundary } return builder.buildBound(isStart, isInclusive); } -} + + @Override + default ClusteringBound asStartBound() + { + assert isStart(); + return this; + } + + @Override + default ClusteringBound asEndBound() + { + assert isEnd(); + return this; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/ClusteringBoundary.java b/src/java/org/apache/cassandra/db/ClusteringBoundary.java index 3e50f5264c..9e0a87c2ef 100644 --- a/src/java/org/apache/cassandra/db/ClusteringBoundary.java +++ b/src/java/org/apache/cassandra/db/ClusteringBoundary.java @@ -37,4 +37,16 @@ public interface ClusteringBoundary extends ClusteringBoundOrBoundary { return from.accessor().factory().boundary(kind, from.getRawValues()); } -} + + @Override + default ClusteringBound asStartBound() + { + return openBound(false); + } + + @Override + default ClusteringBound asEndBound() + { + return closeBound(false); + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/ClusteringComparator.java b/src/java/org/apache/cassandra/db/ClusteringComparator.java index c1aebfad77..316e6f71cb 100644 --- a/src/java/org/apache/cassandra/db/ClusteringComparator.java +++ b/src/java/org/apache/cassandra/db/ClusteringComparator.java @@ -445,6 +445,11 @@ public class ClusteringComparator implements Comparator return accessor.factory().bound(isEnd ? ClusteringPrefix.Kind.INCL_END_BOUND : ClusteringPrefix.Kind.EXCL_START_BOUND, Arrays.copyOf(components, cc)); + + case ByteSource.LTLT_NEXT_COMPONENT: + case ByteSource.GTGT_NEXT_COMPONENT: + throw new AssertionError("Unexpected sstable lower/upper bound - byte comparable representation of artificial sstable bounds is not supported"); + default: throw new AssertionError("Unexpected separator " + Integer.toHexString(sep) + " in ClusteringBound encoding"); } @@ -550,4 +555,4 @@ public class ClusteringComparator implements Comparator { return Objects.hashCode(clusteringTypes); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/ClusteringPrefix.java b/src/java/org/apache/cassandra/db/ClusteringPrefix.java index 0320c0457d..be81e5d57f 100644 --- a/src/java/org/apache/cassandra/db/ClusteringPrefix.java +++ b/src/java/org/apache/cassandra/db/ClusteringPrefix.java @@ -64,19 +64,20 @@ public interface ClusteringPrefix extends IMeasurableMemory, Clusterable { // WARNING: the ordering of that enum matters because we use ordinal() in the serialization - EXCL_END_BOUND (0, -1, v -> ByteSource.LT_NEXT_COMPONENT), - INCL_START_BOUND (0, -1, v -> ByteSource.LT_NEXT_COMPONENT), - EXCL_END_INCL_START_BOUNDARY(0, -1, v -> ByteSource.LT_NEXT_COMPONENT), - STATIC_CLUSTERING (1, -1, v -> v == Version.LEGACY - ? ByteSource.LT_NEXT_COMPONENT + 1 - : ByteSource.EXCLUDED), - CLUSTERING (2, 0, v -> v == Version.LEGACY - ? ByteSource.NEXT_COMPONENT - : ByteSource.TERMINATOR), - INCL_END_EXCL_START_BOUNDARY(3, 1, v -> ByteSource.GT_NEXT_COMPONENT), - INCL_END_BOUND (3, 1, v -> ByteSource.GT_NEXT_COMPONENT), - EXCL_START_BOUND (3, 1, v -> ByteSource.GT_NEXT_COMPONENT); - + // @formatter:off + EXCL_END_BOUND ( 0, -1, v -> ByteSource.LT_NEXT_COMPONENT), + INCL_START_BOUND ( 0, -1, v -> ByteSource.LT_NEXT_COMPONENT), + EXCL_END_INCL_START_BOUNDARY ( 0, -1, v -> ByteSource.LT_NEXT_COMPONENT), + STATIC_CLUSTERING ( 1, -1, v -> v == Version.LEGACY ? ByteSource.LT_NEXT_COMPONENT + 1 + : ByteSource.EXCLUDED), + CLUSTERING ( 2, 0, v -> v == Version.LEGACY ? ByteSource.NEXT_COMPONENT + : ByteSource.TERMINATOR), + INCL_END_EXCL_START_BOUNDARY ( 3, 1, v -> ByteSource.GT_NEXT_COMPONENT), + INCL_END_BOUND ( 3, 1, v -> ByteSource.GT_NEXT_COMPONENT), + EXCL_START_BOUND ( 3, 1, v -> ByteSource.GT_NEXT_COMPONENT), + SSTABLE_LOWER_BOUND (-1, -1, v -> ByteSource.LTLT_NEXT_COMPONENT), + SSTABLE_UPPER_BOUND ( 4, 1, v -> ByteSource.GTGT_NEXT_COMPONENT); + // @formatter:on private final int comparison; @@ -312,6 +313,22 @@ public interface ClusteringPrefix extends IMeasurableMemory, Clusterable */ public String toString(TableMetadata metadata); + /** + * Returns this prefix as a start bound. + * If this prefix is a bound, just returns it asserting that it is a start bound. + * If this prefix is a clustering, returns an included start bound. + * If this prefix is a boundary, returns an open bound of it + */ + ClusteringBound asStartBound(); + + /** + * Returns this prefix as an end bound. + * If this prefix is a bound, just returns it asserting that it is an end bound. + * If this prefix is a clustering, returns an included end bound. + * In this prefix is a boundary, returns a close bound of it. + */ + ClusteringBound asEndBound(); + /* * TODO: we should stop using Clustering for partition keys. Maybe we can add * a few methods to DecoratedKey so we don't have to (note that while using a Clustering @@ -715,4 +732,4 @@ public interface ClusteringPrefix extends IMeasurableMemory, Clusterable return equals(prefix, (ClusteringPrefix) o); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java index f000d63f99..b3be264585 100644 --- a/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java +++ b/src/java/org/apache/cassandra/db/PartitionRangeReadCommand.java @@ -18,10 +18,12 @@ package org.apache.cassandra.db; import java.io.IOException; +import java.util.List; import java.util.concurrent.TimeUnit; import com.google.common.annotations.VisibleForTesting; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.db.virtual.VirtualKeyspaceRegistry; import org.apache.cassandra.db.virtual.VirtualTable; import org.apache.cassandra.config.DatabaseDescriptor; @@ -62,6 +64,7 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR protected static final SelectionDeserializer selectionDeserializer = new Deserializer(); protected final DataRange dataRange; + protected final Slices requestedSlices; private PartitionRangeReadCommand(boolean isDigest, int digestVersion, @@ -77,6 +80,8 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR { super(Kind.PARTITION_RANGE, isDigest, digestVersion, acceptsTransient, metadata, nowInSec, columnFilter, rowFilter, limits, index, trackWarnings); this.dataRange = dataRange; + this.requestedSlices = dataRange.clusteringIndexFilter.getSlices(metadata()); + } private static PartitionRangeReadCommand create(boolean isDigest, @@ -329,20 +334,43 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR inputCollector.addMemtableIterator(RTBoundValidator.validate(iter, RTBoundValidator.Stage.MEMTABLE, false)); } + int selectedSSTablesCnt = 0; for (SSTableReader sstable : view.sstables) { + boolean intersects = intersects(sstable); + boolean hasPartitionLevelDeletions = hasPartitionLevelDeletions(sstable); + boolean hasRequiredStatics = hasRequiredStatics(sstable); + + if (!intersects && !hasPartitionLevelDeletions && !hasRequiredStatics) + continue; + @SuppressWarnings("resource") // We close on exception and on closing the result returned by this method UnfilteredPartitionIterator iter = sstable.partitionIterator(columnFilter(), dataRange(), readCountUpdater); inputCollector.addSSTableIterator(sstable, RTBoundValidator.validate(iter, RTBoundValidator.Stage.SSTABLE, false)); if (!sstable.isRepaired()) controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime()); + + selectedSSTablesCnt++; } + + final int finalSelectedSSTables = selectedSSTablesCnt; + // iterators can be empty for offline tools if (inputCollector.isEmpty()) return EmptyIterators.unfilteredPartition(metadata()); - return checkCacheFilter(UnfilteredPartitionIterators.mergeLazily(inputCollector.finalizeIterators(cfs, nowInSec(), controller.oldestUnrepairedTombstone())), cfs); + List finalizedIterators = inputCollector.finalizeIterators(cfs, nowInSec(), controller.oldestUnrepairedTombstone()); + UnfilteredPartitionIterator merged = UnfilteredPartitionIterators.mergeLazily(finalizedIterators); + return checkCacheFilter(Transformation.apply(merged, new Transformation() + { + @Override + protected void onClose() + { + super.onClose(); + cfs.metric.updateSSTableIteratedInRangeRead(finalSelectedSSTables); + } + }), cfs); } catch (RuntimeException | Error e) { @@ -358,6 +386,12 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR } } + @Override + protected boolean intersects(SSTableReader sstable) + { + return requestedSlices.intersects(sstable.getSSTableMetadata().coveredClustering); + } + /** * Creates a new {@code SSTableReadsListener} to update the SSTables read counts. * @return a new {@code SSTableReadsListener} to update the SSTables read counts. @@ -545,4 +579,4 @@ public class PartitionRangeReadCommand extends ReadCommand implements PartitionR return executionController(); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/ReadCommand.java b/src/java/org/apache/cassandra/db/ReadCommand.java index 95fa95da91..0978b8ef05 100644 --- a/src/java/org/apache/cassandra/db/ReadCommand.java +++ b/src/java/org/apache/cassandra/db/ReadCommand.java @@ -788,6 +788,19 @@ public abstract class ReadCommand extends AbstractReadQuery return msg; } + protected abstract boolean intersects(SSTableReader sstable); + + protected boolean hasRequiredStatics(SSTableReader sstable) { + // If some static columns are queried, we should always include the sstable: the clustering values stats of the sstable + // don't tell us if the sstable contains static values in particular. + return !columnFilter().fetchedColumns().statics.isEmpty() && sstable.header.hasStatic(); + } + + protected boolean hasPartitionLevelDeletions(SSTableReader sstable) + { + return sstable.getSSTableMetadata().hasPartitionLevelDeletions; + } + public abstract Verb verb(); protected abstract void appendCQLWhereClause(StringBuilder sb); @@ -1123,4 +1136,4 @@ public abstract class ReadCommand extends AbstractReadQuery + command.indexSerializedSize(version); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/SerializationHeader.java b/src/java/org/apache/cassandra/db/SerializationHeader.java index a1301da06a..d75adfd8dc 100644 --- a/src/java/org/apache/cassandra/db/SerializationHeader.java +++ b/src/java/org/apache/cassandra/db/SerializationHeader.java @@ -25,7 +25,6 @@ import com.google.common.collect.ImmutableList; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.TypeParser; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.exceptions.UnknownColumnException; @@ -38,6 +37,7 @@ import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.serializers.AbstractTypeSerializer; import org.apache.cassandra.utils.ByteBufferUtil; public class SerializationHeader @@ -398,6 +398,8 @@ public class SerializationHeader public static class Serializer implements IMetadataComponentSerializer { + private final AbstractTypeSerializer typeSerializer = new AbstractTypeSerializer(); + public void serializeForMessaging(SerializationHeader header, ColumnFilter selection, DataOutputPlus out, boolean hasStatic) throws IOException { EncodingStats.serializer.serialize(header.stats, out); @@ -462,10 +464,8 @@ public class SerializationHeader { EncodingStats.serializer.serialize(header.stats, out); - writeType(header.keyType, out); - out.writeUnsignedVInt32(header.clusteringTypes.size()); - for (AbstractType type : header.clusteringTypes) - writeType(type, out); + typeSerializer.serialize(header.keyType, out); + typeSerializer.serializeList(header.clusteringTypes, out); writeColumnsWithTypes(header.staticColumns, out); writeColumnsWithTypes(header.regularColumns, out); @@ -476,17 +476,11 @@ public class SerializationHeader { EncodingStats stats = EncodingStats.serializer.deserialize(in); - AbstractType keyType = readType(in); - int size = in.readUnsignedVInt32(); - List> clusteringTypes = new ArrayList<>(size); - for (int i = 0; i < size; i++) - clusteringTypes.add(readType(in)); + AbstractType keyType = typeSerializer.deserialize(in); + List> clusteringTypes = typeSerializer.deserializeList(in); - Map> staticColumns = new LinkedHashMap<>(); - Map> regularColumns = new LinkedHashMap<>(); - - readColumnsWithType(in, staticColumns); - readColumnsWithType(in, regularColumns); + Map> staticColumns = readColumnsWithType(in); + Map> regularColumns = readColumnsWithType(in); return new Component(keyType, clusteringTypes, staticColumns, regularColumns, stats); } @@ -496,10 +490,8 @@ public class SerializationHeader { int size = EncodingStats.serializer.serializedSize(header.stats); - size += sizeofType(header.keyType); - size += TypeSizes.sizeofUnsignedVInt(header.clusteringTypes.size()); - for (AbstractType type : header.clusteringTypes) - size += sizeofType(type); + size += typeSerializer.serializedSize(header.keyType); + size += typeSerializer.serializedListSize(header.clusteringTypes); size += sizeofColumnsWithTypes(header.staticColumns); size += sizeofColumnsWithTypes(header.regularColumns); @@ -512,7 +504,7 @@ public class SerializationHeader for (Map.Entry> entry : columns.entrySet()) { ByteBufferUtil.writeWithVIntLength(entry.getKey(), out); - writeType(entry.getValue(), out); + typeSerializer.serialize(entry.getValue(), out); } } @@ -522,36 +514,21 @@ public class SerializationHeader for (Map.Entry> entry : columns.entrySet()) { size += ByteBufferUtil.serializedSizeWithVIntLength(entry.getKey()); - size += sizeofType(entry.getValue()); + size += typeSerializer.serializedSize(entry.getValue()); } return size; } - private void readColumnsWithType(DataInputPlus in, Map> typeMap) throws IOException + private Map> readColumnsWithType(DataInputPlus in) throws IOException { - int length = in.readUnsignedVInt32(); + int length = in.readUnsignedVInt32(); + Map> typeMap = new LinkedHashMap<>(length); for (int i = 0; i < length; i++) { ByteBuffer name = ByteBufferUtil.readWithVIntLength(in); - typeMap.put(name, readType(in)); + typeMap.put(name, typeSerializer.deserialize(in)); } - } - - private void writeType(AbstractType type, DataOutputPlus out) throws IOException - { - // TODO: we should have a terser serializaion format. Not a big deal though - ByteBufferUtil.writeWithVIntLength(UTF8Type.instance.decompose(type.toString()), out); - } - - private AbstractType readType(DataInputPlus in) throws IOException - { - ByteBuffer raw = ByteBufferUtil.readWithVIntLength(in); - return TypeParser.parse(UTF8Type.instance.compose(raw)); - } - - private int sizeofType(AbstractType type) - { - return ByteBufferUtil.serializedSizeWithVIntLength(UTF8Type.instance.decompose(type.toString())); + return typeMap; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index 34414bbea6..cd390fa6db 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -713,14 +713,26 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar break; } - if (shouldInclude(sstable)) + boolean intersects = intersects(sstable); + boolean hasRequiredStatics = hasRequiredStatics(sstable); + boolean hasPartitionLevelDeletions = hasPartitionLevelDeletions(sstable); + + if (!intersects && !hasRequiredStatics && !hasPartitionLevelDeletions) + { + nonIntersectingSSTables++; + continue; + } + + if (intersects || hasRequiredStatics) { if (!sstable.isRepaired()) controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime()); // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator @SuppressWarnings("resource") - UnfilteredRowIteratorWithLowerBound iter = makeIterator(cfs, sstable, metricsCollector); + UnfilteredRowIterator iter = intersects ? makeRowIteratorWithLowerBound(cfs, sstable, metricsCollector) + : makeRowIteratorWithSkippedNonStaticContent(cfs, sstable, metricsCollector); + inputCollector.addSSTableIterator(sstable, iter); mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone, iter.partitionLevelDeletion().markedForDeleteAt()); @@ -728,27 +740,30 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar else { nonIntersectingSSTables++; - // sstable contains no tombstone if maxLocalDeletionTime == Integer.MAX_VALUE, so we can safely skip those entirely - if (sstable.mayHaveTombstones()) + + // if the sstable contained range or cell tombstones, it would intersect; since we are here, it means + // that there are no cell or range tombstones we are interested in (due to the filter) + // however, we know that there are partition level deletions in this sstable and we need to make + // an iterator figure out that (see `StatsMetadata.hasPartitionLevelDeletions`) + + // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator + @SuppressWarnings("resource") + UnfilteredRowIterator iter = makeRowIteratorWithSkippedNonStaticContent(cfs, sstable, metricsCollector); + + // if the sstable contains a partition delete, then we must include it regardless of whether it + // shadows any other data seen locally as we can't guarantee that other replicas have seen it + if (!iter.partitionLevelDeletion().isLive()) { - // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator - @SuppressWarnings("resource") - UnfilteredRowIteratorWithLowerBound iter = makeIterator(cfs, sstable, metricsCollector); - // if the sstable contains a partition delete, then we must include it regardless of whether it - // shadows any other data seen locally as we can't guarantee that other replicas have seen it - if (!iter.partitionLevelDeletion().isLive()) - { - if (!sstable.isRepaired()) - controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime()); - inputCollector.addSSTableIterator(sstable, iter); - includedDueToTombstones++; - mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone, - iter.partitionLevelDeletion().markedForDeleteAt()); - } - else - { - iter.close(); - } + if (!sstable.isRepaired()) + controller.updateMinOldestUnrepairedTombstone(sstable.getMinLocalDeletionTime()); + inputCollector.addSSTableIterator(sstable, iter); + includedDueToTombstones++; + mostRecentPartitionTombstone = Math.max(mostRecentPartitionTombstone, + iter.partitionLevelDeletion().markedForDeleteAt()); + } + else + { + iter.close(); } } } @@ -779,30 +794,52 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar } } - private boolean shouldInclude(SSTableReader sstable) + @Override + protected boolean intersects(SSTableReader sstable) { - // If some static columns are queried, we should always include the sstable: the clustering values stats of the sstable - // don't tell us if the sstable contains static values in particular. - // TODO: we could record if a sstable contains any static value at all. - if (!columnFilter().fetchedColumns().statics.isEmpty()) - return true; - - return clusteringIndexFilter().shouldInclude(sstable); + return clusteringIndexFilter().intersects(sstable.metadata().comparator, sstable.getSSTableMetadata().coveredClustering); } - private UnfilteredRowIteratorWithLowerBound makeIterator(ColumnFamilyStore cfs, - SSTableReader sstable, - SSTableReadsListener listener) + private UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs, + SSTableReader sstable, + SSTableReadsListener listener) { return StorageHook.instance.makeRowIteratorWithLowerBound(cfs, - partitionKey(), sstable, + partitionKey(), clusteringIndexFilter(), columnFilter(), listener); } + private UnfilteredRowIterator makeRowIterator(ColumnFamilyStore cfs, + SSTableReader sstable, + ClusteringIndexNamesFilter clusteringIndexFilter, + SSTableReadsListener listener) + { + return StorageHook.instance.makeRowIterator(cfs, + sstable, + partitionKey(), + clusteringIndexFilter.getSlices(cfs.metadata()), + columnFilter(), + clusteringIndexFilter.isReversed(), + listener); + } + + private UnfilteredRowIterator makeRowIteratorWithSkippedNonStaticContent(ColumnFamilyStore cfs, + SSTableReader sstable, + SSTableReadsListener listener) + { + return StorageHook.instance.makeRowIterator(cfs, + sstable, + partitionKey(), + Slices.NONE, + columnFilter(), + clusteringIndexFilter().isReversed(), + listener); + } + /** * Return a wrapped iterator that when closed will update the sstables iterated and READ sample metrics. * Note that we cannot use the Transformations framework because they greedily get the static row, which @@ -895,23 +932,21 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar if (filter == null) break; - if (!shouldInclude(sstable)) + boolean intersects = intersects(sstable); + boolean hasRequiredStatics = hasRequiredStatics(sstable); + boolean hasPartitionLevelDeletions = hasPartitionLevelDeletions(sstable); + + if (!intersects && !hasRequiredStatics) { // This mean that nothing queried by the filter can be in the sstable. One exception is the top-level partition deletion // however: if it is set, it impacts everything and must be included. Getting that top-level partition deletion costs us // some seek in general however (unless the partition is indexed and is in the key cache), so we first check if the sstable // has any tombstone at all as a shortcut. - if (!sstable.mayHaveTombstones()) + if (!hasPartitionLevelDeletions) continue; // no tombstone at all, we can skip that sstable // We need to get the partition deletion and include it if it's live. In any case though, we're done with that sstable. - try (UnfilteredRowIterator iter = StorageHook.instance.makeRowIterator(cfs, - sstable, - partitionKey(), - filter.getSlices(metadata()), - columnFilter(), - filter.isReversed(), - metricsCollector)) + try (UnfilteredRowIterator iter = makeRowIteratorWithSkippedNonStaticContent(cfs, sstable, metricsCollector)) { if (!iter.partitionLevelDeletion().isLive()) { @@ -938,13 +973,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar continue; } - try (UnfilteredRowIterator iter = StorageHook.instance.makeRowIterator(cfs, - sstable, - partitionKey(), - filter.getSlices(metadata()), - columnFilter(), - filter.isReversed(), - metricsCollector)) + try (UnfilteredRowIterator iter = makeRowIterator(cfs, sstable, filter, metricsCollector)) { if (iter.isEmpty()) continue; @@ -1025,7 +1054,7 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar toRemove = new TreeSet<>(result.metadata().comparator); toRemove.add(clustering); } - } + } } try (UnfilteredRowIterator iterator = result.unfilteredIterator(columnFilter(), clusterings, false)) @@ -1339,4 +1368,4 @@ public class SinglePartitionReadCommand extends ReadCommand implements SinglePar return executionController(); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/Slice.java b/src/java/org/apache/cassandra/db/Slice.java index e2c787d6f5..1fc60ba1f7 100644 --- a/src/java/org/apache/cassandra/db/Slice.java +++ b/src/java/org/apache/cassandra/db/Slice.java @@ -19,24 +19,28 @@ package org.apache.cassandra.db; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.List; +import java.util.Objects; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.Comparables; /** * A slice represents the selection of a range of rows. *

* A slice has a start and an end bound that are both (potentially full) clustering prefixes. - * A slice selects every rows whose clustering is bigger than the slice start prefix but smaller - * than the end prefix. Both start and end can be either inclusive or exclusive. + * A slice selects every row whose clustering is included within its start and end bounds. + * Both start and end can be either inclusive or exclusive. */ public class Slice { public static final Serializer serializer = new Serializer(); - /** The slice selecting all rows (of a given partition) */ + /** + * The slice selecting all rows (of a given partition) + */ public static final Slice ALL = new Slice(BufferClusteringBound.BOTTOM, BufferClusteringBound.TOP) { @Override @@ -46,9 +50,9 @@ public class Slice } @Override - public boolean intersects(ClusteringComparator comparator, List minClusteringValues, List maxClusteringValues) + public boolean intersects(ClusteringComparator comparator, Slice other) { - return true; + return !other.isEmpty(comparator); } @Override @@ -91,6 +95,9 @@ public class Slice return new Slice(builder.buildBound(true, true), builder.buildBound(false, true)); } + /** + * Makes a slice covering a single clustering + */ public static Slice make(Clustering clustering) { // This doesn't give us what we want with the clustering prefix @@ -98,14 +105,25 @@ public class Slice return new Slice(ClusteringBound.inclusiveStartOf(clustering), ClusteringBound.inclusiveEndOf(clustering)); } + /** + * Makes a slice covering a range from start to end clusterings, with both start and end included + */ public static Slice make(Clustering start, Clustering end) { // This doesn't give us what we want with the clustering prefix assert start != Clustering.STATIC_CLUSTERING && end != Clustering.STATIC_CLUSTERING; - return new Slice(ClusteringBound.inclusiveStartOf(start), ClusteringBound.inclusiveEndOf(end)); } + /** + * Makes a slice for the given bounds + */ + public static Slice make(ClusteringBoundOrBoundary start, ClusteringBoundOrBoundary end) + { + // This doesn't give us what we want with the clustering prefix + return make(start.asStartBound(), end.asEndBound()); + } + public ClusteringBound start() { return start; @@ -141,31 +159,23 @@ public class Slice * Return whether the slice formed by the two provided bound is empty or not. * * @param comparator the comparator to compare the bounds. - * @param start the start for the slice to consider. This must be a start bound. - * @param end the end for the slice to consider. This must be an end bound. + * @param start the start for the slice to consider. This must be a start bound. + * @param end the end for the slice to consider. This must be an end bound. * @return whether the slice formed by {@code start} and {@code end} is * empty or not. */ public static boolean isEmpty(ClusteringComparator comparator, ClusteringBound start, ClusteringBound end) { assert start.isStart() && end.isEnd(); - - int cmp = comparator.compare(start, end); - - if (cmp < 0) - return false; - else if (cmp > 0) - return true; - else - return start.isExclusive() || end.isExclusive(); + // Note: the comparator orders inclusive starts and exclusive ends as equal, and inclusive ends as being greater than starts. + return comparator.compare(start, end) >= 0; } /** * Returns whether a given clustering or bound is included in this slice. * * @param comparator the comparator for the table this is a slice of. - * @param bound the bound to test inclusion of. - * + * @param bound the bound to test inclusion of. * @return whether {@code bound} is within the bounds of this slice. */ public boolean includes(ClusteringComparator comparator, ClusteringPrefix bound) @@ -176,13 +186,12 @@ public class Slice /** * Returns a slice for continuing paging from the last returned clustering prefix. * - * @param comparator the comparator for the table this is a filter for. + * @param comparator the comparator for the table this is a filter for. * @param lastReturned the last clustering that was returned for the query we are paging for. The - * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned - * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise). - * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results. - * @param reversed whether the query we're paging for is reversed or not. - * + * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned + * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise). + * @param inclusive whether we want to include the {@code lastReturned} in the newly returned page of results. + * @param reversed whether the query we're paging for is reversed or not. * @return a new slice that selects results coming after {@code lastReturned}, or {@code null} if paging * the resulting slice selects nothing (i.e. if it originally selects nothing coming after {@code lastReturned}). */ @@ -229,20 +238,18 @@ public class Slice } /** - * Given the per-clustering column minimum and maximum value a sstable contains, whether or not this slice potentially - * intersects that sstable or not. + * Whether this slice and the provided slice intersects. * * @param comparator the comparator for the table this is a slice of. - * @param minClusteringValues the smallest values for each clustering column that a sstable contains. - * @param maxClusteringValues the biggest values for each clustering column that a sstable contains. - * - * @return whether the slice might intersects with the sstable having {@code minClusteringValues} and - * {@code maxClusteringValues}. + * @param other the other slice to check intersection with. + * @return whether this slice intersects {@code other}. */ - public boolean intersects(ClusteringComparator comparator, List minClusteringValues, List maxClusteringValues) + public boolean intersects(ClusteringComparator comparator, Slice other) { - // If this slice starts after max clustering or ends before min clustering, it can't intersect - return start.compareTo(comparator, maxClusteringValues) <= 0 && end.compareTo(comparator, minClusteringValues) >= 0; + // Construct the intersection of the two slices and check if it is non-empty. + // This also works correctly when one or more of the inputs are be empty (i.e. with end <= start). + return comparator.compare(Comparables.max(start, other.start, comparator), + Comparables.min(end, other.end, comparator)) < 0; } public String toString(ClusteringComparator comparator) @@ -269,12 +276,12 @@ public class Slice @Override public boolean equals(Object other) { - if(!(other instanceof Slice)) + if (!(other instanceof Slice)) return false; - Slice that = (Slice)other; + Slice that = (Slice) other; return this.start().equals(that.start()) - && this.end().equals(that.end()); + && this.end().equals(that.end()); } @Override @@ -294,7 +301,7 @@ public class Slice public long serializedSize(Slice slice, int version, List> types) { return ClusteringBound.serializer.serializedSize(slice.start, version, types) - + ClusteringBound.serializer.serializedSize(slice.end, version, types); + + ClusteringBound.serializer.serializedSize(slice.end, version, types); } public Slice deserialize(DataInputPlus in, int version, List> types) throws IOException @@ -304,4 +311,4 @@ public class Slice return new Slice(start, end); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/Slices.java b/src/java/org/apache/cassandra/db/Slices.java index c8483ccf47..bae83d5980 100644 --- a/src/java/org/apache/cassandra/db/Slices.java +++ b/src/java/org/apache/cassandra/db/Slices.java @@ -20,6 +20,7 @@ package org.apache.cassandra.db; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; +import java.util.stream.Collectors; import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; @@ -55,7 +56,7 @@ public abstract class Slices implements Iterable * Creates a {@code Slices} object that contains a single slice. * * @param comparator the comparator for the table {@code slice} is a slice of. - * @param slice the single slice that the return object should contains. + * @param slice the single slice that the return object should contain. * * @return the newly created {@code Slices} object. */ @@ -69,16 +70,16 @@ public abstract class Slices implements Iterable } /** - * Whether the slices has a lower bound, that is whether it's first slice start is {@code Slice.BOTTOM}. + * Whether the slices instance has a lower bound, that is whether it's first slice start is {@code Slice.BOTTOM}. * - * @return whether the slices has a lower bound. + * @return whether this slices instance has a lower bound. */ public abstract boolean hasLowerBound(); /** - * Whether the slices has an upper bound, that is whether it's last slice end is {@code Slice.TOP}. + * Whether the slices instance has an upper bound, that is whether it's last slice end is {@code Slice.TOP}. * - * @return whether the slices has an upper bound. + * @return whether this slices instance has an upper bound. */ public abstract boolean hasUpperBound(); @@ -96,6 +97,16 @@ public abstract class Slices implements Iterable */ public abstract Slice get(int i); + public ClusteringBound start() + { + return get(0).start(); + } + + public ClusteringBound end() + { + return get(size() - 1).end(); + } + /** * Returns slices for continuing the paging of those slices given the last returned clustering prefix. * @@ -103,7 +114,7 @@ public abstract class Slices implements Iterable * @param lastReturned the last clustering that was returned for the query we are paging for. The * resulting slices will be such that only results coming stricly after {@code lastReturned} are returned * (where coming after means "greater than" if {@code !reversed} and "lesser than" otherwise). - * @param inclusive whether or not we want to include the {@code lastReturned} in the newly returned page of results. + * @param inclusive whether we want to include the {@code lastReturned} in the newly returned page of results. * @param reversed whether the query we're paging for is reversed or not. * * @return new slices that select results coming after {@code lastReturned}. @@ -130,18 +141,13 @@ public abstract class Slices implements Iterable */ public abstract boolean selects(Clustering clustering); - /** - * Given the per-clustering column minimum and maximum value a sstable contains, whether or not this slices potentially - * intersects that sstable or not. + * Checks whether any of the slices intersects witht the given one. * - * @param minClusteringValues the smallest values for each clustering column that a sstable contains. - * @param maxClusteringValues the biggest values for each clustering column that a sstable contains. - * - * @return whether the slices might intersects with the sstable having {@code minClusteringValues} and - * {@code maxClusteringValues}. + * @return {@code true} if there exists a slice which ({@link Slice#intersects(ClusteringComparator, Slice)}) with + * the provided slice */ - public abstract boolean intersects(List minClusteringValues, List maxClusteringValues); + public abstract boolean intersects(Slice slice); public abstract String toCQLString(TableMetadata metadata, RowFilter rowFilter); @@ -157,12 +163,12 @@ public abstract class Slices implements Iterable /** * In simple object that allows to test the inclusion of rows in those slices assuming those rows * are passed (to {@link #includes}) in clustering order (or reverse clustering ordered, depending - * of the argument passed to {@link #inOrderTester}). + * on the argument passed to {@link #inOrderTester}). */ public interface InOrderTester { - public boolean includes(Clustering value); - public boolean isDone(); + boolean includes(Clustering value); + boolean isDone(); } /** @@ -243,17 +249,12 @@ public abstract class Slices implements Iterable if (slices.size() <= 1) return slices; - Collections.sort(slices, new Comparator() - { - @Override - public int compare(Slice s1, Slice s2) - { - int c = comparator.compare(s1.start(), s2.start()); - if (c != 0) - return c; + slices.sort((s1, s2) -> { + int c = comparator.compare(s1.start(), s2.start()); + if (c != 0) + return c; - return comparator.compare(s1.end(), s2.end()); - } + return comparator.compare(s2.end(), s1.end()); }); List slicesCopy = new ArrayList<>(slices.size()); @@ -278,12 +279,7 @@ public abstract class Slices implements Iterable } if (includesStart) - { last = Slice.make(last.start(), s2.end()); - continue; - } - - assert !includesFinish; } slicesCopy.add(last); @@ -302,7 +298,7 @@ public abstract class Slices implements Iterable return; List> types = slices == ALL - ? Collections.>emptyList() + ? Collections.emptyList() : ((ArrayBackedSlices)slices).comparator.subtypes(); for (Slice slice : slices) @@ -317,7 +313,7 @@ public abstract class Slices implements Iterable return size; List> types = slices instanceof SelectAllSlices - ? Collections.>emptyList() + ? Collections.emptyList() : ((ArrayBackedSlices)slices).comparator.subtypes(); for (Slice slice : slices) @@ -441,11 +437,12 @@ public abstract class Slices implements Iterable return Slices.NONE; } - public boolean intersects(List minClusteringValues, List maxClusteringValues) + @Override + public boolean intersects(Slice slice) { - for (Slice slice : this) + for (Slice s : this) { - if (slice.intersects(comparator, minClusteringValues, maxClusteringValues)) + if (s.intersects(comparator, slice)) return true; } return false; @@ -540,15 +537,7 @@ public abstract class Slices implements Iterable @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("{"); - for (int i = 0; i < slices.length; i++) - { - if (i > 0) - sb.append(", "); - sb.append(slices[i].toString(comparator)); - } - return sb.append("}").toString(); + return Arrays.stream(slices).map(s -> s.toString(comparator)).collect(Collectors.joining(", ", "{", "}")); } @Override @@ -636,7 +625,7 @@ public abstract class Slices implements Iterable operator = first.startInclusive ? Operator.LTE : Operator.LT; else operator = first.startInclusive ? Operator.GTE : Operator.GT; - sb.append(' ').append(operator.toString()).append(' ') + sb.append(' ').append(operator).append(' ') .append(column.type.toCQLString(first.startValue)); rowFilter = rowFilter.without(column, operator, first.startValue); } @@ -650,7 +639,7 @@ public abstract class Slices implements Iterable operator = first.endInclusive ? Operator.GTE : Operator.GT; else operator = first.endInclusive ? Operator.LTE : Operator.LT; - sb.append(' ').append(operator.toString()).append(' ') + sb.append(' ').append(operator).append(' ') .append(column.type.toCQLString(first.endValue)); rowFilter = rowFilter.without(column, operator, first.endValue); } @@ -667,7 +656,7 @@ public abstract class Slices implements Iterable return sb.toString(); } - // An somewhat adhoc utility class only used by nameAsCQLString + // Somewhat adhoc utility class only used by nameAsCQLString private static class ComponentOfSlice { public final boolean startInclusive; @@ -768,7 +757,8 @@ public abstract class Slices implements Iterable return trivialTester; } - public boolean intersects(List minClusteringValues, List maxClusteringValues) + @Override + public boolean intersects(Slice slice) { return true; } @@ -844,7 +834,8 @@ public abstract class Slices implements Iterable return trivialTester; } - public boolean intersects(List minClusteringValues, List maxClusteringValues) + @Override + public boolean intersects(Slice slice) { return false; } @@ -866,4 +857,4 @@ public abstract class Slices implements Iterable return ""; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/StorageHook.java b/src/java/org/apache/cassandra/db/StorageHook.java index c998338abe..b2a243970e 100644 --- a/src/java/org/apache/cassandra/db/StorageHook.java +++ b/src/java/org/apache/cassandra/db/StorageHook.java @@ -35,11 +35,11 @@ public interface StorageHook public void reportWrite(TableId tableId, PartitionUpdate partitionUpdate); public void reportRead(TableId tableId, DecoratedKey key); public UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs, - DecoratedKey partitionKey, - SSTableReader sstable, - ClusteringIndexFilter filter, - ColumnFilter selectedColumns, - SSTableReadsListener listener); + SSTableReader sstable, + DecoratedKey partitionKey, + ClusteringIndexFilter filter, + ColumnFilter selectedColumns, + SSTableReadsListener listener); public UnfilteredRowIterator makeRowIterator(ColumnFamilyStore cfs, SSTableReader sstable, DecoratedKey key, @@ -63,8 +63,7 @@ public interface StorageHook public void reportRead(TableId tableId, DecoratedKey key) {} public UnfilteredRowIteratorWithLowerBound makeRowIteratorWithLowerBound(ColumnFamilyStore cfs, - DecoratedKey partitionKey, - SSTableReader sstable, + SSTableReader sstable, DecoratedKey partitionKey, ClusteringIndexFilter filter, ColumnFilter selectedColumns, SSTableReadsListener listener) @@ -88,4 +87,4 @@ public interface StorageHook } }; } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java index 924ff21ec5..564ca1a223 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java @@ -23,7 +23,6 @@ import org.apache.cassandra.db.*; import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.db.partitions.Partition; import org.apache.cassandra.db.rows.*; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.TableMetadata; @@ -142,13 +141,14 @@ public interface ClusteringIndexFilter public UnfilteredRowIterator getUnfilteredRowIterator(ColumnFilter columnFilter, Partition partition); /** - * Whether the provided sstable may contain data that is selected by this filter (based on the sstable metadata). + * Whether the data selected by this filter intersects with the provided slice. * - * @param sstable the sstable for which we want to test the need for inclusion. + * @param comparator the comparator of the table this if a filter on. + * @param slice the slice to check intersection with, * - * @return whether {@code sstable} should be included to answer this filter. + * @return whether the data selected by this filter intersects with {@code slice}. */ - public boolean shouldInclude(SSTableReader sstable); + public boolean intersects(ClusteringComparator comparator, Slice slice); public Kind kind(); @@ -161,4 +161,4 @@ public interface ClusteringIndexFilter public ClusteringIndexFilter deserialize(DataInputPlus in, int version, TableMetadata metadata) throws IOException; public long serializedSize(ClusteringIndexFilter filter, int version); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java index 600aa9fef9..a98e3bde99 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java @@ -18,7 +18,6 @@ package org.apache.cassandra.db.filter; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.*; import org.apache.cassandra.cql3.Operator; @@ -26,7 +25,6 @@ import org.apache.cassandra.db.*; import org.apache.cassandra.db.partitions.*; import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.transform.Transformation; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; import org.apache.cassandra.schema.ColumnMetadata; @@ -142,16 +140,11 @@ public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter return partition.unfilteredIterator(columnFilter, clusteringsInQueryOrder, isReversed()); } - public boolean shouldInclude(SSTableReader sstable) + public boolean intersects(ClusteringComparator comparator, Slice slice) { - ClusteringComparator comparator = sstable.metadata().comparator; - List minClusteringValues = sstable.getSSTableMetadata().minClusteringValues; - List maxClusteringValues = sstable.getSSTableMetadata().maxClusteringValues; - - // If any of the requested clustering is within the bounds covered by the sstable, we need to include the sstable for (Clustering clustering : clusterings) { - if (Slice.make(clustering).intersects(comparator, minClusteringValues, maxClusteringValues)) + if (slice.includes(comparator, clustering)) return true; } return false; @@ -255,4 +248,4 @@ public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter } } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java index 178c96b453..67aeeb7c56 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java @@ -18,18 +18,15 @@ package org.apache.cassandra.db.filter; import java.io.IOException; -import java.util.List; -import java.nio.ByteBuffer; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.partitions.CachedPartition; import org.apache.cassandra.db.partitions.Partition; +import org.apache.cassandra.db.rows.*; import org.apache.cassandra.db.transform.Transformation; -import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.schema.TableMetadata; /** * A filter over a single partition. @@ -126,15 +123,9 @@ public class ClusteringIndexSliceFilter extends AbstractClusteringIndexFilter return partition.unfilteredIterator(columnFilter, slices, reversed); } - public boolean shouldInclude(SSTableReader sstable) + public boolean intersects(ClusteringComparator comparator, Slice slice) { - List minClusteringValues = sstable.getSSTableMetadata().minClusteringValues; - List maxClusteringValues = sstable.getSSTableMetadata().maxClusteringValues; - - if (minClusteringValues.isEmpty() || maxClusteringValues.isEmpty()) - return true; - - return slices.intersects(minClusteringValues, maxClusteringValues); + return slices.intersects(slice); } public String toString(TableMetadata metadata) @@ -176,4 +167,4 @@ public class ClusteringIndexSliceFilter extends AbstractClusteringIndexFilter return new ClusteringIndexSliceFilter(slices, reversed); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java b/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java index 9b477aeeea..ed1e6a6e1a 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteArrayObjectFactory.java @@ -18,14 +18,7 @@ package org.apache.cassandra.db.marshal; -import org.apache.cassandra.db.AbstractArrayClusteringPrefix; -import org.apache.cassandra.db.ArrayClustering; -import org.apache.cassandra.db.ArrayClusteringBound; -import org.apache.cassandra.db.ArrayClusteringBoundary; -import org.apache.cassandra.db.Clustering; -import org.apache.cassandra.db.ClusteringBound; -import org.apache.cassandra.db.ClusteringBoundary; -import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.rows.ArrayCell; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.CellPath; @@ -74,6 +67,13 @@ class ByteArrayObjectFactory implements ValueAccessor.ObjectFactory private static final ArrayClusteringBound TOP_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY); + /** The biggest start bound, i.e. the one that starts after any row. */ + private static final ArrayClusteringBound MAX_START_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.EXCL_START_BOUND, + AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY); + /** The smallest end bound, i.e. the one that end before any row. */ + private static final ArrayClusteringBound MIN_END_BOUND = new ArrayClusteringBound(ClusteringPrefix.Kind.EXCL_END_BOUND, + AbstractArrayClusteringPrefix.EMPTY_VALUES_ARRAY); + public Cell cell(ColumnMetadata column, long timestamp, int ttl, int localDeletionTime, byte[] value, CellPath path) { return new ArrayCell(column, timestamp, ttl, localDeletionTime, value, path); @@ -101,11 +101,19 @@ class ByteArrayObjectFactory implements ValueAccessor.ObjectFactory public ClusteringBound bound(ClusteringPrefix.Kind kind) { - return kind.isStart() ? BOTTOM_BOUND : TOP_BOUND; + switch (kind) + { + case EXCL_END_BOUND: return MIN_END_BOUND; + case INCL_START_BOUND: return BOTTOM_BOUND; + case INCL_END_BOUND: return TOP_BOUND; + case EXCL_START_BOUND: return MAX_START_BOUND; + default: + throw new AssertionError(String.format("Unexpected kind %s for empty bound or boundary", kind)); + } } public ClusteringBoundary boundary(ClusteringPrefix.Kind kind, byte[]... values) { return new ArrayClusteringBoundary(kind, values); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java b/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java index 0ac3db9265..82eba37486 100644 --- a/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java +++ b/src/java/org/apache/cassandra/db/marshal/ByteBufferObjectFactory.java @@ -20,14 +20,7 @@ package org.apache.cassandra.db.marshal; import java.nio.ByteBuffer; -import org.apache.cassandra.db.AbstractBufferClusteringPrefix; -import org.apache.cassandra.db.BufferClustering; -import org.apache.cassandra.db.BufferClusteringBound; -import org.apache.cassandra.db.BufferClusteringBoundary; -import org.apache.cassandra.db.Clustering; -import org.apache.cassandra.db.ClusteringBound; -import org.apache.cassandra.db.ClusteringBoundary; -import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.*; import org.apache.cassandra.db.rows.BufferCell; import org.apache.cassandra.db.rows.Cell; import org.apache.cassandra.db.rows.CellPath; @@ -42,6 +35,13 @@ class ByteBufferObjectFactory implements ValueAccessor.ObjectFactory private static final BufferClusteringBound TOP_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.INCL_END_BOUND, AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY); + /** The biggest start bound, i.e. the one that starts after any row. */ + private static final BufferClusteringBound MAX_START_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.EXCL_START_BOUND, + AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY); + /** The smallest end bound, i.e. the one that end before any row. */ + private static final BufferClusteringBound MIN_END_BOUND = new BufferClusteringBound(ClusteringPrefix.Kind.EXCL_END_BOUND, + AbstractBufferClusteringPrefix.EMPTY_VALUES_ARRAY); + static final ValueAccessor.ObjectFactory instance = new ByteBufferObjectFactory(); private ByteBufferObjectFactory() {} @@ -73,11 +73,19 @@ class ByteBufferObjectFactory implements ValueAccessor.ObjectFactory public ClusteringBound bound(ClusteringPrefix.Kind kind) { - return kind.isStart() ? BOTTOM_BOUND : TOP_BOUND; + switch (kind) + { + case EXCL_END_BOUND: return MIN_END_BOUND; + case INCL_START_BOUND: return BOTTOM_BOUND; + case INCL_END_BOUND: return TOP_BOUND; + case EXCL_START_BOUND: return MAX_START_BOUND; + default: + throw new AssertionError(String.format("Unexpected kind %s for empty bound or boundary", kind)); + } } public ClusteringBoundary boundary(ClusteringPrefix.Kind kind, ByteBuffer... values) { return new BufferClusteringBoundary(kind, values); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java b/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java index 6d37640146..7c3ba150ae 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionStatisticsCollector.java @@ -22,9 +22,10 @@ import org.apache.cassandra.db.rows.Cell; public interface PartitionStatisticsCollector { - public void update(LivenessInfo info); - public void update(DeletionTime deletionTime); - public void update(Cell cell); - public void updateColumnSetPerRow(long columnSetInRow); - public void updateHasLegacyCounterShards(boolean hasLegacyCounterShards); -} + void update(LivenessInfo info); + void updatePartitionDeletion(DeletionTime dt); + void update(DeletionTime deletionTime); + void update(Cell cell); + void updateColumnSetPerRow(long columnSetInRow); + void updateHasLegacyCounterShards(boolean hasLegacyCounterShards); +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java index a051ee1a9d..02fdde0611 100644 --- a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java +++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java @@ -386,4 +386,4 @@ public abstract class UnfilteredPartitionIterators }; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java b/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java new file mode 100644 index 0000000000..ed6e39a5a2 --- /dev/null +++ b/src/java/org/apache/cassandra/db/rows/ArtificialBoundMarker.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.rows; + +import java.util.Objects; + +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.schema.TableMetadata; + +public class ArtificialBoundMarker extends RangeTombstoneBoundMarker +{ + public ArtificialBoundMarker(ClusteringBound bound) + { + super(bound, DeletionTime.LIVE); + assert bound.isArtificial(); + } + + @Override + public boolean equals(Object other) + { + if (this == other) + return true; + + if (!(other instanceof ArtificialBoundMarker)) + return false; + + ArtificialBoundMarker that = (ArtificialBoundMarker) other; + return Objects.equals(bound, that.bound); + } + + @Override + public int hashCode() + { + return Objects.hash(bound); + } + + @Override + public String toString(TableMetadata metadata) + { + return String.format("LowerBoundMarker %s", bound.toString(metadata)); + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/rows/EncodingStats.java b/src/java/org/apache/cassandra/db/rows/EncodingStats.java index 3f54c386d3..a29ce8f85c 100644 --- a/src/java/org/apache/cassandra/db/rows/EncodingStats.java +++ b/src/java/org/apache/cassandra/db/rows/EncodingStats.java @@ -215,6 +215,12 @@ public class EncodingStats implements IMeasurableMemory updateLocalDeletionTime(deletionTime.localDeletionTime()); } + @Override + public void updatePartitionDeletion(DeletionTime dt) + { + update(dt); + } + public void updateTimestamp(long timestamp) { isTimestampSet = true; @@ -285,4 +291,4 @@ public class EncodingStats implements IMeasurableMemory return new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java index 4d1d71bf12..2004c3dfd3 100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java @@ -21,13 +21,19 @@ package org.apache.cassandra.db.rows; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Comparator; -import java.util.List; +import java.util.Optional; -import org.apache.cassandra.db.marshal.ByteBufferAccessor; -import org.apache.cassandra.schema.TableMetadata; -import org.apache.cassandra.db.*; +import com.google.common.annotations.VisibleForTesting; + +import org.apache.cassandra.db.Clusterable; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.RegularAndStaticColumns; +import org.apache.cassandra.db.RowIndexEntry; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ClusteringIndexFilter; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.transform.RTBoundValidator; @@ -35,6 +41,7 @@ import org.apache.cassandra.io.sstable.IndexInfo; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.io.sstable.format.SSTableReadsListener; import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.IteratorWithLowerBound; /** @@ -48,10 +55,11 @@ import org.apache.cassandra.utils.IteratorWithLowerBound; public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilteredRowIterator implements IteratorWithLowerBound { private final SSTableReader sstable; - private final ClusteringIndexFilter filter; + private final Slices slices; + private final boolean isReverseOrder; private final ColumnFilter selectedColumns; private final SSTableReadsListener listener; - private ClusteringBound lowerBound; + private Optional lowerBoundMarker; private boolean firstItemRetrieved; public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey, @@ -59,26 +67,45 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt ClusteringIndexFilter filter, ColumnFilter selectedColumns, SSTableReadsListener listener) + { + this(partitionKey, sstable, filter.getSlices(sstable.metadata()), filter.isReversed(), selectedColumns, listener); + } + + @VisibleForTesting + public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey, + SSTableReader sstable, + Slices slices, + boolean isReverseOrder, + ColumnFilter selectedColumns, + SSTableReadsListener listener) { super(partitionKey); this.sstable = sstable; - this.filter = filter; + this.slices = slices; + this.isReverseOrder = isReverseOrder; this.selectedColumns = selectedColumns; this.listener = listener; - this.lowerBound = null; this.firstItemRetrieved = false; } public Unfiltered lowerBound() { - if (lowerBound != null) - return makeBound(lowerBound); + if (lowerBoundMarker != null) + return lowerBoundMarker.orElse(null); - // The partition index lower bound is more accurate than the sstable metadata lower bound but it is only - // present if the iterator has already been initialized, which we only do when there are tombstones since in - // this case we cannot use the sstable metadata clustering values - ClusteringBound ret = getPartitionIndexLowerBound(); - return ret != null ? makeBound(ret) : makeBound(getMetadataLowerBound()); + // lower bound from cache may be more accurate as it stores information about clusterings range for that exact + // row, so we try it first (without initializing iterator) + ClusteringBound lowerBound = maybeGetLowerBoundFromKeyCache(); + if (lowerBound == null) + // If we couldn't get the lower bound from cache, we try with metadata + lowerBound = maybeGetLowerBoundFromMetadata(); + + if (lowerBound != null) + lowerBoundMarker = Optional.of(makeBound(lowerBound)); + else + lowerBoundMarker = Optional.empty(); + + return lowerBoundMarker.orElse(null); } private Unfiltered makeBound(ClusteringBound bound) @@ -86,21 +113,15 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt if (bound == null) return null; - if (lowerBound != bound) - lowerBound = bound; - - return new RangeTombstoneBoundMarker(lowerBound, DeletionTime.LIVE); + return new ArtificialBoundMarker(bound); } @Override protected UnfilteredRowIterator initializeIterator() { @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator - UnfilteredRowIterator iter = RTBoundValidator.validate( - sstable.rowIterator(partitionKey(), filter.getSlices(metadata()), selectedColumns, filter.isReversed(), listener), - RTBoundValidator.Stage.SSTABLE, - false - ); + UnfilteredRowIterator iter = RTBoundValidator.validate(sstable.rowIterator(partitionKey(), slices, selectedColumns, isReverseOrder, listener), + RTBoundValidator.Stage.SSTABLE, false); return iter; } @@ -113,19 +134,20 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt // Check that the lower bound is not bigger than the first item retrieved firstItemRetrieved = true; + Unfiltered lowerBound = lowerBound(); if (lowerBound != null && ret != null) - assert comparator().compare(lowerBound, ret.clustering()) <= 0 - : String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s", - lowerBound.toString(metadata()), - ret.toString(metadata()), - sstable.getFilename()); + assert comparator().compare(lowerBound.clustering(), ret.clustering()) <= 0 + : String.format("Lower bound [%s ]is bigger than first returned value [%s] for sstable %s", + lowerBound.clustering().toString(metadata()), + ret.toString(metadata()), + sstable.getFilename()); return ret; } private Comparator comparator() { - return filter.isReversed() ? metadata().comparator.reversed() : metadata().comparator; + return isReverseOrder ? metadata().comparator.reversed() : metadata().comparator; } @Override @@ -137,7 +159,7 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt @Override public boolean isReverseOrder() { - return filter.isReversed(); + return isReverseOrder; } @Override @@ -155,7 +177,7 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt @Override public DeletionTime partitionLevelDeletion() { - if (!sstable.mayHaveTombstones()) + if (!sstable.getSSTableMetadata().hasPartitionLevelDeletions) return DeletionTime.LIVE; return super.partitionLevelDeletion(); @@ -170,39 +192,21 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt return super.staticRow(); } - private static ClusteringBound createInclusiveOpen(boolean isReversed, ClusteringPrefix from) - { - return from.accessor().factory().inclusiveOpen(isReversed, from.getRawValues()); - } - /** * @return the lower bound stored on the index entry for this partition, if available. */ - private ClusteringBound getPartitionIndexLowerBound() + private ClusteringBound maybeGetLowerBoundFromKeyCache() { - // NOTE: CASSANDRA-11206 removed the lookup against the key-cache as the IndexInfo objects are no longer - // in memory for not heap backed IndexInfo objects (so, these are on disk). - // CASSANDRA-11369 is there to fix this afterwards. - - // Creating the iterator ensures that rowIndexEntry is loaded if available (partitions bigger than - // DatabaseDescriptor.column_index_size) - if (!canUseMetadataLowerBound()) - maybeInit(); - - RowIndexEntry rowIndexEntry = sstable.getCachedPosition(partitionKey(), false); + RowIndexEntry rowIndexEntry = sstable.getCachedPosition(partitionKey(), false); if (rowIndexEntry == null || !rowIndexEntry.indexOnHeap()) return null; try (RowIndexEntry.IndexInfoRetriever onHeapRetriever = rowIndexEntry.openWithIndex(null)) { - IndexInfo column = onHeapRetriever.columnsIndex(filter.isReversed() ? rowIndexEntry.columnsIndexCount() - 1 : 0); - ClusteringPrefix lowerBoundPrefix = filter.isReversed() ? column.lastName : column.firstName; - assert lowerBoundPrefix.getRawValues().length <= metadata().comparator.size() : - String.format("Unexpected number of clustering values %d, expected %d or fewer for %s", - lowerBoundPrefix.getRawValues().length, - metadata().comparator.size(), - sstable.getFilename()); - return createInclusiveOpen(filter.isReversed(), lowerBoundPrefix); + IndexInfo columns = onHeapRetriever.columnsIndex(isReverseOrder() ? rowIndexEntry.columnsIndexCount() - 1 : 0); + ClusteringBound bound = isReverseOrder() ? columns.lastName.asEndBound() : columns.firstName.asStartBound(); + assertBoundSize(bound); + return bound.artificialLowerBound(isReverseOrder()); } catch (IOException e) { @@ -212,51 +216,53 @@ public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilt /** * Whether we can use the clustering values in the stats of the sstable to build the lower bound. - *

- * Currently, the clustering values of the stats file records for each clustering component the min and max - * value seen, null excluded. In other words, having a non-null value for a component in those min/max clustering - * values does _not_ guarantee that there isn't an unfiltered in the sstable whose clustering has either no value for - * that component (it's a prefix) or a null value. - *

- * This is problematic as this means we can't in general build a lower bound from those values since the "min" - * values doesn't actually guarantee minimality. - *

- * However, we can use those values if we can guarantee that no clustering in the sstable 1) is a true prefix and - * 2) uses null values. Nat having true prefixes means having no range tombstone markers since rows use - * {@link Clustering} which is always "full" (all components are always present). As for null values, we happen to - * only allow those in compact tables (for backward compatibility), so we can simply exclude those tables. - *

- * Note that the information we currently have at our disposal make this condition less precise that it could be. - * In particular, {@link SSTableReader#mayHaveTombstones} could return {@code true} (making us not use the stats) - * because of cell tombstone or even expiring cells even if the sstable has no range tombstone markers, even though - * it's really only markers we want to exclude here (more precisely, as said above, we want to exclude anything - * whose clustering is not "full", but that's only markers). It wouldn't be very hard to collect whether a sstable - * has any range tombstone marker however so it's a possible improvement. */ private boolean canUseMetadataLowerBound() { - // Side-note: pre-2.1 sstable stat file had clustering value arrays whose size may not match the comparator size - // and that would break getMetadataLowerBound. We don't support upgrade from 2.0 to 3.0 directly however so it's - // not a true concern. Besides, !sstable.mayHaveTombstones already ensure this is a 3.0 sstable anyway. - return !sstable.mayHaveTombstones() && !sstable.metadata().isCompactTable(); + if (sstable.metadata().isCompactTable()) + return false; + + Slices requestedSlices = slices; + + if (requestedSlices.isEmpty()) + return true; + + // Simply exclude the cases where lower bound would not be used anyway, that is, the start of covered range of + // clusterings in sstable is lower than the requested slice. In such case, we need to access that sstable's + // iterator anyway so there is no need to use a lower bound optimization extra complexity. + if (!isReverseOrder()) + { + return !requestedSlices.hasLowerBound() || + metadata().comparator.compare(requestedSlices.start(), sstable.getSSTableMetadata().coveredClustering.start()) < 0; + } + else + { + return !requestedSlices.hasUpperBound() || + metadata().comparator.compare(requestedSlices.end(), sstable.getSSTableMetadata().coveredClustering.end()) > 0; + } } /** * @return a global lower bound made from the clustering values stored in the sstable metadata, note that * this currently does not correctly compare tombstone bounds, especially ranges. */ - private ClusteringBound getMetadataLowerBound() + private ClusteringBound maybeGetLowerBoundFromMetadata() { if (!canUseMetadataLowerBound()) return null; final StatsMetadata m = sstable.getSSTableMetadata(); - List vals = filter.isReversed() ? m.maxClusteringValues : m.minClusteringValues; - assert vals.size() <= metadata().comparator.size() : + ClusteringBound bound = m.coveredClustering.open(isReverseOrder); + assertBoundSize(bound); + return bound.artificialLowerBound(isReverseOrder); + } + + private void assertBoundSize(ClusteringPrefix lowerBound) + { + assert lowerBound.size() <= metadata().comparator.size() : String.format("Unexpected number of clustering values %d, expected %d or fewer for %s", - vals.size(), + lowerBound.size(), metadata().comparator.size(), sstable.getFilename()); - return ByteBufferAccessor.instance.factory().inclusiveOpen(filter.isReversed(), vals.toArray(new ByteBuffer[vals.size()])); } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java index 6ca74f00f7..1539023163 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReaderBuilder.java @@ -88,6 +88,12 @@ public abstract class SSTableReaderBuilder this.openReason = openReason; this.header = header; this.readerFactory = descriptor.getFormat().getReaderFactory(); + + if (statsMetadata.firstKey != null && statsMetadata.lastKey != null) + { + this.first = metadata.partitioner.decorateKey(statsMetadata.firstKey); + this.last = metadata.partitioner.decorateKey(statsMetadata.lastKey); + } } public abstract SSTableReader build(); @@ -140,8 +146,13 @@ public abstract class SSTableReaderBuilder metadata.partitioner, metadata.params.minIndexInterval, metadata.params.maxIndexInterval); - first = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); - last = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); + if (first == null || last == null) + { + // this is a legacy code in order to be able to load older sstables + // since version 'nc', first and last keys are stored in stats file + first = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); + last = metadata.partitioner.decorateKey(ByteBufferUtil.readWithLength(iStream)); + } } catch (IOException e) { diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java index f82a7c2fc6..7270fd2e32 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java @@ -327,7 +327,9 @@ public abstract class SSTableWriter extends SSTable implements Transactional repairedAt, pendingRepair, isTransient, - header); + header, + SSTable.getMinimalKey(first).getKey(), + SSTable.getMinimalKey(last).getKey()); } protected StatsMetadata statsMetadata() diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java index aa41b149c2..781c2791d8 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/Version.java +++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java @@ -17,19 +17,19 @@ */ package org.apache.cassandra.io.sstable.format; +import java.util.Objects; import java.util.regex.Pattern; /** * A set of feature flags associated with a SSTable format - * + *

* versions are denoted as [major][minor]. Minor versions must be forward-compatible: * new fields are allowed in e.g. the metadata component, but fields can't be removed * or have their size changed. - * + *

* Minor versions were introduced with version "hb" for Cassandra 1.0.3; prior to that, * we always incremented the major version. - * */ public abstract class Version { @@ -37,6 +37,7 @@ public abstract class Version protected final String version; protected final SSTableFormat format; + protected Version(SSTableFormat format, String version) { this.format = format; @@ -62,12 +63,31 @@ public abstract class Version /** * The old bloomfilter format serializes the data as BIG_ENDIAN long's, the new one uses the * same format as in memory (serializes as bytes). + * * @return True if the bloomfilter file is old serialization format */ public abstract boolean hasOldBfFormat(); + /** + * @deprecated it is replaced by {@link #hasImprovedMinMax()} since 'nc' and to be completetly removed since 'oa' + */ + @Deprecated public abstract boolean hasAccurateMinMax(); + /** + * @deprecated it is replaced by {@link #hasImprovedMinMax()} since 'nc' and to be completetly removed since 'oa' + */ + @Deprecated + public abstract boolean hasLegacyMinMax(); + + public abstract boolean hasOriginatingHostId(); + + public abstract boolean hasImprovedMinMax(); + + public abstract boolean hasPartitionLevelDeletionsPresenceMarker(); + + public abstract boolean hasKeyRange(); + public String getVersion() { return version; @@ -89,6 +109,7 @@ public abstract class Version } abstract public boolean isCompatible(); + abstract public boolean isCompatibleForStreaming(); @Override @@ -98,16 +119,13 @@ public abstract class Version } @Override - public boolean equals(Object o) + public boolean equals(Object other) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == other) return true; + if (other == null || getClass() != other.getClass()) return false; - Version version1 = (Version) o; - - if (version != null ? !version.equals(version1.version) : version1.version != null) return false; - - return true; + Version otherVersion = (Version) other; + return Objects.equals(version, otherVersion.version); } @Override @@ -115,6 +133,4 @@ public abstract class Version { return version != null ? version.hashCode() : 0; } - - public abstract boolean hasOriginatingHostId(); } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java index c84782f5f6..3510998b64 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java @@ -121,7 +121,7 @@ public class BigFormat implements SSTableFormat // we always incremented the major version. static class BigVersion extends Version { - public static final String current_version = "nb"; + public static final String current_version = "nc"; public static final String earliest_supported_version = "ma"; // ma (3.0.0): swap bf hash order @@ -133,19 +133,24 @@ public class BigFormat implements SSTableFormat // na (4.0-rc1): uncompressed chunks, pending repair session, isTransient, checksummed sstable metadata file, new Bloomfilter format // nb (4.0.0): originating host id + // nc (4.1): improved min/max, partition level deletion presence marker, key range (CASSANDRA-18134) // // NOTE: when adding a new version, please add that to LegacySSTableTest, too. private final boolean isLatestVersion; - public final int correspondingMessagingVersion; + private final int correspondingMessagingVersion; private final boolean hasCommitLogLowerBound; private final boolean hasCommitLogIntervals; private final boolean hasAccurateMinMax; + private final boolean hasLegacyMinMax; private final boolean hasOriginatingHostId; - public final boolean hasMaxCompressedLength; + private final boolean hasMaxCompressedLength; private final boolean hasPendingRepair; private final boolean hasMetadataChecksum; private final boolean hasIsTransient; + private final boolean hasImprovedMinMax; + private final boolean hasPartitionLevelDeletionPresenceMarker; + private final boolean hasKeyRange; /** * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped) @@ -162,13 +167,17 @@ public class BigFormat implements SSTableFormat hasCommitLogLowerBound = version.compareTo("mb") >= 0; hasCommitLogIntervals = version.compareTo("mc") >= 0; - hasAccurateMinMax = version.compareTo("md") >= 0; - hasOriginatingHostId = version.matches("(m[e-z])|(n[b-z])"); + hasAccurateMinMax = version.matches("(m[d-z])|(n[a-z])"); // deprecated in 'nc' and to be removed in 'oa' + hasLegacyMinMax = version.matches("(m[a-z])|(n[a-z])"); // deprecated in 'nc' and to be removed in 'oa' + hasOriginatingHostId = version.matches("(m[e-z])") || version.compareTo("nb") >= 0; hasMaxCompressedLength = version.compareTo("na") >= 0; hasPendingRepair = version.compareTo("na") >= 0; hasIsTransient = version.compareTo("na") >= 0; hasMetadataChecksum = version.compareTo("na") >= 0; hasOldBfFormat = version.compareTo("na") < 0; + hasImprovedMinMax = version.compareTo("nc") >= 0; + hasPartitionLevelDeletionPresenceMarker = version.compareTo("nc") >= 0; + hasKeyRange = version.compareTo("nc") >= 0; } @Override @@ -177,6 +186,12 @@ public class BigFormat implements SSTableFormat return isLatestVersion; } + @Override + public int correspondingMessagingVersion() + { + return correspondingMessagingVersion; + } + @Override public boolean hasCommitLogLowerBound() { @@ -189,6 +204,13 @@ public class BigFormat implements SSTableFormat return hasCommitLogIntervals; } + @Override + public boolean hasMaxCompressedLength() + { + return hasMaxCompressedLength; + } + + @Override public boolean hasPendingRepair() { return hasPendingRepair; @@ -200,24 +222,55 @@ public class BigFormat implements SSTableFormat return hasIsTransient; } - @Override - public int correspondingMessagingVersion() - { - return correspondingMessagingVersion; - } - @Override public boolean hasMetadataChecksum() { return hasMetadataChecksum; } + @Override + public boolean hasOldBfFormat() + { + return hasOldBfFormat; + } + @Override public boolean hasAccurateMinMax() { return hasAccurateMinMax; } + @Override + public boolean hasLegacyMinMax() + { + return hasLegacyMinMax; + } + + @Override + public boolean hasOriginatingHostId() + { + return hasOriginatingHostId; + } + + @Override + public boolean hasImprovedMinMax() + { + return hasImprovedMinMax; + } + + @Override + public boolean hasPartitionLevelDeletionsPresenceMarker() + { + return hasPartitionLevelDeletionPresenceMarker; + } + + @Override + public boolean hasKeyRange() + { + return hasKeyRange; + } + + @Override public boolean isCompatible() { return version.compareTo(earliest_supported_version) >= 0 && version.charAt(0) <= current_version.charAt(0); @@ -228,22 +281,5 @@ public class BigFormat implements SSTableFormat { return isCompatible() && version.charAt(0) == current_version.charAt(0); } - - public boolean hasOriginatingHostId() - { - return hasOriginatingHostId; - } - - @Override - public boolean hasMaxCompressedLength() - { - return hasMaxCompressedLength; - } - - @Override - public boolean hasOldBfFormat() - { - return hasOldBfFormat; - } } } diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java index 0adb9df227..6bb0f6eff1 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java @@ -304,7 +304,7 @@ public class BigTableWriter extends SSTableWriter @Override public RangeTombstoneMarker applyToMarker(RangeTombstoneMarker marker) { - collector.updateClusteringValues(marker.clustering()); + collector.updateClusteringValuesByBoundOrBoundary(marker.clustering()); if (marker.isBoundary()) { RangeTombstoneBoundaryMarker bm = (RangeTombstoneBoundaryMarker)marker; @@ -327,7 +327,7 @@ public class BigTableWriter extends SSTableWriter @Override public DeletionTime applyToDeletion(DeletionTime deletionTime) { - collector.update(deletionTime); + collector.updatePartitionDeletion(deletionTime); return deletionTime; } } @@ -662,4 +662,4 @@ public class BigTableWriter extends SSTableWriter return accumulate; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java index 7d4e26bd7d..6ab1551898 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java @@ -18,37 +18,42 @@ package org.apache.cassandra.io.sstable.metadata; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Collections; import java.util.EnumMap; -import java.util.List; import java.util.Map; import java.util.UUID; -import com.google.common.base.Preconditions; - import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus; import com.clearspring.analytics.stream.cardinality.ICardinality; -import org.apache.cassandra.db.*; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ClusteringBoundOrBoundary; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.DeletionTime; +import org.apache.cassandra.db.LivenessInfo; +import org.apache.cassandra.db.SerializationHeader; +import org.apache.cassandra.db.Slice; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; import org.apache.cassandra.db.partitions.PartitionStatisticsCollector; import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.Unfiltered; import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MurmurHash; import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.streamhist.TombstoneHistogram; import org.apache.cassandra.utils.streamhist.StreamingTombstoneHistogramBuilder; +import org.apache.cassandra.utils.streamhist.TombstoneHistogram; public class MetadataCollector implements PartitionStatisticsCollector { public static final double NO_COMPRESSION_RATIO = -1.0; - private static final ByteBuffer[] EMPTY_CLUSTERING = new ByteBuffer[0]; static EstimatedHistogram defaultCellPerPartitionCountHistogram() { @@ -82,15 +87,18 @@ public class MetadataCollector implements PartitionStatisticsCollector NO_COMPRESSION_RATIO, defaultTombstoneDropTimeHistogram(), 0, - Collections.emptyList(), - Collections.emptyList(), + Collections.emptyList(), + Slice.ALL, true, ActiveRepairService.UNREPAIRED_SSTABLE, -1, -1, null, null, - false); + false, + true, + ByteBufferUtil.EMPTY_BYTE_BUFFER, + ByteBufferUtil.EMPTY_BYTE_BUFFER); } protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram(); @@ -103,9 +111,25 @@ public class MetadataCollector implements PartitionStatisticsCollector protected double compressionRatio = NO_COMPRESSION_RATIO; protected StreamingTombstoneHistogramBuilder estimatedTombstoneDropTime = new StreamingTombstoneHistogramBuilder(SSTable.TOMBSTONE_HISTOGRAM_BIN_SIZE, SSTable.TOMBSTONE_HISTOGRAM_SPOOL_SIZE, SSTable.TOMBSTONE_HISTOGRAM_TTL_ROUND_SECONDS); protected int sstableLevel; - private ClusteringPrefix minClustering = null; - private ClusteringPrefix maxClustering = null; + + /** + * The smallest clustering prefix for any {@link Unfiltered} in the sstable. + * + *

This is always either a Clustering, or a start bound (since for any end range tombstone bound, there should + * be a corresponding start bound that is smaller). + */ + private ClusteringPrefix minClustering = ClusteringBound.MAX_START; + /** + * The largest clustering prefix for any {@link Unfiltered} in the sstable. + * + *

This is always either a Clustering, or an end bound (since for any start range tombstone bound, there should + * be a corresponding end bound that is bigger). + */ + private ClusteringPrefix maxClustering = ClusteringBound.MIN_END; + private boolean clusteringInitialized = false; + protected boolean hasLegacyCounterShards = false; + private boolean hasPartitionLevelDeletions = false; protected long totalColumnsSet; protected long totalRows; public int totalTombstones; @@ -201,6 +225,12 @@ public class MetadataCollector implements PartitionStatisticsCollector updateTombstoneCount(); } + public void updatePartitionDeletion(DeletionTime dt) + { + if (!dt.isLive()) + hasPartitionLevelDeletions = true; + update(dt); + } public void update(DeletionTime dt) { if (!dt.isLive()) @@ -251,11 +281,54 @@ public class MetadataCollector implements PartitionStatisticsCollector return this; } - public MetadataCollector updateClusteringValues(ClusteringPrefix clustering) + public void updateClusteringValues(Clustering clustering) { - minClustering = minClustering == null || comparator.compare(clustering, minClustering) < 0 ? clustering : minClustering; - maxClustering = maxClustering == null || comparator.compare(clustering, maxClustering) > 0 ? clustering : maxClustering; - return this; + if (clustering == Clustering.STATIC_CLUSTERING) + return; + + // In case of monotonically growing stream of clusterings, we will usually require only one comparison + // because if we detected X is greater than the current MAX, then it cannot be lower than the current MIN + // at the same time. The only case when we need to update MIN when the current MAX was detected to be updated + // is the case when MIN was not yet initialized and still point the ClusteringBound.MAX_START + if (comparator.compare(clustering, maxClustering) > 0) + { + maxClustering = clustering; + if (minClustering == ClusteringBound.MAX_START) + minClustering = clustering; + } + else if (comparator.compare(clustering, minClustering) < 0) + { + minClustering = clustering; + } + } + + public void updateClusteringValuesByBoundOrBoundary(ClusteringBoundOrBoundary clusteringBoundOrBoundary) + { + // In a SSTable, every opening marker will be closed, so the start of a range tombstone marker will never be + // the maxClustering (the corresponding close might though) and there is no point in doing the comparison + // (and vice-versa for the close). By the same reasoning, a boundary will never be either the min or max + // clustering, and we can save on comparisons. + if (clusteringBoundOrBoundary.isBoundary()) + return; + + // see the comment in updateClusteringValues(Clustering) + if (comparator.compare(clusteringBoundOrBoundary, maxClustering) > 0) + { + if (clusteringBoundOrBoundary.kind().isEnd()) + maxClustering = clusteringBoundOrBoundary; + + // note that since we excluded boundaries above, there is no way that the provided clustering prefix is + // a start and en end at the same time + else if (minClustering == ClusteringBound.MAX_START) + minClustering = clusteringBoundOrBoundary; + } + else if (comparator.compare(clusteringBoundOrBoundary, minClustering) < 0) + { + if (clusteringBoundOrBoundary.kind().isStart()) + minClustering = clusteringBoundOrBoundary; + else if (maxClustering == ClusteringBound.MIN_END) + maxClustering = clusteringBoundOrBoundary; + } } public void updateHasLegacyCounterShards(boolean hasLegacyCounterShards) @@ -263,12 +336,11 @@ public class MetadataCollector implements PartitionStatisticsCollector this.hasLegacyCounterShards = this.hasLegacyCounterShards || hasLegacyCounterShards; } - public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header) + public Map finalizeMetadata(String partitioner, double bloomFilterFPChance, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, ByteBuffer firstKey, ByteBuffer lastKey) { - Preconditions.checkState((minClustering == null && maxClustering == null) - || comparator.compare(maxClustering, minClustering) >= 0); - ByteBuffer[] minValues = minClustering != null ? minClustering.retainable().getBufferArray() : EMPTY_CLUSTERING; - ByteBuffer[] maxValues = maxClustering != null ? maxClustering.retainable().getBufferArray() : EMPTY_CLUSTERING; + assert minClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || minClustering.kind().isStart(); + assert maxClustering.kind() == ClusteringPrefix.Kind.CLUSTERING || maxClustering.kind().isEnd(); + Map components = new EnumMap<>(MetadataType.class); components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance)); components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize, @@ -283,15 +355,18 @@ public class MetadataCollector implements PartitionStatisticsCollector compressionRatio, estimatedTombstoneDropTime.build(), sstableLevel, - makeList(minValues), - makeList(maxValues), + comparator.subtypes(), + Slice.make(minClustering.retainable().asStartBound(), maxClustering.retainable().asEndBound()), hasLegacyCounterShards, repairedAt, totalColumnsSet, totalRows, originatingHostId, pendingRepair, - isTransient)); + isTransient, + hasPartitionLevelDeletions, + firstKey, + lastKey)); components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality)); components.put(MetadataType.HEADER, header.toComponent()); return components; @@ -305,18 +380,6 @@ public class MetadataCollector implements PartitionStatisticsCollector estimatedTombstoneDropTime.releaseBuffers(); } - private static List makeList(ByteBuffer[] values) - { - // In most case, l will be the same size than values, but it's possible for it to be smaller - List l = new ArrayList(values.length); - for (int i = 0; i < values.length; i++) - if (values[i] == null) - break; - else - l.add(values[i]); - return l; - } - public static class MinMaxLongTracker { private final long defaultMin; diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java index ee5505f6fb..ff34577c9a 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java @@ -19,28 +19,34 @@ package org.apache.cassandra.io.sstable.metadata; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import java.util.UUID; +import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.cassandra.db.rows.EncodingStats; +import org.apache.cassandra.db.BufferClusteringBound; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.Slice; import org.apache.cassandra.db.TypeSizes; import org.apache.cassandra.db.commitlog.CommitLogPosition; import org.apache.cassandra.db.commitlog.IntervalSet; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.rows.Cell; +import org.apache.cassandra.db.rows.EncodingStats; import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.util.DataInputPlus; import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.serializers.AbstractTypeSerializer; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.EstimatedHistogram; import org.apache.cassandra.utils.TimeUUID; -import org.apache.cassandra.utils.streamhist.TombstoneHistogram; import org.apache.cassandra.utils.UUIDSerializer; +import org.apache.cassandra.utils.streamhist.TombstoneHistogram; /** * SSTable metadata that always stay on heap. @@ -62,8 +68,7 @@ public class StatsMetadata extends MetadataComponent public final double compressionRatio; public final TombstoneHistogram estimatedTombstoneDropTime; public final int sstableLevel; - public final List minClusteringValues; - public final List maxClusteringValues; + public final Slice coveredClustering; public final boolean hasLegacyCounterShards; public final long repairedAt; public final long totalColumnsSet; @@ -74,6 +79,23 @@ public class StatsMetadata extends MetadataComponent // just holds the current encoding stats to avoid allocating - it is not serialized public final EncodingStats encodingStats; + // Used to serialize min/max clustering. Can be null if the metadata was deserialized from a legacy version + private final List> clusteringTypes; + + /** + * This boolean is used as an approximation of whether a given key can be guaranteed not to have partition + * deletions in this sstable. Obviously, this is pretty imprecise: a single partition deletion in the sstable + * means we have to assume _any_ key may have a partition deletion. This is still likely useful as workloads that + * does not use partition level deletions, or only very rarely, are probably not that rare. + * TODO we could replace this by a small bloom-filter instead; the only downside being that we'd have to care about + * the size of this bloom filters not getting out of hands, and it's a tiny bit unclear if it's worth the added + * complexity. + */ + public final boolean hasPartitionLevelDeletions; + + public final ByteBuffer firstKey; + public final ByteBuffer lastKey; + public StatsMetadata(EstimatedHistogram estimatedPartitionSize, EstimatedHistogram estimatedCellPerPartitionCount, IntervalSet commitLogIntervals, @@ -86,15 +108,18 @@ public class StatsMetadata extends MetadataComponent double compressionRatio, TombstoneHistogram estimatedTombstoneDropTime, int sstableLevel, - List minClusteringValues, - List maxClusteringValues, + List> clusteringTypes, + Slice coveredClustering, boolean hasLegacyCounterShards, long repairedAt, long totalColumnsSet, long totalRows, UUID originatingHostId, TimeUUID pendingRepair, - boolean isTransient) + boolean isTransient, + boolean hasPartitionLevelDeletions, + ByteBuffer firstKey, + ByteBuffer lastKey) { this.estimatedPartitionSize = estimatedPartitionSize; this.estimatedCellPerPartitionCount = estimatedCellPerPartitionCount; @@ -108,8 +133,8 @@ public class StatsMetadata extends MetadataComponent this.compressionRatio = compressionRatio; this.estimatedTombstoneDropTime = estimatedTombstoneDropTime; this.sstableLevel = sstableLevel; - this.minClusteringValues = minClusteringValues; - this.maxClusteringValues = maxClusteringValues; + this.clusteringTypes = clusteringTypes; + this.coveredClustering = coveredClustering; this.hasLegacyCounterShards = hasLegacyCounterShards; this.repairedAt = repairedAt; this.totalColumnsSet = totalColumnsSet; @@ -118,6 +143,9 @@ public class StatsMetadata extends MetadataComponent this.pendingRepair = pendingRepair; this.isTransient = isTransient; this.encodingStats = new EncodingStats(minTimestamp, minLocalDeletionTime, minTTL); + this.hasPartitionLevelDeletions = hasPartitionLevelDeletions; + this.firstKey = firstKey; + this.lastKey = lastKey; } public MetadataType getType() @@ -163,15 +191,18 @@ public class StatsMetadata extends MetadataComponent compressionRatio, estimatedTombstoneDropTime, newLevel, - minClusteringValues, - maxClusteringValues, + clusteringTypes, + coveredClustering, hasLegacyCounterShards, repairedAt, totalColumnsSet, totalRows, originatingHostId, pendingRepair, - isTransient); + isTransient, + hasPartitionLevelDeletions, + firstKey, + lastKey); } public StatsMetadata mutateRepairedMetadata(long newRepairedAt, TimeUUID newPendingRepair, boolean newIsTransient) @@ -188,15 +219,18 @@ public class StatsMetadata extends MetadataComponent compressionRatio, estimatedTombstoneDropTime, sstableLevel, - minClusteringValues, - maxClusteringValues, + clusteringTypes, + coveredClustering, hasLegacyCounterShards, newRepairedAt, totalColumnsSet, totalRows, originatingHostId, newPendingRepair, - newIsTransient); + newIsTransient, + hasPartitionLevelDeletions, + firstKey, + lastKey); } @Override @@ -220,13 +254,15 @@ public class StatsMetadata extends MetadataComponent .append(estimatedTombstoneDropTime, that.estimatedTombstoneDropTime) .append(sstableLevel, that.sstableLevel) .append(repairedAt, that.repairedAt) - .append(maxClusteringValues, that.maxClusteringValues) - .append(minClusteringValues, that.minClusteringValues) + .append(coveredClustering, that.coveredClustering) .append(hasLegacyCounterShards, that.hasLegacyCounterShards) .append(totalColumnsSet, that.totalColumnsSet) .append(totalRows, that.totalRows) .append(originatingHostId, that.originatingHostId) .append(pendingRepair, that.pendingRepair) + .append(hasPartitionLevelDeletions, that.hasPartitionLevelDeletions) + .append(firstKey, that.firstKey) + .append(lastKey, that.lastKey) .build(); } @@ -247,13 +283,15 @@ public class StatsMetadata extends MetadataComponent .append(estimatedTombstoneDropTime) .append(sstableLevel) .append(repairedAt) - .append(maxClusteringValues) - .append(minClusteringValues) + .append(coveredClustering) .append(hasLegacyCounterShards) .append(totalColumnsSet) .append(totalRows) .append(originatingHostId) .append(pendingRepair) + .append(hasPartitionLevelDeletions) + .append(firstKey) + .append(lastKey) .build(); } @@ -261,6 +299,8 @@ public class StatsMetadata extends MetadataComponent { private static final Logger logger = LoggerFactory.getLogger(StatsMetadataSerializer.class); + private final AbstractTypeSerializer typeSerializer = new AbstractTypeSerializer(); + public int serializedSize(Version version, StatsMetadata component) throws IOException { int size = 0; @@ -270,14 +310,19 @@ public class StatsMetadata extends MetadataComponent size += 8 + 8 + 4 + 4 + 4 + 4 + 8 + 8; // mix/max timestamp(long), min/maxLocalDeletionTime(int), min/max TTL, compressionRatio(double), repairedAt (long) size += TombstoneHistogram.serializer.serializedSize(component.estimatedTombstoneDropTime); size += TypeSizes.sizeof(component.sstableLevel); - // min column names - size += 4; - for (ByteBuffer value : component.minClusteringValues) - size += 2 + value.remaining(); // with short length - // max column names - size += 4; - for (ByteBuffer value : component.maxClusteringValues) - size += 2 + value.remaining(); // with short length + + if (version.hasLegacyMinMax()) + { + // min column names + size += 4; + ClusteringBound minClusteringValues = component.coveredClustering.start(); + size += minClusteringValues.size() * 2 /* short length */ + minClusteringValues.dataSize(); + // max column names + size += 4; + ClusteringBound maxClusteringValues = component.coveredClustering.end(); + size += maxClusteringValues.size() * 2 /* short length */ + maxClusteringValues.dataSize(); + } + size += TypeSizes.sizeof(component.hasLegacyCounterShards); size += 8 + 8; // totalColumnsSet, totalRows if (version.hasCommitLogLowerBound()) @@ -304,6 +349,25 @@ public class StatsMetadata extends MetadataComponent size += UUIDSerializer.serializer.serializedSize(component.originatingHostId, version.correspondingMessagingVersion()); } + if (version.hasPartitionLevelDeletionsPresenceMarker()) + { + size += TypeSizes.sizeof(component.hasPartitionLevelDeletions); + } + + if (version.hasImprovedMinMax()) + { + size += typeSerializer.serializedListSize(component.clusteringTypes); + size += Slice.serializer.serializedSize(component.coveredClustering, + version.correspondingMessagingVersion(), + component.clusteringTypes); + } + + if (version.hasKeyRange()) + { + size += ByteBufferUtil.serializedSizeWithVIntLength(component.firstKey); + size += ByteBufferUtil.serializedSizeWithVIntLength(component.lastKey); + } + return size; } @@ -322,12 +386,27 @@ public class StatsMetadata extends MetadataComponent TombstoneHistogram.serializer.serialize(component.estimatedTombstoneDropTime, out); out.writeInt(component.sstableLevel); out.writeLong(component.repairedAt); - out.writeInt(component.minClusteringValues.size()); - for (ByteBuffer value : component.minClusteringValues) - ByteBufferUtil.writeWithShortLength(value, out); - out.writeInt(component.maxClusteringValues.size()); - for (ByteBuffer value : component.maxClusteringValues) - ByteBufferUtil.writeWithShortLength(value, out); + + if (version.hasLegacyMinMax()) + { + ClusteringBound minClusteringValues = component.coveredClustering.start(); + out.writeInt(countUntilNull(minClusteringValues.getBufferArray())); + for (ByteBuffer value : minClusteringValues.getBufferArray()) + { + if (value == null) + break; + ByteBufferUtil.writeWithShortLength(value, out); + } + ClusteringBound maxClusteringValues = component.coveredClustering.end(); + out.writeInt(countUntilNull(maxClusteringValues.getBufferArray())); + for (ByteBuffer value : maxClusteringValues.getBufferArray()) + { + if (value == null) + break; + ByteBufferUtil.writeWithShortLength(value, out); + } + } + out.writeBoolean(component.hasLegacyCounterShards); out.writeLong(component.totalColumnsSet); @@ -368,6 +447,27 @@ public class StatsMetadata extends MetadataComponent out.writeByte(0); } } + + if (version.hasPartitionLevelDeletionsPresenceMarker()) + { + out.writeBoolean(component.hasPartitionLevelDeletions); + } + + if (version.hasImprovedMinMax()) + { + assert component.clusteringTypes != null; + typeSerializer.serializeList(component.clusteringTypes, out); + Slice.serializer.serialize(component.coveredClustering, + out, + version.correspondingMessagingVersion(), + component.clusteringTypes); + } + + if (version.hasKeyRange()) + { + ByteBufferUtil.writeWithVIntLength(component.firstKey, out); + ByteBufferUtil.writeWithVIntLength(component.lastKey, out); + } } public StatsMetadata deserialize(Version version, DataInputPlus in) throws IOException @@ -407,24 +507,25 @@ public class StatsMetadata extends MetadataComponent int sstableLevel = in.readInt(); long repairedAt = in.readLong(); - // for legacy sstables, we skip deserializing the min and max clustering value - // to prevent erroneously excluding sstables from reads (see CASSANDRA-14861) - int colCount = in.readInt(); - List minClusteringValues = new ArrayList<>(colCount); - for (int i = 0; i < colCount; i++) + List> clusteringTypes = null; + Slice coveredClustering = Slice.ALL; + if (version.hasLegacyMinMax()) { - ByteBuffer val = ByteBufferUtil.readWithShortLength(in); - if (version.hasAccurateMinMax()) - minClusteringValues.add(val); - } + // We always deserialize the min/max clustering values if they are there, but we ignore them for + // legacy sstables where !hasAccurateMinMax due to CASSANDRA-14861. + int colCount = in.readInt(); + ByteBuffer[] minClusteringValues = new ByteBuffer[colCount]; + for (int i = 0; i < colCount; i++) + minClusteringValues[i] = ByteBufferUtil.readWithShortLength(in); + + colCount = in.readInt(); + ByteBuffer[] maxClusteringValues = new ByteBuffer[colCount]; + for (int i = 0; i < colCount; i++) + maxClusteringValues[i] = ByteBufferUtil.readWithShortLength(in); - colCount = in.readInt(); - List maxClusteringValues = new ArrayList<>(colCount); - for (int i = 0; i < colCount; i++) - { - ByteBuffer val = ByteBufferUtil.readWithShortLength(in); if (version.hasAccurateMinMax()) - maxClusteringValues.add(val); + coveredClustering = Slice.make(BufferClusteringBound.inclusiveStartOf(minClusteringValues), + BufferClusteringBound.inclusiveEndOf(maxClusteringValues)); } boolean hasLegacyCounterShards = in.readBoolean(); @@ -438,7 +539,7 @@ public class StatsMetadata extends MetadataComponent if (version.hasCommitLogIntervals()) commitLogIntervals = commitLogPositionSetSerializer.deserialize(in); else - commitLogIntervals = new IntervalSet(commitLogLowerBound, commitLogUpperBound); + commitLogIntervals = new IntervalSet<>(commitLogLowerBound, commitLogUpperBound); TimeUUID pendingRepair = null; if (version.hasPendingRepair() && in.readByte() != 0) @@ -452,6 +553,28 @@ public class StatsMetadata extends MetadataComponent if (version.hasOriginatingHostId() && in.readByte() != 0) originatingHostId = UUIDSerializer.serializer.deserialize(in, 0); + // If not recorded, the only time we can guarantee there is no partition level deletion is if there is no + // deletion at all. Otherwise, we have to assume there may be some. + boolean hasPartitionLevelDeletions = minLocalDeletionTime != Cell.NO_DELETION_TIME; + if (version.hasPartitionLevelDeletionsPresenceMarker()) + { + hasPartitionLevelDeletions = in.readBoolean(); + } + + if (version.hasImprovedMinMax()) + { + clusteringTypes = typeSerializer.deserializeList(in); + coveredClustering = Slice.serializer.deserialize(in, version.correspondingMessagingVersion(), clusteringTypes); + } + + ByteBuffer firstKey = null; + ByteBuffer lastKey = null; + if (version.hasKeyRange()) + { + firstKey = ByteBufferUtil.readWithVIntLength(in); + lastKey = ByteBufferUtil.readWithVIntLength(in); + } + return new StatsMetadata(partitionSizes, columnCounts, commitLogIntervals, @@ -464,15 +587,25 @@ public class StatsMetadata extends MetadataComponent compressionRatio, tombstoneHistogram, sstableLevel, - minClusteringValues, - maxClusteringValues, + clusteringTypes, + coveredClustering, hasLegacyCounterShards, repairedAt, totalColumnsSet, totalRows, originatingHostId, pendingRepair, - isTransient); + isTransient, + hasPartitionLevelDeletions, + firstKey, + lastKey); } + + private int countUntilNull(ByteBuffer[] bufferArray) + { + int i = ArrayUtils.indexOf(bufferArray, null); + return i < 0 ? bufferArray.length : i; + } + } } diff --git a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java index bba4cd3213..8b077e745a 100644 --- a/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/KeyspaceMetrics.java @@ -82,8 +82,10 @@ public class KeyspaceMetrics public final LatencyMetrics rangeLatency; /** (Local) write metrics */ public final LatencyMetrics writeLatency; - /** Histogram of the number of sstable data files accessed per read */ + /** Histogram of the number of sstable data files accessed per single partition read */ public final Histogram sstablesPerReadHistogram; + /** Histogram of the number of sstable data files accessed per partition range read */ + public final Histogram sstablesPerRangeReadHistogram; /** Tombstones scanned in queries on this Keyspace */ public final Histogram tombstoneScannedHistogram; /** Live cells scanned in queries on this Keyspace */ @@ -231,6 +233,7 @@ public class KeyspaceMetrics // create histograms for TableMetrics to replicate updates to sstablesPerReadHistogram = createKeyspaceHistogram("SSTablesPerReadHistogram", true); + sstablesPerRangeReadHistogram = createKeyspaceHistogram("SSTablesPerRangeReadHistogram", true); tombstoneScannedHistogram = createKeyspaceHistogram("TombstoneScannedHistogram", false); liveScannedHistogram = createKeyspaceHistogram("LiveScannedHistogram", false); colUpdateTimeDeltaHistogram = createKeyspaceHistogram("ColUpdateTimeDeltaHistogram", false); @@ -398,4 +401,4 @@ public class KeyspaceMetrics return new MetricName(groupName, "keyspace", metricName, keyspaceName, mbeanName.toString()); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/metrics/TableMetrics.java b/src/java/org/apache/cassandra/metrics/TableMetrics.java index 04102f7995..5dd8e13702 100644 --- a/src/java/org/apache/cassandra/metrics/TableMetrics.java +++ b/src/java/org/apache/cassandra/metrics/TableMetrics.java @@ -100,8 +100,10 @@ public class TableMetrics public final Gauge estimatedPartitionCount; /** Histogram of estimated number of columns. */ public final Gauge estimatedColumnCountHistogram; - /** Histogram of the number of sstable data files accessed per read */ + /** Histogram of the number of sstable data files accessed per single partition read */ public final TableHistogram sstablesPerReadHistogram; + /** Histogram of the number of sstable data files accessed per partition range read */ + public final TableHistogram sstablesPerRangeReadHistogram; /** (Local) read metrics */ public final LatencyMetrics readLatency; /** (Local) range slice metrics */ @@ -548,6 +550,7 @@ public class TableMetrics SSTableReader::getEstimatedCellPerPartitionCount), null); sstablesPerReadHistogram = createTableHistogram("SSTablesPerReadHistogram", cfs.keyspace.metric.sstablesPerReadHistogram, true); + sstablesPerRangeReadHistogram = createTableHistogram("SSTablesPerRangeReadHistogram", cfs.keyspace.metric.sstablesPerRangeReadHistogram, true); compressionRatio = createTableGauge("CompressionRatio", new Gauge() { public Double getValue() @@ -997,6 +1000,11 @@ public class TableMetrics sstablesPerReadHistogram.update(count); } + public void updateSSTableIteratedInRangeRead(int count) + { + sstablesPerRangeReadHistogram.update(count); + } + /** * Release all associated metrics. */ @@ -1425,4 +1433,4 @@ public class TableMetrics return total; } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java b/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java new file mode 100644 index 0000000000..1be4d61f68 --- /dev/null +++ b/src/java/org/apache/cassandra/serializers/AbstractTypeSerializer.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.serializers; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.apache.cassandra.db.TypeSizes; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.TypeParser; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.io.util.DataInputPlus; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.utils.ByteBufferUtil; + +public class AbstractTypeSerializer +{ + public void serialize(AbstractType type, DataOutputPlus out) throws IOException + { + ByteBufferUtil.writeWithVIntLength(UTF8Type.instance.decompose(type.toString()), out); + } + + public void serializeList(List> types, DataOutputPlus out) throws IOException + { + out.writeUnsignedVInt32(types.size()); + for (AbstractType type : types) + serialize(type, out); + } + + public AbstractType deserialize(DataInputPlus in) throws IOException + { + ByteBuffer raw = ByteBufferUtil.readWithVIntLength(in); + return TypeParser.parse(UTF8Type.instance.compose(raw)); + } + + public List> deserializeList(DataInputPlus in) throws IOException + { + int size = (int) in.readUnsignedVInt(); + List> types = new ArrayList<>(size); + for (int i = 0; i < size; i++) + types.add(deserialize(in)); + return types; + } + + public long serializedSize(AbstractType type) + { + return ByteBufferUtil.serializedSizeWithVIntLength(UTF8Type.instance.decompose(type.toString())); + } + + public long serializedListSize(List> types) + { + long size = TypeSizes.sizeofUnsignedVInt(types.size()); + for (AbstractType type : types) + size += serializedSize(type); + return size; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java index b7164e8d7e..48f63e3897 100644 --- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java +++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java @@ -17,20 +17,12 @@ */ package org.apache.cassandra.tools; -import static org.apache.cassandra.tools.Util.BLUE; -import static org.apache.cassandra.tools.Util.CYAN; -import static org.apache.cassandra.tools.Util.RESET; -import static org.apache.cassandra.tools.Util.WHITE; -import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; -import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationWords; - import java.io.DataInputStream; import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.nio.ByteBuffer; import java.nio.file.Files; -import java.util.Arrays; import java.util.Comparator; import java.util.EnumSet; import java.util.List; @@ -38,7 +30,17 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import com.google.common.collect.MinMaxPriorityQueue; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; + import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.ClusteringComparator; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.marshal.AbstractType; @@ -65,15 +67,13 @@ import org.apache.cassandra.schema.TableMetadataRef; import org.apache.cassandra.tools.Util.TermHistogram; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; -import com.google.common.collect.MinMaxPriorityQueue; +import static org.apache.cassandra.tools.Util.BLUE; +import static org.apache.cassandra.tools.Util.CYAN; +import static org.apache.cassandra.tools.Util.RESET; +import static org.apache.cassandra.tools.Util.WHITE; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; +import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationWords; /** * Shows the contents of sstable metadata @@ -354,22 +354,12 @@ public class SSTableMetadataViewer field("TTL max", stats.maxTTL, toDurationString(stats.maxTTL, TimeUnit.SECONDS)); if (validation != null && header != null) - printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType()); + printMinMaxToken(descriptor, FBUtilities.newPartitioner(descriptor), header.getKeyType(), stats); - if (header != null && header.getClusteringTypes().size() == stats.minClusteringValues.size()) + if (header != null) { - List> clusteringTypes = header.getClusteringTypes(); - List minClusteringValues = stats.minClusteringValues; - List maxClusteringValues = stats.maxClusteringValues; - String[] minValues = new String[clusteringTypes.size()]; - String[] maxValues = new String[clusteringTypes.size()]; - for (int i = 0; i < clusteringTypes.size(); i++) - { - minValues[i] = clusteringTypes.get(i).getString(minClusteringValues.get(i)); - maxValues[i] = clusteringTypes.get(i).getString(maxClusteringValues.get(i)); - } - field("minClusteringValues", Arrays.toString(minValues)); - field("maxClusteringValues", Arrays.toString(maxValues)); + ClusteringComparator comparator = new ClusteringComparator(header.getClusteringTypes()); + field("Covered clusterings", stats.coveredClustering.toString(comparator)); } field("Estimated droppable tombstones", stats.getEstimatedDroppableTombstoneRatio((int) (currentTimeMillis() / 1000) - this.gc)); @@ -474,19 +464,30 @@ public class SSTableMetadataViewer } } - private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType keyType) + private void printMinMaxToken(Descriptor descriptor, IPartitioner partitioner, AbstractType keyType, StatsMetadata statsMetadata) throws IOException { - File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY)); - if (!summariesFile.exists()) - return; - - try (DataInputStream iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath()))) + if (descriptor.version.hasKeyRange()) { - Pair firstLast = new IndexSummary.IndexSummarySerializer() - .deserializeFirstLastKey(iStream, partitioner); - field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey())); - field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey())); + if (statsMetadata.firstKey == null || statsMetadata.lastKey == null) + return; + + field("First token", partitioner.getToken(statsMetadata.firstKey), keyType.getString(statsMetadata.firstKey)); + field("Last token", partitioner.getToken(statsMetadata.lastKey), keyType.getString(statsMetadata.lastKey)); + } + else + { + File summariesFile = new File(descriptor.filenameFor(Component.SUMMARY)); + if (!summariesFile.exists()) + return; + + try (DataInputStream iStream = new DataInputStream(Files.newInputStream(summariesFile.toPath()))) + { + Pair firstLast = new IndexSummary.IndexSummarySerializer() + .deserializeFirstLastKey(iStream, partitioner); + field("First token", firstLast.left.getToken(), keyType.getString(firstLast.left.getKey())); + field("Last token", firstLast.right.getToken(), keyType.getString(firstLast.right.getKey())); + } } } diff --git a/src/java/org/apache/cassandra/utils/Comparables.java b/src/java/org/apache/cassandra/utils/Comparables.java new file mode 100644 index 0000000000..612ade8bbe --- /dev/null +++ b/src/java/org/apache/cassandra/utils/Comparables.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.utils; + +import java.util.Comparator; + +/** + * Utility methods linked to comparing comparable values. + */ +public class Comparables +{ + /** + * Returns the maximum of 2 comparable values. On ties, returns the first argument. + */ + public static > T max(T a, T b) + { + return a.compareTo(b) < 0 ? b : a; + } + + /** + * Returns the maximum of 2 values given a comparator of those values. On ties, returns the first argument. + */ + public static T max(T a, T b, Comparator comparator) + { + return comparator.compare(a, b) < 0 ? b : a; + } + + /** + * Returns the minimum of 2 comparable values. On ties, returns the first argument. + */ + public static > T min(T a, T b) + { + return a.compareTo(b) > 0 ? b : a; + } + + /** + * Returns the minimum of 2 values given a comparator of those values. On ties, returns the first argument. + */ + public static T min(T a, T b, Comparator comparator) + { + return comparator.compare(a, b) > 0 ? b : a; + } +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/MergeIterator.java b/src/java/org/apache/cassandra/utils/MergeIterator.java index 6713dd0a43..27238968be 100644 --- a/src/java/org/apache/cassandra/utils/MergeIterator.java +++ b/src/java/org/apache/cassandra/utils/MergeIterator.java @@ -395,10 +395,11 @@ public abstract class MergeIterator extends AbstractIterator implem private boolean isLowerBound() { + assert item != null; return item == lowerBound; } - public void consume(Reducer reducer) + public void consume(Reducer reducer) { if (isLowerBound()) { @@ -488,4 +489,4 @@ public abstract class MergeIterator extends AbstractIterator implem return (Out) source.next(); } } -} +} \ No newline at end of file diff --git a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java index be4cec0eae..83bb828f30 100644 --- a/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java +++ b/src/java/org/apache/cassandra/utils/bytecomparable/ByteSource.java @@ -75,6 +75,10 @@ public interface ByteSource int LT_NEXT_COMPONENT = 0x20; int GT_NEXT_COMPONENT = 0x60; + // Unsupported, for artificial bounds + int LTLT_NEXT_COMPONENT = 0x1F; // LT_NEXT_COMPONENT - 1 + int GTGT_NEXT_COMPONENT = 0x61; // GT_NEXT_COMPONENT + 1 + // Special value for components that should be excluded from the normal min/max span. (static rows) int EXCLUDED = 0x18; @@ -850,4 +854,4 @@ public interface ByteSource ? (Peekable) p : new Peekable(p); } -} +} \ No newline at end of file diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-CompressionInfo.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-CompressionInfo.db new file mode 100644 index 0000000000000000000000000000000000000000..2abf0a8898f0ddb1eda4d572f2bd3ea8e5668cff GIT binary patch literal 207 zcmZSJ^@%cZ&d)6>qegoml_CjcR JQwXha8vw|A7D503 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..8db5dadcd0f1535fc14c32802594b18d90cc5893 GIT binary patch literal 8713 zcmd5?Yj9NM89tk2b5r=Z1u)p2NK=|b%kG{%dkMwtVl9n`y0%_WN+E-zZH(C*BMawXV4g=8;`+R42&pB)V>5u)P zGkNlSxA*zp_j$h)1g{{h6-?F+MNuMxNl=7Rp>x1$wMll_;dHqx3kCX(vR?UZWEB(g zBay|m%j%k%Yl02UVgEhNZHq!Ri<XTu&32yk^VoXTg;> z2uUu$c~LMOMrO^kU^<2iBXfB*m~JDp#Y!L>sKoAjaz7}*I^l_4b4ZkSS0432 z1M4I8|NenMO1+qp#mY~xS&8ry{5C_B8IXRKL&w;{zRWwxRd{Fzso)uuY!8}FvM+Fz zwjkzJ(a0&Bx`S)BN!ZOSod*#x`~3xc{7cyw*B?MrO|(0$Hb>mhNL;?H{-cB_&3R2f}1aX=|o zj`?_cj9GSII#JFK zcLA_{HfO&|zeFVaPXLrPF2HSD0XmG#mwNy@jm%df09{7ro_c_8BXh_{Ae-@HMR}TC zGAn1W&YNxsAzN2xDQDOv+G#J){gksL#GX>pOxZM_V!b}d6JFj;nKbO<#q06EglBSb zDpTrE+B~7$$FJbq`57XPELCSw&g`JGom5qL_bh^A?;;gWvJ3P}9Qp7OfU?E~cvP)O zM&_|+!E|a&it+?Q`8t_Kg^%CmQhA;{YlFoc5uR*}^UEpL0o){$jNi3V%{oP_|UPv^6sX8E~6=*z+h> zp9xiF%dVa(bMR@CV6t@(1e2W4J@OfppwC!t&Uz|!EP775*&M`g44xA!x=XdE_T2S} zIaP^2yTZf$$;(}5^DUm(6<&xT9YzJ@(+ZJ7PwabCap)>cGbM*Zi>r9@%8#aY0D;GV z=r9sXF9LLGM2s#w0nlY+Ub7dV+rSj9mwBf6xOxA|_xc6srp){fZQPKSbm|_(0XvIX z2fx)t@S0r~$!fG-`7x4V%7FA(%FhLl*%ST3L@L0aG&bb%#N78$I9P=H(-u!`dOiI( z-hMRIIS5haiFr}3L|Ge)=w46ts%5-GGkIt;4MeArSbY_UE{%xMv-81p8=1GD9OP+4 z%yyrd!U?rQi2d@tZxzz-O!NW!eYmv4tU14*EAaKdxB_a%=)ifgbhH~GFYMVaNxg8*f1BvMb=0nnj6XplU+fH^fHZN(GM z?gi@7m>8bd1<;+M4yzBK&b9sTLm8zT)*Zhrm?2t7sqA+1r?S$6Loh1Nc-3kCiXhxg z@t-naH}(dx*KSQXlSbwO#K+E{NQ{>>KZY~KCzKJIa;EqMwIsHtzDgirCZQzidjst0 z`xg+{wB}&)aWHkw0roizVp?x7WiNrc-T*UJP?Zdc@}A1O)$AFec;snpR?McSJf;LfSzGDs1? zb{!p&TS*`jiZJ&UPu%GT$C1yqDQ7ek;!N`ab?Qb^EXAsb*ebeot6;im_y*xNGA-9# zg+=mYEK#+Tc+JAqg+XCOp=is(tEiO8*-8VqsQ3;4NwcVU-3`!Bnw_WZh*b4k zf!beJcaQN9Py#4luI&I?HC_SKp`!v_P9>49MS^=+Z6Z!gI&u$}&Co@Ls^PqXV=sJ=i)T&+!@?o;b*oKzXN(NtbyG&LQ!NkkshQNP=q*4|u6h-9{y2cCCnZ{j*3M{%f_YNO#?$&jm{gms(Kh|-G+@L%mQ_rXjkx6nN zG$~V=7kRV*aUVk`VWaXCi`qHVSstgp8O9}`+$~_iY-K8H)<&`kYT)c|Yr~<$tFBBw8;21iNw*xP+4L6HQ(skmmcFgv$X=wJwp1VW*`$PiBq`C)tFTkI4pe)Q7h#u< zi$kr%1nv~|BeeonUf1`)H!9$d62`R6pCJm_4+s@(I!^&!*I)KQ580Z-FJ>JJ<&@)y zzu~hkg=pa4 z&H{C&4Lk?XrC)<2+IIw~JB@mjK=#nI3YKq5PFxN_{}D~t*aRPcGhJD|?L#ga|MLWg z)Go`P5bpNpGDEll=LoJ)fIUt!>B(KO3kw^2wu(N&R zC=rw(^O#F5H4746CU+3Vz3lQL(fW=y8TYbxXBDS=*~e1S%VzV!7hqWAty1xDw+L?P zLty;$#EwGxC}oqfa#LGGmYyz#WcHDX17SEx^N9ddj%z!>WugD*dJuJ_+5)mK?(}iN z^baUH!SX~Y{4`@6%Q+4uF2p*sm13OaG*04r)ua@&Yfv9U0M7t)>zIhdf8QaL9U4od z7SzoW@t>-F=>{W?7k8@mHC9^)Vjrd}z`q6R=<1ER_zEUgKL++$zV5~1_S;Y%)otQq z;E)L(C(63P*l0IEhi;<)9|q`5Fca^n^Yia`lnca7D)pz=fU3`sd3k2!)1c%UMXgkT zMW}bcEfbq@7m8XOvZHCVDhpEi;kBkhau0uvp&u{gt6fKJ^)IK#w? zKvT#4gO9i9<5td7xK8}f862X@*O-uL;+GjBlifW&i*D!r$e3cB=E@Uv8g@sXni>iv z$A_we?Luc<%qp^#5R#*WS^?@P0#fx1)FkA3{zKljANZL3S@59aDW zVn9w63QJUP8nQNIh)D7nQ;@CX9>hzzOicyF-tiiL(~4|rPJqY$7@1B;bdEL!l5Dju v9b3qqOh-o@8?NS7renk3X2HpBngkaky_HhS$5xMdsSp38!c_D^$&CL2<_ggw literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32 new file mode 100644 index 0000000000..a3554fabfc --- /dev/null +++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Digest.crc32 @@ -0,0 +1 @@ +516351458 \ No newline at end of file diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-QeG?9VvSl?u?nlO3ah5gs!pkUv&FN6SH^X%f7KC+p%{uyn_|tl#PPNyY|Gf1 zirJof@Y=ZMHWjf8yRa*UUDdeel6llZE!2vk*5ldY`DV+w<~x!t$(CeGvNi2wt5d2O zmt>xCF)qdxV_d_{^W(PS{CbC(Bl1PQ$k(*xt18!yXr6LWF3OdoT(?H$+81(R7j|J+ z47-|fp%!YP7HY*%Yq))W+*jP$6>W*OL|dY*X-8X~Qcbx;^OTEnQLY^2y1O;*EAH%d zm^mU}nnx|vLai8T4Ntprd7vP8ffsniz^m>n`Xm=%VOTa?_C-{g1@bN1G}&b zyJFbYjLZHcYM~Zt#Zc?PYvWRjEuI(8i|2Eaw`yE!u~7@PP%DR8k1s8rZ>YtNG31;( z%bBjWXXg4Hcm0}iN#?_(=w6B=6ulTepmj?=h7kGhJ47`@gr54+qXihXI=FEx1i`ik- zk6LW(!Y=H}VOJF{Q=%!+lxRwfnG%P;&n;eV{Ks1C`LCA_)ME2Ap2pL1Jng76t^1Q| zv5^ePknAWvO!I4Mu}zz%P19z~wAnG*gi9?pe&H8>#qdilwh_^YXhiHDlXKLJ%l;(e zVqAa&_ z)I6xgrd*VZa&?i(IcmzK7MpTWF3OdoTxzi?7v-W{Iil6ibg5iwvEd88;HyJn>PKp^ z4M+y0m;q_(I62+(t09+KY|2HsDA(zdXO*9(7MpTWF3J`8WUHoJYO(DrVmOA!Fnmyp zZAvsHni5x;+y1ZrTlW=xl8bRMF2v}Rm$wb(K)8JCPJCgW0zZMj;mG0U~Mbu#6qkb zV)Zj^#-$cp$Yp9aHOEZNYOyU>%Qa@XcHMAv7;<@-D&@j(43A;>pcY%oCFPQGB`Vx9 z>I==FVhj^*?y;^Kx znXpV)mKl~k+*XT?U-*SzG5k`CovO;lJY(K^-@7g1vOmeV7#HJ;F)p>(>;109+jv{* z3aG}V7CZITLmuQ=$9q>T_By`mXoq(B_kA-iwb(K)8JCPJCgW0zZMj;mG0T-Izp2Vq zxzu9ktFycM&AZywINMi4EjD6#{IkbDCvv_Sms)Hgm#Nv*95Xem#kO26*O=v+C`NDZ zoSSlam@4JMa14)OxLRz>)pCtlu5V9Y62J3Oxzu6{>4o${`W$?z#b#WLi*ZFhW}${$ zYOyI7<)U0c%Jpcrcy>@4ygW}(LQg;mXpDe1YN^FWE!0A-7;34-wp=aOnB|%?%bmuV z)_p~v$^fN_P?ys B@eu$3 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust/nc-1-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..ef1355cb6e1d7bb5066bf18a7c4aaa0c789f11db GIT binary patch literal 9718 zcmeI%Yfw~G902fp?!CLbH&{@4j8q7s3@pS$U{Xn31G0j6B&AJU*o{XD%(A0MIy8oT z*cu{7LK`1N4dZ~p2&F(o9~3CHK!&26U}$FB8k@uniC55FVdein^?~V=Oy|y=x%bEa z?45J&J&*sg9LI@k*ZU~A`6F095xL~cg_p~lauGN#Ah+imjVBjcWXRP>#Zpe5UXvr0 zlt_g-gQU^r88z|yjrwGRPH)ibQY3>h-L~{=_2o;m!*Z1qaB$j2vR zkXtX|=QGrbo+oj1y#wo$xCHAyaPJ`HerN7Mz93gFhC%az7ts33zoQy}alyM3jSr#i z1qT=xOC2^tD|QW{{wj;|OJUru)jSUEpxrhG?d02z=95dNx(~)(g`{q1_vipLo>!?R zkHNT4!`nBYAC63Hh7SC0$42PTzPv{0*sO!6ptaY-c0i||-V+6#6L9`9=)F@e7U;^Z z`UL2PwBvrzU)5B+1wH;r|2AN<&Lz7II%n%vIR8m~$_3l}w^k!_M^$kC^Lq2}r{MGa z$l1+hz=DyOj{uAQz3L?BiHQU7{V4vds)qZA-FkP+7Es^5=3Lpwz>dl9cN>76U#z?d z`>*=m?FQ_>+qJ|NHK@NrG8M!9!0Sw0y9UI4zqF{~`|`h?kYob!;Ed4#m=8on{MxMo z@wmaC+xG!Kd7|koY|qZo*eE!j3sbdqRiJ)Fc5BZF@Tnh$I_iKsYd>g$?KNKv>W^NG z6ZLCXXX1gA3hqoF0)FGEz>r?x>$|E8f)~xlK58k?18(qW3_rMNA)>jUc4pC>t*>}Z z|7T9egU?tm|1UY*{2t12;Z*<8njyzn4u>8P%i+-D;Qn(CH-8(J!?7F=3@DbvE$1AL zyuLFt#KJk%{=q562KuUQCEh(B^*>fF|Ifu*Y_B@*-R;vKbrclaOr42%E-dVk@QBE$ z4bd^N?20`DHV)WaxWCr}c?px3Ew)&(1)FVJY~x-w$%`%Xnp(J!&kJcB*A-%2)38B` z$C2QC&34nir@KBh6Y(X_zb7wd_9;|iAKv9yMz zHTQQ~!@hjMFJH>L%N9%WSdzz*JeK4wnnDWfnGT`(Bp@{ZLI^FW2%&}cKxh$X2rarELW^rbXo-CgTFM+kOV5VT GvI+oAk`#FW literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..c212d3df38a5d052d5c1c934fa727db82e56dfe3 GIT binary patch literal 7529 zcmaJ`eQ;FO6@M>zN%p~zn~;ViXn4sI-Q7Uh?+>VwK&>T!nrNj^Yaw;8q#X)aY@wwj zV#Y40G#^x>_>u4xELzwg1*J-|8x*KRV}@Y@LvgdEG%(gCQCpBQufKEOe!N|4_#+eU zIp>~pf9IaxIqycO6oeK*wAHJs+8~I6DwGI~0h`V4aJt+cuW#l!0TT83(TQdz2xWrU zAWW~Dv0%$D?mfQik|;e|c=MEy@!6Tarq65_qlBf^tc!cx}MrvH8qhap$I$u=u%NwOUBNxo#8HB}_3TNaU6r<)Gkmbp=;03NqPU|u;7 zn>pU(l3h!2mi0vuDPIYUOslQtpbvf9r3%Q*{?t4uqxl5e(?`b0ggS&g*7Hm_>`OT^ z3Y#IV0&IZw*9ara(Y(esD5di#gjH-4&#|K6+Hi$pf4mSv=dy$~Mc?=#M0x9B*()oq zHxrf+Dt>>Sp}cv6KTmO|m_5_pD;uUZ^c|PeAob|%(qOlP+%+D-Z{8HUrp+Ysw?pL9 z8BrwL)u)>4 zlA$PlezD@z($Q&cw~%IAWl*+uVex7;uY`sb9fLe0D2ZrN_-*{%=p=p{JFk6pMw(wT z$1~z|UQCDcm)Ps+BNX$#FuWPtbF^%@JmLtGL{cE({*w?|c3wl!?qsur0g`mWSn(v= z9Dac$`wu|kO}07u1W69ILE=lcIaWt#mY2fs%WO*)4=hvzZS&yz)yUu2Sme5jUK z68cAI>HQKc{U%TSSv3cJ|9_wQnCBo$O~?b!!GvfI&Yl}12Y)uibKr8{PdUh8CF$x$ z!ufm;yctgqYf10~MRg- zDwjQ$sd>9v3yJ9>OqD=gAw*I5UabUt=7PmxiTxY`J2us#KglTmu zWbBc|YS&3xEr*QNCRX2W!)n=#6Je=U6hwPHJT~MCOF7ZoC&$mas+`qlF?oyZ2Pgr%!=Qd%&1+%p`G>WAZMLuJy_+ zY*E#%?57zBo+P#d*vedn;KDS zt*#dV&diqS*p2m~t!}L5`2VCu{3oxH7*Bb9l#Mhk8$G@!zB(J&E1nI9^IpI_p52O{ zPg2x(fFuk?TU(T=-lKK4jdVy_NpxXxH6U(xis4;868Tm@9R(pzF`=e?<#>}691E@QHV_Q4EQZd|A6)85QlZN za1ey$w9nzANvu`k_C0V80FMjF5IXejWOugpyFC=+f?*_gPqvn&o`?3*+I<5Ej;AP7 ztB(GaFKhWos*Ybq)3LN^jC4HE9Z!eTt>YIuPz#(Hzq>a|S$bZ=@gD*C09c&~q4%R& z3h<#O0yf3qCsia`x(>0=&sXai#GC>-J#O(sXx?}PTexf~dv09=nQ2oyWC2%=j#hD1 z#XjIWwfWy+DQ!Wq)n(V2YUSNRF%KtY>{JPQVa_FN%<9TilMYUx$j_cGJ(^+0L zlW6N=1nkeF1aw+o$)Tl|${7B%G?A8`gRYj6XSEXf^=-YuDc@q81qDEn#nWxO>gO^P;)T}yFe<&HsQ>SiBi=|?&L z^VJEPX_FT5$Y896@a5dqP%g`8ALMPpQIB}zZDCVxJPc|JJ%JafE$lvlwoqhBX$yTQ z6NBlvf#SE9WyNC@y+-SyC*Z^M+5sXidxTGu-1nJoO3Voqtm5c`2|x0WtNCQ*&DlPg zXNQl%d}*ApHkzG*y<=qOgMoOx*q!eR>)A7Wu+1lwFEmkl`nsd^wTQ=(&&s6r(P}DC z+$knuRFT1QW-sc+qtmC|F4dA~0DCIl6sw_mB)Wi7QhYiq@-m;stZ`F*$>xjE*!;BF z$3Vtl4ll!vci2wRz45cQbTyM?!$pvpE@4k=z)>ZtnXHg*oS!boTPq0MH^9ghT@Jg> zP`e$&w@@HMHrzRC4^$$bVM~MMftl9E!=d7OnV&iw1*Nly#@{Q zP6Et0NY2;tAn#`pX|DBlVkqO}Yauy}kNlg2J6Sa^pBfPhM%hYsW!! zo;uPjZ=*_0-b9(Tu-H)EI%8q6;?=np54o+geZ8{5;#k2E*Xm<74J5$-s8$F~m+Z4S z{)sx^9~`|=_~(|60l%jj@UNnR2)Hu{uwHg1{Upb$XD0A+v>aN2&Pc>QK|IMW5Z1j~ zlofBnk}CeB0Rrx{3U&|P{h@3yx6bOy21)BD{$TC0emTq8`e0L!EG{Zm?3d}APCJL! zSP`rmhS$Q`+wzjgJy`ogWJ9L8$?h1=h#ie?(ef%tOiR}7&W6mH8ApY(T?eAVJIgYv zp6qyh#1ak?)Pe$8$1HVLHGi7Q+y1)_o*H_i($d3Ib9Xc~D~rZRO|9;u!?TqLcOmkJq*m=Tx^7yT$R|zX?0=|5a&M{=|g^@BE)D?~P)8ux<>jJKN$|yKD`Z zBkyGN)m1~j4_D(1#(fQg{P8s4Y9#^tAt@1q)N~Vs-$JR=tB%fA!|?F^u;#_?a?sac z%n~3Tok`&A2f->%ogMi+RzjkK@EjlCj~V$z{5g`X+an~C;$cv=;?eCv2#+Uyuiod# zH0ygKq<^#o`o0wXPYKTloT@sT?vlF5dg>f>O~(w-jXuP5T5Ezht+{FdM7jbKbielU z)IBY~jYFUG^7IBFKPUtrd~FQmt7sSQ*Zm})JJ z?7`DP8rg%xA4$TOi9ND_j(&C>QW3B}M`W>|azY7rwWsIGNTj+e5O! zbbgse?3I~ucT|IyosHeN(=T_rz1 z$W&+YOQJqx{k#t`O}rW;atDQDFo!04G+p)4xj*m6%RSipSBYIHPyPFlr;_&uy5)?J ztk`u%TGd>q#!&jtp|B*O1Y!(+9sK|x!h0+tY8y8@pj$H1*CS`_$yQvty(r$D97o+D z1dPui5#z3*^z$P@S)wQR6q_w%=9w~> zi`g7EF=0+%KP z^WaB(?Va9>>k=RWS5Y`I~03Mc+Z)fkC5yH3#liQ;Wr+8+NOMu9BH literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32 new file mode 100644 index 0000000000..3e0bf3e604 --- /dev/null +++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Digest.crc32 @@ -0,0 +1 @@ +3677152410 \ No newline at end of file diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-Qe~5HS$Na1wTg1uByWjo(=f0g&4y}~# z<7W(#vBgs5!p%!YzQ0w|v(~Un{#x)vC zwj^7UEy>ofldYao&A24hnsK2PYM~Zt#Zap|e`0>FIG%{ML|dXQ(blk|t)5a%xkU4n zi*iw}9Ob(3-TYi}yw=0a75O4xmoj7u%HInkVGPK=ooyYr{!=Zb2v zDHr9UTw%&3nx|Zpi*n^C*M%GNb49h-l#6mvt|;YFiw#Mp-F54qH0gBN&# zR}Q@DxuRNZ)Iu%P%ArrfD-~+Uzmfgi9?p ze&H8>#qdilwh_^YXhiHiCg-Xdm-9)+#kd$(jB%;OW?YPmapibc|4FNHsl~=G{KBuE z3PZnAi;Y-_g;+!9)OMe*8JAjY8JCPp#ubxssl~QjE!UXkx}A7DzCWwVr54-BXk;`p z=1@>AHew+bV&xF4|D?^h)M5*{OwFd|n5kJUw&iNM#w^!fH(afqY|5n;n{r_|hQ~0x zQ;RL-l5$D85*2QyTxzl53%=kh2VZKj4M+y0m;osVEX(+$HRMu@O}Qu+!|+Zmwkgq+Xi9uE6WntDlzOh{O)kd8xENQCajC^d zEW|>r7-IEOSA{Dwoa_GzuGahu+JGLT4QOAr*y9_&oKuS}A`_8`$gX9MlvKr zvK*4B#Wt%vMzc!J!t_aZ%2Aqdsl~=G{1W!Zg#BuBVwZ8HRE!+#kd$3QTw|7N zuN$s58-rHdiJe|}bBEj^!NO321-00o#i2Zlb_P4cZdPLrxyEX-gKc8SLD|$)R0Rp_VPCkszOyfb$fjqM=dtxqFj_KSGm+;J6FVT43A;BT5QYJa*bK8 z@`c5GVU}{K#TGa0HEyt&78jGdv7Ecnsl_%Ul1|bYD4kWfBEvbG+j2HH_t||O+CZ|e zTI|WqAHP(KJ+yyPn1LD3|J@RV_At z;TL|z@JlWBLF%q?%edwF+iOJ8A};5XjEiwGt{CG|i*3i->yF14`QkyYo@!ibu@CZJ z009ud^S8VGQ!Tb9W_M2v%A)Kr)=V=lwb(K)8JCPJCgW0zZMj;mG0TZ|8*uxmu?WK!*sk`BB?7S|UajC^-T#Sox%xuc#@z?XrF#Rszze)$;5Ab&wb;Y3Uvro_%#-*q??kJ` z<`^6!#xZPciyK=Fxzu9Y=pJUHvvL+!j=OPicS9{U<)U1aEApGInsSNeMe`Vr;V}$X zi*31Dt})BibhU$ literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..4a3ab075fc509ccea6cd53a5ad6e61b304f51bc3 GIT binary patch literal 9727 zcmeI%drVVT90%}wZd=||D5yLln+T!=OL+-oDid(PUN9b$O_y<@mBM(XR7Iv?akxL& z!a5^E64Z??#DIYaQ-Lv;MTHIBK*Gjm24P88jf=xMWT+kNR$BS}&X&yVFUihLZtnf? zJH5G&bI$#iavUdJz1CO8%`YM1hKOr{xbouqj<`xVE-=6UTcsBlTB^%e3S~l`R-?=l z%FBfkwN6l~wR&az9=#@2r`G5+>NG*8FVGj1mS}Ww)s;UbwRygtGjg$BJdp3)+}y0I zH<2B@b$}SAADilI;X6Jv%O=cYZm*r69T%H4imPR@oC92FTh0wG;yI#4io|iY>$CmY z&bDLS+EaF*>_pjxvYfIzWlzdplvhw*MLC#qIOPqLlPSMQIiK=g%5{`GDUV_k>Uog$ zR9#Q`EH=NK@_uXyb$p4fpIDnovv7M^1h(BqYv2(n!%U{5*5wM%7{@b63>yv&~&yOSG_ZzB{>`d$< zxPOw#?@*o3z~h-8z?NixgW5}_xc^HpUqW@+m$k@tH(QV$GUkvSXEn&qvmVHDHSUk@ ziPuowvwIn`*SHVziqUlBRfG6>2hZT=9bxz!wcj}LGxF1$@%g>vV?_1B8w%t%QrnRm z?!+K>oyYfQtP9;w;^cN4wkL5(miNGeg;e<;e*kfZTs)_O=7Y|l`IUcPJ_73!p9<7J zB;Bt#!n(A=aTBy``zX3zyV8ORShw#oPD48;Zkd91_Ul38$u(Os1nX{+ls;&W=s?t; zm&@-QhIQZ8_pU>)j7;u;4*GuU2I$ZsZ5woK&c372iC0uxq0^5QM?vQWo_-Q~*Iic= zbnS(fB#N^|o<23a1(>XH&FzNHOGtp@pR}Z%v5kLAGctEb4#z*QF^+!< zKF?2_*i;ED(G%lAU}?ahA_aP8W-oj`vOf%U@cm)G*2A)I=p`O%gLPz(0-{CUd;J2R(S~m!MWqVy=$b$LUCrnjZ;8xGJ@O=xGM06B3nHS92 zt`*Pef6sJ0_>5)u|B~V6-=Q2APF+8mGsGIpaOeiH42Nz9_n$M|{9#yzV;K%~D3;+C zbA}^tZOdL~;+&g)=VVhOLxw5U{q3QEhAQ!YE|%5ysN)W6)z^09gxO4OiS(pO6&4;5 z8MQt-CYJ57d%*eu8w-#2d>~F?;h-Xmlk_*FPiXTOI~-aJL1+F(?0){j$3`OE;q~VLN7nrFPjk7-VnJ7~6^fKunNP)J?kUp{@xawADYWz>@#b3!EKBql&OO8l( f2}NQ57621Ucb4cix`fiAoy8@VBzWVC#+bhVApW&? literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_clust_counter/nc-1-big-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..9b24e0450c732103bdd22afb63db34808745ed6f GIT binary patch literal 47 jcmZQzU}#`qU|z`!txfzhD;|Ns9D42%pv*;M%X892Dp^79pv5{rPiijjfQP>~@o(5G#kTB z3}ePrgLR6yWIq&@EUg^@;d5rRprW{XxQL*^>=ioNmuzL>?oO)q(Q z_u>1!>w9iD2G#yq#Q+gvCm>V6R3DHru?IPLR*xJq8-c9Q;qe%*`3c1% z`xhcdO~fEC9xFml7{>2Ac?Q4lblW+Uzi#pe4&z)*9U328|1-fb zE~yJ%4=vp`hWZ;&W2}Smz&^(`bg*XQ6m&@30Gdy7t#Sm$!?^rG=!ncjG@chJZodQL zv7K*RhhDNeuLnBmn=NahQ%4M4&{<`Bk3eg#rEP&OI=UkRx+3x9!_YhblsloDFLvZY zcNQOvhyJp)>2>JoPewNaljZXAe&~t~8{qsW9fhah{3o8b6mb5tddI|Ra6LPDe0@DI zXCaOQz>=lI%6#aVnLY6RNdK_4!1W_=MT9dO*sR`A}S894OW=3ij{72k$m zhy4$~n%AoY`3rcf39bjx$J7HV5Rd!9sf6!q>CN1HD~P9*x)R}jAS3<8paR6zV?Pe; z0)F&R_i@;st*)#LIG(GEH0?H!-&Ec=GzomQ`P34HzW2{{7#qpVxq-V zRy^_5|Cc?S*e(AaPJ9w`wSNzX9tY#!!=dNl-^0N*iDCSEIR73FOptr;;mFHdwJV)W zNZap>bjm(r`^+#`zj;p9ApGXyE#j@z6DM9t*w-r;eXm+vxP{vH0nF(z;R0bHhXx}^WcB738$;c8Yy8P-+xOgSMA` asKnJli&WI^FREk()2ft?y+M4 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple/nc-1-big-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..9b24e0450c732103bdd22afb63db34808745ed6f GIT binary patch literal 47 jcmZQzU}#`qU|k#6 literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Data.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Data.db new file mode 100644 index 0000000000000000000000000000000000000000..1431e3e27d5573a09206afa2dfc71aa95d0792d3 GIT binary patch literal 138 zcmdO5WMG)Yz-Una|Ns9621bTY!U`Jv3=Ap^j13GrnR?z`<$gz(JYi%>llXgzf%VIk z3lko!&|t`9U}R)qG_+#K*Wzb5C>m+Qz-~VAQH2eItdR|a{*^-@{-lKuY#3yWZ5Zqi iAOiq7yeF6d literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32 b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32 new file mode 100644 index 0000000000..18641177fb --- /dev/null +++ b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Digest.crc32 @@ -0,0 +1 @@ +4045937701 \ No newline at end of file diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Filter.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Filter.db new file mode 100644 index 0000000000000000000000000000000000000000..8868e5c18008783c106cc95dcb2a5942cf662a48 GIT binary patch literal 24 fcmZQzU|?lnU|?!-Qe5=Lb}!nk1*kThv{#{d8(*aT1j literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Statistics.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Statistics.db new file mode 100644 index 0000000000000000000000000000000000000000..ead79f6f295c72384167f8557cea21a3f4aa1f87 GIT binary patch literal 4831 zcmeI${ZA8T902fpt}X9$P*8b^jG>4UEKFobrglOV*v4SI%x=1jgLbwIT2iVa)37-7 zhi+rl!I+6J8(WA00~u@x%sDd^HkE;-8JiKplCE1cTb)COv!lCWPrlzLOZErYCwcPR z=f2--?|JUIyQgIsM!aHGw1l}|lK6)3RmWGBuW$2J#4xgID+#>GBpZx7_<6(au8)H+5q4=LcFptHW)8cJn!iIz9CySZlbAoSRmIrHg!y@Y zH*tKA;%pJVpV>>;BGq>&Uo6J+Uwrinic7v}ME1GWj_gaUMMR}165eYP2Bf92+B=)jl(bUy{FWTP-1BFY_v4oiplYx$p2{22psfEu4gBoUu2b-QwmCA=;C$8>D%&g|zoD#eXbSlF^@%fWz-L=O?t;nzZ&t+rTJ|B*8oW&YkQl9o19j2TyI`1!>s)8|B}k&Rh+R# s%T>xtinpvQ-czmNj}`6WDj)y9{4B#xy-8!tGF0xY(z_4re8-&sFX@Z8Jpcdz literal 0 HcmV?d00001 diff --git a/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Summary.db b/test/data/legacy-sstables/nc/legacy_tables/legacy_nc_simple_counter/nc-1-big-Summary.db new file mode 100644 index 0000000000000000000000000000000000000000..9b24e0450c732103bdd22afb63db34808745ed6f GIT binary patch literal 47 jcmZQzU}#`qU|[] cells; + Clustering[] clusterings; + ClusteringBound[] clusteringBounds; + ClusteringBoundary[] clusteringBoundaries; + MetadataCollector collector; + + @Setup + public void setup() + { + TableMetadata.Builder tableMetadataBuilder = TableMetadata.builder("k", "t") + .addPartitionKeyColumn("pk", LongType.instance) + .addRegularColumn("rc", LongType.instance); + for (int i = 0; i < clusteringKeyNum; i++) + tableMetadataBuilder.addClusteringColumn("ck" + i, LongType.instance); + TableMetadata tableMetadata = tableMetadataBuilder.build(); + collector = new MetadataCollector(tableMetadata.comparator); + + ColumnMetadata columnMetadata = tableMetadata.regularColumns().iterator().next(); + ThreadLocalRandom current = ThreadLocalRandom.current(); + datasetSize = Pow2.roundToPowerOfTwo(datasetSize); + cells = new Cell[datasetSize]; + for (int i = 0; i < datasetSize; i++) + { + cells[i] = new BufferCell(columnMetadata, current.nextLong(0, Long.MAX_VALUE), current.nextInt(1, Integer.MAX_VALUE), Cell.NO_DELETION_TIME, null, null); + } + clusterings = new Clustering[datasetSize]; + clusteringBounds = new ClusteringBound[datasetSize]; + clusteringBoundaries = new ClusteringBoundary[datasetSize]; + ByteBuffer[] cks = new ByteBuffer[clusteringKeyNum]; + Kind[] clusteringBoundKinds = new Kind[]{ Kind.INCL_START_BOUND, Kind.INCL_END_BOUND, Kind.EXCL_START_BOUND, Kind.EXCL_END_BOUND }; + Kind[] clusteringBoundaryKinds = new Kind[]{ Kind.INCL_END_EXCL_START_BOUNDARY, Kind.EXCL_END_INCL_START_BOUNDARY }; + for (int i = 0; i < datasetSize; i++) + { + for (int j = 0; j < clusteringKeyNum; j++) + cks[j] = LongType.instance.decompose(current.nextLong()); + clusterings[i] = Clustering.make(Arrays.copyOf(cks, cks.length)); + clusteringBounds[i] = ClusteringBound.create(clusteringBoundKinds[i % clusteringBoundKinds.length], clusterings[i]); + clusteringBoundaries[i] = ClusteringBoundary.create(clusteringBoundaryKinds[i % clusteringBoundaryKinds.length], clusterings[i]); + } + + System.gc(); + // shuffle array contents to ensure a more 'natural' layout + for (int i = 0; i < datasetSize; i++) + { + int to = current.nextInt(0, datasetSize); + Cell temp = cells[i]; + cells[i] = cells[to]; + cells[to] = temp; + } + + for (int i = 0; i < datasetSize; i++) + { + int to = current.nextInt(0, datasetSize); + Clustering temp = clusterings[i]; + clusterings[i] = clusterings[to]; + clusterings[to] = temp; + } + } + + @Benchmark + public void updateCell() + { + collector.update(nextCell()); + } + + @Benchmark + public void updateClustering() + { + collector.updateClusteringValues(nextClustering()); + } + + @Benchmark + public void updateClusteringBound() + { + collector.updateClusteringValuesByBoundOrBoundary(nextClusteringBound()); + } + + @Benchmark + public void updateClusteringBoundary() + { + collector.updateClusteringValuesByBoundOrBoundary(nextClusteringBoundary()); + } + + public Cell nextCell() + { + return cells[datumIndex++ & (cells.length - 1)]; + } + + public Clustering nextClustering() + { + return clusterings[datumIndex++ & (clusterings.length - 1)]; + } + + public ClusteringBound nextClusteringBound() + { + return clusteringBounds[datumIndex++ & (clusteringBounds.length - 1)]; + } + + public ClusteringBoundary nextClusteringBoundary() + { + return clusteringBoundaries[datumIndex++ & (clusteringBoundaries.length - 1)]; + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java index ff0e546951..83d12f507e 100644 --- a/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/miscellaneous/SSTablesIteratedTest.java @@ -20,12 +20,16 @@ */ package org.apache.cassandra.cql3.validation.miscellaneous; +import java.util.Arrays; + import org.junit.Test; import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.metrics.ClearableHistogram; import static org.junit.Assert.assertEquals; @@ -50,6 +54,22 @@ public class SSTablesIteratedTest extends CQLTester numSSTablesIterated); } + private void executeAndCheckRangeQuery(String query, int numSSTables, Object[]... rows) throws Throwable + { + logger.info("Executing query: {} with parameters: {}", query, Arrays.toString(rows)); + ColumnFamilyStore cfs = getCurrentColumnFamilyStore(KEYSPACE_PER_TEST); + + ((ClearableHistogram) cfs.metric.sstablesPerRangeReadHistogram.cf).clear(); // resets counts + + assertRows(execute(query), rows); + + long numSSTablesIterated = cfs.metric.sstablesPerRangeReadHistogram.cf.getSnapshot().getMax(); // max sstables read + assertEquals(String.format("Expected %d sstables iterated but got %d instead, with %d live sstables", + numSSTables, numSSTablesIterated, cfs.getLiveSSTables().size()), + numSSTables, + numSSTablesIterated); + } + @Override protected String createTable(String query) { @@ -431,13 +451,19 @@ public class SSTablesIteratedTest extends CQLTester } flush(); + // The code has to read the 1st and 3rd sstables to see that everything before the 2nd sstable is deleted, and + // overall has to read the 3 sstables executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 1", 3, row(1, 1001, "1001")); executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 2", 3, row(1, 1001, "1001"), row(1, 1002, "1002")); executeAndCheck("SELECT * FROM %s WHERE id=1", 3, allRows); - executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000 LIMIT 1", 2, row(1, 1001, "1001")); + + // The 1st and 3rd sstables have data only up to 1000, so they will be skipped + executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000 LIMIT 1", 1, row(1, 1001, "1001")); + executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000", 1, allRows); + + // The condition makes no difference to the code, and all 3 sstables have to read executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 LIMIT 1", 3, row(1, 1001, "1001")); - executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000", 2, allRows); executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000", 3, allRows); } @@ -515,14 +541,30 @@ public class SSTablesIteratedTest extends CQLTester allRows[idx] = row(1, i, Integer.toString(i), Integer.toString(i)); } - executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 1", 2, row(1, 1, "1", "1")); - executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 2", 2, row(1, 1, "1", "1"), row(1, 2, "2", null)); + // The 500th first rows are in the first sstable (and there is no partition deletion/static row), so the 'lower + // bound' optimization will kick in and we'll only read the 1st sstable. + executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 1", 1, row(1, 1, "1", "1")); + executeAndCheck("SELECT * FROM %s WHERE id=1 LIMIT 2", 1, row(1, 1, "1", "1"), row(1, 2, "2", null)); + // Getting everything obviously requires reading both sstables executeAndCheck("SELECT * FROM %s WHERE id=1", 2, allRows); + + // The 'lower bound' optimization don't help us because while the row to fetch is in the 1st sstable, the lower + // bound for the 2nd sstable is 501, which is lower than 1000. executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 1000 LIMIT 1", 2, row(1, 1001, "1001", "1001")); - executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 LIMIT 1", 2, row(1, 1, "1", "1")); + + // Somewhat similar to the previous one: the row is in th 2nd sstable in this case, but as the lower bound for + // the first sstable is 1, this doesn't help. executeAndCheck("SELECT * FROM %s WHERE id=1 AND col > 500 LIMIT 1", 2, row(1, 751, "751", "751")); - executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 500 LIMIT 1", 2, row(1, 1, "1", "1")); + + // The 'col <= ?' condition in both queries doesn't impact the read path, which can still make use of the lower + // bound optimization and read only the first sstable. + executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 LIMIT 1", 1, row(1, 1, "1", "1")); + executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 500 LIMIT 1", 1, row(1, 1, "1", "1")); + + // Making sure the 'lower bound' optimization also work in reverse queries (in which it's more of a 'upper + // bound' optimization). + executeAndCheck("SELECT * FROM %s WHERE id=1 AND col <= 2000 ORDER BY col DESC LIMIT 1", 1, row(1, 2000, "2000", null)); } @Test @@ -996,7 +1038,7 @@ public class SSTablesIteratedTest extends CQLTester @Test public void testCompactAndNonCompactTableWithCounter() throws Throwable { - for (String with : new String[]{"", " WITH COMPACT STORAGE"}) + for (String with : new String[]{ "", " WITH COMPACT STORAGE" }) { createTable("CREATE TABLE %s (pk int, c int, count counter, PRIMARY KEY(pk, c))" + with); @@ -1038,7 +1080,7 @@ public class SSTablesIteratedTest extends CQLTester executeAndCheck("SELECT s, v FROM %s WHERE pk = 3 AND c = 3", 3, row(3, set(1))); executeAndCheck("SELECT v FROM %s WHERE pk = 1 AND c = 1", 3, row(set(3))); executeAndCheck("SELECT v FROM %s WHERE pk = 2 AND c = 1", 2, row(set(3))); - executeAndCheck("SELECT v FROM %s WHERE pk = 3 AND c = 3", 3, row(set(1))); + executeAndCheck("SELECT v FROM %s WHERE pk = 3 AND c = 3", 1, row(set(1))); executeAndCheck("SELECT s FROM %s WHERE pk = 1", 3, row((Integer) null)); executeAndCheck("SELECT s FROM %s WHERE pk = 2", 2, row(1), row(1)); executeAndCheck("SELECT DISTINCT s FROM %s WHERE pk = 2", 2, row(1)); @@ -1087,7 +1129,7 @@ public class SSTablesIteratedTest extends CQLTester @Test public void testCompactAndNonCompactTableWithPartitionTombstones() throws Throwable { - for (Boolean compact : new Boolean[] {Boolean.FALSE, Boolean.TRUE}) + for (Boolean compact : new Boolean[]{ Boolean.FALSE, Boolean.TRUE }) { String with = compact ? " WITH COMPACT STORAGE" : ""; createTable("CREATE TABLE %s (pk int PRIMARY KEY, v1 int, v2 int)" + with); @@ -1642,4 +1684,157 @@ public class SSTablesIteratedTest extends CQLTester executeAndCheck("SELECT v1 FROM %s WHERE pk = 5", 1); executeAndCheck("SELECT v2 FROM %s WHERE pk = 5", 1); } -} + + @Test + public void testSkippingBySliceInSinglePartitionReads() throws Throwable + { + createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))"); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 1, 1); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 2); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 3); + flush(); + assertEquals(1, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 4); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 5); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 6); + flush(); + assertEquals(2, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 7); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 8); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 9); + flush(); + assertEquals(3, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 10); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 11); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 6, 12); + flush(); + assertEquals(4, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + // point query - test whether sstables are skipped due to not covering the requested slice + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 0", 0); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 1", 1, row(1, 1, 1)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 2", 2, row(1, 2, 4)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 3", 3, row(1, 3, 7)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 4", 3, row(1, 4, 10)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 5", 2, row(1, 5, 11)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 6", 1, row(1, 6, 12)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c = 7", 0); + + // range query - test whether sstables are skipped due to not covering the requeste slice + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > -10 AND c <= 0", 0); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > -10 AND c < 1", 0); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > -10 AND c <= 1", 1, row(1, 1, 1)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > 1 AND c < 3", 2, row(1, 2, 4)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c >= 6", 1, row(1, 6, 12)); + executeAndCheck("SELECT * FROM %s WHERE pk = 1 AND c > 6", 0); + } + + @Test + public void testSkippingBySliceInPartitionRangeReads() throws Throwable + { + createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))"); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 1, 1); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 2); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 3); + flush(); + assertEquals(1, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 2, 4); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 5); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 6); + flush(); + assertEquals(2, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 3, 7); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 8); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 9); + flush(); + assertEquals(3, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 4, 10); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 5, 11); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", 1, 6, 12); + flush(); + assertEquals(4, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + // point query - test whether sstables are skipped due to not covering the requested slice + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 0 ALLOW FILTERING", 0); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 1 ALLOW FILTERING", 1, row(1, 1, 1)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 2 ALLOW FILTERING", 2, row(1, 2, 4)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 3 ALLOW FILTERING", 3, row(1, 3, 7)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 4 ALLOW FILTERING", 3, row(1, 4, 10)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 5 ALLOW FILTERING", 2, row(1, 5, 11)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 6 ALLOW FILTERING", 1, row(1, 6, 12)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c = 7 ALLOW FILTERING", 0); + + // range query - test whether sstables are skipped due to not covering the requeste slice + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > -10 AND c <= 0 ALLOW FILTERING", 0); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > -10 AND c < 1 ALLOW FILTERING", 0); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > -10 AND c <= 1 ALLOW FILTERING", 1, row(1, 1, 1)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > 1 AND c < 3 ALLOW FILTERING", 2, row(1, 2, 4)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c >= 6 ALLOW FILTERING", 1, row(1, 6, 12)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE c > 6 ALLOW FILTERING", 0); + } + + @Test + public void testSkippingByKeyRangeInPartitionRangeReads() throws Throwable + { + DecoratedKey[] keys = new DecoratedKey[8]; + for (int i = 0; i < keys.length; i++) + keys[i] = DatabaseDescriptor.getPartitioner().decorateKey(Int32Type.instance.decompose(i)); + Arrays.sort(keys); + + int[] k = new int[keys.length]; + String[] t = new String[keys.length]; + for (int i = 0; i < keys.length; i++) + { + DecoratedKey key = keys[i]; + k[i] = Int32Type.instance.compose(key.getKey()); + t[i] = key.getToken().getTokenValue().toString(); + } + + createTable("CREATE TABLE %s (pk int, c int, v int, PRIMARY KEY(pk, c))"); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[1], 1, 1); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[2], 2, 2); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[3], 3, 3); + flush(); + assertEquals(1, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[2], 2, 4); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[3], 3, 5); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[4], 4, 6); + flush(); + assertEquals(2, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[3], 3, 7); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[4], 4, 8); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[5], 5, 9); + flush(); + assertEquals(3, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[4], 4, 10); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[5], 5, 11); + execute("INSERT INTO %s (pk, c, v) VALUES (?, ?, ?) USING TIMESTAMP 1000", k[6], 6, 12); + flush(); + assertEquals(4, getCurrentColumnFamilyStore(KEYSPACE_PER_TEST).getLiveSSTables().size()); + + // range query - test whether sstables are skipped due to not covering the requested slice + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN (pk) <= " + t[0], 0); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN (pk) < " + t[1], 0); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[1] + " AND TOKEN (pk) < " + t[2], 1, row(k[1], 1, 1)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[2] + " AND TOKEN (pk) < " + t[3], 2, row(k[2], 2, 4)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[3] + " AND TOKEN (pk) < " + t[4], 3, row(k[3], 3, 7)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[3] + " AND TOKEN (pk) <= " + t[4], 4, row(k[3], 3, 7), row(k[4], 4, 10)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[4] + " AND TOKEN (pk) < " + t[5], 3, row(k[4], 4, 10)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[5] + " AND TOKEN (pk) < " + t[6], 2, row(k[5], 5, 11)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[6] + " AND TOKEN (pk) < " + t[7], 1, row(k[6], 6, 12)); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) > " + t[6], 0); + executeAndCheckRangeQuery("SELECT * FROM %s WHERE TOKEN(pk) >= " + t[7], 0); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java index fcf5c51da0..36b587c024 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java @@ -363,8 +363,8 @@ public class CompactionsTest for (SSTableReader sstable : cfs.getLiveSSTables()) { StatsMetadata stats = sstable.getSSTableMetadata(); - assertEquals(ByteBufferUtil.bytes("0"), stats.minClusteringValues.get(0)); - assertEquals(ByteBufferUtil.bytes("b"), stats.maxClusteringValues.get(0)); + assertEquals(ByteBufferUtil.bytes("0"), stats.coveredClustering.start().bufferAt(0)); + assertEquals(ByteBufferUtil.bytes("b"), stats.coveredClustering.end().bufferAt(0)); } assertEquals(keys, k); @@ -565,4 +565,4 @@ public class CompactionsTest CompactionManager.instance.setConcurrentCompactors(1); assertEquals(1, CompactionManager.instance.getCoreCompactorThreads()); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/filter/SliceTest.java b/test/unit/org/apache/cassandra/db/filter/SliceTest.java index c83880c03e..18233f09f5 100644 --- a/test/unit/org/apache/cassandra/db/filter/SliceTest.java +++ b/test/unit/org/apache/cassandra/db/filter/SliceTest.java @@ -18,19 +18,30 @@ * */ package org.apache.cassandra.db.filter; - import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayList; +import java.util.List; -import org.apache.cassandra.db.*; import org.junit.Test; +import org.apache.cassandra.db.BufferClusteringBound; +import org.apache.cassandra.db.Clustering; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ClusteringComparator; +import org.apache.cassandra.db.ClusteringPrefix; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.Int32Type; import org.apache.cassandra.utils.ByteBufferUtil; -import static org.apache.cassandra.db.ClusteringPrefix.Kind.*; -import static org.junit.Assert.*; +import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_END_BOUND; +import static org.apache.cassandra.db.ClusteringPrefix.Kind.EXCL_START_BOUND; +import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_END_BOUND; +import static org.apache.cassandra.db.ClusteringPrefix.Kind.INCL_START_BOUND; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class SliceTest { @@ -48,221 +59,221 @@ public class SliceTest // filter falls entirely before sstable Slice slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // same case, but with empty start slice = Slice.make(makeBound(sk), makeBound(ek, 1, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // same case, but with missing components for start slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // same case, but with missing components for start and end slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // end of slice matches start of sstable for the first component, but not the second component slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0))); // same case, but with missing components for start slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0))); // same case, but with missing components for start and end slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0)); - assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0))); // first two components match, but not the last slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 0)); - assertFalse(slice.intersects(cc, columnNames(1, 1, 1), columnNames(3, 1, 1))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3, 1, 1))); // all three components in slice end match the start of the sstable slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 1, 1), columnNames(3, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3, 1, 1))); // filter falls entirely after sstable slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek, 4, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // same case, but with empty end slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // same case, but with missing components for end slice = Slice.make(makeBound(sk, 4, 0, 0), makeBound(ek, 1)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // same case, but with missing components for start and end slice = Slice.make(makeBound(sk, 4, 0), makeBound(ek, 1)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); // start of slice matches end of sstable for the first component, but not the second component slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 0, 0))); // start of slice matches end of sstable for the first two components, but not the last component slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 1, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 0))); // all three components in the slice start match the end of the sstable slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 2, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(0, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1))); // slice covers entire sstable (with no matching edges) slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 2, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // same case, but with empty ends slice = Slice.make(makeBound(sk), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // same case, but with missing components slice = Slice.make(makeBound(sk, 0), makeBound(ek, 2, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // slice covers entire sstable (with matching start) slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // slice covers entire sstable (with matching end) slice = Slice.make(makeBound(sk, 0, 0, 0), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // slice covers entire sstable (with matching start and end) slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // slice falls entirely within sstable (with matching start) slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // same case, but with a missing end component slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // slice falls entirely within sstable (with matching end) slice = Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // same case, but with a missing start component slice = Slice.make(makeBound(sk, 1, 1), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(1, 1, 1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 1, 1, 1))); // slice falls entirely within sstable slice = Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2))); // same case, but with a missing start component slice = Slice.make(makeBound(sk, 1, 1), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2))); // same case, but with a missing start and end components slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2))); // same case, but with an equal first component and missing start and end components slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2))); // slice falls entirely within sstable (slice start and end are the same) slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2))); // slice starts within sstable, empty end slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // same case, but with missing end components slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 3)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // slice starts within sstable (matching sstable start), empty end slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // same case, but with missing end components slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 3)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // slice starts within sstable (matching sstable end), empty end slice = Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // same case, but with missing end components slice = Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // slice ends within sstable, empty end slice = Slice.make(makeBound(sk), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // same case, but with missing start components slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 1, 1)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // slice ends within sstable (matching sstable start), empty start slice = Slice.make(makeBound(sk), makeBound(ek, 1, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // same case, but with missing start components slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // slice ends within sstable (matching sstable end), empty start slice = Slice.make(makeBound(sk), makeBound(ek, 2, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // same case, but with missing start components slice = Slice.make(makeBound(sk, 0), makeBound(ek, 2, 0, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 0, 0))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 0, 0))); // empty min/max column names slice = Slice.make(makeBound(sk), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(), columnNames())); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek))); slice = Slice.make(makeBound(sk, 1), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(), columnNames())); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek))); slice = Slice.make(makeBound(sk), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(), columnNames())); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek))); slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(), columnNames())); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek))); slice = Slice.make(makeBound(sk), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 2))); slice = Slice.make(makeBound(sk), makeBound(ek, 2)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk, 2), makeBound(ek, 3)); - assertFalse(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); // basic check on reversed slices slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 0, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(2, 0, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 2, 0, 0), makeBound(ek, 3, 0, 0))); slice = Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 0, 0, 0)); - assertFalse(slice.intersects(cc, columnNames(1, 1, 0), columnNames(3, 0, 0))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 1, 0), makeBound(ek, 3, 0, 0))); slice = Slice.make(makeBound(sk, 1, 1, 1), makeBound(ek, 1, 1, 0)); - assertTrue(slice.intersects(cc, columnNames(1, 0, 0), columnNames(2, 2, 2))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1, 0, 0), makeBound(ek, 2, 2, 2))); } @Test @@ -279,32 +290,32 @@ public class SliceTest // slice does intersect Slice slice = Slice.make(makeBound(sk), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(1), columnNames(1, 2))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2))); slice = Slice.make(makeBound(sk), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk, 1), makeBound(ek)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk, 1), makeBound(ek, 1)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk, 0), makeBound(ek, 1, 2, 3)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk, 1, 2, 3), makeBound(ek, 2)); - assertTrue(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); // slice does not intersect slice = Slice.make(makeBound(sk, 2), makeBound(ek, 3, 4, 5)); - assertFalse(slice.intersects(cc, columnNames(), columnNames(1))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk), makeBound(ek, 1))); slice = Slice.make(makeBound(sk, 0), makeBound(ek, 0, 1, 2)); - assertFalse(slice.intersects(cc, columnNames(1), columnNames(1, 2))); + assertSlicesDoNotIntersect(cc, slice, Slice.make(makeBound(sk, 1), makeBound(ek, 1, 2))); } @Test @@ -353,14 +364,6 @@ public class SliceTest return BufferClusteringBound.create(kind, values); } - private static List columnNames(Integer ... components) - { - List names = new ArrayList<>(components.length); - for (int component : components) - names.add(ByteBufferUtil.bytes(component)); - return names; - } - private static Slice s(int start, int finish) { return Slice.make(makeBound(INCL_START_BOUND, start), @@ -382,4 +385,25 @@ public class SliceTest for (int i = 0; i < expected.length; i++) assertEquals(expected[i], slices.get(i)); } -} + + private void assertSlicesIntersect(ClusteringComparator cc, Slice s1, Slice s2) + { + assertSlicesIntersectInternal(cc, s1, s2); + assertSlicesIntersectInternal(cc, Slice.ALL, s1); + assertSlicesIntersectInternal(cc, Slice.ALL, s2); + assertSlicesDoNotIntersect(cc, Slice.make(ClusteringBound.exclusiveStartOf(Clustering.EMPTY), ClusteringBound.exclusiveEndOf(Clustering.EMPTY)), s1); + assertSlicesDoNotIntersect(cc, Slice.make(ClusteringBound.exclusiveStartOf(Clustering.EMPTY), ClusteringBound.exclusiveEndOf(Clustering.EMPTY)), s2); + } + + private void assertSlicesIntersectInternal(ClusteringComparator cc, Slice s1, Slice s2) + { + assertTrue(String.format("Slice %s should intersect with slice %s", s1.toString(cc), s2.toString(cc)), s1.intersects(cc, s2)); + assertTrue(String.format("Slice %s should intersect with slice %s", s2.toString(cc), s1.toString(cc)), s2.intersects(cc, s1)); + } + + private void assertSlicesDoNotIntersect(ClusteringComparator cc, Slice s1, Slice s2) + { + assertFalse(String.format("Slice %s should not intersect with slice %s", s1.toString(cc), s2.toString(cc)), s1.intersects(cc, s2)); + assertFalse(String.format("Slice %s should not intersect with slice %s", s2.toString(cc), s1.toString(cc)), s2.intersects(cc, s1)); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java index 2812353f36..334edcbbbf 100644 --- a/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java +++ b/test/unit/org/apache/cassandra/db/lifecycle/LogTransactionTest.java @@ -42,6 +42,7 @@ import org.junit.Test; import org.apache.cassandra.Util; import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.Directories; import org.apache.cassandra.db.SerializationHeader; import org.apache.cassandra.db.compaction.OperationType; @@ -1275,9 +1276,10 @@ public class LogTransactionTest extends AbstractTransactionalTest FileHandle dFile = new FileHandle.Builder(descriptor.filenameFor(Component.DATA)).complete(); FileHandle iFile = new FileHandle.Builder(descriptor.filenameFor(Component.PRIMARY_INDEX)).complete(); + DecoratedKey key = MockSchema.readerBounds(generation); SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata().comparator) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, -1, null, false, header, key.getKey().slice(), key.getKey().slice()) .get(MetadataType.STATS); SSTableReader reader = SSTableReader.internalOpen(descriptor, components, @@ -1290,7 +1292,7 @@ public class LogTransactionTest extends AbstractTransactionalTest metadata, SSTableReader.OpenReason.NORMAL, header); - reader.first = reader.last = MockSchema.readerBounds(generation); + reader.first = reader.last = key; return reader; } diff --git a/test/unit/org/apache/cassandra/db/rows/RowsTest.java b/test/unit/org/apache/cassandra/db/rows/RowsTest.java index cfeebfd2b1..8742fd889c 100644 --- a/test/unit/org/apache/cassandra/db/rows/RowsTest.java +++ b/test/unit/org/apache/cassandra/db/rows/RowsTest.java @@ -200,6 +200,12 @@ public class RowsTest { this.hasLegacyCounterShards |= hasLegacyCounterShards; } + + @Override + public void updatePartitionDeletion(DeletionTime dt) + { + update(dt); + } } private static long secondToTs(int now) @@ -648,4 +654,4 @@ public class RowsTest liveCell(b, 1), liveCell(a)); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java new file mode 100644 index 0000000000..efe93f6af9 --- /dev/null +++ b/test/unit/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBoundTest.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.db.rows; + +import java.math.BigInteger; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.db.ClusteringBound; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.db.SinglePartitionReadCommand; +import org.apache.cassandra.db.Slice; +import org.apache.cassandra.db.Slices; +import org.apache.cassandra.db.filter.ClusteringIndexSliceFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.lifecycle.View; +import org.apache.cassandra.db.marshal.Int32Type; +import org.apache.cassandra.db.marshal.IntegerType; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.mockito.Mockito; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class UnfilteredRowIteratorWithLowerBoundTest +{ + private static final String KEYSPACE = "ks"; + private static final String TABLE = "tbl"; + private static final String SLICES_TABLE = "tbl_slices"; + private static TableMetadata tableMetadata; + private static TableMetadata slicesTableMetadata; + + @BeforeClass + public static void defineSchema() throws ConfigurationException + { + DatabaseDescriptor.daemonInitialization(); + + tableMetadata = + TableMetadata.builder(KEYSPACE, TABLE) + .addPartitionKeyColumn("k", UTF8Type.instance) + .addStaticColumn("s", UTF8Type.instance) + .addClusteringColumn("i", IntegerType.instance) + .addRegularColumn("v", UTF8Type.instance) + .build(); + + slicesTableMetadata = TableMetadata.builder(KEYSPACE, SLICES_TABLE) + .addPartitionKeyColumn("k", UTF8Type.instance) + .addClusteringColumn("c1", Int32Type.instance) + .addClusteringColumn("c2", Int32Type.instance) + .addRegularColumn("v", IntegerType.instance) + .build(); + + SchemaLoader.prepareServer(); + SchemaLoader.createKeyspace(KEYSPACE, KeyspaceParams.simple(1), tableMetadata, slicesTableMetadata); + } + + @Before + public void truncate() + { + Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE).truncateBlocking(); + Keyspace.open(KEYSPACE).getColumnFamilyStore(SLICES_TABLE).truncateBlocking(); + } + + @Test + public void testLowerBoundApplicableSingleColumnAsc() + { + String query = "INSERT INTO %s.%s (k, i) VALUES ('k1', %s)"; + SSTableReader sstable = createSSTable(tableMetadata, KEYSPACE, TABLE, query); + assertEquals(Slice.make(Util.clustering(tableMetadata.comparator, BigInteger.valueOf(0)), + Util.clustering(tableMetadata.comparator, BigInteger.valueOf(9))), + sstable.getSSTableMetadata().coveredClustering); + DecoratedKey key = tableMetadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1")); + + Slice slice1 = Slice.make(Util.clustering(tableMetadata.comparator, BigInteger.valueOf(3)).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(tableMetadata, key, slice1, sstable, false)); + assertTrue(lowerBoundApplicable(tableMetadata, key, slice1, sstable, true)); + + Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(tableMetadata.comparator, BigInteger.valueOf(3)).asEndBound()); + assertTrue(lowerBoundApplicable(tableMetadata, key, slice2, sstable, false)); + assertFalse(lowerBoundApplicable(tableMetadata, key, slice2, sstable, true)); + + // corner cases + Slice slice3 = Slice.make(Util.clustering(tableMetadata.comparator, BigInteger.valueOf(0)).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(tableMetadata, key, slice3, sstable, false)); + assertTrue(lowerBoundApplicable(tableMetadata, key, slice3, sstable, true)); + + Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(tableMetadata.comparator, BigInteger.valueOf(9)).asEndBound()); + assertTrue(lowerBoundApplicable(tableMetadata, key, slice4, sstable, false)); + assertFalse(lowerBoundApplicable(tableMetadata, key, slice4, sstable, true)); + } + + @Test + public void testLowerBoundApplicableSingleColumnDesc() + { + String TABLE_REVERSED = "tbl_reversed"; + String createTable = String.format( + "CREATE TABLE %s.%s (k text, i varint, v int, primary key (k, i)) WITH CLUSTERING ORDER BY (i DESC)", + KEYSPACE, TABLE_REVERSED); + QueryProcessor.executeOnceInternal(createTable); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_REVERSED); + TableMetadata metadata = cfs.metadata(); + String query = "INSERT INTO %s.%s (k, i) VALUES ('k1', %s)"; + SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE_REVERSED, query); + assertEquals(Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(9)), + Util.clustering(metadata.comparator, BigInteger.valueOf(0))), + sstable.getSSTableMetadata().coveredClustering); + DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1")); + + Slice slice1 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(8)).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false)); + assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true)); + + Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(8)).asEndBound()); + assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false)); + assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true)); + + // corner cases + Slice slice3 = Slice.make(Util.clustering(metadata.comparator, BigInteger.valueOf(9)).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false)); + assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true)); + + Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, BigInteger.valueOf(0)).asEndBound()); + assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false)); + assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true)); + } + + @Test + public void testLowerBoundApplicableMultipleColumnsAsc() + { + String query = "INSERT INTO %s.%s (k, c1, c2) VALUES ('k1', 0, %s)"; + SSTableReader sstable = createSSTable(slicesTableMetadata, KEYSPACE, SLICES_TABLE, query); + assertEquals(Slice.make(Util.clustering(slicesTableMetadata.comparator, 0, 0), + Util.clustering(slicesTableMetadata.comparator, 0, 9)), + sstable.getSSTableMetadata().coveredClustering); + DecoratedKey key = slicesTableMetadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1")); + + Slice slice1 = Slice.make(Util.clustering(slicesTableMetadata.comparator, 0, 3).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice1, sstable, false)); + assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice1, sstable, true)); + + Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(slicesTableMetadata.comparator, 0, 3).asEndBound()); + assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice2, sstable, false)); + assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice2, sstable, true)); + + // corner cases + Slice slice3 = Slice.make(Util.clustering(slicesTableMetadata.comparator, 0, 0).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice3, sstable, false)); + assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice3, sstable, true)); + + Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(slicesTableMetadata.comparator, 0, 9).asEndBound()); + assertTrue(lowerBoundApplicable(slicesTableMetadata, key, slice4, sstable, false)); + assertFalse(lowerBoundApplicable(slicesTableMetadata, key, slice4, sstable, true)); + } + + @Test + public void testLowerBoundApplicableMultipleColumnsDesc() + { + String TABLE_REVERSED = "tbl_slices_reversed"; + String createTable = String.format( + "CREATE TABLE %s.%s (k text, c1 int, c2 int, v int, primary key (k, c1, c2)) WITH CLUSTERING ORDER BY (c1 ASC, c2 DESC)", + KEYSPACE, TABLE_REVERSED); + QueryProcessor.executeOnceInternal(createTable); + ColumnFamilyStore cfs = Keyspace.open(KEYSPACE).getColumnFamilyStore(TABLE_REVERSED); + TableMetadata metadata = cfs.metadata(); + + String query = "INSERT INTO %s.%s (k, c1, c2) VALUES ('k1', 0, %s)"; + SSTableReader sstable = createSSTable(metadata, KEYSPACE, TABLE_REVERSED, query); + assertEquals(Slice.make(Util.clustering(metadata.comparator, 0, 9), + Util.clustering(metadata.comparator, 0, 0)), + sstable.getSSTableMetadata().coveredClustering); + DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1")); + + Slice slice1 = Slice.make(Util.clustering(metadata.comparator, 0, 8).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(metadata, key, slice1, sstable, false)); + assertTrue(lowerBoundApplicable(metadata, key, slice1, sstable, true)); + + Slice slice2 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, 0, 8).asEndBound()); + assertTrue(lowerBoundApplicable(metadata, key, slice2, sstable, false)); + assertFalse(lowerBoundApplicable(metadata, key, slice2, sstable, true)); + + // corner cases + Slice slice3 = Slice.make(Util.clustering(metadata.comparator, 0, 9).asStartBound(), ClusteringBound.TOP); + assertFalse(lowerBoundApplicable(metadata, key, slice3, sstable, false)); + assertTrue(lowerBoundApplicable(metadata, key, slice3, sstable, true)); + + Slice slice4 = Slice.make(ClusteringBound.BOTTOM, Util.clustering(metadata.comparator, 0, 0).asEndBound()); + assertTrue(lowerBoundApplicable(metadata, key, slice4, sstable, false)); + assertFalse(lowerBoundApplicable(metadata, key, slice4, sstable, true)); + } + + private SSTableReader createSSTable(TableMetadata metadata, String keyspace, String table, String query) + { + ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table); + for (int i = 0; i < 10; i++) + QueryProcessor.executeInternal(String.format(query, keyspace, table, i)); + cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.UNIT_TESTS); + DecoratedKey key = metadata.partitioner.decorateKey(ByteBufferUtil.bytes("k1")); + ColumnFamilyStore.ViewFragment view = cfs.select(View.select(SSTableSet.LIVE, key)); + assertEquals(1, view.sstables.size()); + return view.sstables.get(0); + } + + private boolean lowerBoundApplicable(TableMetadata metadata, DecoratedKey key, Slice slice, SSTableReader sstable, boolean isReversed) + { + Slices.Builder slicesBuilder = new Slices.Builder(metadata.comparator); + slicesBuilder.add(slice); + Slices slices = slicesBuilder.build(); + ClusteringIndexSliceFilter filter = new ClusteringIndexSliceFilter(slices, isReversed); + + SinglePartitionReadCommand cmd = SinglePartitionReadCommand.create(metadata, + FBUtilities.nowInSeconds(), + ColumnFilter.all(metadata), + RowFilter.NONE, + DataLimits.NONE, + key, + filter); + + try (UnfilteredRowIteratorWithLowerBound iter = new UnfilteredRowIteratorWithLowerBound(key, + sstable, + slices, + isReversed, + ColumnFilter.all(metadata), + Mockito.mock(SSTableReadsListener.class))) + { + return iter.lowerBound() != null; + } + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 063afe52db..655e552584 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -94,7 +94,7 @@ public class LegacySSTableTest * See {@link #testGenerateSstables()} to generate sstables. * Take care on commit as you need to add the sstable files using {@code git add -f} */ - public static final String[] legacyVersions = {"nb", "na", "me", "md", "mc", "mb", "ma"}; + public static final String[] legacyVersions = {"nc", "nb", "na", "me", "md", "mc", "mb", "ma"}; // 1200 chars static final String longString = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java index 01ecb94b19..215f02e485 100644 --- a/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/SSTableMetadataTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.junit.BeforeClass; @@ -27,7 +28,6 @@ import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; import org.apache.cassandra.db.RowUpdateBuilder; @@ -35,6 +35,7 @@ import org.apache.cassandra.db.marshal.AsciiType; import org.apache.cassandra.db.marshal.IntegerType; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.TableMetadata; import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; @@ -218,11 +219,11 @@ public class SSTableMetadataTest assertEquals(1, store.getLiveSSTables().size()); for (SSTableReader sstable : store.getLiveSSTables()) { - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100"); - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "7col149"); - // make sure stats don't reference native or off-heap data - assertBuffersAreRetainable(sstable.getSSTableMetadata().minClusteringValues); - assertBuffersAreRetainable(sstable.getSSTableMetadata().maxClusteringValues); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0col100"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "7col149"); + // make sure the clustering values are minimised + assertTrue(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0).capacity() < 50); + assertTrue(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0).capacity() < 50); } String key = "row2"; @@ -240,11 +241,11 @@ public class SSTableMetadataTest assertEquals(1, store.getLiveSSTables().size()); for (SSTableReader sstable : store.getLiveSSTables()) { - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0col100"); - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "9col298"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0col100"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "9col298"); // make sure stats don't reference native or off-heap data - assertBuffersAreRetainable(sstable.getSSTableMetadata().minClusteringValues); - assertBuffersAreRetainable(sstable.getSSTableMetadata().maxClusteringValues); + assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.start().getBufferArray())); + assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.end().getBufferArray())); } key = "row3"; @@ -258,11 +259,11 @@ public class SSTableMetadataTest assertEquals(1, store.getLiveSSTables().size()); for (SSTableReader sstable : store.getLiveSSTables()) { - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().minClusteringValues.get(0)), "0"); - assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().maxClusteringValues.get(0)), "9col298"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.start().bufferAt(0)), "0"); + assertEquals(ByteBufferUtil.string(sstable.getSSTableMetadata().coveredClustering.end().bufferAt(0)), "9col298"); // make sure stats don't reference native or off-heap data - assertBuffersAreRetainable(sstable.getSSTableMetadata().minClusteringValues); - assertBuffersAreRetainable(sstable.getSSTableMetadata().maxClusteringValues); + assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.start().getBufferArray())); + assertBuffersAreRetainable(Arrays.asList(sstable.getSSTableMetadata().coveredClustering.end().getBufferArray())); } } @@ -278,54 +279,4 @@ public class SSTableMetadataTest } } - /*@Test - public void testLegacyCounterShardTracking() - { - ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore("Counter1"); - - // A cell with all shards - CounterContext.ContextState state = CounterContext.ContextState.allocate(1, 1, 1); - state.writeGlobal(CounterId.fromInt(1), 1L, 1L); - state.writeLocal(CounterId.fromInt(2), 1L, 1L); - state.writeRemote(CounterId.fromInt(3), 1L, 1L); - - ColumnFamily cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); - new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - Util.flush(cfs); - assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); - cfs.truncateBlocking(); - - // A cell with global and remote shards - state = CounterContext.ContextState.allocate(0, 1, 1); - state.writeLocal(CounterId.fromInt(2), 1L, 1L); - state.writeRemote(CounterId.fromInt(3), 1L, 1L); - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); - new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - Util.flush(cfs); - assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); - cfs.truncateBlocking(); - - // A cell with global and local shards - state = CounterContext.ContextState.allocate(1, 1, 0); - state.writeGlobal(CounterId.fromInt(1), 1L, 1L); - state.writeLocal(CounterId.fromInt(2), 1L, 1L); - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); - new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - Util.flush(cfs); - assertTrue(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); - cfs.truncateBlocking(); - - // A cell with global only - state = CounterContext.ContextState.allocate(1, 0, 0); - state.writeGlobal(CounterId.fromInt(1), 1L, 1L); - cells = ArrayBackedSortedColumns.factory.create(cfs.metadata); - cells.addColumn(new BufferCounterCell(cellname("col"), state.context, 1L, Long.MIN_VALUE)); - new Mutation(Util.dk("k").getKey(), cells).applyUnsafe(); - Util.flush(cfs); - assertFalse(cfs.getLiveSSTables().iterator().next().getSSTableMetadata().hasLegacyCounterShards); - cfs.truncateBlocking(); - } */ -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 9b5ff62013..1e85f34106 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -20,6 +20,7 @@ package org.apache.cassandra.io.sstable.metadata; import org.apache.cassandra.io.sstable.SequenceBasedSSTableId; import org.apache.cassandra.io.util.*; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; @@ -40,6 +41,7 @@ import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.Version; import org.apache.cassandra.io.sstable.format.big.BigFormat; +import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.Throwables; import static org.junit.Assert.assertEquals; @@ -124,7 +126,9 @@ public class MetadataSerializerTest String partitioner = RandomPartitioner.class.getCanonicalName(); double bfFpChance = 0.1; - return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList())); + ByteBuffer first = ByteBufferUtil.bytes(1); + ByteBuffer last = ByteBufferUtil.bytes(2); + return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, false, SerializationHeader.make(cfm, Collections.emptyList()), first, last); } private void testVersions(String... versions) throws Throwable @@ -159,7 +163,7 @@ public class MetadataSerializerTest @Test public void testNVersions() throws Throwable { - testVersions("na", "nb"); + testVersions("na", "nb", "nc"); } public void testOldReadsNew(String oldV, String newV) throws IOException @@ -193,13 +197,34 @@ public class MetadataSerializerTest public void pendingRepairCompatibility() { Arrays.asList("ma", "mb", "mc", "md", "me").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPendingRepair())); - Arrays.asList("na", "nb").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPendingRepair())); + Arrays.asList("na", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPendingRepair())); } @Test public void originatingHostCompatibility() { Arrays.asList("ma", "mb", "mc", "md", "na").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasOriginatingHostId())); - Arrays.asList("me", "nb").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasOriginatingHostId())); + Arrays.asList("me", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasOriginatingHostId())); + } + + @Test + public void improvedMinMaxCompatibility() + { + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasImprovedMinMax())); + Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasImprovedMinMax())); + } + + @Test + public void legacyMinMaxCompatiblity() + { + Arrays.asList("oa").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasLegacyMinMax())); + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb", "nc").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasLegacyMinMax())); + } + + @Test + public void partitionLevelDeletionPresenceMarkerCompatibility() + { + Arrays.asList("ma", "mb", "mc", "md", "me", "na", "nb").forEach(v -> assertFalse(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); + Arrays.asList("nc", "oa").forEach(v -> assertTrue(BigFormat.instance.getVersion(v).hasPartitionLevelDeletionsPresenceMarker())); } } diff --git a/test/unit/org/apache/cassandra/schema/MockSchema.java b/test/unit/org/apache/cassandra/schema/MockSchema.java index 691abede2a..7c095649c1 100644 --- a/test/unit/org/apache/cassandra/schema/MockSchema.java +++ b/test/unit/org/apache/cassandra/schema/MockSchema.java @@ -41,6 +41,7 @@ import org.apache.cassandra.dht.Murmur3Partitioner; import org.apache.cassandra.io.sstable.Component; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.io.sstable.IndexSummary; +import org.apache.cassandra.io.sstable.SSTable; import org.apache.cassandra.io.sstable.SSTableId; import org.apache.cassandra.io.sstable.format.SSTableFormat; import org.apache.cassandra.io.sstable.format.SSTableReader; @@ -182,14 +183,18 @@ public class MockSchema SerializationHeader header = SerializationHeader.make(cfs.metadata(), Collections.emptyList()); MetadataCollector collector = new MetadataCollector(cfs.metadata().comparator); collector.update(new DeletionTime(timestamp, minLocalDeletionTime)); + + BufferDecoratedKey first = readerBounds(firstToken); + BufferDecoratedKey last = readerBounds(lastToken); + StatsMetadata metadata = (StatsMetadata) collector.sstableLevel(level) - .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header) + .finalizeMetadata(cfs.metadata().partitioner.getClass().getCanonicalName(), 0.01f, UNREPAIRED_SSTABLE, null, false, header, SSTable.getMinimalKey(first).getKey().slice(), SSTable.getMinimalKey(last).getKey().slice()) .get(MetadataType.STATS); SSTableReader reader = SSTableReader.internalOpen(descriptor, components, cfs.metadata, fileHandle.sharedCopy(), fileHandle.sharedCopy(), indexSummary.sharedCopy(), new AlwaysPresentFilter(), 1L, metadata, SSTableReader.OpenReason.NORMAL, header); - reader.first = readerBounds(firstToken); - reader.last = readerBounds(lastToken); + reader.first = first ; + reader.last = last; if (!keepRef) reader.selfRef().release(); return reader; diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java index f5cf2b639a..f544558cf1 100644 --- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java +++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java @@ -408,9 +408,10 @@ public class ByteSourceComparisonTest extends ByteSourceTestBase BiFunction decompose, boolean testLegacy) { + EnumSet skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND); for (Version v : Version.values()) - for (ClusteringPrefix.Kind k1 : ClusteringPrefix.Kind.values()) - for (ClusteringPrefix.Kind k2 : ClusteringPrefix.Kind.values()) + for (ClusteringPrefix.Kind k1 : EnumSet.complementOf(skippedKinds)) + for (ClusteringPrefix.Kind k2 : EnumSet.complementOf(skippedKinds)) { if (!testLegacy && v == Version.LEGACY) continue; @@ -473,7 +474,7 @@ public class ByteSourceComparisonTest extends ByteSourceTestBase return factory.staticClustering(); default: - throw new AssertionError(); + throw new AssertionError(k1); } } @@ -1175,4 +1176,4 @@ public class ByteSourceComparisonTest extends ByteSourceTestBase { return ByteBuffer.allocate(paddedCapacity); } -} +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java index 5a59ddfe1f..b257fb6599 100644 --- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java +++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceConversionTest.java @@ -327,7 +327,8 @@ public class ByteSourceConversionTest extends ByteSourceTestBase { ValueAccessor accessor = ByteBufferAccessor.instance; ClusteringComparator comp = new ClusteringComparator(); - for (ClusteringPrefix.Kind kind : ClusteringPrefix.Kind.values()) + EnumSet skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND); + for (ClusteringPrefix.Kind kind : EnumSet.complementOf(skippedKinds)) { if (kind.isBoundary()) continue; @@ -350,7 +351,8 @@ public class ByteSourceConversionTest extends ByteSourceTestBase BiFunction decompose) { boolean checkEquals = t1 != DecimalType.instance && t2 != DecimalType.instance; - for (ClusteringPrefix.Kind k1 : ClusteringPrefix.Kind.values()) + EnumSet skippedKinds = EnumSet.of(ClusteringPrefix.Kind.SSTABLE_LOWER_BOUND, ClusteringPrefix.Kind.SSTABLE_UPPER_BOUND); + for (ClusteringPrefix.Kind k1 : EnumSet.complementOf(skippedKinds)) { ClusteringComparator comp = new ClusteringComparator(t1, t2); V[] b = accessor.createArray(2); @@ -781,4 +783,4 @@ public class ByteSourceConversionTest extends ByteSourceTestBase { return ByteBuffer.allocate(paddedCapacity); } -} +} \ No newline at end of file