diskBoundaries;
private int locationIndex;
protected Directories.DataDirectory currentDirectory;
-
+ protected String sstableDirectoryPath;
public CompactionAwareWriter(ColumnFamilyStore cfs,
Directories directories,
ILifecycleTransaction txn,
@@ -151,9 +150,10 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
return realAppend(partition);
}
- public final File getSStableDirectory() throws IOException
+ // hot path, called per partition
+ public final String getSStableDirectoryPath() throws IOException
{
- return getDirectories().getLocationForDisk(currentDirectory);
+ return sstableDirectoryPath;
}
@Override
@@ -173,35 +173,36 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
* specific strategy has decided a new sstable is needed.
* Guaranteed to be called before the first call to realAppend.
*/
- protected void maybeSwitchWriter(DecoratedKey key)
+ public final SSTableWriter maybeSwitchWriter(DecoratedKey key)
{
- if (maybeSwitchLocation(key))
- return;
-
- if (shouldSwitchWriterInCurrentLocation(key))
- switchCompactionWriter(currentDirectory, key);
+ SSTableWriter newWriter = maybeSwitchLocation(key);
+ if (newWriter == null && shouldSwitchWriterInCurrentLocation(key))
+ {
+ newWriter = switchCompactionWriter(currentDirectory, key);
+ }
+ return newWriter;
}
/**
* Switches the file location and writer and returns true if the new key should be placed in a different data
* directory.
*/
- protected boolean maybeSwitchLocation(DecoratedKey key)
+ private SSTableWriter maybeSwitchLocation(DecoratedKey key)
{
if (diskBoundaries == null)
{
if (locationIndex < 0)
{
Directories.DataDirectory defaultLocation = getWriteDirectory(nonExpiredSSTables, getExpectedWriteSize());
- switchCompactionWriter(defaultLocation, key);
+ SSTableWriter writer = switchCompactionWriter(defaultLocation, key);
locationIndex = 0;
- return true;
+ return writer;
}
- return false;
+ return null;
}
if (locationIndex > -1 && key.compareTo(diskBoundaries.get(locationIndex)) < 0)
- return false;
+ return null;
int prevIdx = locationIndex;
while (locationIndex == -1 || key.compareTo(diskBoundaries.get(locationIndex)) > 0)
@@ -209,8 +210,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
Directories.DataDirectory newLocation = locations.get(locationIndex);
if (prevIdx >= 0)
logger.debug("Switching write location from {} to {}", locations.get(prevIdx), newLocation);
- switchCompactionWriter(newLocation, key);
- return true;
+ return switchCompactionWriter(newLocation, key);
}
/**
@@ -223,14 +223,14 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
* Implementations of this method should finish the current sstable writer and start writing to this directory.
*
* Called once before starting to append and then whenever we see a need to start writing to another directory.
- *
- * @param directory
- * @param nextKey
*/
- protected void switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+ protected SSTableWriter switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
{
currentDirectory = directory;
- sstableWriter.switchWriter(sstableWriter(directory, nextKey));
+ sstableDirectoryPath = getDirectories().getLocationForDisk(currentDirectory).path();
+ SSTableWriter newWriter = sstableWriter(directory, nextKey);
+ sstableWriter.switchWriter(newWriter);
+ return newWriter;
}
protected SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index d0fb70587c..455817aaff 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -26,6 +26,7 @@ import org.apache.cassandra.db.compaction.LeveledManifest;
import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
public class MajorLeveledCompactionWriter extends CompactionAwareWriter
{
@@ -87,12 +88,12 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
}
@Override
- public void switchCompactionWriter(Directories.DataDirectory location, DecoratedKey nextKey)
+ public SSTableWriter switchCompactionWriter(Directories.DataDirectory location, DecoratedKey nextKey)
{
averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1));
partitionsWritten = 0;
sstablesWritten = 0;
- super.switchCompactionWriter(location, nextKey);
+ return super.switchCompactionWriter(location, nextKey);
}
protected int sstableLevel()
diff --git a/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java b/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java
index 6203ee2bf5..9f0d260818 100644
--- a/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java
+++ b/src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java
@@ -53,14 +53,14 @@ public class MaxThreshold extends Threshold
}
@Override
- protected long failValue(ClientState state)
+ public long failValue(ClientState state)
{
long failValue = failThreshold.applyAsLong(state);
return failValue <= 0 ? Long.MAX_VALUE : failValue;
}
@Override
- protected long warnValue(ClientState state)
+ public long warnValue(ClientState state)
{
long warnValue = warnThreshold.applyAsLong(state);
return warnValue <= 0 ? Long.MAX_VALUE : warnValue;
diff --git a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
index fd7880e367..aec80ba1cd 100644
--- a/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
+++ b/src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java
@@ -288,7 +288,7 @@ public abstract class UnfilteredPartitionIterators
}
/**
- * Digests the the provided iterator.
+ * Digests the provided iterator.
*
* Caller must close the provided iterator.
*
diff --git a/src/java/org/apache/cassandra/db/rows/BTreeRow.java b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
index bf6b5b5061..8bccda8aa7 100644
--- a/src/java/org/apache/cassandra/db/rows/BTreeRow.java
+++ b/src/java/org/apache/cassandra/db/rows/BTreeRow.java
@@ -80,13 +80,15 @@ public class BTreeRow extends AbstractRow
private static final Comparator COLUMN_COMPARATOR = (cd1, cd2) -> cd1.column.compareTo(cd2.column);
- // We need to filter the tombstones of a row on every read (twice in fact: first to remove purgeable tombstone, and then after reconciliation to remove
- // all tombstone since we don't return them to the client) as well as on compaction. But it's likely that many rows won't have any tombstone at all, so
- // we want to speed up that case by not having to iterate/copy the row in this case. We could keep a single boolean telling us if we have tombstones,
- // but that doesn't work for expiring columns. So instead we keep the deletion time for the first thing in the row to be deleted. This allow at any given
- // time to know if we have any deleted information or not. If we any "true" tombstone (i.e. not an expiring cell), this value will be forced to
- // Long.MIN_VALUE, but if we don't and have expiring cells, this will the time at which the first expiring cell expires. If we have no tombstones and
- // no expiring cells, this will be Cell.MAX_DELETION_TIME;
+ // We need to filter the tombstones of a row on every read (twice in fact: first to remove purgeable tombstone,
+ // and then after reconciliation to remove all tombstone since we don't return them to the client) as well as on
+ // compaction. But it's likely that many rows won't have any tombstone at all, so we want to speed up that case
+ // by not having to iterate/copy the row in this case. We could keep a single boolean telling us if we have
+ // tombstones, but that doesn't work for expiring columns. So instead we keep the deletion time for the first
+ // thing in the row to be deleted. This allows at any given time to know if we have any deleted information or not.
+ // If we have any "true" tombstone (i.e. not an expiring cell), this value will be forced to Long.MIN_VALUE,
+ // but if we don't and have expiring cells, this will the time at which the first expiring cell expires. If we
+ // have no tombstones and no expiring cells, this will be Cell.MAX_DELETION_TIME;
private final long minLocalDeletionTime;
private BTreeRow(Clustering clustering,
diff --git a/src/java/org/apache/cassandra/db/rows/Cell.java b/src/java/org/apache/cassandra/db/rows/Cell.java
index d3d0eb0416..d4c015b0cc 100644
--- a/src/java/org/apache/cassandra/db/rows/Cell.java
+++ b/src/java/org/apache/cassandra/db/rows/Cell.java
@@ -258,7 +258,7 @@ public abstract class Cell extends ColumnData
* where not all field are always present (in fact, only the [ flags ] are guaranteed to be present). The fields have the following
* meaning:
* - [ flags ] is the cell flags. It is a byte for which each bit represents a flag whose meaning is explained below (*_MASK constants)
- * - [ timestamp ] is the cell timestamp. Present unless the cell has the USE_TIMESTAMP_MASK.
+ * - [ timestamp ] is the cell timestamp. Present unless the cell has the USE_ROW_TIMESTAMP_MASK.
* - [ deletion time]: the local deletion time for the cell. Present if either the cell is deleted (IS_DELETED_MASK)
* or it is expiring (IS_EXPIRING_MASK) but doesn't have the USE_ROW_TTL_MASK.
* - [ ttl ]: the ttl for the cell. Present if the row is expiring (IS_EXPIRING_MASK) but doesn't have the
@@ -269,13 +269,13 @@ public abstract class Cell extends ColumnData
* - [ value ]: the cell value, unless it has the HAS_EMPTY_VALUE_MASK.
* - [ path ]: the cell path if the column this is a cell of is complex.
*/
- static class Serializer
+ public static class Serializer
{
- private final static int IS_DELETED_MASK = 0x01; // Whether the cell is a tombstone or not.
- private final static int IS_EXPIRING_MASK = 0x02; // Whether the cell is expiring.
- private final static int HAS_EMPTY_VALUE_MASK = 0x04; // Wether the cell has an empty value. This will be the case for tombstone in particular.
- private final static int USE_ROW_TIMESTAMP_MASK = 0x08; // Wether the cell has the same timestamp than the row this is a cell of.
- private final static int USE_ROW_TTL_MASK = 0x10; // Wether the cell has the same ttl than the row this is a cell of.
+ public final static int IS_DELETED_MASK = 0x01; // Whether the cell is a tombstone or not.
+ public final static int IS_EXPIRING_MASK = 0x02; // Whether the cell is expiring.
+ public final static int HAS_EMPTY_VALUE_MASK = 0x04; // Wether the cell has an empty value. This will be the case for tombstone in particular.
+ public final static int USE_ROW_TIMESTAMP_MASK = 0x08; // Wether the cell has the same timestamp than the row this is a cell of.
+ public final static int USE_ROW_TTL_MASK = 0x10; // Wether the cell has the same ttl than the row this is a cell of.
public void serialize(Cell cell, ColumnMetadata column, DataOutputPlus out, LivenessInfo rowLiveness, SerializationHeader header) throws IOException
{
@@ -319,11 +319,11 @@ public abstract class Cell extends ColumnData
public Cell deserialize(DataInputPlus in, LivenessInfo rowLiveness, ColumnMetadata column, SerializationHeader header, DeserializationHelper helper, ValueAccessor accessor) throws IOException
{
int flags = in.readUnsignedByte();
- boolean hasValue = (flags & HAS_EMPTY_VALUE_MASK) == 0;
- boolean isDeleted = (flags & IS_DELETED_MASK) != 0;
- boolean isExpiring = (flags & IS_EXPIRING_MASK) != 0;
- boolean useRowTimestamp = (flags & USE_ROW_TIMESTAMP_MASK) != 0;
- boolean useRowTTL = (flags & USE_ROW_TTL_MASK) != 0;
+ boolean hasValue = hasValue(flags);
+ boolean isDeleted = isDeleted(flags);
+ boolean isExpiring = isExpiring(flags);
+ boolean useRowTimestamp = useRowTimestamp(flags);
+ boolean useRowTTL = useRowTTL(flags);
long timestamp = useRowTimestamp ? rowLiveness.timestamp() : header.readTimestamp(in);
@@ -390,11 +390,11 @@ public abstract class Cell extends ColumnData
public boolean skip(DataInputPlus in, ColumnMetadata column, SerializationHeader header) throws IOException
{
int flags = in.readUnsignedByte();
- boolean hasValue = (flags & HAS_EMPTY_VALUE_MASK) == 0;
- boolean isDeleted = (flags & IS_DELETED_MASK) != 0;
- boolean isExpiring = (flags & IS_EXPIRING_MASK) != 0;
- boolean useRowTimestamp = (flags & USE_ROW_TIMESTAMP_MASK) != 0;
- boolean useRowTTL = (flags & USE_ROW_TTL_MASK) != 0;
+ boolean hasValue = hasValue(flags);
+ boolean isDeleted = isDeleted(flags);
+ boolean isExpiring = isExpiring(flags);
+ boolean useRowTimestamp = useRowTimestamp(flags);
+ boolean useRowTTL = useRowTTL(flags);
if (!useRowTimestamp)
header.skipTimestamp(in);
@@ -413,5 +413,30 @@ public abstract class Cell extends ColumnData
return true;
}
+
+ public static boolean useRowTTL(int cellFlags)
+ {
+ return (cellFlags & USE_ROW_TTL_MASK) != 0;
+ }
+
+ public static boolean useRowTimestamp(int cellFlags)
+ {
+ return (cellFlags & USE_ROW_TIMESTAMP_MASK) != 0;
+ }
+
+ public static boolean isExpiring(int cellFlags)
+ {
+ return (cellFlags & IS_EXPIRING_MASK) != 0;
+ }
+
+ public static boolean isDeleted(int cellFlags)
+ {
+ return (cellFlags & IS_DELETED_MASK) != 0;
+ }
+
+ public static boolean hasValue(int cellFlags)
+ {
+ return (cellFlags & HAS_EMPTY_VALUE_MASK) == 0;
+ }
}
}
diff --git a/src/java/org/apache/cassandra/db/rows/Cells.java b/src/java/org/apache/cassandra/db/rows/Cells.java
index 48331a73a6..621a91d19c 100644
--- a/src/java/org/apache/cassandra/db/rows/Cells.java
+++ b/src/java/org/apache/cassandra/db/rows/Cells.java
@@ -36,7 +36,7 @@ public abstract class Cells
private Cells() {}
/**
- * Collect statistics ont a given cell.
+ * Collect statistics on a given cell.
*
* @param cell the cell for which to collect stats.
* @param collector the stats collector.
diff --git a/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java b/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java
index 2db62044fe..377880ae1c 100644
--- a/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java
+++ b/src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java
@@ -159,7 +159,7 @@ public interface RangeTombstoneMarker extends Unfiltered, IMeasurableMemory
return DeletionTime.LIVE;
DeletionTime biggestDeletionTime = openMarkers[biggestOpenMarker];
- // it's only open in the merged iterator if it doesn't supersedes the partition level deletion
+ // it's only open in the merged iterator if it doesn't supersede the partition level deletion
return !biggestDeletionTime.supersedes(partitionDeletion) ? DeletionTime.LIVE : biggestDeletionTime;
}
@@ -172,7 +172,7 @@ public interface RangeTombstoneMarker extends Unfiltered, IMeasurableMemory
continue;
// Note that we can have boundaries that are both open and close, but in that case all we care about
- // is what it the open deletion after the marker, so we favor the opening part in this case.
+ // is what is the open deletion after the marker, so we favor the opening part in this case.
if (marker.isOpen(reversed))
openMarkers[i] = marker.openDeletionTime(reversed);
else
@@ -192,7 +192,7 @@ public interface RangeTombstoneMarker extends Unfiltered, IMeasurableMemory
{
DeletionTime openMarker = currentOpenDeletionTimeInMerged();
// We only have an open marker in the merged stream if it's not shadowed by the partition deletion (which can be LIVE itself), so
- // if have an open marker, we know it's the "active" deletion for the merged stream.
+ // if we have an open marker, we know it's the "active" deletion for the merged stream.
return openMarker.isLive() ? partitionDeletion : openMarker;
}
}
diff --git a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
index 2fcba1bce8..ca1edfdbaf 100644
--- a/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
+++ b/src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java
@@ -99,19 +99,19 @@ public class UnfilteredSerializer
/*
* Unfiltered flags constants.
*/
- private final static int END_OF_PARTITION = 0x01; // Signal the end of the partition. Nothing follows a field with that flag.
- private final static int IS_MARKER = 0x02; // Whether the encoded unfiltered is a marker or a row. All following markers applies only to rows.
- private final static int HAS_TIMESTAMP = 0x04; // Whether the encoded row has a timestamp (i.e. if row.partitionKeyLivenessInfo().hasTimestamp() == true).
- private final static int HAS_TTL = 0x08; // Whether the encoded row has some expiration info (i.e. if row.partitionKeyLivenessInfo().hasTTL() == true).
- private final static int HAS_DELETION = 0x10; // Whether the encoded row has some deletion info.
- private final static int HAS_ALL_COLUMNS = 0x20; // Whether the encoded row has all of the columns from the header present.
- private final static int HAS_COMPLEX_DELETION = 0x40; // Whether the encoded row has some complex deletion for at least one of its columns.
- private final static int EXTENSION_FLAG = 0x80; // If present, another byte is read containing the "extended flags" above.
+ public final static int END_OF_PARTITION = 0x01; // Signal the end of the partition. Nothing follows a field with that flag.
+ public final static int IS_MARKER = 0x02; // Whether the encoded unfiltered is a marker or a row. All following markers applies only to rows.
+ public final static int HAS_TIMESTAMP = 0x04; // Whether the encoded row has a timestamp (i.e. if row.partitionKeyLivenessInfo().hasTimestamp() == true).
+ public final static int HAS_TTL = 0x08; // Whether the encoded row has some expiration info (i.e. if row.partitionKeyLivenessInfo().hasTTL() == true).
+ public final static int HAS_DELETION = 0x10; // Whether the encoded row has some deletion info.
+ public final static int HAS_ALL_COLUMNS = 0x20; // Whether the encoded row has all of the columns from the header present.
+ public final static int HAS_COMPLEX_DELETION = 0x40; // Whether the encoded row has some complex deletion for at least one of its columns.
+ public final static int EXTENSION_FLAG = 0x80; // If present, another byte is read containing the "extended flags" above.
/*
* Extended flags
*/
- private final static int IS_STATIC = 0x01; // Whether the encoded row is a static. If there is no extended flag, the row is assumed not static.
+ public final static int IS_STATIC = 0x01; // Whether the encoded row is a static. If there is no extended flag, the row is assumed not static.
/**
* A shadowable tombstone cannot replace a previous row deletion otherwise it could resurrect a
* previously deleted cell not updated by a subsequent update, SEE CASSANDRA-11500
@@ -119,7 +119,7 @@ public class UnfilteredSerializer
* @deprecated See CASSANDRA-11500
*/
@Deprecated(since = "4.0")
- private final static int HAS_SHADOWABLE_DELETION = 0x02; // Whether the row deletion is shadowable. If there is no extended flag (or no row deletion), the deletion is assumed not shadowable.
+ public final static int HAS_SHADOWABLE_DELETION = 0x02; // Whether the row deletion is shadowable. If there is no extended flag (or no row deletion), the deletion is assumed not shadowable.
public void serialize(Unfiltered unfiltered, SerializationHelper helper, DataOutputPlus out, int version)
throws IOException
@@ -220,14 +220,14 @@ public class UnfilteredSerializer
LivenessInfo pkLiveness = row.primaryKeyLivenessInfo();
Row.Deletion deletion = row.deletion();
- if ((flags & HAS_TIMESTAMP) != 0)
+ if (hasTimestamp(flags))
header.writeTimestamp(pkLiveness.timestamp(), out);
- if ((flags & HAS_TTL) != 0)
+ if (hasTTL(flags))
{
header.writeTTL(pkLiveness.ttl(), out);
header.writeLocalDeletionTime(pkLiveness.localExpirationTime(), out);
}
- if ((flags & HAS_DELETION) != 0)
+ if (hasDeletion(flags))
header.writeDeletionTime(deletion.time(), out);
if ((flags & HAS_ALL_COLUMNS) == 0)
@@ -251,7 +251,7 @@ public class UnfilteredSerializer
if (cd.column.isSimple())
Cell.serializer.serialize((Cell) cd, column, out, pkLiveness, header);
else
- writeComplexColumn((ComplexColumnData) cd, column, (flags & HAS_COMPLEX_DELETION) != 0, pkLiveness, header, out);
+ writeComplexColumn((ComplexColumnData) cd, column, hasComplexDeletion(flags), pkLiveness, header, out);
}
catch (IOException e)
{
@@ -412,7 +412,7 @@ public class UnfilteredSerializer
public void writeEndOfPartition(DataOutputPlus out) throws IOException
{
- out.writeByte((byte)1);
+ out.writeByte((byte)END_OF_PARTITION);
}
public long serializedSizeEndOfPartition()
@@ -502,12 +502,12 @@ public class UnfilteredSerializer
else
{
assert !isStatic(extendedFlags); // deserializeStaticRow should be used for that.
- if ((flags & HAS_DELETION) != 0)
+ if (hasDeletion(flags))
{
assert header.isForSSTable();
- boolean hasTimestamp = (flags & HAS_TIMESTAMP) != 0;
- boolean hasTTL = (flags & HAS_TTL) != 0;
- boolean deletionIsShadowable = (extendedFlags & HAS_SHADOWABLE_DELETION) != 0;
+ boolean hasTimestamp = hasTimestamp(flags);
+ boolean hasTTL = hasTTL(flags);
+ boolean deletionIsShadowable = deletionIsShadowable(extendedFlags);
Clustering clustering = Clustering.serializer.deserialize(in, helper.version, header.clusteringTypes());
long nextPosition = in.readUnsignedVInt() + in.getFilePointer();
in.readUnsignedVInt(); // skip previous unfiltered size
@@ -572,12 +572,12 @@ public class UnfilteredSerializer
try
{
boolean isStatic = isStatic(extendedFlags);
- boolean hasTimestamp = (flags & HAS_TIMESTAMP) != 0;
- boolean hasTTL = (flags & HAS_TTL) != 0;
- boolean hasDeletion = (flags & HAS_DELETION) != 0;
- boolean deletionIsShadowable = (extendedFlags & HAS_SHADOWABLE_DELETION) != 0;
- boolean hasComplexDeletion = (flags & HAS_COMPLEX_DELETION) != 0;
- boolean hasAllColumns = (flags & HAS_ALL_COLUMNS) != 0;
+ boolean hasTimestamp = hasTimestamp(flags);
+ boolean hasTTL = hasTTL(flags);
+ boolean hasDeletion = hasDeletion(flags);
+ boolean deletionIsShadowable = deletionIsShadowable(extendedFlags);
+ boolean hasComplexDeletion = hasComplexDeletion(flags);
+ boolean hasAllColumns = hasAllColumns(flags);
Columns headerColumns = header.columns(isStatic);
if (header.isForSSTable())
@@ -734,7 +734,17 @@ public class UnfilteredSerializer
public static Unfiltered.Kind kind(int flags)
{
- return (flags & IS_MARKER) != 0 ? Unfiltered.Kind.RANGE_TOMBSTONE_MARKER : Unfiltered.Kind.ROW;
+ return isTombstoneMarker(flags) ? Unfiltered.Kind.RANGE_TOMBSTONE_MARKER : Unfiltered.Kind.ROW;
+ }
+
+ public static boolean isTombstoneMarker(int flags)
+ {
+ return (flags & IS_MARKER) != 0;
+ }
+
+ public static boolean isRow(int flags)
+ {
+ return (flags & IS_MARKER) == 0;
}
public static boolean isStatic(int extendedFlags)
@@ -742,7 +752,12 @@ public class UnfilteredSerializer
return (extendedFlags & IS_STATIC) != 0;
}
- private static boolean isExtended(int flags)
+ public static boolean deletionIsShadowable(int extendedFlags)
+ {
+ return (extendedFlags & HAS_SHADOWABLE_DELETION) != 0;
+ }
+
+ public static boolean isExtended(int flags)
{
return (flags & EXTENSION_FLAG) != 0;
}
@@ -756,4 +771,29 @@ public class UnfilteredSerializer
{
return row.isStatic() || row.deletion().isShadowable();
}
+
+ public static boolean hasTTL(int flags)
+ {
+ return (flags & HAS_TTL) != 0;
+ }
+
+ public static boolean hasTimestamp(int flags)
+ {
+ return (flags & HAS_TIMESTAMP) != 0;
+ }
+
+ public static boolean hasAllColumns(int flags)
+ {
+ return (flags & HAS_ALL_COLUMNS) != 0;
+ }
+
+ public static boolean hasComplexDeletion(int flags)
+ {
+ return (flags & HAS_COMPLEX_DELETION) != 0;
+ }
+
+ public static boolean hasDeletion(int flags)
+ {
+ return (flags & HAS_DELETION) != 0;
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/ComparableObjectToken.java b/src/java/org/apache/cassandra/dht/ComparableObjectToken.java
index 8aada75663..040e54dbdd 100644
--- a/src/java/org/apache/cassandra/dht/ComparableObjectToken.java
+++ b/src/java/org/apache/cassandra/dht/ComparableObjectToken.java
@@ -21,7 +21,7 @@ abstract class ComparableObjectToken> extends Token
{
private static final long serialVersionUID = 1L;
- final C token; // Package-private to allow access from subtypes, which should all reside in the dht package.
+ C token; // Package-private to allow access from subtypes, which should all reside in the dht package.
protected ComparableObjectToken(C token)
{
diff --git a/src/java/org/apache/cassandra/dht/IPartitioner.java b/src/java/org/apache/cassandra/dht/IPartitioner.java
index af2d9051b6..ef6d2b23fb 100644
--- a/src/java/org/apache/cassandra/dht/IPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/IPartitioner.java
@@ -41,6 +41,19 @@ public interface IPartitioner
return DatabaseDescriptor.getPartitioner();
}
+ /**
+ * @return a new instance of a reusable key
+ */
+ default ReusableDecoratedKey createReusableKey(int initialSize)
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ default boolean supportsReusableKeys()
+ {
+ return false;
+ }
+
/**
* Transform key to object representation of the on-disk format.
*
diff --git a/src/java/org/apache/cassandra/dht/LocalPartitioner.java b/src/java/org/apache/cassandra/dht/LocalPartitioner.java
index 1fec57065a..2609402af7 100644
--- a/src/java/org/apache/cassandra/dht/LocalPartitioner.java
+++ b/src/java/org/apache/cassandra/dht/LocalPartitioner.java
@@ -243,4 +243,38 @@ public class LocalPartitioner implements IPartitioner
{
return AccordBytesSplitter::new;
}
+
+ private class ReusableLocalToken extends LocalToken
+ {
+ void setToken(ByteBuffer token)
+ {
+ this.token = token;
+ }
+ }
+
+ private class ReusableLocalKey extends ReusableDecoratedKey
+ {
+ public ReusableLocalKey(int initialSize)
+ {
+ super(new ReusableLocalToken(), initialSize);
+ }
+
+ @Override
+ protected void recalculateToken()
+ {
+ ((ReusableLocalToken)getToken()).setToken(key);
+ }
+ }
+
+ @Override
+ public ReusableDecoratedKey createReusableKey(int initialSize)
+ {
+ return new ReusableLocalKey(initialSize);
+ }
+
+ @Override
+ public boolean supportsReusableKeys()
+ {
+ return true;
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
index 74a605946a..e76a7df8ad 100644
--- a/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
+++ b/src/java/org/apache/cassandra/dht/Murmur3Partitioner.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -35,6 +36,7 @@ import com.google.common.primitives.Longs;
import accord.primitives.Ranges;
import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
import org.apache.cassandra.db.PreHashedDecoratedKey;
import org.apache.cassandra.db.TypeSizes;
import org.apache.cassandra.db.marshal.AbstractType;
@@ -177,7 +179,7 @@ public class Murmur3Partitioner implements IPartitioner
{
static final long serialVersionUID = -5833580143318243006L;
- public final long token;
+ long token;
public LongToken(long token)
{
@@ -320,6 +322,18 @@ public class Murmur3Partitioner implements IPartitioner
return new LongToken(normalize(hash[0]));
}
+ private long calculateTokenValue(ByteBuffer key, long[] hash)
+ {
+ if (key.remaining() == 0)
+ {
+ hash[0] = MINIMUM.token;
+ hash[1] = 0;
+ return MINIMUM.token;
+ }
+ populateHash(key, hash);
+ return normalize(hash[0]);
+ }
+
@Override
public boolean isFixedLength()
{
@@ -386,10 +400,15 @@ public class Murmur3Partitioner implements IPartitioner
private long[] getHash(ByteBuffer key)
{
long[] hash = new long[2];
- MurmurHash.hash3_x64_128(key, key.position(), key.remaining(), 0, hash);
+ populateHash(key, hash);
return hash;
}
+ private void populateHash(ByteBuffer key, long[] hash)
+ {
+ MurmurHash.hash3_x64_128(key, key.position(), key.remaining(), 0, hash);
+ }
+
public LongToken getRandomToken()
{
return getRandomToken(ThreadLocalRandom.current());
@@ -565,4 +584,79 @@ public class Murmur3Partitioner implements IPartitioner
{
return ignore -> splitter;
}
+
+ private static class ReusableLongToken extends LongToken
+ {
+ public ReusableLongToken()
+ {
+ super(MINIMUM.token);
+ }
+
+ void setToken(long token)
+ {
+ this.token = token;
+ }
+ }
+
+ private static class ReusableMurmurKey extends ReusableDecoratedKey
+ {
+ protected final long[] hash = new long[2];
+ private long tokenValue = MINIMUM.token;
+ ReusableMurmurKey(int initialSize)
+ {
+ super(new ReusableLongToken(), initialSize);
+ }
+
+ protected void recalculateToken()
+ {
+ Token token = getToken();
+ ((ReusableLongToken) token).setToken(Murmur3Partitioner.instance.calculateTokenValue(key, hash));
+ tokenValue = token.getLongValue();
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ return (obj instanceof ReusableMurmurKey) ? equals((ReusableMurmurKey) obj) : super.equals(obj);
+ }
+
+ public boolean equals(ReusableMurmurKey obj)
+ {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+
+ if (tokenValue != obj.tokenValue)
+ return false;
+ return Arrays.equals(keyBytes, 0, keyLength, obj.keyBytes, 0, obj.getKeyLength()); // we compare faster than BB.equals for array backed BB
+ }
+
+ @Override
+ public int compareTo(PartitionPosition pos)
+ {
+ return (pos instanceof ReusableMurmurKey) ? compareTo((ReusableMurmurKey) pos) : super.compareTo(pos);
+ }
+
+ public int compareTo(ReusableMurmurKey obj)
+ {
+ if (this == obj)
+ return 0;
+
+ int cmp = Long.compare(tokenValue, obj.tokenValue);
+ return cmp == 0 ? Arrays.compareUnsigned(keyBytes, 0, keyLength, obj.keyBytes, 0, obj.keyLength) : cmp;
+ }
+ }
+
+ @Override
+ public ReusableDecoratedKey createReusableKey(int initialSize)
+ {
+ return new ReusableMurmurKey(initialSize);
+ }
+
+ @Override
+ public boolean supportsReusableKeys()
+ {
+ return true;
+ }
}
diff --git a/src/java/org/apache/cassandra/dht/ReusableDecoratedKey.java b/src/java/org/apache/cassandra/dht/ReusableDecoratedKey.java
new file mode 100644
index 0000000000..7a9723e17c
--- /dev/null
+++ b/src/java/org/apache/cassandra/dht/ReusableDecoratedKey.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public abstract class ReusableDecoratedKey extends BufferDecoratedKey
+{
+ protected int keyLength = 0;
+ protected byte[] keyBytes;
+
+ public ReusableDecoratedKey(Token token, int initialSize)
+ {
+ super(token, ByteBuffer.wrap(new byte[initialSize]).limit(0));
+ keyBytes = key.array();
+ }
+
+ abstract void recalculateToken();
+
+ public void copyKey(ByteBuffer newKey)
+ {
+ int length = newKey.remaining();
+ maybeResizeKey(length);
+ ByteBufferUtil.copyBytes(newKey, newKey.position(), key, 0, length);
+ keyLength = length;
+ key.limit(length);
+ recalculateToken();
+ }
+
+ /** WARNING: retains ref to external buffer */
+ public void shadowKey(ByteBuffer newKey, byte[] newKeyBytes, int newKeyLength)
+ {
+ key = newKey;
+ keyBytes = newKeyBytes;
+ keyLength = newKeyLength;
+ recalculateToken();
+ }
+
+ /** WARNING: retains ref to external buffer */
+ public void shadowKey(int newKeyLength)
+ {
+ keyLength = newKeyLength;
+ recalculateToken();
+ }
+
+ @Override
+ public int getKeyLength()
+ {
+ return keyLength;
+ }
+
+ public byte[] keyBytes()
+ {
+ return keyBytes;
+ }
+
+ public void reset()
+ {
+ keyLength = 0;
+ key.limit(0);
+ recalculateToken();
+ }
+
+ private void maybeResizeKey(int length)
+ {
+ int capacity = keyBytes.length;
+ if (capacity > length)
+ return;
+ keyBytes = new byte[Math.max(length, capacity * 2)];
+ key = ByteBuffer.wrap(keyBytes);
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
index e6fbad882e..a320684dd8 100644
--- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java
@@ -500,7 +500,7 @@ public abstract class AbstractSSTableIterator
{
// We use a same reasoning as in handlePreSliceData regarding the strictness of the inequality below.
// We want to exclude deserialized unfiltered equal to end, because 1) we won't miss any rows since those
- // woudn't be equal to a slice bound and 2) a end bound can be equal to a start bound
+ // wouldn't be equal to a slice bound and 2) a end bound can be equal to a start bound
// (EXCL_END(x) == INCL_START(x) for instance) and in that case we don't want to return start bound because
// it's fundamentally excluded. And if the bound is a end (for a range tombstone), it means it's exactly
// our slice end, but in that case we will properly close the range tombstone anyway as part of our "close
diff --git a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
index 67d13c2327..2487e66fd2 100644
--- a/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java
@@ -47,7 +47,7 @@ import org.apache.cassandra.service.ActiveRepairService;
/**
* Base class for the sstable writers used by CQLSSTableWriter.
*/
-abstract class AbstractSSTableSimpleWriter implements Closeable
+public abstract class AbstractSSTableSimpleWriter implements Closeable
{
protected final File directory;
protected final TableMetadataRef metadata;
@@ -172,7 +172,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
}
}
- PartitionUpdate.Builder getUpdateFor(ByteBuffer key) throws IOException
+ public PartitionUpdate.Builder getUpdateFor(ByteBuffer key) throws IOException
{
return getUpdateFor(metadata.get().partitioner.decorateKey(key));
}
diff --git a/src/java/org/apache/cassandra/io/sstable/ClusteringDescriptor.java b/src/java/org/apache/cassandra/io/sstable/ClusteringDescriptor.java
new file mode 100644
index 0000000000..2afbc4db77
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/ClusteringDescriptor.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.readUnfilteredClustering;
+
+public class ClusteringDescriptor extends ResizableByteBuffer
+{
+ private static final byte MAX_START_KIND = (byte) ClusteringBound.MAX_START.kind().ordinal();
+ private static final byte MIN_END_KIND = (byte) ClusteringBound.MIN_END.kind().ordinal();
+
+ static final byte EXCL_END_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.EXCL_END_BOUND.ordinal();
+ static final byte INCL_START_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.INCL_START_BOUND.ordinal();
+ static final byte INCL_END_EXCL_START_BOUNDARY_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY.ordinal();
+
+ static final byte STATIC_CLUSTERING_KIND = (byte)ClusteringPrefix.Kind.STATIC_CLUSTERING.ordinal();
+ static final byte ROW_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.CLUSTERING.ordinal();
+
+ static final byte EXCL_END_INCL_START_BOUNDARY_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY.ordinal();
+ static final byte INCL_END_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.INCL_END_BOUND.ordinal();
+ static final byte EXCL_START_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.EXCL_START_BOUND.ordinal();
+
+ protected final AbstractType[] clusteringTypes;
+ protected ClusteringPrefix.Kind clusteringKind;
+ protected byte clusteringKindEncoded;
+ protected int clusteringColumnsBound;
+
+ public ClusteringDescriptor(AbstractType[] clusteringTypes)
+ {
+ this.clusteringTypes = clusteringTypes;
+ }
+
+ protected void loadClustering(RandomAccessReader dataReader, byte clusteringKind, int clusteringColumnsBound) throws IOException
+ {
+ set(ClusteringPrefix.Kind.values()[clusteringKind], clusteringKind, clusteringColumnsBound);
+ if (clusteringKind != STATIC_CLUSTERING_KIND)
+ readUnfilteredClustering(dataReader, clusteringTypes, this.clusteringColumnsBound, this);
+ else
+ resetBuffer();
+ }
+
+ public final ClusteringDescriptor resetMinEnd() {
+ set(MIN_END_KIND, 0);
+ resetBuffer();
+ return this;
+ }
+
+ public final ClusteringDescriptor resetMaxStart() {
+ set(MAX_START_KIND, 0);
+ resetBuffer();
+ return this;
+ }
+
+ public final boolean isMaxStart()
+ {
+ return clusteringKindEncoded == MAX_START_KIND && clusteringColumnsBound == 0;
+ }
+
+ public final void resetClustering()
+ {
+ set(ClusteringPrefix.Kind.CLUSTERING, 0);
+
+ resetBuffer();
+ }
+
+ public final void copy(ClusteringDescriptor newClustering)
+ {
+ set(newClustering.clusteringKind, newClustering.clusteringColumnsBound());
+ overwrite(newClustering.clusteringBytes(), newClustering.clusteringLength());
+ }
+
+ private void set(ClusteringPrefix.Kind clusteringKind, int clusteringColumnsBound) {
+ set(clusteringKind, (byte) clusteringKind.ordinal(), clusteringColumnsBound);
+ }
+
+ private void set(byte clusteringKindEncoded, int clusteringColumnsBound) {
+ set(ClusteringPrefix.Kind.values()[clusteringKindEncoded], clusteringKindEncoded, clusteringColumnsBound);
+ }
+
+ private void set(ClusteringPrefix.Kind clusteringKind, byte clusteringKindEncoded, int clusteringColumnsBound)
+ {
+ this.clusteringKindEncoded = clusteringKindEncoded;
+ this.clusteringKind = clusteringKind;
+ this.clusteringColumnsBound = clusteringColumnsBound;
+ }
+
+ // Expose and rename parent data
+ public final ByteBuffer clusteringBuffer() {
+ return buffer();
+ }
+
+ public final int clusteringLength() {
+ return length();
+ }
+
+ public final byte[] clusteringBytes() {
+ return bytes();
+ }
+
+ public final AbstractType[] clusteringTypes()
+ {
+ return clusteringTypes;
+ }
+
+ public final byte clusteringKindEncoded() {
+ return clusteringKindEncoded;
+ }
+
+ public final ClusteringPrefix.Kind clusteringKind() {
+ return clusteringKind;
+ }
+
+ public final void clusteringKind(ClusteringPrefix.Kind kind)
+ {
+ clusteringKind = kind;
+ clusteringKindEncoded = (byte)kind.ordinal();
+ }
+
+ public final int clusteringColumnsBound() {
+ return clusteringColumnsBound;
+ }
+
+ public final boolean isStartBound()
+ {
+ return (clusteringKindEncoded == INCL_START_BOUND_CLUSTERING_KIND || clusteringKindEncoded == EXCL_START_BOUND_CLUSTERING_KIND);
+ }
+
+ public final boolean isEndBound()
+ {
+ return (clusteringKindEncoded == INCL_END_BOUND_CLUSTERING_KIND || clusteringKindEncoded == EXCL_END_BOUND_CLUSTERING_KIND);
+ }
+
+ public final boolean isBoundary()
+ {
+ return (clusteringKindEncoded == EXCL_END_INCL_START_BOUNDARY_CLUSTERING_KIND || clusteringKindEncoded == INCL_END_EXCL_START_BOUNDARY_CLUSTERING_KIND);
+ }
+
+ public final ClusteringPrefix toClusteringPrefix(List> clusteringTypesList) {
+ if (clusteringKindEncoded == ROW_CLUSTERING_KIND) {
+ return Clustering.serializer.deserialize(clusteringBuffer(), 0, clusteringTypesList);
+ }
+ else if (clusteringColumnsBound == 0) {
+ return ByteArrayAccessor.factory.bound(clusteringKind);
+ }
+ else {
+ byte[][] values;
+ try (DataInputBuffer buffer = new DataInputBuffer(clusteringBuffer(), true))
+ {
+ values = ClusteringPrefix.serializer.deserializeValuesWithoutSize(buffer, clusteringColumnsBound, 0, clusteringTypesList);
+ }
+ catch (IOException e)
+ {
+ throw new RuntimeException("Reading from an in-memory buffer shouldn't trigger an IOException", e);
+ }
+ return ByteArrayAccessor.factory.boundOrBoundary(clusteringKind, values);
+ }
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
index 8976ed4130..fecb6940a4 100644
--- a/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java
@@ -56,6 +56,12 @@ public class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator imp
return ImmutableSet.of(sstable);
}
+ @Override
+ public boolean isFullRange()
+ {
+ return false;
+ }
+
public long getCurrentPosition()
{
return 0;
diff --git a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
index 671bccb824..2cf6280469 100644
--- a/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java
@@ -39,6 +39,7 @@ public interface ISSTableScanner extends UnfilteredPartitionIterator
public long getCurrentPosition();
public long getBytesScanned();
public Set getBackingSSTables();
+ public boolean isFullRange();
public static void closeAllAndPropagate(Collection scanners, Throwable throwable)
{
diff --git a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
index 350a98eb9e..607d876c0f 100644
--- a/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
+++ b/src/java/org/apache/cassandra/io/sstable/IndexInfo.java
@@ -31,6 +31,7 @@ import org.apache.cassandra.io.sstable.format.Version;
import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
import org.apache.cassandra.io.util.DataInputPlus;
import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
import org.apache.cassandra.utils.ObjectSizes;
/**
@@ -87,6 +88,17 @@ public class IndexInfo
return new IndexInfo.Serializer(version, header.clusteringTypes());
}
+ public String toString(TableMetadata metadata)
+ {
+ return "IndexInfo{" +
+ "offset=" + offset +
+ ", width=" + width +
+ ", firstName=" + firstName.toString(metadata) +
+ ", lastName=" + lastName.toString(metadata) +
+ ", endOpenMarker=" + endOpenMarker +
+ '}';
+ }
+
public static class Serializer implements ISerializer
{
// This is the default index size that we use to delta-encode width when serializing so we get better vint-encoding.
diff --git a/src/java/org/apache/cassandra/io/sstable/PartitionDescriptor.java b/src/java/org/apache/cassandra/io/sstable/PartitionDescriptor.java
new file mode 100644
index 0000000000..425f2413d3
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/PartitionDescriptor.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime.ReusableDeletionTime;
+import org.apache.cassandra.dht.ReusableDecoratedKey;
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+public final class PartitionDescriptor extends ResizableByteBuffer
+{
+ private long position;
+ private final ReusableDeletionTime deletionTime = ReusableDeletionTime.live();
+ private final ReusableDecoratedKey decoratedKey;
+
+ public PartitionDescriptor(ReusableDecoratedKey decoratedKey)
+ {
+ this.decoratedKey = decoratedKey;
+ this.decoratedKey.shadowKey(buffer(), bytes(), length());
+ }
+
+ /**
+ * Loads the following structure:
+ *
+ * struct partition_header header {
+ * be16 key_length; // e.g. 8 if long
+ * byte key[key_length];
+ * struct deletion_time deletion_time {
+ * be32 local_deletion_time;
+ * be64 marked_for_delete_at;
+ * };
+ * };
+ *
+ */
+ void load(RandomAccessReader dataReader, DeletionTime.Serializer serializer) throws IOException
+ {
+ position = dataReader.getPosition();
+
+ boolean resize = loadShortLength(dataReader);
+ if (!resize)
+ {
+ decoratedKey.shadowKey(length());
+ }
+ else
+ {
+ decoratedKey.shadowKey(buffer(), bytes(), length());
+ }
+ serializer.deserialize(dataReader, deletionTime);
+ }
+
+ public long position()
+ {
+ return position;
+ }
+
+ public DeletionTime deletionTime()
+ {
+ return deletionTime;
+ }
+
+ public DecoratedKey key()
+ {
+ return decoratedKey;
+ }
+
+ public ByteBuffer keyBuffer() {
+ return super.buffer();
+ }
+
+ public int keyLength() {
+ return super.length();
+ }
+
+ public byte[] keyBytes() {
+ return super.bytes();
+ }
+
+ public void resetPartition()
+ {
+ resetBuffer();
+ decoratedKey.reset();
+ deletionTime.resetLive();
+ position = 0;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "PartitionDescriptor{ key=" + decoratedKey +
+ ", position=" + position +
+ ", deletionTime=" + deletionTime +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableCursorKeyReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableCursorKeyReader.java
new file mode 100644
index 0000000000..d0bbe73821
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableCursorKeyReader.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.Throwables;
+
+@NotThreadSafe
+public class SSTableCursorKeyReader implements AutoCloseable
+{
+ private final FileHandle indexFile;
+ private final RandomAccessReader indexFileReader;
+ private final long initialPosition;
+
+ public static class Entry extends ResizableByteBuffer
+ {
+ private long dataPosition = -1;
+ private long keyPosition = -1;
+
+ public void load(RandomAccessReader indexReader) throws IOException
+ {
+ keyPosition = indexReader.getFilePointer();
+ super.loadShortLength(indexReader);
+ if (length() != 0)
+ {
+ dataPosition = indexReader.readUnsignedVInt();
+ // skip row index entries
+ int size = indexReader.readUnsignedVInt32();
+ if (size > 0)
+ indexReader.skipBytesFully(size);
+ }
+ else
+ {
+ dataPosition = -1;
+ }
+ }
+
+ public long dataPosition()
+ {
+ return dataPosition;
+ }
+
+ public long keyPosition()
+ {
+ return keyPosition;
+ }
+
+ public ByteBuffer getKey()
+ {
+ return buffer();
+ }
+ }
+
+ private SSTableCursorKeyReader(FileHandle indexFile,
+ RandomAccessReader indexFileReader)
+ {
+ this.indexFile = indexFile;
+ this.indexFileReader = indexFileReader;
+ this.initialPosition = indexFileReader.getFilePointer();
+ }
+
+ public static SSTableCursorKeyReader create(RandomAccessReader indexFileReader) throws IOException
+ {
+ return new SSTableCursorKeyReader(null, indexFileReader);
+ }
+
+ @SuppressWarnings({ "resource", "RedundantSuppression" }) // iFile and reader are closed in the BigTableKeyReader#close method
+ public static SSTableCursorKeyReader create(FileHandle indexFile) throws IOException
+ {
+ FileHandle iFile = null;
+ RandomAccessReader reader = null;
+ try
+ {
+ iFile = indexFile.sharedCopy();
+ reader = iFile.createReader();
+ return new SSTableCursorKeyReader(iFile, reader);
+ }
+ catch (RuntimeException ex)
+ {
+ Throwables.closeNonNullAndAddSuppressed(ex, reader, iFile);
+ throw ex;
+ }
+ }
+
+ @Override
+ public void close()
+ {
+ FileUtils.closeQuietly(indexFileReader);
+ FileUtils.closeQuietly(indexFile);
+ }
+
+ public boolean advance(Entry entry) throws IOException
+ {
+ if (indexFileReader.isEOF())
+ {
+ return false;
+ }
+ entry.load(indexFileReader);
+ return true;
+ }
+
+ public boolean isExhausted()
+ {
+ return indexFileReader.isEOF();
+ }
+
+ public long indexPosition()
+ {
+ return indexFileReader.getFilePointer();
+ }
+
+ public void seek(long position) throws IOException
+ {
+ if (position > indexLength())
+ throw new IndexOutOfBoundsException("The requested position exceeds the index length");
+ indexFileReader.seek(position);
+ }
+
+ public long indexLength()
+ {
+ return indexFileReader.length();
+ }
+
+ public void reset() throws IOException
+ {
+ indexFileReader.seek(initialPosition);
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("BigTable-SSTableCursorKeyReader(%s), indexPosition=%d", indexFile.path(), indexFileReader.getFilePointer());
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableCursorReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableCursorReader.java
new file mode 100644
index 0000000000..f9dfc2ffcb
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableCursorReader.java
@@ -0,0 +1,891 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.tools.Util;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.*;
+
+public class SSTableCursorReader implements AutoCloseable
+{
+ private static final ColumnMetadata[] COLUMN_METADATA_TYPE = new ColumnMetadata[0];
+ private final boolean hasStaticColumns;
+
+ public interface State
+ {
+ /** start of file, after partition end but before EOF */
+ int PARTITION_START = 1;
+ int STATIC_ROW_START = 1 << 1;
+ int ROW_START = 1 << 2;
+ /** common to row/static row cells */
+ int CELL_HEADER_START = 1 << 3;
+ int CELL_VALUE_START = 1 << 4;
+ int CELL_END = 1 << 5;
+ int TOMBSTONE_START = 1 << 6;
+
+ /** common to rows/tombstones. Call continue(); for next unfiltered, or maybe partition end */
+ int UNFILTERED_END = 1 << 7;
+ /** at {@link UnfilteredSerializer#isEndOfPartition(int)} */
+ int PARTITION_END = 1 << 8;
+ /** EOF */
+ int DONE = 1 << 9;
+
+ /* Special case for seeking in file */
+ int SEEK = 1 << 10;
+ static boolean isState(int state, int mask) {
+ return (state & mask) != 0;
+ }
+ }
+
+ public class CellCursor {
+ public ReusableLivenessInfo rowLiveness;
+ public Columns columns;
+
+ public int columnsSize;
+ public int columnsIndex;
+ public int cellFlags;
+ public final ReusableLivenessInfo cellLiveness = new ReusableLivenessInfo();
+ public CellPath cellPath;
+ public AbstractType cellType;
+ public ColumnMetadata cellColumn;
+ private ColumnMetadata[] columnsArray;
+ private AbstractType[] cellTypeArray;
+
+ void init (Columns columns, ReusableLivenessInfo rowLiveness)
+ {
+ if (this.columns != columns)
+ {
+ // This will be a problem with changing columns
+ this.columns = columns;
+ columnsArray = columns.toArray(COLUMN_METADATA_TYPE);
+ cellTypeArray = new AbstractType[columnsArray.length];
+ for (int i = 0; i < columnsArray.length; i++)
+ {
+ ColumnMetadata cellColumn = columnsArray[i];
+ cellTypeArray[i] = serializationHeader.getType(cellColumn);
+ }
+ columnsSize = columns.size();
+ }
+ this.rowLiveness = rowLiveness;
+ columnsIndex = 0;
+ cellFlags = 0;
+ cellPath = null;
+ cellType = null;
+ }
+
+ public boolean hasNext()
+ {
+ return columnsIndex < columnsSize;
+ }
+
+ /**
+ * For Cell deserialization see {@link Cell.Serializer#deserialize}
+ *
+ * @return true if the cell has a value, false otherwise
+ */
+ boolean readCellHeader() throws IOException
+ {
+ if (!(columnsIndex < columnsSize)) throw new IllegalStateException();
+
+ // HOTSPOT: suprisingly expensive
+ int currIndex = columnsIndex++;
+ cellColumn = columnsArray[currIndex];
+ cellType = cellTypeArray[currIndex];
+ cellFlags = dataReader.readUnsignedByte();
+ // TODO: specialize common case where flags == HAS_VALUE | USE_ROW_TS?
+ boolean hasValue = Cell.Serializer.hasValue(cellFlags);
+ boolean isDeleted = Cell.Serializer.isDeleted(cellFlags);
+ boolean isExpiring = Cell.Serializer.isExpiring(cellFlags);
+ boolean useRowTimestamp = Cell.Serializer.useRowTimestamp(cellFlags);
+ boolean useRowTTL = Cell.Serializer.useRowTTL(cellFlags);
+
+ long timestamp = useRowTimestamp ? rowLiveness.timestamp() : serializationHeader.readTimestamp(dataReader);
+
+ long localDeletionTime = useRowTTL
+ ? rowLiveness.localExpirationTime()
+ : (isDeleted || isExpiring ? serializationHeader.readLocalDeletionTime(dataReader) : Cell.NO_DELETION_TIME);
+
+ int ttl = useRowTTL ? rowLiveness.ttl() : (isExpiring ? serializationHeader.readTTL(dataReader) : Cell.NO_TTL);
+ localDeletionTime = Cell.decodeLocalDeletionTime(localDeletionTime, ttl, deserializationHelper);
+
+ cellLiveness.reset(timestamp, ttl, localDeletionTime);
+ cellPath = cellColumn.isComplex()
+ ? cellColumn.cellPathSerializer().deserialize(dataReader)
+ : null;
+ return hasValue;
+ }
+ }
+
+ private final Ref ssTableReaderRef;
+ private final AbstractType[] clusteringColumnTypes;
+ private final DeserializationHelper deserializationHelper;
+ private final SerializationHeader serializationHeader;
+
+ // need to be closed
+ private final SSTableReader ssTableReader;
+ private final RandomAccessReader dataReader;
+ private final DeletionTime.Serializer deletionTimeSerializer;
+
+ private final CellCursor staticRowCellCursor = new CellCursor();
+ private final CellCursor rowCellCursor = new CellCursor();
+
+
+ private CellCursor cellCursor;
+
+ // SHARED STATIC_ROW/ROW/TOMB
+ private int basicUnfilteredFlags = 0;
+ private int extendedFlags = 0;
+
+ private int state = PARTITION_START;
+
+ public static SSTableCursorReader fromDescriptor(Descriptor desc) throws IOException
+ {
+ TableMetadata metadata = Util.metadataFromSSTable(desc);
+ SSTableReader reader = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata));
+ return new SSTableCursorReader(reader, metadata, reader.ref());
+ }
+
+ public SSTableCursorReader(SSTableReader reader)
+ {
+ this(reader, reader.metadata(), null);
+ }
+
+ private SSTableCursorReader(SSTableReader reader, TableMetadata metadata, Ref readerRef)
+ {
+ ssTableReader = reader;
+ ssTableReaderRef = readerRef;
+ Version version = reader.descriptor.version;
+ deletionTimeSerializer = DeletionTime.getSerializer(version);
+ ImmutableList clusteringColumns = metadata.clusteringColumns();
+ int clusteringColumnCount = clusteringColumns.size();
+ clusteringColumnTypes = new AbstractType[clusteringColumnCount];
+ for (int i = 0; i < clusteringColumnTypes.length; i++)
+ {
+ clusteringColumnTypes[i] = clusteringColumns.get(i).type;
+ }
+ deserializationHelper = new DeserializationHelper(metadata, version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL, null);
+ serializationHeader = reader.header;
+
+ dataReader = reader.openDataReader();
+ hasStaticColumns = metadata.hasStaticColumns();
+ }
+
+ @Override
+ public void close()
+ {
+ dataReader.close();
+ if (ssTableReaderRef != null)
+ ssTableReaderRef.close();
+ }
+
+ private void resetOnPartitionStart()
+ {
+ basicUnfilteredFlags = 0;
+ extendedFlags = 0;
+ }
+
+ public int seekPartition(long position)
+ {
+ state = SEEK;
+ if (position == 0)
+ {
+ dataReader.seek(position);
+ state = PARTITION_START;
+ }
+ else {
+ // verify partition start is after a partition end marker
+ dataReader.seek(position - 1);
+ try
+ {
+ basicUnfilteredFlags = dataReader.readUnsignedByte();
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ // end of partition
+ if (!UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags)) {
+ throw new IllegalArgumentException("Seeking to a partition at: " + position + " did not result in a valid state");
+ }
+ state = dataReader.isEOF() ? DONE : PARTITION_START;
+ }
+ resetOnPartitionStart();
+ return state;
+ }
+
+ public int seekUnfiltered(long position)
+ {
+ state = SEEK;
+ // partition elements (Unfiltered) have flags
+ dataReader.seek(position);
+ int state = 0;
+ try
+ {
+ state = checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+ }
+ catch (IOException e)
+ {
+ return corruptSSTable(e);
+ }
+ if (!isState(state , ROW_START | TOMBSTONE_START | DONE)) throw new IllegalStateException();
+ return state;
+ }
+
+ // struct partition {
+ // struct partition_header header
+ // optional row
+ // struct unfiltered unfiltereds[];
+ //};
+ public int readPartitionHeader(PartitionDescriptor header)
+ {
+ if (state != PARTITION_START) throw new IllegalStateException();
+ resetOnPartitionStart();
+ try
+ {
+ header.load(dataReader, deletionTimeSerializer);
+ return checkNextFlagsAfterPartitionStart(false);
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ // struct static_row {
+ // byte flags; // preloaded
+ // byte extended_flags; // preloaded
+ // varint row_body_size;
+ // varint prev_unfiltered_size; // for backward traversing, ignored
+ // optional liveness_info;
+ // optional deletion_time;
+ // *** We read the columns in a separate method ***
+ // optional missing_columns;
+ // cell[] cells; // potentially only some
+ //};
+ public int readStaticRowHeader(UnfilteredDescriptor unfilteredDescriptor)
+ {
+ if (state != STATIC_ROW_START) throw new IllegalStateException();
+ try
+ {
+ unfilteredDescriptor.loadStaticRow(dataReader, serializationHeader, deserializationHelper, basicUnfilteredFlags, extendedFlags);
+ }
+ catch (IOException e)
+ {
+ return corruptSSTable(e);
+ }
+
+ staticRowCellCursor.init(unfilteredDescriptor.rowColumns(), unfilteredDescriptor.livenessInfo());
+ cellCursor = staticRowCellCursor;
+ if (!staticRowCellCursor.hasNext())
+ {
+ try
+ {
+ return checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+ else
+ {
+ return state = State.CELL_HEADER_START;
+ }
+ }
+
+ public int copyCellValue(DataOutputPlus writer, byte[] buffer) throws IOException
+ {
+ if (state != CELL_VALUE_START) throw new IllegalStateException();
+ if (cellCursor.cellType == null) throw new IllegalStateException();
+ int length = cellCursor.cellType.valueLengthIfFixed();
+ copyCellContents(writer, buffer, length);
+
+ try
+ {
+ if (!cellCursor.hasNext())
+ {
+ try
+ {
+ return checkNextFlagsAfterCellValuesEnd();
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+ return state = State.CELL_END;
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ // TODO: move to cell cursor? maybe avoid copy through buffer?
+ private void copyCellContents(DataOutputPlus writer, byte[] transferBuffer, int length) throws IOException
+ {
+ if (length >= 0)
+ {
+ try
+ {
+ dataReader.readFully(transferBuffer, 0, length);
+ }
+ catch (Exception e)
+ {
+ corruptSSTable(e);
+ }
+ writer.write(transferBuffer, 0, length);
+ }
+ else
+ {
+ try
+ {
+ length = dataReader.readUnsignedVInt32();
+ }
+ catch (IOException e)
+ {
+ corruptSSTable(e);
+ }
+ if (length < 0)
+ corruptSSTable("Corrupt (negative) value length encountered");
+ writer.writeUnsignedVInt32(length);
+ int remaining = length;
+ while (remaining > 0)
+ {
+ int readLength = Math.min(remaining, transferBuffer.length);
+ try
+ {
+ dataReader.readFully(transferBuffer, 0, readLength);
+ }
+ catch (Exception e)
+ {
+ corruptSSTable(e);
+ }
+ writer.write(transferBuffer, 0, readLength);
+ remaining -= readLength;
+ }
+ }
+ }
+
+ // struct row {
+ // byte flags;
+ // optional clustering_blocks;
+ // varint row_body_size;
+ // varint prev_unfiltered_size; // for backward traversing, ignored
+ // optional liveness_info;
+ // optional deletion_time;
+ // *** We read the columns in a separate step ***
+ // optional missing_columns;
+ // cell[] cells; // potentially only some
+ //};
+ public int readRowHeader(UnfilteredDescriptor unfilteredDescriptor)
+ {
+ if (state != State.ROW_START) throw new IllegalStateException();
+ if (!UnfilteredSerializer.isRow(basicUnfilteredFlags)) throw new IllegalStateException();
+ try
+ {
+ unfilteredDescriptor.loadRow(dataReader, serializationHeader, deserializationHelper, basicUnfilteredFlags);
+
+ rowCellCursor.init(unfilteredDescriptor.rowColumns(), unfilteredDescriptor.livenessInfo());
+ cellCursor = rowCellCursor;
+ if (!rowCellCursor.hasNext())
+ {
+ return checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+ }
+ else
+ {
+ return state = State.CELL_HEADER_START;
+ }
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ // TODO: introduce cell header class
+ public int readCellHeader()
+ {
+ if (state != State.CELL_HEADER_START) throw new IllegalStateException();
+ try
+ {
+ if (cellCursor.readCellHeader())
+ {
+ return state = State.CELL_VALUE_START;
+ }
+ if (!cellCursor.hasNext())
+ return checkNextFlagsAfterCellValuesEnd();
+ return state = State.CELL_END;
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ public int skipCellValue()
+ {
+ if (state != State.CELL_VALUE_START) throw new IllegalStateException();
+ try
+ {
+ cellCursor.cellType.skipValue(dataReader);
+ return !cellCursor.hasNext() ? checkNextFlagsAfterCellValuesEnd() : (state = State.CELL_HEADER_START);
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ /**
+ * See: {@link org.apache.cassandra.db.rows.UnfilteredSerializer#serialize(RangeTombstoneMarker, SerializationHelper, DataOutputPlus, long, int)}
+ *
+ * struct range_tombstone_marker {
+ * byte flags = IS_MARKER;
+ * byte kind_ordinal;
+ * be16 bound_values_count;
+ * struct clustering_block[] clustering_blocks;
+ * varint marker_body_size;
+ * varint prev_unfiltered_size;
+ * };
+ * struct range_tombstone_bound_marker : range_tombstone_marker {
+ * struct delta_deletion_time deletion_time;
+ * };
+ * struct range_tombstone_boundary_marker : range_tombstone_marker {
+ * struct delta_deletion_time end_deletion_time;
+ * struct delta_deletion_time start_deletion_time;
+ * };
+ *
+ *
+ */
+ public int readTombstoneMarker(UnfilteredDescriptor unfilteredDescriptor)
+ {
+ try
+ {
+ if (state != TOMBSTONE_START) throw new IllegalStateException();
+ if (!UnfilteredSerializer.isTombstoneMarker(basicUnfilteredFlags)) throw new IllegalStateException();
+ unfilteredDescriptor.loadTombstone(dataReader, serializationHeader, basicUnfilteredFlags);
+ return checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+
+ /**
+ * {@link ClusteringPrefix.Serializer#deserializeValuesWithoutSize}
+ */
+ static void readUnfilteredClustering(RandomAccessReader dataReader, AbstractType[] types, int clusteringColumnsBound, ResizableByteBuffer clustering) throws IOException
+ {
+ clustering.resetBuffer();
+ if (clusteringColumnsBound == 0) {
+ return;
+ }
+ long clusteringBlockHeader = 0;
+ int fixedLengthClusteringLength = 0;
+ for (int clusteringIndex = 0; clusteringIndex < clusteringColumnsBound; clusteringIndex++)
+ {
+ // struct clustering_block {
+ // varint clustering_block_header;
+ // simple_cell[] clustering_cells;
+ // };
+ if (clusteringIndex % 32 == 0)
+ {
+ if (fixedLengthClusteringLength != 0) {
+ clustering.loadPart(dataReader, fixedLengthClusteringLength);
+ fixedLengthClusteringLength = 0;
+ }
+ clusteringBlockHeader = dataReader.readUnsignedVInt();
+ clustering.writeUnsignedVInt(clusteringBlockHeader);
+ }
+
+ // load value if present
+ if ((clusteringBlockHeader & 0b11) == 0)
+ {
+ AbstractType type = types[clusteringIndex];
+ if (type.isValueLengthFixed())
+ {
+ fixedLengthClusteringLength += type.valueLengthIfFixed();
+ }
+ else
+ {
+ if (fixedLengthClusteringLength != 0) {
+ clustering.loadPart(dataReader, fixedLengthClusteringLength);
+ fixedLengthClusteringLength = 0;
+ }
+ int varLength = dataReader.readUnsignedVInt32();
+ clustering.writeUnsignedVInt(varLength);
+ clustering.loadPart(dataReader, varLength);
+ }
+ }
+ clusteringBlockHeader = clusteringBlockHeader >>> 2;
+ }
+ if (fixedLengthClusteringLength != 0) clustering.loadPart(dataReader, fixedLengthClusteringLength);
+ if (clusteringBlockHeader != 0) {
+ throw new IOException("Clustering block upper bits (those not associated with keys) expected to be 0:" + clusteringBlockHeader);
+ }
+ }
+
+ private static void skipClustering(RandomAccessReader dataReader, AbstractType[] types, int clusteringColumnsBound) throws IOException
+ {
+ long clusteringBlockHeader = 0;
+ for (int clusteringIndex = 0; clusteringIndex < clusteringColumnsBound; clusteringIndex++)
+ {
+ // struct clustering_block {
+ // varint clustering_block_header;
+ // simple_cell[] clustering_cells;
+ // };
+ if (clusteringIndex % 32 == 0)
+ {
+ clusteringBlockHeader = dataReader.readUnsignedVInt();
+ }
+ // skip value if present
+ if ((clusteringBlockHeader & 0b11) == 0)
+ {
+ AbstractType type = types[clusteringIndex];
+ int len = type.isValueLengthFixed() ? type.valueLengthIfFixed() : dataReader.readUnsignedVInt32();
+ dataReader.skipBytes(len);
+ }
+ clusteringBlockHeader = clusteringBlockHeader >>> 2;
+ }
+ if (clusteringBlockHeader != 0) {
+ throw new IOException("Clustering block upper bits (those not associated with keys) expected to be 0:" + clusteringBlockHeader);
+ }
+ }
+
+ /**
+ * {@link UnfilteredSerializer#deserializeRowBody(DataInputPlus, SerializationHeader, DeserializationHelper, int, int, Row.Builder)}
+ */
+ static void readLivenessInfo(RandomAccessReader dataReader, SerializationHeader serializationHeader, DeserializationHelper deserializationHelper, int flags, ReusableLivenessInfo livenessInfo) throws IOException
+ {
+ long timestamp = LivenessInfo.NO_TIMESTAMP;
+ int ttl = LivenessInfo.NO_TTL;
+ long localExpirationTime = LivenessInfo.NO_EXPIRATION_TIME;
+ if (UnfilteredSerializer.hasTimestamp(flags))
+ {
+ // struct liveness_info {
+ // varint64 delta_timestamp;
+ // optional delta_ttl;
+ // optional delta_local_deletion_time;
+ //};
+ timestamp = serializationHeader.readTimestamp(dataReader);
+ if (UnfilteredSerializer.hasTTL(flags))
+ {
+ ttl = serializationHeader.readTTL(dataReader);
+ localExpirationTime = Cell.decodeLocalDeletionTime(serializationHeader.readLocalDeletionTime(dataReader), ttl, deserializationHelper);
+ }
+ }
+ livenessInfo.reset(timestamp, ttl, localExpirationTime);
+ }
+
+ // SKIPPING
+ public int skipPartition()
+ {
+ if (state == PARTITION_END)
+ return continueReading();
+
+ if (state == PARTITION_START)
+ {
+ try
+ {
+ int partitionKeyLength = dataReader.readUnsignedShort();
+ dataReader.skipBytes(partitionKeyLength);
+
+ // PARTITION DELETION TIME
+ deletionTimeSerializer.skip(dataReader);
+ checkNextFlagsAfterPartitionStart(true);
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+ else if (!isState(state, STATIC_ROW_START | ROW_START | TOMBSTONE_START | PARTITION_END))
+ {
+ throw new IllegalStateException("Unexpected state: " + state);
+ }
+
+ while (!isState(state,PARTITION_START | DONE))
+ {
+ switch (state)
+ {
+ case STATIC_ROW_START:
+ state = skipStaticRow(true);
+ break;
+ case ROW_START:
+ case TOMBSTONE_START:
+ state = skipUnfiltered(true);
+ break;
+ }
+ }
+ return state;
+ }
+
+ public int skipStaticRow(boolean autoContinue)
+ {
+ if (state != State.STATIC_ROW_START) throw new IllegalStateException();
+
+ try
+ {
+ long rowSize = dataReader.readUnsignedVInt();
+ dataReader.seek(dataReader.getPosition() + rowSize);
+ return checkNextFlagsAfterStaticRowOrUnfilteredStart(autoContinue);
+ }
+ catch (IOException e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ public int skipUnfiltered(boolean autoContinue)
+ {
+ if (!isState(state, ROW_START | TOMBSTONE_START))
+ throw new IllegalStateException();
+
+ AbstractType[] types = clusteringColumnTypes;
+ int clusteringColumnsBound = types.length;
+ // tombstone markers have `kind` & `clusteringColumnsBound`
+ try
+ {
+ if (!UnfilteredSerializer.isRow(basicUnfilteredFlags))
+ {
+ dataReader.readByte();// byte kind =
+ clusteringColumnsBound = dataReader.readUnsignedShort();
+ }
+ /**
+ * {@link org.apache.cassandra.db.ClusteringPrefix.Deserializer}
+ */
+ skipClustering(dataReader, types, clusteringColumnsBound);
+ // same for row/tombstone
+ long rowSize = dataReader.readUnsignedVInt();
+ dataReader.seek(dataReader.getPosition() + rowSize);
+
+ return checkNextFlagsAfterStaticRowOrUnfilteredStart(autoContinue);
+ }
+ catch (Exception e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ public int skipRowCells(long unfilteredDataStart, long unfilteredSize, boolean autoContinue)
+ {
+ if (!(isState(state,CELL_HEADER_START | CELL_VALUE_START | CELL_END))) throw new IllegalStateException();
+
+ try
+ {
+ dataReader.seek(unfilteredDataStart + unfilteredSize);
+ return checkNextFlagsAfterStaticRowOrUnfilteredStart(autoContinue);
+ }
+ catch (IOException e)
+ {
+ return corruptSSTable(e);
+ }
+ }
+
+ public int continueReading() {
+ // TODO: can be optimized by pre-calculating next state when the flags are read
+ switch (state)
+ {
+ case PARTITION_END:
+ state = dataReader.isEOF() ? DONE : PARTITION_START;
+ break;
+ case UNFILTERED_END:
+ if (UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags))
+ {
+ state = PARTITION_END;
+ }
+ else
+ {
+ state = UnfilteredSerializer.isRow(basicUnfilteredFlags) ? ROW_START : TOMBSTONE_START;
+ }
+ break;
+ case CELL_END:
+ if (cellCursor.hasNext())
+ {
+ state = CELL_HEADER_START;
+ }
+ else
+ {
+ state = UNFILTERED_END;
+ }
+ break;
+ default:
+ throw new IllegalStateException("Cannot continue reading in current state: " + state);
+ }
+ return state;
+ }
+
+ private int checkNextFlagsAfterPartitionStart(boolean autoContinue) throws IOException
+ {
+ long preFlagsPosition = dataReader.getPosition();
+ basicUnfilteredFlags = dataReader.readUnsignedByte();
+ if (UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags))
+ {
+ state = !autoContinue ? PARTITION_END :
+ dataReader.isEOF() ? DONE : PARTITION_START;
+ }
+ else if (UnfilteredSerializer.isExtended(basicUnfilteredFlags))
+ {
+ state = STATIC_ROW_START;
+ extendedFlags = dataReader.readUnsignedByte();
+ validateStaticRowFlags(preFlagsPosition);
+ }
+ else
+ {
+ state = UnfilteredSerializer.isRow(basicUnfilteredFlags) ? ROW_START : TOMBSTONE_START;
+ }
+ return state;
+ }
+
+ private void validateStaticRowFlags(long preFlagsPosition)
+ {
+ if (!UnfilteredSerializer.isStatic(extendedFlags))
+ {
+ corruptSSTable("Row at: " + preFlagsPosition + " has extended flags but is not static, extendedFlags: " + extendedFlags);
+ }
+ if (!UnfilteredSerializer.isRow(basicUnfilteredFlags))
+ {
+ corruptSSTable("Static row at: " + preFlagsPosition + " is not a row, flags: " + basicUnfilteredFlags);
+ }
+ if (!hasStaticColumns)
+ {
+ corruptSSTable("Row at: " + preFlagsPosition + " is static, but table has no static columns " + ssTableReader.metadata());
+ }
+ if (UnfilteredSerializer.deletionIsShadowable(extendedFlags))
+ {
+ throw new UnsupportedOperationException("Static row at: " + preFlagsPosition + " has deletionIsShadowable, which is deprecated since 4.0");
+ }
+ }
+
+ private int checkNextFlagsAfterStaticRowOrUnfilteredStart(boolean autoContinue) throws IOException
+ {
+ int flags = this.basicUnfilteredFlags = dataReader.readUnsignedByte();
+ if (UnfilteredSerializer.isExtended(flags))
+ {
+ corruptSSTable("Unexpected static row (flags=" + flags + ") mid-partition, at position: " + (dataReader.getPosition() - 1));
+ }
+
+ if (!autoContinue) {
+ return this.state = UNFILTERED_END;
+ }
+ else
+ {
+ return this.state = nextStateMidPartition(flags);
+ }
+ }
+
+ private int checkNextFlagsAfterCellValuesEnd() throws IOException
+ {
+ int flags = this.basicUnfilteredFlags = dataReader.readUnsignedByte();
+ if (UnfilteredSerializer.isExtended(flags))
+ {
+ corruptSSTable("Unexpected static row (flags=" + flags + ") mid-partition, at position: " + (dataReader.getPosition() - 1));
+ }
+ return this.state = CELL_END;
+ }
+
+ private int corruptSSTable(Exception e)
+ {
+ ssTableReader.markSuspect();
+ if (e instanceof CorruptSSTableException)
+ throw (CorruptSSTableException) e;
+
+ throw new CorruptSSTableException(e, ssTableReader.getFilename());
+ }
+
+ protected int corruptSSTable(String message)
+ {
+ return corruptSSTable(new IllegalStateException(message));
+ }
+
+ private int nextStateMidPartition(int basicUnfilteredFlags)
+ {
+ if (UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags))
+ {
+ return dataReader.isEOF() ? DONE : PARTITION_START;
+ }
+ else if (UnfilteredSerializer.isRow(basicUnfilteredFlags))
+ {
+ return ROW_START;
+ }
+ else
+ {
+ return TOMBSTONE_START;
+ }
+ }
+
+ public boolean isEOF() {
+ return state == DONE || dataReader.isEOF();
+ }
+
+ public int state()
+ {
+ return state;
+ }
+
+ public long position() {
+ return dataReader.getFilePointer();
+ }
+
+ public long uncompressedLength()
+ {
+ return ssTableReader.uncompressedLength();
+ }
+
+ public SSTableReader ssTableReader()
+ {
+ return ssTableReader;
+ }
+
+ public CellCursor cellCursor()
+ {
+ return cellCursor;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableCursorWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableCursorWriter.java
new file mode 100644
index 0000000000..2a170acb73
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableCursorWriter.java
@@ -0,0 +1,689 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import com.google.common.primitives.Ints;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.DeletionTime.ReusableDeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.guardrails.Threshold;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SortedTableWriter;
+import org.apache.cassandra.io.sstable.format.big.BigFormatPartitionWriter;
+import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.db.rows.UnfilteredSerializer.*;
+
+public class SSTableCursorWriter implements AutoCloseable
+{
+ private static final UnfilteredSerializer SERIALIZER = UnfilteredSerializer.serializer;
+ private static final ColumnMetadata[] EMPTY_COL_META = new ColumnMetadata[0];
+ private final SortedTableWriter ssTableWriter;
+ private final SequentialWriter dataWriter;
+ private final SortedTableWriter.AbstractIndexWriter indexWriter;
+ private final DeletionTime.Serializer deletionTimeSerializer;
+ private final MetadataCollector metadataCollector;
+ private final SerializationHeader serializationHeader;
+ /**
+ * See: {@link BloomFilter#reusableIndexes}
+ */
+ private final long[] reusableIndexes = new long[21];
+ private final boolean hasStaticColumns;
+
+ private long partitionStart;
+ // ROW contents, needed because of the order of writing and the var int fields
+ private int rowFlags; // discovered as we go along
+ private int rowExtendedFlags;
+ private final DataOutputBuffer rowHeaderBuffer = new DataOutputBuffer(); // holds the contents between FLAGS and SIZE
+ private final DataOutputBuffer rowBuffer = new DataOutputBuffer();
+ private final ReusableDeletionTime openMarker = ReusableDeletionTime.live();
+
+ private final ColumnMetadata[] staticColumns;
+ private final ColumnMetadata[] regularColumns;
+ private final IntArrayList missingColumns = new IntArrayList();
+ private ColumnMetadata[] columns; // points to static/regular
+ private int columnsWrittenCount = 0;
+ private int nextCellIndex = 0;
+ // Index info
+ private final DataOutputBuffer rowIndexEntries = new DataOutputBuffer();
+ private final IntArrayList rowIndexEntriesOffsets = new IntArrayList();
+ private final ClusteringDescriptor rowIndexEntryLastClustering;
+ private int indexBlockStartOffset;
+ private int rowIndexEntryOffset;
+ private final int indexBlockThreshold;
+
+ private SSTableCursorWriter(
+ Descriptor desc,
+ SortedTableWriter ssTableWriter,
+ SequentialWriter dataWriter,
+ SortedTableWriter.AbstractIndexWriter indexWriter,
+ MetadataCollector metadataCollector,
+ SerializationHeader serializationHeader)
+ {
+ this.ssTableWriter = ssTableWriter;
+ this.dataWriter = dataWriter;
+ this.indexWriter = indexWriter;
+ this.deletionTimeSerializer = DeletionTime.getSerializer(desc.version);
+ this.metadataCollector = metadataCollector;
+ this.serializationHeader = serializationHeader;
+ hasStaticColumns = serializationHeader.hasStatic();
+ staticColumns = hasStaticColumns ? serializationHeader.columns(true).toArray(EMPTY_COL_META) : EMPTY_COL_META;
+ regularColumns = serializationHeader.columns(false).toArray(EMPTY_COL_META);
+ this.indexBlockThreshold = DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY);
+ rowIndexEntryLastClustering = new ClusteringDescriptor(serializationHeader.clusteringTypes().toArray(AbstractType[]::new));
+ }
+
+ public SSTableCursorWriter(SortedTableWriter ssTableWriter)
+ {
+ this(ssTableWriter.descriptor,
+ ssTableWriter,
+ ssTableWriter.dataWriter,
+ ssTableWriter.indexWriter,
+ ssTableWriter.metadataCollector,
+ ssTableWriter.partitionWriter.getHeader());
+ }
+
+ @Override
+ public void close()
+ {
+ SSTableReader finish = ssTableWriter.finish(false);
+ if (finish != null) {
+ Ref ref = finish.ref();
+ if (ref != null) ref.close();
+ }
+ ssTableWriter.close();
+ }
+
+ public long getPartitionStart()
+ {
+ return partitionStart;
+ }
+
+ public long getPosition()
+ {
+ return dataWriter.position();
+ }
+
+ public int writePartitionStart(byte[] partitionKey, int partitionKeyLength, DeletionTime partitionDeletionTime) throws IOException
+ {
+ rowIndexEntries.clear();
+ rowIndexEntriesOffsets.clear();
+ rowIndexEntryOffset = 0;
+ openMarker.resetLive();
+
+ partitionStart = dataWriter.position();
+ writePartitionHeader(partitionKey, partitionKeyLength, partitionDeletionTime);
+ updateIndexBlockStartOffset(dataWriter.position());
+ return indexBlockStartOffset;
+ }
+
+ public void writePartitionEnd(byte[] partitionKey, int partitionKeyLength, DeletionTime partitionDeletionTime, int headerLength) throws IOException
+ {
+ SERIALIZER.writeEndOfPartition(dataWriter);
+ long partitionEnd = dataWriter.position();
+ long partitionSize = partitionEnd - partitionStart;
+ addPartitionMetadata(partitionKey, partitionKeyLength, partitionSize, partitionDeletionTime);
+
+ /** {@link SortedTableWriter#endPartition(DecoratedKey, DeletionTime)}
+ lastWrittenKey = key; // tracked for verification, see {@link SortedTableWriter#verifyPartition(DecoratedKey)}, checking the key size and sorting
+ // first/last are retained for metadata {@link org.apache.cassandra.io.sstable.format.SSTableWriter#finalizeMetadata()}. They are also exposed via
+ // getters from the writer, but usage is unclear.
+ last = lastWrittenKey;
+ if (first == null)
+ first = lastWrittenKey;
+ // this is implemented differently for BIG/BTI
+ createRowIndexEntry(key, partitionLevelDeletion, partitionEnd - 1);
+ */
+ appendBIGIndex(partitionKey, partitionKeyLength, partitionStart, headerLength, partitionDeletionTime, partitionEnd);
+ }
+
+ private void appendBIGIndex(byte[] key, int keyLength, long partitionStart, int headerLength, DeletionTime partitionDeletionTime, long partitionEnd) throws IOException
+ {
+ /**
+ * {@link BigTableWriter#createRowIndexEntry(DecoratedKey, DeletionTime, long)}
+ * {@link BigTableWriter.IndexWriter#append(DecoratedKey, RowIndexEntry, long, ByteBuffer)}
+ *
+ */
+ BigTableWriter.IndexWriter indexWriter = (BigTableWriter.IndexWriter) this.indexWriter;
+ SequentialWriter indexFileWriter = indexWriter.writer;
+ ((BloomFilter)indexWriter.bf).add(key, 0, keyLength, reusableIndexes);
+ long indexStart = indexFileWriter.position();
+ try
+ {
+ ByteArrayUtil.writeWithShortLength(key, 0, keyLength, indexFileWriter);
+
+ indexFileWriter.writeUnsignedVInt(partitionStart);
+
+ // if the list of entries has one or fewer entries, no point in index entries.
+ /** See: {@link org.apache.cassandra.io.sstable.format.big.RowIndexEntry#create} */
+ if (rowIndexEntriesOffsets.size() <= 1)
+ {
+ /**
+ * {@link RowIndexEntry#serialize(DataOutputPlus, ByteBuffer)}
+ */
+ indexFileWriter.writeUnsignedVInt32(0); // size
+ }
+ else {
+ // add last block
+ long indexBlockSize = (partitionEnd - partitionStart - 1) - indexBlockStartOffset;
+ if (indexBlockSize != 0) {
+ addIndexBlock(partitionEnd - 1, indexBlockSize);
+ }
+ // if we have intermeddiate index info elements we also need to serialize the partitionDeletionTime
+ /** {@link RowIndexEntry.IndexedEntry#serialize(DataOutputPlus, ByteBuffer) */
+ // size up to the offsets?
+ int endOfEntries = rowIndexEntries.getLength();
+ // Write the headerLength, partitionDeletionTime and rowIndexEntriesOffsets.size() after the entries,
+ // just to calculate size.
+ rowIndexEntries.writeUnsignedVInt((long)headerLength);
+ deletionTimeSerializer.serialize(partitionDeletionTime, rowIndexEntries);
+
+ rowIndexEntries.writeUnsignedVInt32(rowIndexEntriesOffsets.size()); // number of entries
+
+ // bytes until offsets
+ int entriesAndOffsetsSize = rowIndexEntries.getLength() + rowIndexEntriesOffsets.size() * 4;
+ assert entriesAndOffsetsSize > 0;
+ indexFileWriter.writeUnsignedVInt32(entriesAndOffsetsSize); // size != 0
+ // copy the header elements
+ indexFileWriter.write(rowIndexEntries.getData(), endOfEntries, rowIndexEntries.getLength() - endOfEntries);
+ indexFileWriter.write(rowIndexEntries.getData(), 0, endOfEntries);
+ for (int i = 0; i < rowIndexEntriesOffsets.size(); i++)
+ {
+ int offset = rowIndexEntriesOffsets.get(i);
+ indexFileWriter.writeInt(offset);
+ }
+ }
+ }
+ catch (IOException e)
+ {
+ throw new FSWriteError(e, indexFileWriter.getPath());
+ }
+ indexWriter.summary.maybeAddEntry(key, 0, keyLength, indexStart);
+ }
+
+ final long guardrailsPartitionSizeWarning = Guardrails.partitionSize.warnValue(null);
+ final long guardrailsPartitionTombstonesWarning = Guardrails.partitionTombstones.warnValue(null);
+
+ /**
+ * update metadata like {@link SortedTableWriter#endPartition} and {@link SortedTableWriter#startPartition}
+ */
+ private void addPartitionMetadata(byte[] partitionKey, int partitionKeyLength, long partitionSize, DeletionTime partitionDeletionTime)
+ {
+ if (partitionSize > guardrailsPartitionSizeWarning)
+ guardPartitionThreshold(Guardrails.partitionSize, partitionKey, partitionKeyLength, partitionSize);
+
+ if (metadataCollector.totalTombstones > guardrailsPartitionTombstonesWarning)
+ guardPartitionThreshold(Guardrails.partitionTombstones, partitionKey, partitionKeyLength, metadataCollector.totalTombstones);
+
+ metadataCollector.updatePartitionDeletion(partitionDeletionTime);
+ metadataCollector.addPartitionSizeInBytes(partitionSize);
+ metadataCollector.addKey(partitionKey, 0, partitionKeyLength);
+ metadataCollector.addCellPerPartitionCount();
+ }
+
+ private void guardPartitionThreshold(Threshold guardrail, byte[] partitionKey, int partitionKeyLength, long size)
+ {
+ if (guardrail.triggersOn(size, null))
+ {
+ String message = String.format("%s.%s:%s on sstable %s",
+ ssTableWriter.metadata().keyspace,
+ ssTableWriter.metadata().name,
+ ssTableWriter.metadata().partitionKeyType.getString(ByteBuffer.wrap(partitionKey, 0, partitionKeyLength)),
+ ssTableWriter.getFilename());
+ guardrail.guard(size, message, true, null);
+ }
+ }
+
+ private void writePartitionHeader(byte[] partitionKey, int partitionKeyLength, DeletionTime partitionDeletionTime) throws IOException
+ {
+ dataWriter.writeShort(partitionKeyLength);
+ dataWriter.write(partitionKey, 0, partitionKeyLength);
+ deletionTimeSerializer.serialize(partitionDeletionTime, dataWriter);
+ }
+
+ public boolean writeEmptyStaticRow() throws IOException
+ {
+ if (!hasStaticColumns)
+ return false;
+ rowFlags = UnfilteredSerializer.EXTENSION_FLAG;
+ rowExtendedFlags = UnfilteredSerializer.IS_STATIC;
+ columns = staticColumns;
+ // TOD: we should be able to skip the use of the row buffers in this special case, maybe it doesn't matter
+ rowHeaderBuffer.clear();
+ // NOTE: if we are to write this value (which is not used), this is where we should compute it.
+ rowHeaderBuffer.writeUnsignedVInt32(0);
+ rowBuffer.clear();
+ columnsWrittenCount = 0;
+ missingColumns.clear();
+ writeRowEnd(null, false);
+
+ updateIndexBlockStartOffset(dataWriter.position());
+ return true;
+ }
+
+ public void writeRowStart(LivenessInfo livenessInfo, DeletionTime deletionTime, boolean isStatic) throws IOException
+ {
+ if (isStatic) {
+ rowFlags = UnfilteredSerializer.EXTENSION_FLAG;
+ rowExtendedFlags = UnfilteredSerializer.IS_STATIC;
+ columns = staticColumns;
+ }
+ else {
+ rowFlags = 0;
+ rowExtendedFlags = 0;
+ columns = regularColumns;
+ }
+ // NOTE: Data after this point needs a computed ahead of write size. This, combined with the cost of rewriting
+ // the size after the writing completes, means we have to buffer the row timestamps (most likely to differ in length)
+ // and the row columns data (will differ if they use their own timestamps, probably). Unfortunate.
+ // rest of header
+ rowHeaderBuffer.clear();
+ missingColumns.clear();
+ rowBuffer.clear();
+ columnsWrittenCount = 0;
+ nextCellIndex = 0;
+
+ // NOTE: if we are to write this value (which is not used), this is where we should compute it.
+ rowHeaderBuffer.writeUnsignedVInt32(0);
+
+ // copy TS/TTL/deletion data
+ rowFlags |= writeRowTimeData(livenessInfo, deletionTime, rowHeaderBuffer);
+ }
+
+ /**
+ * See {@link UnfilteredSerializer#serialize(Row, SerializationHelper, DataOutputPlus, long, int)}
+ */
+ private int writeRowTimeData(LivenessInfo livenessInfo, DeletionTime deletionTime, DataOutputPlus writer) throws IOException
+ {
+ int flags = 0;
+ boolean writtenLivenessMetadata = false;
+
+ if (!livenessInfo.isEmpty())
+ {
+ flags |= HAS_TIMESTAMP;
+ serializationHeader.writeTimestamp(livenessInfo.timestamp(), writer);
+ metadataCollector.update(livenessInfo);
+ writtenLivenessMetadata = true;
+ }
+ if (livenessInfo.isExpiring())
+ {
+ flags |= HAS_TTL;
+ serializationHeader.writeTTL(livenessInfo.ttl(), writer);
+ serializationHeader.writeLocalDeletionTime(livenessInfo.localExpirationTime(), writer);
+ if (!writtenLivenessMetadata) metadataCollector.update(livenessInfo);
+ }
+ if (!deletionTime.isLive())
+ {
+ flags |= HAS_DELETION;
+ writeDeletionTime(deletionTime, writer);
+ }
+
+ /**
+ * Metadata calls matching: {@link org.apache.cassandra.db.rows.Rows#collectStats}
+ * But the collection of data is conditional and the cell metadata is collected elsewhere.
+ */
+ return flags;
+ }
+
+ private void writeDeletionTime(DeletionTime deletionTime, DataOutputPlus writer) throws IOException
+ {
+ serializationHeader.writeDeletionTime(deletionTime, writer);
+ metadataCollector.update(deletionTime);
+ }
+
+ public void writeCellHeader(int cellFlags, ReusableLivenessInfo cellLiveness, ColumnMetadata cellColumn) throws IOException
+ {
+ for (; nextCellIndex < columns.length; nextCellIndex++) {
+ if (columns[nextCellIndex].compareTo(cellColumn) == 0)
+ break;
+ missingColumns.addInt(nextCellIndex);
+ }
+ if (nextCellIndex == columns.length)
+ throw new IllegalStateException("Column not found: " + cellColumn +" or cell writes out of order, or bug.");
+ nextCellIndex++;
+ writeCellHeader(cellFlags, cellLiveness, rowBuffer);
+ }
+
+ private void writeCellHeader(int cellFlags, ReusableLivenessInfo cellLiveness, DataOutputPlus writer) throws IOException
+ {
+ columnsWrittenCount++;
+ writer.writeByte(cellFlags);
+ if (!Cell.Serializer.useRowTimestamp(cellFlags)) {
+ long timestamp = cellLiveness.timestamp();
+ serializationHeader.writeTimestamp(timestamp, writer);
+ }
+ if (!Cell.Serializer.useRowTTL(cellFlags)) {
+ boolean isDeleted = Cell.Serializer.isDeleted(cellFlags);
+ boolean isExpiring = Cell.Serializer.isExpiring(cellFlags);
+ if (isDeleted || isExpiring) {
+ // TODO: is this conversion from LET to LDT correct?
+ serializationHeader.writeLocalDeletionTime(cellLiveness.localExpirationTime(), writer);
+ }
+ if (isExpiring) {
+ serializationHeader.writeTTL(cellLiveness.ttl(), writer);
+ }
+ }
+ /**
+ * matching {@link org.apache.cassandra.db.rows.Cells#collectStats};
+ */
+ metadataCollector.updateCellLiveness(cellLiveness);
+ }
+
+ public int writeCellValue(SSTableCursorReader cursor, byte[] copyColumnValueBuffer) throws IOException
+ {
+ return cursor.copyCellValue(rowBuffer, copyColumnValueBuffer);
+ }
+
+ public void writeCellValue(DataOutputBuffer tempCellBuffer) throws IOException
+ {
+ rowBuffer.write(tempCellBuffer.getData(), 0, tempCellBuffer.getLength());
+ }
+
+ public void writeRowEnd(UnfilteredDescriptor rHeader, boolean updateClusteringMetadata) throws IOException
+ {
+ boolean isExtended = isExtended(rowFlags);
+ boolean isStatic = isExtended && UnfilteredSerializer.isStatic(rowExtendedFlags);
+ int columnsLength = columns.length;
+ if (columnsWrittenCount == columnsLength)
+ {
+ rowFlags |= HAS_ALL_COLUMNS;
+ }
+ else if (columnsWrittenCount == 0) {
+ // Same as Columns.serializer.serializeSubset(Columns.NONE, serializationHeader.columns(isStatic), rowHeaderBuffer)
+ if (columnsLength < 64) {
+ // all the bits are set, because all the columns are missing, value is always positive
+ rowHeaderBuffer.writeUnsignedVInt(-1L >>> (64 - columnsLength));
+ }
+ else {
+ // no columns are present, nothing to write
+ rowHeaderBuffer.writeUnsignedVInt32(columnsLength);
+ }
+ }
+ else if (columnsWrittenCount < columnsLength)
+ {
+ for (; nextCellIndex < columnsLength; nextCellIndex++)
+ missingColumns.addInt(nextCellIndex);
+
+ if (columnsLength < 64) {
+ // set a bit for every missing column
+ long mask = 0;
+ for (int missingIndex : missingColumns) {
+ mask |= (1L << missingIndex);
+ }
+ rowHeaderBuffer.writeUnsignedVInt(mask);
+ }
+ else {
+ encodeLargeColumnsSubset();
+ }
+ }
+ long unfilteredStartPosition = dataWriter.position();
+ /** See: {@link UnfilteredSerializer#serialize} */
+ dataWriter.writeByte(rowFlags);
+ if (isExtended)
+ {
+ dataWriter.writeByte(rowExtendedFlags);
+ }
+
+ if (!isStatic)
+ {
+ byte[] clustering = rHeader.clusteringBytes();
+ int clusteringLength = rHeader.clusteringLength();
+ dataWriter.write(clustering, 0, clusteringLength);
+ }
+
+ // Now that we know the size, write it + the rest of the data
+ dataWriter.writeUnsignedVInt32(rowHeaderBuffer.getLength() + rowBuffer.getLength());
+
+ dataWriter.write(rowHeaderBuffer.getData(), 0, rowHeaderBuffer.getLength());
+ dataWriter.write(rowBuffer.getData(), 0, rowBuffer.getLength());
+
+ long unfilteredEndPosition = getPosition();
+
+ /**
+ * Matching the: {@link org.apache.cassandra.db.rows.Rows#collectStats} along with above cell level metadata updates
+ */
+ metadataCollector.updateColumnSetPerRow(columnsWrittenCount);
+
+ if (isStatic)
+ {
+ updateIndexBlockStartOffset(dataWriter.position());
+ }
+ else
+ {
+ updateMetadataAndIndexBlock(rHeader, unfilteredStartPosition, unfilteredEndPosition, updateClusteringMetadata);
+ }
+ }
+
+ /**
+ * See: {@link org.apache.cassandra.io.sstable.format.SortedTableWriter#addRangeTomstoneMarker}
+ */
+ public void writeRangeTombstone(UnfilteredDescriptor rangeTombstone, boolean updateClusteringMetadata) throws IOException
+ {
+ int tombstoneKind = rangeTombstone.clusteringKindEncoded();
+ ClusteringPrefix.Kind kind = ClusteringPrefix.Kind.values()[tombstoneKind];
+ long unfilteredStartPosition = getPosition();
+ /** See: {@link org.apache.cassandra.db.rows.UnfilteredSerializer#serialize */
+ dataWriter.writeByte((byte)IS_MARKER);
+ /** See: {@link org.apache.cassandra.db.ClusteringBoundOrBoundary.Serializer#serialize} */
+ dataWriter.writeByte(tombstoneKind);
+ dataWriter.writeShort(rangeTombstone.clusteringColumnsBound());
+
+ int clusteringLength = rangeTombstone.clusteringLength();
+ if (clusteringLength != 0)
+ {
+ byte[] clustering = rangeTombstone.clusteringBytes();
+ dataWriter.write(clustering, 0, clusteringLength);
+ }
+ rowHeaderBuffer.clear();
+ // TODO: previousUnfilteredSize
+ rowHeaderBuffer.writeUnsignedVInt32(0);
+
+ if (kind.isBoundary())
+ {
+ writeDeletionTime(rangeTombstone.deletionTime(), rowHeaderBuffer);
+ writeDeletionTime(rangeTombstone.deletionTime2(), rowHeaderBuffer);
+ openMarker.reset(rangeTombstone.deletionTime2());
+ }
+ else
+ {
+ writeDeletionTime(rangeTombstone.deletionTime(), rowHeaderBuffer);
+ if (kind.isOpen(false))
+ openMarker.reset(rangeTombstone.deletionTime());
+ else
+ openMarker.resetLive();
+ }
+
+ dataWriter.writeUnsignedVInt32(rowHeaderBuffer.getLength());
+ dataWriter.write(rowHeaderBuffer.getData(), 0, rowHeaderBuffer.getLength());
+
+ long unfilteredEndPosition = getPosition();
+
+ /** {@link org.apache.cassandra.io.sstable.format.big.BigFormatPartitionWriter#addUnfiltered(Unfiltered)} */
+ // if we hit the index block size that we have to index after, go ahead and index it.
+ updateMetadataAndIndexBlock(rangeTombstone, unfilteredStartPosition, unfilteredEndPosition, updateClusteringMetadata);
+ }
+
+ private void updateMetadataAndIndexBlock(
+ UnfilteredDescriptor unfilteredDescriptor,
+ long unfilteredStartPosition,
+ long unfilteredEndPosition,
+ boolean updateClusteringMetadata) throws IOException
+ {
+ if (updateClusteringMetadata) updateClusteringMetadata(unfilteredDescriptor);
+ // write the first clustering into rowIndexEntries buffer (we will need it unless we never write the first entry)
+ if (unfilteredStartPosition == indexBlockStartOffset || (rowIndexEntryOffset == rowIndexEntries.position())) {
+ writeClusteringToRowIndexEntries(unfilteredDescriptor);
+ }
+ else
+ {
+ rowIndexEntryLastClustering.copy(unfilteredDescriptor);
+ }
+ /** {@link BigFormatPartitionWriter#addUnfiltered(Unfiltered)} */
+ // if we hit the index block size that we have to index after, go ahead and index it.
+ long indexBlockSize = currentOffsetInPartition(unfilteredEndPosition) - indexBlockStartOffset;
+ if (indexBlockSize >= this.indexBlockThreshold)
+ addIndexBlock(unfilteredEndPosition, indexBlockSize);
+ }
+
+ public void updateClusteringMetadata(UnfilteredDescriptor unfilteredDescriptor)
+ {
+ metadataCollector.updateClusteringValues(unfilteredDescriptor);
+ }
+
+ /**
+ * See:
+ * {@link BigFormatPartitionWriter#addIndexBlock()}
+ * - {@link org.apache.cassandra.io.sstable.IndexInfo.Serializer#serialize(org.apache.cassandra.io.sstable.IndexInfo, org.apache.cassandra.io.util.DataOutputPlus)}
+ */
+ private void addIndexBlock(long endOfRowPosition, long indexBlockSize) throws IOException
+ {
+ if (rowIndexEntriesOffsets.isEmpty() && rowIndexEntryOffset != 0) {
+ throw new IllegalStateException();
+ }
+
+ // serialize the index info
+ /** {@link org.apache.cassandra.io.sstable.IndexInfo.Serializer#serialize(org.apache.cassandra.io.sstable.IndexInfo, org.apache.cassandra.io.util.DataOutputPlus)}*/
+ rowIndexEntriesOffsets.addInt(rowIndexEntryOffset);
+
+ // first clustering is already in, write last entry
+ if (rowIndexEntryLastClustering.clusteringLength() == 0) {
+ // first entry is the last entry, copy it
+ byte[] entriesData = rowIndexEntries.getData();
+ long endOfFirstEntry = rowIndexEntries.position();
+ rowIndexEntries.write(entriesData, rowIndexEntryOffset, (int) (endOfFirstEntry - rowIndexEntryOffset));
+ }
+ else
+ {
+ writeClusteringToRowIndexEntries(rowIndexEntryLastClustering);
+ rowIndexEntryLastClustering.resetClustering();
+ }
+ rowIndexEntries.writeUnsignedVInt((long)indexBlockStartOffset);
+ rowIndexEntries.writeVInt(indexBlockSize - IndexInfo.Serializer.WIDTH_BASE);
+
+ boolean isDeleteTimePresent = !openMarker.isLive();
+ rowIndexEntries.writeBoolean(isDeleteTimePresent);
+ if (isDeleteTimePresent)
+ deletionTimeSerializer.serialize(openMarker, rowIndexEntries);
+ // next block starts
+ rowIndexEntryOffset = Ints.checkedCast(rowIndexEntries.position());
+ updateIndexBlockStartOffset(endOfRowPosition);
+ }
+
+ private void updateIndexBlockStartOffset(long endOfRowPosition)
+ {
+ indexBlockStartOffset = (int) (endOfRowPosition - partitionStart);
+ }
+
+ private void writeClusteringToRowIndexEntries(ClusteringDescriptor clustering) throws IOException
+ {
+ ClusteringPrefix.Kind kind = clustering.clusteringKind();
+ rowIndexEntries.writeByte(kind.ordinal());
+ if (kind != ClusteringPrefix.Kind.CLUSTERING)
+ rowIndexEntries.writeShort(clustering.clusteringColumnsBound());
+ rowIndexEntries.write(clustering.clusteringBytes(), 0, clustering.clusteringLength());
+ }
+
+ private long currentOffsetInPartition(long position)
+ {
+ return position - partitionStart;
+ }
+
+ private void encodeLargeColumnsSubset() throws IOException
+ {
+ // no columns are present, nothing to write
+ rowHeaderBuffer.writeUnsignedVInt32(missingColumns.size());
+ if (missingColumns.size() > columns.length / 2)
+ {
+ // write present columns
+ int presentIndex = 0;
+ int missingIndex = 0;
+ for (int i = 0; i < missingColumns.size(); i++)
+ {
+ missingIndex = missingColumns.get(i);
+ for (; presentIndex < missingIndex; presentIndex++)
+ rowHeaderBuffer.writeUnsignedVInt32(presentIndex);
+ presentIndex = missingIndex + 1;
+ }
+ if (missingIndex < columns.length-1) {
+ for (; presentIndex < missingIndex; presentIndex++)
+ rowHeaderBuffer.writeUnsignedVInt32(presentIndex);
+ }
+ }
+ else
+ {
+ // write missing columns
+ for (int missingIndex : missingColumns) {
+ rowHeaderBuffer.writeUnsignedVInt32(missingIndex);
+ }
+ }
+ }
+
+ public void setLast(ByteBuffer key)
+ {
+ IPartitioner partitioner = ssTableWriter.getPartitioner();
+ DecoratedKey last = partitioner.decorateKey(ByteBufferUtil.clone(key));
+ ssTableWriter.setLast(last);
+ }
+
+ public void setFirst(ByteBuffer key)
+ {
+ IPartitioner partitioner = ssTableWriter.getPartitioner();
+ DecoratedKey first = partitioner.decorateKey(ByteBufferUtil.clone(key));
+ ssTableWriter.setFirst(first);
+ ssTableWriter.setLast(first);
+ }
+
+ public IPartitioner partitioner()
+ {
+ return ssTableWriter.getPartitioner();
+ }
+
+ public DeletionTime openMarker() {
+ return openMarker;
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
index d90cc898bc..59b52ca77a 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java
@@ -68,7 +68,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
private final BlockingQueue writeQueue = newBlockingQueue(0);
private final DiskWriter diskWriter = new DiskWriter();
- SSTableSimpleUnsortedWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
+ public SSTableSimpleUnsortedWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
{
this(null, directory, metadata, columns, maxSSTableSizeInMiB);
}
@@ -198,7 +198,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
}
}
- static class SyncException extends RuntimeException
+ public static class SyncException extends RuntimeException
{
SyncException(IOException ioe)
{
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
index 05120fa65d..7b5fb8a89e 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java
@@ -64,7 +64,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
* @param maxSSTableSizeInMiB defines the max SSTable size if the value is positive.
* Any non-positive value indicates the sstable size is unlimited.
*/
- protected SSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
+ public SSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
{
this(null, directory, metadata, columns, maxSSTableSizeInMiB);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/UnfilteredDescriptor.java b/src/java/org/apache/cassandra/io/sstable/UnfilteredDescriptor.java
new file mode 100644
index 0000000000..df04c4ab8c
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/sstable/UnfilteredDescriptor.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DeletionTime.ReusableDeletionTime;
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+public class UnfilteredDescriptor extends ClusteringDescriptor
+{
+ private final ReusableLivenessInfo rowLivenessInfo = new ReusableLivenessInfo();
+ private final ReusableDeletionTime deletionTime = ReusableDeletionTime.live();
+ private final ReusableDeletionTime deletionTime2 = ReusableDeletionTime.live();
+
+ private long position;
+ private int flags;
+ private int extendedFlags;
+
+ private long unfilteredSize;
+ private long unfilteredDataStart;
+ private long prevUnfilteredSize;
+ Columns rowColumns;
+
+ public UnfilteredDescriptor(AbstractType[] clusteringTypes)
+ {
+ super(clusteringTypes);
+ }
+
+ void loadTombstone(RandomAccessReader dataReader,
+ SerializationHeader serializationHeader,
+ int flags) throws IOException
+ {
+ this.flags = flags;
+ this.extendedFlags = 0;
+ rowColumns = null;
+ byte clusteringKind = dataReader.readByte();
+ if (clusteringKind == STATIC_CLUSTERING_KIND || clusteringKind == ROW_CLUSTERING_KIND) {
+ // STATIC_CLUSTERING or CLUSTERING -> no deletion info, should not happen
+ throw new IllegalStateException();
+ }
+
+ int columnsBound = dataReader.readUnsignedShort();
+ loadClustering(dataReader, clusteringKind, columnsBound);
+ unfilteredSize = dataReader.readUnsignedVInt();
+ prevUnfilteredSize = dataReader.readUnsignedVInt(); // debug only, unused otherwise
+ if (clusteringKind == EXCL_END_INCL_START_BOUNDARY_CLUSTERING_KIND ||
+ clusteringKind == INCL_END_EXCL_START_BOUNDARY_CLUSTERING_KIND)
+ {
+ // boundary
+ // CLOSE
+ serializationHeader.readDeletionTime(dataReader, deletionTime);
+ // OPEN
+ serializationHeader.readDeletionTime(dataReader, deletionTime2);
+ }
+ else
+ {
+ // bound
+ // CLOSE|OPEN
+ serializationHeader.readDeletionTime(dataReader, deletionTime);
+ }
+ }
+
+ void loadRow(RandomAccessReader dataReader,
+ SerializationHeader serializationHeader,
+ DeserializationHelper deserializationHelper,
+ int flags) throws IOException {
+ // body = whatever is covered by size, so inclusive of the prev_row_size inclusive of flags
+ position = dataReader.getPosition() - 1;
+ this.flags = flags;
+ this.extendedFlags = 0;
+
+ loadClustering(dataReader, ROW_CLUSTERING_KIND, this.clusteringTypes.length);
+
+ rowColumns = serializationHeader.columns(false);
+
+ loadCommonRowFields(dataReader, serializationHeader, deserializationHelper, flags);
+ }
+
+ void loadStaticRow(RandomAccessReader dataReader,
+ SerializationHeader serializationHeader,
+ DeserializationHelper deserializationHelper,
+ int flags,
+ int extendedFlags) throws IOException {
+ // body = whatever is covered by size, so inclusive of the prev_row_size inclusive of flags
+ position = dataReader.getPosition() - 2;
+ this.flags = flags;
+ this.extendedFlags = extendedFlags;
+ // no clustering
+ loadClustering(dataReader, STATIC_CLUSTERING_KIND, 0);
+ rowColumns = serializationHeader.columns(true);
+
+ loadCommonRowFields(dataReader, serializationHeader, deserializationHelper, flags);
+ }
+
+ private void loadCommonRowFields(RandomAccessReader dataReader,
+ SerializationHeader serializationHeader,
+ DeserializationHelper deserializationHelper,
+ int flags) throws IOException
+ {
+ unfilteredSize = dataReader.readUnsignedVInt();
+ unfilteredDataStart = dataReader.getPosition();
+ prevUnfilteredSize = dataReader.readUnsignedVInt(); // debug only, unused otherwise
+
+ SSTableCursorReader.readLivenessInfo(dataReader,
+ serializationHeader,
+ deserializationHelper,
+ flags,
+ rowLivenessInfo);
+
+ if (UnfilteredSerializer.hasDeletion(flags))
+ {
+ // struct delta_deletion_time {
+ // varint delta_marked_for_delete_at;
+ // varint delta_local_deletion_time;
+ //};
+ serializationHeader.readDeletionTime(dataReader, deletionTime);
+ }
+ else
+ {
+ deletionTime.resetLive();
+ }
+ if (!UnfilteredSerializer.hasAllColumns(flags))
+ {
+ // TODO: re-implement GC free
+ rowColumns = Columns.serializer.deserializeSubset(rowColumns, dataReader);
+ }
+ }
+
+ public void resetUnfiltered()
+ {
+ resetClustering();
+ position = 0;
+ flags = 0;
+ extendedFlags = 0;
+ unfilteredSize = 0;
+ unfilteredDataStart = 0;
+ prevUnfilteredSize = 0;
+ rowColumns = null;
+ }
+
+ public long position()
+ {
+ return position;
+ }
+
+ public ReusableLivenessInfo livenessInfo()
+ {
+ return rowLivenessInfo;
+ }
+
+ public ReusableDeletionTime deletionTime()
+ {
+ return deletionTime;
+ }
+
+ public ReusableDeletionTime openDeletionTime()
+ {
+ return isBoundary() ? deletionTime2 : isEndBound() ? null : deletionTime;
+ }
+
+ public ReusableDeletionTime deletionTime2()
+ {
+ return deletionTime2;
+ }
+
+ public int flags()
+ {
+ return flags;
+ }
+
+ public long size()
+ {
+ return unfilteredSize;
+ }
+
+ public long dataStart()
+ {
+ return unfilteredDataStart;
+ }
+
+ public Columns rowColumns()
+ {
+ return rowColumns;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "UnfilteredDescriptor{" +
+ "rowLivenessInfo=" + rowLivenessInfo +
+ ", deletionTime=" + deletionTime +
+ ", position=" + position +
+ ", flags=" + flags +
+ ", extFlags=" + extendedFlags +
+ ", unfilteredSize=" + unfilteredSize +
+ ", prevUnfilteredSize=" + prevUnfilteredSize +
+ ", unfilteredDataStart=" + unfilteredDataStart +
+ ", rowColumns=" + rowColumns +
+ ", clusteringTypes=" + Arrays.toString(clusteringTypes()) +
+ '}';
+ }
+}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index bcd607dc25..93915b1ba2 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -926,7 +926,16 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
/**
* Returns a {@link KeyReader} over all keys in the sstable.
*/
- public abstract KeyReader keyReader() throws IOException;
+ public final KeyReader keyReader() throws IOException {
+ return keyReader(false);
+ }
+
+ /**
+ * Returns a {@link KeyReader} over all keys in the sstable.
+ *
+ * @param detailed should the iterator also provide details per partition entry(e.g. row entry details)
+ */
+ public abstract KeyReader keyReader(boolean detailed) throws IOException;
/**
* Returns a {@link KeyReader} over all keys in the sstable after a given key.
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
index 28035a85da..0b54fec2c5 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java
@@ -280,4 +280,12 @@ implements ISSTableScanner
}
}
}
+
+ @Override
+ public boolean isFullRange()
+ {
+ return dataRange == null || (dataRange.startKey().equals(sstable.getFirst()) &&
+ dataRange.stopKey().equals(sstable.getLast()) &&
+ dataRange.isUnrestricted(sstable.metadata()));
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java
index 69145e1e53..cf40f4e34b 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java
@@ -129,6 +129,13 @@ implements ISSTableScanner
return ImmutableSet.of(sstable);
}
+ @Override
+ public boolean isFullRange()
+ {
+ // hasNext will init start and end
+ return hasNext() && currentStartPosition == 0 && currentEndPosition == sstable.uncompressedLength();
+ }
+
public TableMetadata metadata()
{
return sstable.metadata();
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
index c09459b37d..8543da23e2 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java
@@ -77,8 +77,8 @@ public abstract class SSTableWriter extends SSTable implements Transactional
protected boolean isTransient;
protected long maxDataAge = -1;
protected final long keyCount;
- protected final MetadataCollector metadataCollector;
- protected final SerializationHeader header;
+ public final MetadataCollector metadataCollector;
+ public final SerializationHeader header;
protected final List observers;
protected final MmappedRegionsCache mmappedRegionsCache;
protected final TransactionalProxy txnProxy = txnProxy();
@@ -334,7 +334,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional
}
}
- protected Map finalizeMetadata()
+ protected final Map finalizeMetadata()
{
return metadataCollector.finalizeMetadata(getPartitioner().getClass().getCanonicalName(),
metadata().params.bloomFilterFpChance,
@@ -595,4 +595,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional
return new SSTableZeroCopyWriter(this, txn, owner);
}
}
+
+ public void setFirst(DecoratedKey key) {
+ first = key;
+ }
+
+ public void setLast(DecoratedKey key) {
+ last = key;
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
index 46b65140c5..5322cb4cc1 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java
@@ -43,11 +43,11 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
private final SerializationHelper helper;
private final Version version;
- private long previousRowStart;
- private long initialPosition;
+ private long previousRowStartOffset;
+ private long partitionStartPosition;
private long headerLength;
- protected long startPosition;
+ protected long indexBlockStartOffset;
protected int written;
protected ClusteringPrefix firstClustering;
@@ -78,9 +78,9 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
protected void reset()
{
- this.initialPosition = writer.position();
- this.startPosition = -1;
- this.previousRowStart = 0;
+ this.partitionStartPosition = writer.position();
+ this.indexBlockStartOffset = -1;
+ this.previousRowStartOffset = 0;
this.written = 0;
this.firstClustering = null;
this.lastClustering = null;
@@ -106,7 +106,7 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
if (!header.hasStatic())
{
- this.headerLength = writer.position() - initialPosition;
+ this.headerLength = writer.position() - partitionStartPosition;
state = State.AWAITING_ROWS;
return;
}
@@ -121,7 +121,7 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version.correspondingMessagingVersion());
- this.headerLength = writer.position() - initialPosition;
+ this.headerLength = writer.position() - partitionStartPosition;
state = State.AWAITING_ROWS;
}
@@ -129,21 +129,21 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
{
checkState(state == State.AWAITING_ROWS);
- long pos = currentPosition();
+ long offset = currentOffsetInPartition();
if (firstClustering == null)
{
- // Beginning of an index block. Remember the start and position
+ // Beginning of an index block. Remember the start clustering and position
firstClustering = unfiltered.clustering();
- startOpenMarker = openMarker;
- startPosition = pos;
+ startOpenMarker = openMarker; // first entry is always LIVE (for BTI format)
+ indexBlockStartOffset = offset;
}
long unfilteredPosition = writer.position();
- unfilteredSerializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version.correspondingMessagingVersion());
+ unfilteredSerializer.serialize(unfiltered, helper, writer, offset - previousRowStartOffset, version.correspondingMessagingVersion());
lastClustering = unfiltered.clustering();
- previousRowStart = pos;
+ previousRowStartOffset = offset;
++written;
if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
@@ -159,19 +159,30 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
state = State.COMPLETED;
- long endPosition = currentPosition();
+ long partitionLength = currentOffsetInPartition();
unfilteredSerializer.writeEndOfPartition(writer);
- return endPosition;
+ return partitionLength;
}
- protected long currentPosition()
+ protected long currentOffsetInPartition()
{
- return writer.position() - initialPosition;
+ return writer.position() - partitionStartPosition;
}
- public long getInitialPosition()
+ public long getPartitionStartPosition()
{
- return initialPosition;
+ return partitionStartPosition;
+ }
+
+ /** Some bullshit access for now */
+ public SerializationHeader getHeader()
+ {
+ return header;
+ }
+
+ public SerializationHelper getHelper()
+ {
+ return helper;
}
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
index eb6231f9e7..50c9744856 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java
@@ -175,7 +175,7 @@ public abstract class SortedTableVerifier imp
{
try
{
- outputHandler.debug("Deserializing bloom filter for %s", sstable);
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Deserializing bloom filter for %s", sstable);
deserializeBloomFilter(sstable);
}
catch (Throwable t)
@@ -217,7 +217,7 @@ public abstract class SortedTableVerifier imp
protected int verifyOwnedRanges()
{
List> ownedRanges = Collections.emptyList();
- outputHandler.debug("Checking that all tokens are owned by the current node");
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Checking that all tokens are owned by the current node");
try (KeyIterator iter = sstable.keyIterator())
{
ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace));
@@ -287,7 +287,7 @@ public abstract class SortedTableVerifier imp
throw new CompactionInterruptedException(verifyInfo.getCompactionInfo());
long rowStart = dataFile.getFilePointer();
- outputHandler.debug("Reading row at %d", rowStart);
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Reading row at %d", rowStart);
DecoratedKey key = null;
try
@@ -332,8 +332,11 @@ public abstract class SortedTableVerifier imp
long dataSize = nextRowPositionFromIndex - dataStartFromIndex;
// avoid an NPE if key is null
- String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey());
- outputHandler.debug("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize));
+ if (outputHandler.isDebugEnabled())
+ {
+ String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey());
+ outputHandler.debug("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize));
+ }
try
{
@@ -352,7 +355,7 @@ public abstract class SortedTableVerifier imp
prevKey = key;
- outputHandler.debug("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex);
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex);
dataFile.seek(nextRowPositionFromIndex);
}
catch (Throwable th)
@@ -374,7 +377,7 @@ public abstract class SortedTableVerifier imp
{
try
{
- outputHandler.debug("Deserializing index for %s", sstable);
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Deserializing index for %s", sstable);
deserializeIndex(sstable);
}
catch (Throwable t)
@@ -384,7 +387,7 @@ public abstract class SortedTableVerifier imp
}
}
- private void deserializeIndex(SSTableReader sstable) throws IOException
+ protected void deserializeIndex(SSTableReader sstable) throws IOException
{
try (KeyReader it = sstable.keyReader())
{
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
index 5ccaf26710..d600ee665d 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java
@@ -79,9 +79,9 @@ public abstract class SortedTableWriter o.startPartition(key, partitionWriter.getInitialPosition(), partitionWriter.getInitialPosition()));
+ notifyObservers(o -> o.startPartition(key, partitionWriter.getPartitionStartPosition(), partitionWriter.getPartitionStartPosition()));
}
protected void onStaticRow(Row row)
@@ -331,7 +333,7 @@ public abstract class SortedTableWriter
components;
- protected final IFilter bf;
+ public final IFilter bf;
protected AbstractIndexWriter(Builder b)
{
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
index 801982d5ec..fcbcc18c06 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java
@@ -48,11 +48,11 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
@VisibleForTesting
public static final int DEFAULT_GRANULARITY = 64 * 1024;
- // used, if the row-index-entry reaches config column_index_cache_size
- private DataOutputBuffer buffer;
- // used to track the size of the serialized size of row-index-entry (unused for buffer)
+ // used, if the row-index-entry reaches config switchIndexInfoToBufferThreshold
+ private DataOutputBuffer rowIndexEntryBuffer;
+ // used to track the total serialized size of indexSamples (unused for buffer)
private int indexSamplesSerializedSize;
- // used, until the row-index-entry reaches config column_index_cache_size
+ // used, until the row-index-entry reaches switchIndexInfoToBufferThreshold (from config column_index_cache_size, or default 64k)
private final List indexSamples = new ArrayList<>();
private DataOutputBuffer reusableBuffer;
@@ -62,8 +62,10 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
private final ISerializer idxSerializer;
- private final int cacheSizeThreshold;
- private final int indexSize;
+ /** Beyond this limit we switch from storing IndexInfo in the list to directly serializing them into a buffer */
+ private final int switchIndexInfoToBufferThreshold;
+ /** If a partition grows beyond this size we store inter-partition index data in IndexInfo */
+ private final int indexBlockThreshold;
BigFormatPartitionWriter(SerializationHeader header,
SequentialWriter writer,
@@ -82,8 +84,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
{
super(header, writer, version);
this.idxSerializer = indexInfoSerializer;
- this.cacheSizeThreshold = cacheSizeThreshold;
- this.indexSize = indexSize;
+ this.switchIndexInfoToBufferThreshold = cacheSizeThreshold;
+ this.indexBlockThreshold = indexSize;
}
public void reset()
@@ -93,9 +95,9 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
this.indexSamplesSerializedSize = 0;
this.indexSamples.clear();
- if (this.buffer != null)
- this.reusableBuffer = this.buffer;
- this.buffer = null;
+ if (this.rowIndexEntryBuffer != null)
+ this.reusableBuffer = this.rowIndexEntryBuffer;
+ this.rowIndexEntryBuffer = null;
}
public int getColumnIndexCount()
@@ -105,12 +107,12 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
public ByteBuffer buffer()
{
- return buffer != null ? buffer.buffer() : null;
+ return rowIndexEntryBuffer != null ? rowIndexEntryBuffer.buffer() : null;
}
public List indexSamples()
{
- if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= cacheSizeThreshold)
+ if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= switchIndexInfoToBufferThreshold)
{
return indexSamples;
}
@@ -129,8 +131,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
{
IndexInfo cIndexInfo = new IndexInfo(firstClustering,
lastClustering,
- startPosition,
- currentPosition() - startPosition,
+ indexBlockStartOffset,
+ currentOffsetInPartition() - indexBlockStartOffset,
!openMarker.isLive() ? openMarker : null);
// indexOffsets is used for both shallow (ShallowIndexedEntry) and non-shallow IndexedEntry.
@@ -156,8 +158,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
else
{
indexOffsets[columnIndexCount] =
- buffer != null
- ? Ints.checkedCast(buffer.position())
+ rowIndexEntryBuffer != null
+ ? Ints.checkedCast(rowIndexEntryBuffer.position())
: indexSamplesSerializedSize;
}
}
@@ -165,16 +167,20 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
// First, we collect the IndexInfo objects until we reach Config.column_index_cache_size in an ArrayList.
// When column_index_cache_size is reached, we switch to byte-buffer mode.
- if (buffer == null)
+ if (rowIndexEntryBuffer == null)
{
indexSamplesSerializedSize += idxSerializer.serializedSize(cIndexInfo);
- if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) > cacheSizeThreshold)
+ if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.INT_SIZE > switchIndexInfoToBufferThreshold)
{
- buffer = reuseOrAllocateBuffer();
+ rowIndexEntryBuffer = reuseOrAllocateBuffer();
+ // serialize pre-existing samples
for (IndexInfo indexSample : indexSamples)
{
- idxSerializer.serialize(indexSample, buffer);
+ /** {@link IndexInfo.Serializer#serialize} */
+ idxSerializer.serialize(indexSample, rowIndexEntryBuffer);
}
+ // release pre-existing samples
+ indexSamples.clear();
}
else
{
@@ -182,9 +188,10 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
}
}
// don't put an else here since buffer may be allocated in preceding if block
- if (buffer != null)
+ if (rowIndexEntryBuffer != null)
{
- idxSerializer.serialize(cIndexInfo, buffer);
+ /** {@link IndexInfo.Serializer#serialize} */
+ idxSerializer.serialize(cIndexInfo, rowIndexEntryBuffer);
}
firstClustering = null;
@@ -201,7 +208,7 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
return buffer;
}
// don't use the standard RECYCLER as that only recycles up to 1MB and requires proper cleanup
- return new DataOutputBuffer(cacheSizeThreshold * 2);
+ return new DataOutputBuffer(switchIndexInfoToBufferThreshold * 2);
}
@Override
@@ -210,7 +217,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
super.addUnfiltered(unfiltered);
// if we hit the column index size that we have to index after, go ahead and index it.
- if (currentPosition() - startPosition >= indexSize)
+ long sizeSinceLastIndexBlock = currentOffsetInPartition() - indexBlockStartOffset;
+ if (sizeSinceLastIndexBlock >= this.indexBlockThreshold)
addIndexBlock();
}
@@ -231,10 +239,10 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
// we have to write the offsts to these here. The offsets have already been collected
// in indexOffsets[]. buffer is != null, if it exceeds Config.column_index_cache_size.
// In the other case, when buffer==null, the offsets are serialized in RowIndexEntry.IndexedEntry.serialize().
- if (buffer != null)
+ if (rowIndexEntryBuffer != null)
{
for (int i = 0; i < columnIndexCount; i++)
- buffer.writeInt(indexOffsets[i]);
+ rowIndexEntryBuffer.writeInt(indexOffsets[i]);
}
// we should always have at least one computed index block, but we only write it out if there is more than that.
@@ -245,8 +253,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
public int indexInfoSerializedSize()
{
- return buffer != null
- ? buffer.buffer().limit()
+ return rowIndexEntryBuffer != null
+ ? rowIndexEntryBuffer.buffer().limit()
: indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
index 8f4dd50d61..24b53ae94e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java
@@ -178,7 +178,7 @@ public class BigSSTableReaderLoadingBuilder extends SortedTableReaderLoadingBuil
checkNotNull(serializationHeader);
RowIndexEntry.IndexSerializer serializer = new RowIndexEntry.Serializer(descriptor.version, serializationHeader, tableMetrics);
- return BigTableKeyReader.create(indexFile, serializer);
+ return BigTableKeyReader.create(indexFile, serializer, false);
}
/**
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java
index 04b07af2ce..d3fba91046 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java
@@ -36,19 +36,23 @@ public class BigTableKeyReader implements KeyReader
private final RandomAccessReader indexFileReader;
private final IndexSerializer rowIndexEntrySerializer;
private final long initialPosition;
-
+ private final boolean detailed;
private ByteBuffer key;
private long dataPosition;
private long keyPosition;
+ /** only if detailed */
+ private RowIndexEntry rowIndexEntry;
private BigTableKeyReader(FileHandle indexFile,
RandomAccessReader indexFileReader,
- IndexSerializer rowIndexEntrySerializer)
+ IndexSerializer rowIndexEntrySerializer,
+ boolean detailed)
{
this.indexFile = indexFile;
this.indexFileReader = indexFileReader;
this.rowIndexEntrySerializer = rowIndexEntrySerializer;
this.initialPosition = indexFileReader.getFilePointer();
+ this.detailed = detailed;
}
public static BigTableKeyReader create(RandomAccessReader indexFileReader, IndexSerializer serializer) throws IOException
@@ -58,7 +62,7 @@ public class BigTableKeyReader implements KeyReader
public static BigTableKeyReader create(FileHandle indexFile, RandomAccessReader indexFileReader, IndexSerializer serializer) throws IOException
{
- BigTableKeyReader iterator = new BigTableKeyReader(indexFile, indexFileReader, serializer);
+ BigTableKeyReader iterator = new BigTableKeyReader(indexFile, indexFileReader, serializer, false);
try
{
iterator.advance();
@@ -72,7 +76,7 @@ public class BigTableKeyReader implements KeyReader
}
@SuppressWarnings({ "resource", "RedundantSuppression" }) // iFile and reader are closed in the BigTableKeyReader#close method
- public static BigTableKeyReader create(FileHandle indexFile, IndexSerializer serializer) throws IOException
+ public static BigTableKeyReader create(FileHandle indexFile, IndexSerializer serializer, boolean detailed) throws IOException
{
FileHandle iFile = null;
RandomAccessReader reader = null;
@@ -81,7 +85,7 @@ public class BigTableKeyReader implements KeyReader
{
iFile = indexFile.sharedCopy();
reader = iFile.createReader();
- iterator = new BigTableKeyReader(iFile, reader, serializer);
+ iterator = new BigTableKeyReader(iFile, reader, serializer, detailed);
iterator.advance();
return iterator;
}
@@ -116,7 +120,14 @@ public class BigTableKeyReader implements KeyReader
{
keyPosition = indexFileReader.getFilePointer();
key = ByteBufferUtil.readWithShortLength(indexFileReader);
- dataPosition = rowIndexEntrySerializer.deserializePositionAndSkip(indexFileReader);
+ if (detailed) {
+ rowIndexEntry = rowIndexEntrySerializer.deserialize(indexFileReader);
+ dataPosition = rowIndexEntry.getPosition();
+ }
+ else
+ {
+ dataPosition = rowIndexEntrySerializer.deserializePositionAndSkip(indexFileReader);
+ }
return true;
}
else
@@ -152,6 +163,15 @@ public class BigTableKeyReader implements KeyReader
return dataPosition;
}
+ public RowIndexEntry rowIndexEntry() {
+ assert detailed;
+ return rowIndexEntry;
+ }
+
+ public FileHandle indexFile() {
+ return indexFile;
+ }
+
public long indexPosition()
{
return indexFileReader.getFilePointer();
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
index d44bd7f714..2f31d4c358 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java
@@ -141,9 +141,9 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
}
@Override
- public KeyReader keyReader() throws IOException
+ public KeyReader keyReader(boolean detailed) throws IOException
{
- return BigTableKeyReader.create(ifile, rowIndexEntrySerializer);
+ return BigTableKeyReader.create(ifile, rowIndexEntrySerializer, detailed);
}
@Override
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
index 83243529c4..a275239629 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java
@@ -188,6 +188,7 @@ public class BigTableScanner extends SSTableScanner implem
{
try
{
- outputHandler.debug("Deserializing index summary for %s", sstable);
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Deserializing index summary for %s", sstable);
deserializeIndexSummary(sstable);
}
catch (Throwable t)
@@ -99,6 +107,74 @@ public class BigTableVerifier extends SortedTableVerifier implem
super.verifyIndex();
}
+ @Override
+ protected void deserializeIndex(SSTableReader sstable) throws IOException
+ {
+ DeserializationHelper deserializationHelper = new DeserializationHelper(sstable.metadata(),
+ sstable.descriptor.version.correspondingMessagingVersion(),
+ DeserializationHelper.Flag.LOCAL);
+ ClusteringComparator comparator = sstable.metadata().comparator;
+ try (BigTableKeyReader it = (BigTableKeyReader)sstable.keyReader(true))
+ {
+ if (it.isExhausted())
+ return;
+ ByteBuffer key;
+ boolean isFirst = true;
+ do
+ {
+ key = it.key();
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("Key %s", sstable.metadata().partitionKeyType.getString(key));
+
+ if (isFirst && !Objects.equals(key, sstable.getFirst().getKey()))
+ throw new CorruptSSTableException(new IOException("First partition does not match index"), it.toString());
+ else
+ isFirst = false;
+ RowIndexEntry rowIndexEntry = it.rowIndexEntry();
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("rowIndexEntry %s", rowIndexEntry.toString());
+
+ long partitionBase = it.dataPosition();
+ int blockCount = rowIndexEntry.blockCount();
+ if (blockCount > 0)
+ {
+ long expectedNextOffset = 0;
+ RowIndexEntry.IndexInfoRetriever indexInfoRetriever = rowIndexEntry.openWithIndex(it.indexFile());
+ for (int blockIndex = 0; blockIndex < blockCount; blockIndex++)
+ {
+ IndexInfo indexInfo = indexInfoRetriever.columnsIndex(blockIndex);
+ if (outputHandler.isDebugEnabled()) outputHandler.debug("indexInfo %s", indexInfo.toString(sstable.metadata()));
+
+ long dataFileOffset = partitionBase + indexInfo.offset;
+ if (expectedNextOffset != 0)
+ {
+ if (expectedNextOffset != indexInfo.offset)
+ throw new CorruptSSTableException(new IOException("Row entry indexInfo offset + width should match next block offset:" + indexInfo.offset + " expected: " + expectedNextOffset), it.toString());
+ }
+ expectedNextOffset = indexInfo.offset + indexInfo.width;
+ dataFile.seek(dataFileOffset);
+
+ Unfiltered unfiltered = UnfilteredSerializer.serializer.deserialize(dataFile,
+ sstable.header,
+ deserializationHelper,
+ BTreeRow.sortedBuilder());
+ if (!Objects.equals(indexInfo.firstName, unfiltered.clustering()))
+ throw new CorruptSSTableException(new IOException("Unfiltered clustering in index does not match data:{info=" + indexInfo.toString(sstable.metadata()) + ", row:" + unfiltered.clustering().toString(sstable.metadata()) + "}"), it.toString());
+
+ if (comparator.compare(indexInfo.firstName, indexInfo.lastName) > 0)
+ throw new CorruptSSTableException(new IOException("First name is > Last name:{info=" + indexInfo.toString(sstable.metadata()) + "}"), it.toString());
+
+ }
+ }
+ }
+ // The verifier checks that the partition key/position in the index and data match, here we waant to verify
+ // the entries clustering keys match.
+ while (it.advance()); // no-op, just check if index is readable
+
+ if (!Objects.equals(key, sstable.getLast().getKey()))
+ throw new CorruptSSTableException(new IOException("Last partition does not match index"), it.toString());
+ }
+ dataFile.reset();
+ }
+
private void logDuplicates(DecoratedKey key, Row first, int duplicateRows, long minTimestamp, long maxTimestamp)
{
String keyString = sstable.metadata().partitionKeyType.getString(key.getKey());
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
index 6cf01e7fbc..4f04eee100 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java
@@ -87,7 +87,7 @@ public class BigTableWriter extends SortedTableWriter o.startPartition(key, partitionWriter.getInitialPosition(), indexWriter.writer.position()));
+ notifyObservers(o -> o.startPartition(key, partitionWriter.getPartitionStartPosition(), indexWriter.writer.position()));
}
@Override
@@ -97,7 +97,7 @@ public class BigTableWriter extends SortedTableWriter 1;
if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
{
return new IndexedEntry(position, in, deletionTime, headerLength, columnsIndexCount,
@@ -663,6 +667,21 @@ public class RowIndexEntry extends AbstractRowIndexEntry
in.readUnsignedVInt();
}
+
+ @Override
+ public String toString()
+ {
+ return "IndexedEntry{" +
+ "position=" + position +
+ ", deletionTime=" + deletionTime +
+ ", headerLength=" + headerLength +
+ ", columnsIndex=" + Arrays.toString(columnsIndex) +
+ ", offsets=" + Arrays.toString(offsets) +
+ ", indexedPartSize=" + indexedPartSize +
+ ", idxInfoSerializer=" + idxInfoSerializer +
+ ", version=" + version +
+ '}';
+ }
}
/**
@@ -678,6 +697,7 @@ public class RowIndexEntry extends AbstractRowIndexEntry
BASE_SIZE = ObjectSizes.measure(new ShallowIndexedEntry(0, 0, DeletionTime.LIVE, 0, 10, 0, null, BigFormat.getInstance().getLatestVersion()));
}
+ // only for cache serialization
private final long indexFilePosition;
private final DeletionTime deletionTime;
@@ -701,8 +721,6 @@ public class RowIndexEntry extends AbstractRowIndexEntry
{
super(dataFilePosition);
- assert columnIndexCount > 1;
-
this.indexFilePosition = indexFilePosition;
this.headerLength = headerLength;
this.deletionTime = deletionTime;
@@ -810,6 +828,23 @@ public class RowIndexEntry extends AbstractRowIndexEntry
in.readUnsignedVInt();
}
+
+ @Override
+ public String toString()
+ {
+ return "ShallowIndexedEntry{" +
+ "position=" + position +
+ ", indexFilePosition=" + indexFilePosition +
+ ", deletionTime=" + deletionTime +
+ ", headerLength=" + headerLength +
+ ", columnsIndexCount=" + columnsIndexCount +
+ ", indexedPartSize=" + indexedPartSize +
+ ", offsetsOffset=" + offsetsOffset +
+ ", idxInfoSerializer=" + idxInfoSerializer +
+ ", fieldsSerializedSize=" + fieldsSerializedSize +
+ ", version=" + version +
+ '}';
+ }
}
private static final class ShallowInfoRetriever extends FileIndexInfoRetriever
@@ -900,4 +935,12 @@ public class RowIndexEntry extends AbstractRowIndexEntry
super(message);
}
}
+
+ @Override
+ public String toString()
+ {
+ return "RowIndexEntry{" +
+ "position=" + position +
+ '}';
+ }
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java
index ccf69f7378..6bb024ab7e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java
@@ -79,7 +79,7 @@ class BtiFormatPartitionWriter extends SortedTablePartitionWriter
super.addUnfiltered(unfiltered);
// if we hit the column index size that we have to index after, go ahead and index it.
- if (currentPosition() - startPosition >= rowIndexBlockSize)
+ if (currentOffsetInPartition() - indexBlockStartOffset >= rowIndexBlockSize)
addIndexBlock();
}
@@ -112,7 +112,7 @@ class BtiFormatPartitionWriter extends SortedTablePartitionWriter
protected void addIndexBlock() throws IOException
{
- IndexInfo cIndexInfo = new IndexInfo(startPosition, startOpenMarker);
+ IndexInfo cIndexInfo = new IndexInfo(indexBlockStartOffset, startOpenMarker);
rowTrie.add(firstClustering, lastClustering, cIndexInfo);
firstClustering = null;
++rowIndexBlockCount;
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
index 791bcb9d18..1eee4515e3 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java
@@ -317,7 +317,7 @@ public class BtiTableReader extends SSTableReaderWithFilter
}
@Override
- public PartitionIterator keyReader() throws IOException
+ public PartitionIterator keyReader(boolean detailed) throws IOException
{
return PartitionIterator.create(partitionIndex, metadata().partitioner, rowIndexFile, dfile, descriptor.version);
}
diff --git a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
index 33dec69874..e88f7228f2 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java
@@ -73,7 +73,7 @@ public class BtiTableWriter extends SortedTableWriter commitLogIntervals = IntervalSet.empty();
protected final MinMaxLongTracker timestampTracker = new MinMaxLongTracker();
@@ -122,6 +124,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
* be a corresponding start bound that is smaller).
*/
private ClusteringPrefix minClustering = ClusteringBound.MAX_START;
+ private final ClusteringDescriptor minClusteringDescriptor;
+
/**
* The largest clustering prefix for any {@link Unfiltered} in the sstable.
*
@@ -129,6 +133,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
* be a corresponding end bound that is bigger).
*/
private ClusteringPrefix maxClustering = ClusteringBound.MIN_END;
+ private final ClusteringDescriptor maxClusteringDescriptor;
protected boolean hasLegacyCounterShards = false;
private boolean hasPartitionLevelDeletions = false;
@@ -159,6 +164,9 @@ public class MetadataCollector implements PartitionStatisticsCollector
{
this.comparator = comparator;
this.originatingHostId = originatingHostId;
+ AbstractType[] clusteringTypes = comparator.subtypes().toArray(AbstractType[]::new);
+ this.minClusteringDescriptor = new ClusteringDescriptor(clusteringTypes).resetMaxStart();
+ this.maxClusteringDescriptor = new ClusteringDescriptor(clusteringTypes).resetMinEnd();
}
public MetadataCollector(Iterable sstables, ClusteringComparator comparator)
@@ -185,6 +193,14 @@ public class MetadataCollector implements PartitionStatisticsCollector
return this;
}
+ public MetadataCollector addKey(byte[] key, int offset, int length)
+ {
+ long hashed = MurmurHash.hash2_64(key, offset, length, 0);
+ cardinality.offerHashed(hashed);
+ totalTombstones = 0;
+ return this;
+ }
+
public MetadataCollector addPartitionSizeInBytes(long partitionSize)
{
estimatedPartitionSize.add(partitionSize);
@@ -236,6 +252,15 @@ public class MetadataCollector implements PartitionStatisticsCollector
updateTombstoneCount();
}
+ /**
+ * Cell level stats, if we accept that LDT and LET are the same...
+ */
+ public void updateCellLiveness(LivenessInfo newInfo)
+ {
+ ++currentPartitionCells;
+ update(newInfo);
+ }
+
public void updatePartitionDeletion(DeletionTime dt)
{
if (!dt.isLive())
@@ -299,6 +324,32 @@ public class MetadataCollector implements PartitionStatisticsCollector
return this;
}
+ public void updateClusteringValues(ClusteringDescriptor newClustering) {
+ // In a SSTable, every opening marker will be closed, so the start of a range tombstone marker will never be
+ // the maxClustering (the corresponding close might though) and there is no point in doing the comparison
+ // (and vice-versa for the close). By the same reasoning, a boundary will never be either the min or max
+ // clustering, and we can save on comparisons.
+ if (newClustering == null || newClustering.clusteringKind().isBoundary())
+ return;
+
+ // In case of monotonically growing stream of clusterings, we will usually require only one comparison
+ // because if we detected X is greater than the current MAX, then it cannot be lower than the current MIN
+ // at the same time. The only case when we need to update MIN when the current MAX was detected to be updated
+ // is the case when MIN was not yet initialized and still point the ClusteringBound.MAX_START
+ if (ClusteringComparator.compare(newClustering, maxClusteringDescriptor) > 0)
+ {
+ maxClusteringDescriptor.copy(newClustering);
+ if (minClusteringDescriptor.isMaxStart()) // min is unset
+ {
+ minClusteringDescriptor.copy(newClustering);
+ }
+ }
+ else if (ClusteringComparator.compare(newClustering, minClusteringDescriptor) < 0)
+ {
+ minClusteringDescriptor.copy(newClustering);
+ }
+ }
+
public void updateClusteringValues(Clustering clustering)
{
if (clustering == Clustering.STATIC_CLUSTERING)
@@ -361,6 +412,13 @@ public class MetadataCollector implements PartitionStatisticsCollector
Map components = new EnumMap<>(MetadataType.class);
components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
+ Slice coveredClustering;
+ if (!minClusteringDescriptor.isMaxStart()) // min is end only if the descriptors are unused
+ {
+ minClustering = minClusteringDescriptor.toClusteringPrefix(comparator.subtypes());
+ maxClustering = maxClusteringDescriptor.toClusteringPrefix(comparator.subtypes());
+ }
+ coveredClustering = Slice.make(minClustering.retainable().asStartBound(), maxClustering.retainable().asEndBound());
components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
estimatedCellPerPartitionCount,
commitLogIntervals,
@@ -374,7 +432,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
estimatedTombstoneDropTime.build(),
sstableLevel,
comparator.subtypes(),
- Slice.make(minClustering.retainable().asStartBound(), maxClustering.retainable().asEndBound()),
+ coveredClustering,
hasLegacyCounterShards,
repairedAt,
totalColumnsSet,
diff --git a/src/java/org/apache/cassandra/io/util/ResizableByteBuffer.java b/src/java/org/apache/cassandra/io/util/ResizableByteBuffer.java
new file mode 100644
index 0000000000..35a1e63217
--- /dev/null
+++ b/src/java/org/apache/cassandra/io/util/ResizableByteBuffer.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.utils.vint.VIntCoding;
+import org.jctools.util.Pow2;
+
+public class ResizableByteBuffer
+{
+ private int length = 0;
+ private byte[] bytes = new byte[64];
+ private ByteBuffer buffer = ByteBuffer.wrap(bytes);
+
+ /**
+ * Loads the length (UnsignedShort) from the data reader, and then loads the new data into the buffer.
+ *
+ * @param dataReader data source
+ * @return true if the underlying array/buffer had to be resized to accomodate the new data, false otherwise
+ * @throws IOException
+ */
+ public final boolean loadShortLength(RandomAccessReader dataReader) throws IOException
+ {
+ int newLength = dataReader.readUnsignedShort();
+ return load(dataReader, newLength);
+ }
+
+ /**
+ * Loads the new data (of the given length) into the buffer. Buffer is rewritten.
+ *
+ * @param dataReader data source
+ * @param newLength new data length
+ * @return true if the underlying array/buffer had to be resized to accomodate the new data, false otherwise
+ * @throws IOException
+ */
+ public final boolean load(RandomAccessReader dataReader, int newLength) throws IOException
+ {
+ boolean resize = maybeResizeForOverwrite(newLength);
+ dataReader.readFully(bytes, 0, newLength);
+ setLength(newLength);
+ return resize;
+ }
+
+ public final void resetBuffer() {
+ setLength(0);
+ }
+
+ protected final byte[] bytes()
+ {
+ return bytes;
+ }
+
+ protected final ByteBuffer buffer()
+ {
+ return buffer;
+ }
+
+ protected final int length()
+ {
+ return length;
+ }
+
+ protected final void overwrite(byte[] newBytes, int newLength)
+ {
+ maybeResizeForOverwrite(newLength);
+ System.arraycopy(newBytes, 0, bytes, 0, newLength);
+ setLength(newLength);
+ }
+
+ private void setLength(int newLength)
+ {
+ int currentLength = length;
+ if (newLength != currentLength)
+ {
+ length = newLength;
+ buffer.limit(newLength);
+ }
+ }
+
+ /**
+ * Loads new partial data (of the given length) into the buffer. New data is appended.
+ *
+ * @param dataReader data source
+ * @param partLength new data length
+ * @return true if the underlying array/buffer had to be resized to accomodate the new data, false otherwise
+ * @throws IOException
+ */
+ public final boolean loadPart(RandomAccessReader dataReader, int partLength) throws IOException
+ {
+ int newLength = length + partLength;
+ boolean resize = maybeResizeForAppend(newLength);
+ dataReader.readFully(bytes, length, partLength);
+ setLength(newLength);
+ return resize;
+ }
+
+ public final boolean writeUnsignedVInt(long value)
+ {
+ int currentPosition = length;
+ int newLength = currentPosition + 9;
+ boolean resize = maybeResizeForAppend(newLength);
+ VIntCoding.writeUnsignedVInt(value, buffer.position(currentPosition).limit(newLength));
+ setLength(buffer.position());
+ buffer.position(0);
+ return resize;
+ }
+
+ private boolean maybeResizeForOverwrite(int newLength)
+ {
+ if (newLength > bytes.length)
+ {
+ bytes = new byte[Pow2.roundToPowerOfTwo(newLength)];
+ buffer = ByteBuffer.wrap(bytes);
+ return true;
+ }
+ return false;
+ }
+
+ private boolean maybeResizeForAppend(int newLength)
+ {
+ if (newLength > bytes.length)
+ {
+ int currLength = length;
+ byte[] currBytes = bytes;
+ bytes = new byte[Pow2.roundToPowerOfTwo(newLength)];
+ if (currLength != 0)
+ System.arraycopy(currBytes,0 , bytes, 0, currLength);
+ buffer = ByteBuffer.wrap(bytes);
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/src/java/org/apache/cassandra/tools/JsonTransformer.java b/src/java/org/apache/cassandra/tools/JsonTransformer.java
index 9ffa6be00a..cd960cb4ca 100644
--- a/src/java/org/apache/cassandra/tools/JsonTransformer.java
+++ b/src/java/org/apache/cassandra/tools/JsonTransformer.java
@@ -436,7 +436,14 @@ public final class JsonTransformer
else
{
AbstractType type = column.cellValueType();
- json.writeRawValue(type.toJSONString(clustering.get(i), clustering.accessor(), ProtocolVersion.CURRENT));
+ try
+ {
+ json.writeRawValue(type.toJSONString(clustering.get(i), clustering.accessor(), ProtocolVersion.CURRENT));
+ }
+ catch (Exception e)
+ {
+ json.writeString("unsupported conversion to JSON");
+ }
}
}
json.writeEndArray();
@@ -552,7 +559,13 @@ public final class JsonTransformer
else
{
json.writeFieldName("value");
- json.writeRawValue(cellType.toJSONString(cell.value(), cell.accessor(), ProtocolVersion.CURRENT));
+ try
+ {
+ json.writeRawValue(cellType.toJSONString(cell.value(), cell.accessor(), ProtocolVersion.CURRENT));
+ }
+ catch (Exception e) {
+ json.writeString("unsupported conversion to JSON");
+ }
}
if (liveInfo.isEmpty() || cell.timestamp() != liveInfo.timestamp())
{
diff --git a/src/java/org/apache/cassandra/utils/BloomFilter.java b/src/java/org/apache/cassandra/utils/BloomFilter.java
index a95d131a39..2575b4b3ad 100644
--- a/src/java/org/apache/cassandra/utils/BloomFilter.java
+++ b/src/java/org/apache/cassandra/utils/BloomFilter.java
@@ -101,6 +101,22 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
return indexes;
}
+ private long[] indexes(byte[] key, int offset, int length)
+ {
+ // we use the same array both for storing the hash result, and for storing the indexes we return,
+ // so that we do not need to allocate two arrays.
+ long[] indexes = reusableIndexes.get();
+ return indexes(key, offset, length, indexes);
+ }
+
+ private long[] indexes(byte[] key, int offset, int length, long[] indexes)
+ {
+ MurmurHash.hash3_x64_128(key, offset, length, 0, indexes);
+
+ setIndexes(indexes[1], indexes[0], hashCount, bitset.capacity(), indexes);
+ return indexes;
+ }
+
@Inline
private void setIndexes(long base, long inc, int count, long max, long[] results)
{
@@ -121,6 +137,24 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
}
}
+ public void add(byte[] key, int offset, int length)
+ {
+ long[] indexes = indexes(key, offset, length);
+ for (int i = 0; i < hashCount; i++)
+ {
+ bitset.set(indexes[i]);
+ }
+ }
+
+ public void add(byte[] key, int offset, int length, long[] indexes)
+ {
+ indexes(key, offset, length, indexes);
+ for (int i = 0; i < hashCount; i++)
+ {
+ bitset.set(indexes[i]);
+ }
+ }
+
@Override
public final boolean isPresent(FilterKey key)
{
diff --git a/src/java/org/apache/cassandra/utils/ByteArrayUtil.java b/src/java/org/apache/cassandra/utils/ByteArrayUtil.java
index 13fb44c3bb..31f9434816 100644
--- a/src/java/org/apache/cassandra/utils/ByteArrayUtil.java
+++ b/src/java/org/apache/cassandra/utils/ByteArrayUtil.java
@@ -46,6 +46,11 @@ public class ByteArrayUtil
return FastByteOperations.compareUnsigned(o1, off1, len, o2, off2, len);
}
+ public static int compareUnsigned(byte[] o1, int off1, int len1, byte[] o2, int off2, int len2)
+ {
+ return FastByteOperations.compareUnsigned(o1, off1, len1, o2, off2, len2);
+ }
+
public static byte[] bytes(byte b)
{
return new byte[] {b};
@@ -234,6 +239,16 @@ public class ByteArrayUtil
out.write(buffer);
}
+ public static void writeWithShortLength(byte[] buffer, int offset, int length, DataOutput out) throws IOException
+ {
+ assert length <= buffer.length;
+ assert offset <= length;
+ assert length <= FBUtilities.MAX_UNSIGNED_SHORT
+ : String.format("Attempted serializing to buffer exceeded maximum of %s bytes: %s", FBUtilities.MAX_UNSIGNED_SHORT, length);
+ out.writeShort(length);
+ out.write(buffer, offset, length);
+ }
+
public static void writeWithVIntLength(byte[] bytes, DataOutputPlus out) throws IOException
{
out.writeUnsignedVInt32(bytes.length);
diff --git a/src/java/org/apache/cassandra/utils/MurmurHash.java b/src/java/org/apache/cassandra/utils/MurmurHash.java
index 80cf5cd39f..e5d488d663 100644
--- a/src/java/org/apache/cassandra/utils/MurmurHash.java
+++ b/src/java/org/apache/cassandra/utils/MurmurHash.java
@@ -93,6 +93,66 @@ public class MurmurHash
return h;
}
+ /**
+ * We can go all in and do a unified unsafe implementation to serve arrays/direct memory and layer the byte buffer
+ * types on top. Left for later.
+ */
+ public static long hash2_64(byte[] key, int offset, int length, long seed)
+ {
+ long m64 = 0xc6a4a7935bd1e995L;
+ int r64 = 47;
+
+ long h64 = (seed & 0xffffffffL) ^ (m64 * length);
+
+ int lenLongs = length >> 3;
+
+ for (int i = 0; i < lenLongs; ++i)
+ {
+ int i_8 = i << 3;
+
+ long k64 = ((long) key[offset+i_8+0] & 0xff) + (((long) key[offset+i_8+1] & 0xff)<<8) +
+ (((long) key[offset+i_8+2] & 0xff)<<16) + (((long) key[offset+i_8+3] & 0xff)<<24) +
+ (((long) key[offset+i_8+4] & 0xff)<<32) + (((long) key[offset+i_8+5] & 0xff)<<40) +
+ (((long) key[offset+i_8+6] & 0xff)<<48) + (((long) key[offset+i_8+7] & 0xff)<<56);
+
+ k64 *= m64;
+ k64 ^= k64 >>> r64;
+ k64 *= m64;
+
+ h64 ^= k64;
+ h64 *= m64;
+ }
+
+ int rem = length & 0x7;
+
+ switch (rem)
+ {
+ case 0:
+ break;
+ case 7:
+ h64 ^= (long) key[offset + length - rem + 6] << 48;
+ case 6:
+ h64 ^= (long) key[offset + length - rem + 5] << 40;
+ case 5:
+ h64 ^= (long) key[offset + length - rem + 4] << 32;
+ case 4:
+ h64 ^= (long) key[offset + length - rem + 3] << 24;
+ case 3:
+ h64 ^= (long) key[offset + length - rem + 2] << 16;
+ case 2:
+ h64 ^= (long) key[offset + length - rem + 1] << 8;
+ case 1:
+ h64 ^= (long) key[offset + length - rem];
+ h64 *= m64;
+ }
+
+ h64 ^= h64 >>> r64;
+ h64 *= m64;
+ h64 ^= h64 >>> r64;
+
+ return h64;
+ }
+
public static long hash2_64(ByteBuffer key, int offset, int length, long seed)
{
long m64 = 0xc6a4a7935bd1e995L;
@@ -153,12 +213,22 @@ public class MurmurHash
{
int i_8 = index << 3;
int blockOffset = offset + i_8;
- return ((long) key.get(blockOffset + 0) & 0xff) + (((long) key.get(blockOffset + 1) & 0xff) << 8) +
+ return ((long) key.get(blockOffset + 0) & 0xff) + (((long) key.get(blockOffset + 1) & 0xff) << 8) +
(((long) key.get(blockOffset + 2) & 0xff) << 16) + (((long) key.get(blockOffset + 3) & 0xff) << 24) +
(((long) key.get(blockOffset + 4) & 0xff) << 32) + (((long) key.get(blockOffset + 5) & 0xff) << 40) +
(((long) key.get(blockOffset + 6) & 0xff) << 48) + (((long) key.get(blockOffset + 7) & 0xff) << 56);
}
+ protected static long getBlock(byte[] key, int offset, int index)
+ {
+ int i_8 = index << 3;
+ int blockOffset = offset + i_8;
+ return ((long) key[blockOffset + 0] & 0xff) + (((long) key[blockOffset + 1] & 0xff) << 8) +
+ (((long) key[blockOffset + 2] & 0xff) << 16) + (((long) key[blockOffset + 3] & 0xff) << 24) +
+ (((long) key[blockOffset + 4] & 0xff) << 32) + (((long) key[blockOffset + 5] & 0xff) << 40) +
+ (((long) key[blockOffset + 6] & 0xff) << 48) + (((long) key[blockOffset + 7] & 0xff) << 56);
+ }
+
protected static long rotl64(long v, int n)
{
return ((v << n) | (v >>> (64 - n)));
@@ -251,6 +321,82 @@ public class MurmurHash
result[1] = h2;
}
+ public static void hash3_x64_128(byte[] key, int offset, int length, long seed, long[] result)
+ {
+ final int nblocks = length >> 4; // Process as 128-bit blocks.
+
+ long h1 = seed;
+ long h2 = seed;
+
+ long c1 = 0x87c37b91114253d5L;
+ long c2 = 0x4cf5ad432745937fL;
+
+ //----------
+ // body
+
+ for(int i = 0; i < nblocks; i++)
+ {
+ long k1 = getBlock(key, offset, i * 2 + 0);
+ long k2 = getBlock(key, offset, i * 2 + 1);
+
+ k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1;
+
+ h1 = rotl64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
+
+ k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2;
+
+ h2 = rotl64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
+ }
+
+ //----------
+ // tail
+
+ // Advance offset to the unprocessed tail of the data.
+ offset += nblocks * 16;
+
+ long k1 = 0;
+ long k2 = 0;
+
+ switch(length & 15)
+ {
+ case 15: k2 ^= ((long) key[offset+14]) << 48;
+ case 14: k2 ^= ((long) key[offset+13]) << 40;
+ case 13: k2 ^= ((long) key[offset+12]) << 32;
+ case 12: k2 ^= ((long) key[offset+11]) << 24;
+ case 11: k2 ^= ((long) key[offset+10]) << 16;
+ case 10: k2 ^= ((long) key[offset+9]) << 8;
+ case 9: k2 ^= ((long) key[offset+8]) << 0;
+ k2 *= c2; k2 = rotl64(k2,33); k2 *= c1; h2 ^= k2;
+
+ case 8: k1 ^= ((long) key[offset+7]) << 56;
+ case 7: k1 ^= ((long) key[offset+6]) << 48;
+ case 6: k1 ^= ((long) key[offset+5]) << 40;
+ case 5: k1 ^= ((long) key[offset+4]) << 32;
+ case 4: k1 ^= ((long) key[offset+3]) << 24;
+ case 3: k1 ^= ((long) key[offset+2]) << 16;
+ case 2: k1 ^= ((long) key[offset+1]) << 8;
+ case 1: k1 ^= ((long) key[offset]);
+ k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1;
+ };
+
+ //----------
+ // finalization
+
+ h1 ^= length; h2 ^= length;
+
+ h1 += h2;
+ h2 += h1;
+
+ h1 = fmix(h1);
+ h2 = fmix(h2);
+
+ h1 += h2;
+ h2 += h1;
+
+ result[0] = h1;
+ result[1] = h2;
+ }
+
protected static long invRotl64(long v, int n)
{
return ((v >>> n) | (v << (64 - n)));
diff --git a/src/java/org/apache/cassandra/utils/OutputHandler.java b/src/java/org/apache/cassandra/utils/OutputHandler.java
index 76eb34558f..e299b86669 100644
--- a/src/java/org/apache/cassandra/utils/OutputHandler.java
+++ b/src/java/org/apache/cassandra/utils/OutputHandler.java
@@ -36,7 +36,7 @@ public interface OutputHandler
void debug(String msg);
default void debug(String msg, Object ... args)
{
- debug(String.format(msg, args));
+ if (isDebugEnabled()) debug(String.format(msg, args));
}
// called when the user needs to be warn
@@ -55,6 +55,8 @@ public interface OutputHandler
warn(String.format(msg, args));
}
+ boolean isDebugEnabled();
+
class LogOutput implements OutputHandler
{
private static Logger logger = LoggerFactory.getLogger(LogOutput.class);
@@ -66,7 +68,7 @@ public interface OutputHandler
public void debug(String msg)
{
- logger.trace(msg);
+ logger.debug(msg);
}
public void warn(String msg)
@@ -78,6 +80,18 @@ public interface OutputHandler
{
logger.warn(msg, th);
}
+
+ public boolean isDebugEnabled() { return logger.isDebugEnabled();}
+ }
+
+ class NullOutput implements OutputHandler
+ {
+ public void output(String msg) {}
+ public void debug(String msg, Object ... args) {}
+ public void debug(String msg) {}
+ public void warn(String msg) {}
+ public void warn(Throwable th, String msg) {}
+ public boolean isDebugEnabled() { return false;}
}
class SystemOutput implements OutputHandler
@@ -120,5 +134,10 @@ public interface OutputHandler
if (printStack && th != null)
th.printStackTrace(warnOut);
}
+
+ public boolean isDebugEnabled()
+ {
+ return debug;
+ }
}
}
\ No newline at end of file
diff --git a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
index a444f4147b..4a8b77bc76 100644
--- a/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
+++ b/src/java/org/apache/cassandra/utils/vint/VIntCoding.java
@@ -230,6 +230,7 @@ public class VIntCoding
{
return getUnsignedVInt(input, readerIndex, input.limit());
}
+
public static long getUnsignedVInt(ByteBuffer input, int readerIndex, int readerLimit)
{
if (readerIndex < 0)
diff --git a/test/bin/jmh b/test/bin/jmh
index ad8f44bd3c..53cf166de9 100755
--- a/test/bin/jmh
+++ b/test/bin/jmh
@@ -136,6 +136,6 @@ CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/test/conf/"
CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/build/test/classes/"
CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/build/test/lib/jars/*"
-exec $NUMACTL "$JAVA" -cp "$CLASSPATH" org.openjdk.jmh.Main -jvmArgs="$cassandra_parms $JVM_OPTS" "$@"
+exec $NUMACTL "$JAVA" -Xmx8g -XX:+UseParallelGC -cp "$CLASSPATH" org.openjdk.jmh.Main -jvmArgs="-Djmh.shutdownTimeout=120 $cassandra_parms $JVM_OPTS" "$@"
# vi:ai sw=4 ts=4 tw=0 et
diff --git a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
index 5727a020b6..529c9f3d50 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java
@@ -338,6 +338,12 @@ public class FailingRepairTest extends TestBaseImpl implements Serializable
return Collections.emptySet();
}
+ @Override
+ public boolean isFullRange()
+ {
+ return false;
+ }
+
public TableMetadata metadata()
{
return null;
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationReadRaceTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationReadRaceTestBase.java
index d6a1fe3adf..47e044bf14 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationReadRaceTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationReadRaceTestBase.java
@@ -299,7 +299,7 @@ public abstract class AccordMigrationReadRaceTestBase extends AccordTestBase
@Test
public void testRangeRouting() throws Throwable
{
- String cql = "SELECT * FROM " + qualifiedAccordTableName + " WHERE token(pk) > " + Murmur3Partitioner.MINIMUM.token;
+ String cql = "SELECT * FROM " + qualifiedAccordTableName + " WHERE token(pk) > " + Murmur3Partitioner.MINIMUM.getLongValue();
testSplitAndRetry(cql, this::loadData, result -> {
assertThat(result).isDeepEqualTo(dataFlat);
});
diff --git a/test/distributed/org/apache/cassandra/distributed/test/accord/InteropTokenRangeTest.java b/test/distributed/org/apache/cassandra/distributed/test/accord/InteropTokenRangeTest.java
index ee89e982de..202fdb2c87 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/accord/InteropTokenRangeTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/accord/InteropTokenRangeTest.java
@@ -120,7 +120,7 @@ public class InteropTokenRangeTest extends TestBaseImpl
{
NavigableSet set = new TreeSet<>();
while (result.hasNext())
- set.add(Murmur3Partitioner.instance.getToken(result.next().get("pk")).token);
+ set.add(Murmur3Partitioner.instance.getToken(result.next().get("pk")).getLongValue());
return set;
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTokenConflictTest.java b/test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTokenConflictTest.java
index 72da2a8d6c..2d27394ac3 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTokenConflictTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTokenConflictTest.java
@@ -405,10 +405,10 @@ public class SingleNodeTokenConflictTest extends StatefulASTBase
for (ByteBuffer bb : values)
{
var token = Murmur3Partitioner.instance.getToken(bb);
- if (token.token > Long.MIN_VALUE + 1)
- neighbors.add(keyForToken(token.token - 1));
- if (token.token < Long.MAX_VALUE)
- neighbors.add(keyForToken(token.token + 1));
+ if (token.getLongValue() > Long.MIN_VALUE + 1)
+ neighbors.add(keyForToken(token.getLongValue() - 1));
+ if (token.getLongValue() < Long.MAX_VALUE)
+ neighbors.add(keyForToken(token.getLongValue() + 1));
}
return neighbors.build();
}
diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java
index 3a7059c91f..6ddb44f1be 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java
@@ -1005,9 +1005,9 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
SinglePartitionReadCommand sprc = (SinglePartitionReadCommand) command;
Murmur3Partitioner.LongToken requestToken = (Murmur3Partitioner.LongToken) sprc.partitionKey().getToken();
- assert node.cluster.state.get().isReadReplicaFor(requestToken.token, node.matcher) :
+ assert node.cluster.state.get().isReadReplicaFor(requestToken.getLongValue(), node.matcher) :
String.format("Node %s is not a read replica for %s. Replicas: %s",
- node, requestToken.token, node.cluster.state.get().readReplicasFor(requestToken.token));
+ node, requestToken.getLongValue(), node.cluster.state.get().readReplicasFor(requestToken.getLongValue()));
}
@Override
@@ -1048,7 +1048,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
Murmur3Partitioner.LongToken leftToken = (Murmur3Partitioner.LongToken) prrc.dataRange().keyRange().left.getToken();
Murmur3Partitioner.LongToken rightToken = (Murmur3Partitioner.LongToken) prrc.dataRange().keyRange().right.getToken();
- assert node.cluster.state.get().isReadReplicaFor(leftToken.token, rightToken.token, node.matcher);
+ assert node.cluster.state.get().isReadReplicaFor(leftToken.getLongValue(), rightToken.getLongValue(), node.matcher);
}
@Override
@@ -1085,8 +1085,8 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
{
Mutation command = request.payload;
Murmur3Partitioner.LongToken requestToken = (Murmur3Partitioner.LongToken) command.key().getToken();
- assert node.cluster.state.get().isWriteTargetFor(requestToken.token, node.matcher) : String.format("Node %s is not a write target for %s. Write placements: %s",
- node.node.idx(), requestToken, node.cluster.state.get().writePlacementsFor(requestToken.token));
+ assert node.cluster.state.get().isWriteTargetFor(requestToken.getLongValue(), node.matcher) : String.format("Node %s is not a write target for %s. Write placements: %s",
+ node.node.idx(), requestToken, node.cluster.state.get().writePlacementsFor(requestToken.getLongValue()));
}
public Message respondTo(Message request)
@@ -1145,7 +1145,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
public static long token(Object... pk)
{
- return ((Murmur3Partitioner.LongToken) DatabaseDescriptor.getPartitioner().decorateKey(toBytes(pk)).getToken()).token;
+ return ((Murmur3Partitioner.LongToken) DatabaseDescriptor.getPartitioner().decorateKey(toBytes(pk)).getToken()).getLongValue();
}
public static ByteBuffer toBytes(Object... pk)
diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/GossipDeadlockTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/GossipDeadlockTest.java
index 720d57f919..362b94fd1e 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/log/GossipDeadlockTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/log/GossipDeadlockTest.java
@@ -71,7 +71,7 @@ public class GossipDeadlockTest extends TestBaseImpl
ExecutorPlus e = ExecutorFactory.Global.executorFactory().pooled("BounceMove", 2);
long startToken = cluster.get(2).callOnInstance(() -> {
NodeId nodeId = ClusterMetadata.current().myNodeId();
- return ((Murmur3Partitioner.LongToken)ClusterMetadata.current().tokenMap.tokens(nodeId).get(0)).token;
+ return ((Murmur3Partitioner.LongToken)ClusterMetadata.current().tokenMap.tokens(nodeId).get(0)).getLongValue();
});
AtomicBoolean stop = new AtomicBoolean(false);
Future moves = e.submit(() -> {
diff --git a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
index 44fdc7d701..f9ae0e3abe 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java
@@ -555,7 +555,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
Assert.assertEquals(replication, sut.rf.asKeyspaceParams().replication);
Assert.assertEquals(modelState.simulatedPlacements.nodes.stream().map(Node::token).collect(Collectors.toSet()),
- actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).token).collect(Collectors.toSet()));
+ actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).getLongValue()).collect(Collectors.toSet()));
for (Map.Entry e : actualMetadata.placements.asMap().entrySet())
{
diff --git a/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java b/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java
index 4319768169..0c6d90834b 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java
@@ -66,8 +66,8 @@ public class LCSStreamingKeepLevelTest extends TestBaseImpl
{
populate(cluster);
- long tokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(3).getToken()).token;
- long prevTokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(2).getToken()).token;
+ long tokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(3).getToken()).getLongValue();
+ long prevTokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(2).getToken()).getLongValue();
// move node 4 to the middle point between its current position and the previous node
long newToken = (tokenVal + prevTokenVal) / 2;
cluster.get(4).nodetoolResult("move", String.valueOf(newToken)).asserts().success();
diff --git a/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterMoveTest.java b/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterMoveTest.java
index 99b33984e5..389e3acc6d 100644
--- a/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterMoveTest.java
+++ b/test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterMoveTest.java
@@ -46,7 +46,7 @@ public class CMSPlacementAfterMoveTest extends TestBaseImpl
long node1Token = cluster.get(1).callOnInstance(() -> {
ClusterMetadata metadata = ClusterMetadata.current();
ImmutableList tokens = ClusterMetadata.current().tokenMap.tokens(metadata.myNodeId());
- return ((Murmur3Partitioner.LongToken) tokens.get(0)).token;
+ return ((Murmur3Partitioner.LongToken) tokens.get(0)).getLongValue();
});
long newNode4Token = node1Token + 100; // token after node1s token should be in cms
cluster.get(4).nodetoolResult("move", String.valueOf(newNode4Token));
diff --git a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
index 952663c50e..ee8adceb60 100644
--- a/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
+++ b/test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java
@@ -238,9 +238,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader
}
@Override
- public KeyReader keyReader() throws IOException
+ public KeyReader keyReader(boolean detailed) throws IOException
{
- return delegate.keyReader();
+ return delegate.keyReader(detailed);
}
public KeyReader keyReader(PartitionPosition key) throws IOException
diff --git a/test/distributed/org/apache/cassandra/service/accord/AccordJournalBurnTest.java b/test/distributed/org/apache/cassandra/service/accord/AccordJournalBurnTest.java
index e074ed950e..2d27acbc4c 100644
--- a/test/distributed/org/apache/cassandra/service/accord/AccordJournalBurnTest.java
+++ b/test/distributed/org/apache/cassandra/service/accord/AccordJournalBurnTest.java
@@ -304,7 +304,7 @@ public class AccordJournalBurnTest extends BurnTestBase
while (ci.hasNext())
writer.append(ci.next());
- ci.setTargetDirectory(writer.getSStableDirectory().path());
+ ci.setTargetDirectory(writer.getSStableDirectoryPath());
// point of no return
newSStables = writer.finish();
}
diff --git a/test/harry/main/org/apache/cassandra/harry/test/HarryCompactionTest.java b/test/harry/main/org/apache/cassandra/harry/test/HarryCompactionTest.java
new file mode 100644
index 0000000000..90a86c09ef
--- /dev/null
+++ b/test/harry/main/org/apache/cassandra/harry/test/HarryCompactionTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.util.BitSet;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+public class HarryCompactionTest extends CQLTester
+{
+ private static final AtomicInteger idGen = new AtomicInteger(0);
+ public static final int TEST_REPS = 100;
+ public static final int PARTITIONS_RANGE = 100;
+ public static final int COLUMNS_RANGE = 10;
+ private final Generator staticColumnsGenerator = BitSet.generator(3);
+ private final Generator regularColumnsGenerator = BitSet.generator(9);
+
+ private String keyspace;
+ private String table;
+ private String qualifiedTable;
+ private File dataDir;
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ public void perTestSetup() throws IOException
+ {
+ keyspace = "cql_keyspace" + idGen.incrementAndGet();
+ table = "table" + idGen.incrementAndGet();
+ qualifiedTable = keyspace + '.' + table;
+ dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+ assert dataDir.tryCreateDirectories();
+
+ ServerTestUtils.prepareServerNoRegister();
+ StorageService.instance.initServer();
+ requireNetwork();
+ }
+
+ private final Generator simple_schema = rng -> {
+ return new SchemaSpec(rng.next(),
+ 1000,
+ keyspace,
+ table,
+ Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType),
+ ColumnSpec.pk("pk2", ColumnSpec.int64Type)),
+ Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, true),
+ ColumnSpec.ck("ck2", ColumnSpec.int64Type, true)),
+ Arrays.asList(ColumnSpec.regularColumn("r1", ColumnSpec.asciiType),
+ ColumnSpec.regularColumn("r2", ColumnSpec.int64Type),
+ ColumnSpec.regularColumn("r3", ColumnSpec.int8Type),
+ ColumnSpec.regularColumn("r4", ColumnSpec.doubleType),
+ ColumnSpec.regularColumn("r5", ColumnSpec.floatType),
+ ColumnSpec.regularColumn("r6", ColumnSpec.int32Type),
+ ColumnSpec.regularColumn("r7", ColumnSpec.booleanType),
+ ColumnSpec.regularColumn("r8", ColumnSpec.int16Type),
+ ColumnSpec.regularColumn("r9", ColumnSpec.textType)),
+ Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType),
+ ColumnSpec.staticColumn("s2", ColumnSpec.int64Type),
+ ColumnSpec.staticColumn("s3", ColumnSpec.asciiType)));
+ };
+
+ @Test
+ public void testFlushAndCompact1() throws IOException {
+ testFlushAndCompact(1);
+ }
+
+ @Test
+ public void testFlushAndCompact2() throws IOException {
+ testFlushAndCompact(2);
+ }
+
+ @Test
+ public void testFlushAndCompact3() throws IOException {
+ testFlushAndCompact(3);
+ }
+
+ @Test
+ public void testFlushAndCompact4() throws IOException {
+ testFlushAndCompact(4);
+ }
+
+ @Test
+ public void testFlushAndCompact5() throws IOException
+ {
+ testFlushAndCompact(5);
+ }
+
+ public void testFlushAndCompact(int flushcount) throws IOException
+ {
+ for (int i = 0; i < TEST_REPS; i++)
+ testFlushAndCompactOnce(flushcount);
+ }
+
+ public void testFlushAndCompactOnce(int flushcount) throws IOException
+ {
+ perTestSetup();
+ withRandom( rng -> {
+
+ SchemaSpec schema = simple_schema.generate(rng);
+ schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", schema.keyspace));
+ createTable(schema.compile());
+
+ HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+ history.customThrowing(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ }, "disable compaction");
+
+ AtomicReference sstableWriter = new AtomicReference<>();
+ ThrowingRunnable flushAndChangeWriter = () -> {
+ HarrySSTableWriter prev = sstableWriter.get();
+ if (prev != null)
+ {
+ prev.close();
+ StorageService.instance.bulkLoad(dataDir.absolutePath());
+ dataDir.forEach(file -> file.delete());
+ }
+
+ Invariants.require(sstableWriter.getAndSet(HarrySSTableWriter.builder()
+ .forTable(schema.compile())
+ .inDirectory(dataDir)
+ .build()) == prev);
+ };
+ flushAndChangeWriter.run();
+
+ for (int sstablesFlushed = 0; sstablesFlushed < flushcount; sstablesFlushed++)
+ {
+ for (int i = 0; i < PARTITIONS_RANGE; i++)
+ {
+ for (int j = 0; j < COLUMNS_RANGE; j++)
+ {
+ history.insert(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 2 * COLUMNS_RANGE)); // some overlap, but not all
+ history.deleteRow(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 2 * COLUMNS_RANGE)); // some overlap, but not all
+ history.deleteColumns(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 20),
+ regularColumnsGenerator.generate(rng),
+ staticColumnsGenerator.generate(rng));
+ }
+ history.deletePartition(rng.nextInt(0, 2 * PARTITIONS_RANGE)); // some overlap, but not all
+ }
+
+ history.customThrowing(flushAndChangeWriter, "flush sstable" + sstablesFlushed);
+ }
+
+ history.customThrowing(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.forceMajorCompaction();
+ }, "major compaction");
+
+ for (int i = 0; i < 2*PARTITIONS_RANGE; i++)
+ history.selectPartition(i);
+
+ replay(schema, history, sstableWriter::get);
+ });
+ }
+
+ public void replay(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier writer)
+ {
+ CQLVisitExecutor executor = create(schema, historyBuilder, writer);
+ for (Visit visit : historyBuilder)
+ executor.execute(visit);
+ }
+
+ public CQLVisitExecutor create(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier writer)
+ {
+ DataTracker tracker = new DataTracker.SequentialDataTracker();
+ return new CQLTesterVisitExecutor(schema, tracker,
+ new QuiescentChecker(schema.valueGenerators, tracker, historyBuilder),
+ statement -> {
+ if (logger.isTraceEnabled())
+ logger.trace(statement.toString());
+ return execute(statement.cql(), statement.bindings());
+ })
+ {
+ @Override
+ protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+ {
+ try
+ {
+ writer.get().addRow(statement.cql(), statement.bindings());
+ }
+ catch (IOException e)
+ {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ protected void executeValidatingVisit(Visit visit, List selects, CompiledStatement compiledStatement)
+ {
+ super.executeValidatingVisit(visit, selects, compiledStatement);
+ }
+
+ @Override
+ public void execute(Visit visit)
+ {
+ if (visit.visitedPartitions.size() > 1)
+ throw new IllegalStateException("SSTable Generator does not support batch statements and transactions");
+
+ super.execute(visit);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/test/harry/main/org/apache/cassandra/harry/test/HarryCompactionWithRangeDeletionsTest.java b/test/harry/main/org/apache/cassandra/harry/test/HarryCompactionWithRangeDeletionsTest.java
new file mode 100644
index 0000000000..4855bd3de6
--- /dev/null
+++ b/test/harry/main/org/apache/cassandra/harry/test/HarryCompactionWithRangeDeletionsTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.util.BitSet;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+public class HarryCompactionWithRangeDeletionsTest extends CQLTester
+{
+ private static final AtomicInteger idGen = new AtomicInteger(0);
+ public static final int TEST_REPS = 100;
+ public static final int PARTITIONS_RANGE = 100;
+ public static final int ROWS_RANGE = 10;
+ public static final int STATIC_COLS = 3;
+ public static final int REG_COLS = 9;
+ private final Generator staticColumnsGenerator = BitSet.generator(STATIC_COLS);
+ private final Generator regularColumnsGenerator = BitSet.generator(REG_COLS);
+
+ private String keyspace;
+ private String table;
+ private String qualifiedTable;
+ private File dataDir;
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ public void perTestSetup() throws IOException
+ {
+ keyspace = "cql_keyspace" + idGen.incrementAndGet();
+ table = "table" + idGen.incrementAndGet();
+ qualifiedTable = keyspace + '.' + table;
+ dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+ assert dataDir.tryCreateDirectories();
+
+ ServerTestUtils.prepareServerNoRegister();
+ StorageService.instance.initServer();
+ requireNetwork();
+ }
+
+ private final Generator schemaSpecGenerator = rng -> {
+ return new SchemaSpec(rng.next(),
+ 1000,
+ keyspace,
+ table,
+ Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType),
+ ColumnSpec.pk("pk2", ColumnSpec.int64Type)),
+ Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, true),
+ ColumnSpec.ck("ck2", ColumnSpec.int64Type, true)),
+ Arrays.asList(ColumnSpec.regularColumn("r1", ColumnSpec.asciiType),
+ ColumnSpec.regularColumn("r2", ColumnSpec.int64Type),
+ ColumnSpec.regularColumn("r3", ColumnSpec.int8Type),
+ ColumnSpec.regularColumn("r4", ColumnSpec.doubleType),
+ ColumnSpec.regularColumn("r5", ColumnSpec.floatType),
+ ColumnSpec.regularColumn("r6", ColumnSpec.int32Type),
+ ColumnSpec.regularColumn("r7", ColumnSpec.booleanType),
+ ColumnSpec.regularColumn("r8", ColumnSpec.int16Type),
+ ColumnSpec.regularColumn("r9", ColumnSpec.textType)),
+ Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType),
+ ColumnSpec.staticColumn("s2", ColumnSpec.int64Type),
+ ColumnSpec.staticColumn("s3", ColumnSpec.asciiType)));
+ };
+
+ @Test
+ public void testFlushAndCompact1() throws IOException {
+ testFlushAndCompact(1);
+ }
+
+ @Test
+ public void testFlushAndCompact2() throws IOException {
+ testFlushAndCompact(2);
+ }
+
+ @Test
+ public void testFlushAndCompact3() throws IOException {
+ testFlushAndCompact(3);
+ }
+
+ @Test
+ public void testFlushAndCompact4() throws IOException {
+ testFlushAndCompact(4);
+ }
+
+ @Test
+ public void testFlushAndCompact5() throws IOException
+ {
+ testFlushAndCompact(5);
+ }
+
+ public void testFlushAndCompact(int flushcount) throws IOException
+ {
+ for (int i = 0; i < TEST_REPS; i++)
+ testFlushAndCompactOnce(flushcount);
+ }
+
+ public void testFlushAndCompactOnce(int flushcount) throws IOException
+ {
+ perTestSetup();
+ withRandom(205413964293041L, rng -> {
+
+ SchemaSpec schema = schemaSpecGenerator.generate(rng);
+ schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", schema.keyspace));
+ createTable(schema.compile());
+
+ HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+ history.customThrowing(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ }, "disable compaction");
+
+ AtomicReference sstableWriter = new AtomicReference<>();
+ ThrowingRunnable flushAndChangeWriter = () -> {
+ HarrySSTableWriter prev = sstableWriter.get();
+ if (prev != null)
+ {
+ prev.close();
+ StorageService.instance.bulkLoad(dataDir.absolutePath());
+ dataDir.forEach(file -> file.delete());
+ }
+
+ Invariants.require(sstableWriter.getAndSet(HarrySSTableWriter.builder()
+ .forTable(schema.compile())
+ .inDirectory(dataDir)
+ .build()) == prev);
+ };
+ flushAndChangeWriter.run();
+
+ for (int sstablesFlushed = 0; sstablesFlushed < flushcount; sstablesFlushed++)
+ {
+ for (int i = 0; i < PARTITIONS_RANGE; i++)
+ {
+ for (int j = 0; j < ROWS_RANGE; j++)
+ {
+ history.insert(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 2 * ROWS_RANGE)); // some overlap, but not all
+ }
+ }
+ int lowerBoundRowIdx = rng.nextInt(ROWS_RANGE);
+ int upperBoundRowIdx = rng.nextInt(lowerBoundRowIdx, 2 * ROWS_RANGE);
+ history.deleteRowRange(rng.nextInt(0, 2 * PARTITIONS_RANGE),
+ lowerBoundRowIdx,
+ upperBoundRowIdx,
+ rng.nextInt(REG_COLS),
+ rng.nextBoolean(),
+ rng.nextBoolean());
+
+ history.customThrowing(flushAndChangeWriter, "flush sstable" + sstablesFlushed);
+ }
+
+ history.customThrowing(() -> {
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.forceMajorCompaction();
+ }, "major compaction");
+
+ for (int i = 0; i < 2 * PARTITIONS_RANGE; i++)
+ history.selectPartition(i);
+
+ replay(schema, history, sstableWriter::get);
+ });
+ }
+
+ public void replay(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier writer)
+ {
+ CQLVisitExecutor executor = create(schema, historyBuilder, writer);
+ for (Visit visit : historyBuilder)
+ executor.execute(visit);
+ }
+
+ public CQLVisitExecutor create(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier writer)
+ {
+ DataTracker tracker = new DataTracker.SequentialDataTracker();
+ return new CQLTesterVisitExecutor(schema, tracker,
+ new QuiescentChecker(schema.valueGenerators, tracker, historyBuilder),
+ statement -> {
+ if (logger.isTraceEnabled())
+ logger.trace(statement.toString());
+ return execute(statement.cql(), statement.bindings());
+ })
+ {
+ @Override
+ protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+ {
+ try
+ {
+ writer.get().addRow(statement.cql(), statement.bindings());
+ }
+ catch (IOException e)
+ {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ protected void executeValidatingVisit(Visit visit, List selects, CompiledStatement compiledStatement)
+ {
+ super.executeValidatingVisit(visit, selects, compiledStatement);
+ }
+
+ @Override
+ public void execute(Visit visit)
+ {
+ if (visit.visitedPartitions.size() > 1)
+ throw new IllegalStateException("SSTable Generator does not support batch statements and transactions");
+
+ super.execute(visit);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/test/harry/main/org/apache/cassandra/harry/test/HarrySSTableWriterTest.java b/test/harry/main/org/apache/cassandra/harry/test/HarrySSTableWriterTest.java
new file mode 100644
index 0000000000..9b3b8e818c
--- /dev/null
+++ b/test/harry/main/org/apache/cassandra/harry/test/HarrySSTableWriterTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+public class HarrySSTableWriterTest extends CQLTester
+{
+ private static final AtomicInteger idGen = new AtomicInteger(0);
+ private static final int NUMBER_WRITES_IN_RUNNABLE = 10;
+
+ private String keyspace;
+ private String table;
+ private String qualifiedTable;
+ private File dataDir;
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Before
+ public void perTestSetup() throws IOException
+ {
+ keyspace = "cql_keyspace" + idGen.incrementAndGet();
+ table = "table" + idGen.incrementAndGet();
+ qualifiedTable = keyspace + '.' + table;
+ dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+ assert dataDir.tryCreateDirectories();
+
+ ServerTestUtils.prepareServerNoRegister();
+ StorageService.instance.initServer();
+ requireNetwork();
+ }
+
+ private final Generator simple_schema = rng -> {
+ return new SchemaSpec(rng.next(),
+ 1000,
+ keyspace,
+ table,
+ Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType),
+ ColumnSpec.pk("pk2", ColumnSpec.int64Type)),
+ Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, false),
+ ColumnSpec.ck("ck2", ColumnSpec.int64Type, false)),
+ Arrays.asList(ColumnSpec.regularColumn("r1", ColumnSpec.asciiType),
+ ColumnSpec.regularColumn("r2", ColumnSpec.int64Type),
+ ColumnSpec.regularColumn("r3", ColumnSpec.asciiType)),
+ Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType),
+ ColumnSpec.staticColumn("s2", ColumnSpec.int64Type),
+ ColumnSpec.staticColumn("s3", ColumnSpec.asciiType)));
+ };
+
+ @Test
+ public void generateSSTableTest()
+ {
+ withRandom(rng -> {
+
+ SchemaSpec schema = simple_schema.generate(rng);
+ schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", schema.keyspace));
+ createTable(schema.compile());
+
+ HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+ for (int i = 0; i < 100; i++)
+ history.insert(1);
+
+ AtomicReference sstableWriter = new AtomicReference<>();
+ ThrowingRunnable resetWriter = () -> {
+ HarrySSTableWriter prev = sstableWriter.get();
+ if (prev != null)
+ {
+ prev.close();
+ StorageService.instance.bulkLoad(dataDir.absolutePath());
+ }
+
+ Invariants.require(sstableWriter.getAndSet(HarrySSTableWriter.builder()
+ .forTable(schema.compile())
+ .inDirectory(dataDir)
+ .build()) == prev);
+ };
+ resetWriter.run();
+
+ for (int i = 0; i < 100; i++)
+ {
+ for (int j = 0; j < 10; j++)
+ history.insert(i, j);
+ }
+
+ history.customThrowing(resetWriter, "flush sstable");
+
+ for (int i = 0; i < 100; i++)
+ history.selectPartition(i);
+
+ replay(schema, history, sstableWriter::get);
+ });
+ }
+
+ public void replay(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier writer)
+ {
+ CQLVisitExecutor executor = create(schema, historyBuilder, writer);
+ for (Visit visit : historyBuilder)
+ executor.execute(visit);
+ }
+
+ public CQLVisitExecutor create(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier writer)
+ {
+ DataTracker tracker = new DataTracker.SequentialDataTracker();
+ return new CQLTesterVisitExecutor(schema, tracker,
+ new QuiescentChecker(schema.valueGenerators, tracker, historyBuilder),
+ statement -> {
+ if (logger.isTraceEnabled())
+ logger.trace(statement.toString());
+ return execute(statement.cql(), statement.bindings());
+ })
+ {
+ @Override
+ protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+ {
+ try
+ {
+ writer.get().addRow(statement.cql(), statement.bindings());
+ }
+ catch (IOException e)
+ {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ @Override
+ protected void executeValidatingVisit(Visit visit, List selects, CompiledStatement compiledStatement)
+ {
+ super.executeValidatingVisit(visit, selects, compiledStatement);
+ }
+
+ @Override
+ public void execute(Visit visit)
+ {
+ if (visit.visitedPartitions.size() > 1)
+ throw new IllegalStateException("SSTable Generator does not support batch statements and transactions");
+
+ super.execute(visit);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/test/harry/main/org/apache/cassandra/io/sstable/HarrySSTableWriter.java b/test/harry/main/org/apache/cassandra/io/sstable/HarrySSTableWriter.java
new file mode 100644
index 0000000000..db1bef6622
--- /dev/null
+++ b/test/harry/main/org/apache/cassandra/io/sstable/HarrySSTableWriter.java
@@ -0,0 +1,656 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UpdateParameters;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateIndexStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaTransformation;
+import org.apache.cassandra.schema.SchemaTransformations;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
+import org.apache.cassandra.schema.Views;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.transformations.AlterSchema;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JavaDriverUtils;
+
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+
+public class HarrySSTableWriter implements Closeable
+{
+ public static final ByteBuffer UNSET_VALUE = ByteBufferUtil.UNSET_BYTE_BUFFER;
+
+ static
+ {
+ CassandraRelevantProperties.FORCE_LOAD_LOCAL_KEYSPACES.setBoolean(true);
+ DatabaseDescriptor.clientInitialization(false);
+ // Partitioner is not set in client mode.
+ if (DatabaseDescriptor.getPartitioner() == null)
+ DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+ ClusterMetadataService.initializeForClients();
+ }
+
+ private final AbstractSSTableSimpleWriter writer;
+
+ private HarrySSTableWriter(AbstractSSTableSimpleWriter writer)
+ {
+ this.writer = writer;
+ }
+
+ public static Builder builder()
+ {
+ return new Builder();
+ }
+
+ public HarrySSTableWriter addRow(String cql, Object... values) throws IOException
+ {
+ ModificationStatement statement = prepare(cql);
+ List boundNames = statement.getBindVariables();
+ // TODO: avoid materializing this
+ List> typeCodecs = boundNames.stream()
+ .map(bn -> JavaDriverUtils.codecFor(JavaDriverUtils.driverType(bn.type)))
+ .collect(Collectors.toList());
+
+ int size = Math.min(values.length, boundNames.size());
+ List rawValues = new ArrayList<>(size);
+ for (int i = 0; i < size; i++)
+ {
+ Object value = values[i];
+ rawValues.add(serialize(value, typeCodecs.get(i), boundNames.get(i)));
+ }
+
+ return rawAddRow(statement, rawValues, boundNames);
+ }
+
+ private ModificationStatement prepare(String cql)
+ {
+ ModificationStatement.Parsed statement = QueryProcessor.parseStatement(cql,
+ ModificationStatement.Parsed.class,
+ "INSERT/UPDATE/DELETE");
+ ClientState state = ClientState.forInternalCalls();
+ ModificationStatement preparedModificationStatement = statement.prepare(state);
+ preparedModificationStatement.validate(state);
+
+ if (preparedModificationStatement.hasConditions())
+ throw new IllegalArgumentException("Conditional statements are not supported");
+ if (preparedModificationStatement.isCounter())
+ throw new IllegalArgumentException("Counter modification statements are not supported");
+ if (preparedModificationStatement.getBindVariables().isEmpty())
+ throw new IllegalArgumentException("Provided preparedModificationStatement statement has no bind variables");
+
+ return preparedModificationStatement;
+ }
+
+ /**
+ * Adds a new row to the writer given already serialized values.
+ *
+ * This is a shortcut for {@code rawAddRow(Arrays.asList(values))}.
+ *
+ * @param values the row values (corresponding to the bind variables of the
+ * modification statement used when creating by this writer) as binary.
+ * @return this writer.
+ */
+ public HarrySSTableWriter rawAddRow(ModificationStatement modificationStatement, List values, List boundNames) throws InvalidRequestException, IOException
+ {
+ if (values.size() != boundNames.size())
+ throw new InvalidRequestException(String.format("Invalid number of arguments, expecting %d values but got %d", boundNames.size(), values.size()));
+
+ QueryOptions options = QueryOptions.forInternalCalls(null, values);
+ ClientState state = ClientState.forInternalCalls();
+ List keys = modificationStatement.buildPartitionKeyNames(options, state);
+
+ long now = currentTimeMillis();
+ // Note that we asks indexes to not validate values (the last 'false' arg below) because that triggers a 'Keyspace.open'
+ // and that forces a lot of initialization that we don't want.
+ UpdateParameters params = new UpdateParameters(modificationStatement.metadata,
+ ClientState.forInternalCalls(),
+ options,
+ modificationStatement.getTimestamp(TimeUnit.MILLISECONDS.toMicros(now), options),
+ options.getNowInSec((int) TimeUnit.MILLISECONDS.toSeconds(now)),
+ modificationStatement.getTimeToLive(options),
+ Collections.emptyMap());
+
+ try
+ {
+ if (modificationStatement.hasSlices())
+ {
+ Slices slices = modificationStatement.createSlices(options);
+
+ for (ByteBuffer key : keys)
+ {
+ for (Slice slice : slices)
+ modificationStatement.addUpdateForKey(writer.getUpdateFor(key), slice, params);
+ }
+ }
+ else
+ {
+ NavigableSet> clusterings = modificationStatement.createClustering(options, state);
+
+ for (ByteBuffer key : keys)
+ {
+ for (Clustering clustering : clusterings)
+ modificationStatement.addUpdateForKey(writer.getUpdateFor(key), clustering, params);
+ }
+ }
+ return this;
+ }
+ catch (SSTableSimpleUnsortedWriter.SyncException e)
+ {
+ // If we use a BufferedWriter and had a problem writing to disk, the IOException has been
+ // wrapped in a SyncException (see BufferedWriter below). We want to extract that IOE.
+ throw (IOException) e.getCause();
+ }
+ }
+
+ /**
+ * Close this writer.
+ *
+ * This method should be called, otherwise the produced sstables are not
+ * guaranteed to be complete (and won't be in practice).
+ */
+ public void close() throws IOException
+ {
+ writer.close();
+ }
+
+ private ByteBuffer serialize(Object value, TypeCodec codec, ColumnSpecification columnSpecification)
+ {
+ if (value == null || value == UNSET_VALUE)
+ return (ByteBuffer) value;
+
+ try
+ {
+ return codec.serialize(value, ProtocolVersion.CURRENT);
+ }
+ catch (ClassCastException cce)
+ {
+ // For backwards-compatibility with consumers that may be passing
+ // an Integer for a Date field, for example.
+ return ((AbstractType) columnSpecification.type).decompose(value);
+ }
+ }
+
+ /**
+ * A Builder for a CQLSSTableWriter object.
+ */
+ public static class Builder
+ {
+ private static final Logger logger = LoggerFactory.getLogger(Builder.class);
+ private static final long DEFAULT_BUFFER_SIZE_IN_MIB_FOR_UNSORTED = 128L;
+
+ protected SSTableFormat format = null;
+
+ private final List typeStatements;
+ private final List indexStatements;
+
+ private File directory;
+ private CreateTableStatement.Raw schemaStatement;
+ private IPartitioner partitioner;
+ private boolean sorted = false;
+ private long maxSSTableSizeInMiB = -1L;
+ private boolean buildIndexes = true;
+ private Consumer> sstableProducedListener;
+ private boolean openSSTableOnProduced = false;
+
+ protected Builder()
+ {
+ this.typeStatements = new ArrayList<>();
+ this.indexStatements = new ArrayList<>();
+ }
+
+ /**
+ * The directory where to write the sstables.
+ *
+ * This is a mandatory option.
+ *
+ * @param directory the directory to use, which should exists and be writable.
+ * @return this builder.
+ * @throws IllegalArgumentException if {@code directory} doesn't exist or is not writable.
+ */
+ public Builder inDirectory(String directory)
+ {
+ return inDirectory(new File(directory));
+ }
+
+ /**
+ * The directory where to write the sstables (mandatory option).
+ *
+ * This is a mandatory option.
+ *
+ * @param directory the directory to use, which should exists and be writable.
+ * @return this builder.
+ * @throws IllegalArgumentException if {@code directory} doesn't exist or is not writable.
+ */
+ public Builder inDirectory(File directory)
+ {
+ if (!directory.exists())
+ throw new IllegalArgumentException(directory + " doesn't exists");
+ if (!directory.isWritable())
+ throw new IllegalArgumentException(directory + " exists but is not writable");
+
+ this.directory = directory;
+ return this;
+ }
+
+ public Builder withType(String typeDefinition) throws SyntaxException
+ {
+ typeStatements.add(QueryProcessor.parseStatement(typeDefinition, CreateTypeStatement.Raw.class, "CREATE TYPE"));
+ return this;
+ }
+
+ /**
+ * The schema (CREATE TABLE statement) for the table for which sstable are to be created.
+ *
+ * Please note that the provided CREATE TABLE statement must use a fully-qualified
+ * table name, one that include the keyspace name.
+ *
+ * This is a mandatory option.
+ *
+ * @param schema the schema of the table for which sstables are to be created.
+ * @return this builder.
+ * @throws IllegalArgumentException if {@code schema} is not a valid CREATE TABLE statement
+ * or does not have a fully-qualified table name.
+ */
+ public Builder forTable(String schema)
+ {
+ this.schemaStatement = QueryProcessor.parseStatement(schema, CreateTableStatement.Raw.class, "CREATE TABLE");
+ return this;
+ }
+
+ /**
+ * The schema (CREATE INDEX statement) for index to be created for the table. Only SAI indexes are supported.
+ *
+ * @param indexes CQL statements representing SAI indexes to be created.
+ * @return this builder
+ */
+ public Builder withIndexes(String... indexes)
+ {
+ for (String index : indexes)
+ indexStatements.add(QueryProcessor.parseStatement(index, CreateIndexStatement.Raw.class, "CREATE INDEX"));
+
+ return this;
+ }
+
+ /**
+ * The partitioner to use.
+ *
+ * By default, {@code Murmur3Partitioner} will be used. If this is not the partitioner used
+ * by the cluster for which the SSTables are created, you need to use this method to
+ * provide the correct partitioner.
+ *
+ * @param partitioner the partitioner to use.
+ * @return this builder.
+ */
+ public Builder withPartitioner(IPartitioner partitioner)
+ {
+ this.partitioner = partitioner;
+ return this;
+ }
+
+ /**
+ * Defines the maximum SSTable size in mebibytes when using the sorted writer.
+ * By default, i.e. not specified, there is no maximum size limit for the produced SSTable
+ *
+ * @param size the maximum sizein mebibytes of each individual SSTable allowed
+ * @return this builder
+ */
+ public Builder withMaxSSTableSizeInMiB(int size)
+ {
+ if (size <= 0)
+ {
+ logger.warn("A non-positive value for maximum SSTable size is specified, " +
+ "which disables the size limiting effectively. Please supply a positive value in order " +
+ "to enforce size limiting for the produced SSTables.");
+ }
+ this.maxSSTableSizeInMiB = size;
+ return this;
+ }
+
+ /**
+ * The size of the buffer to use.
+ *
+ * This defines how much data will be buffered before being written as
+ * a new SSTable. This corresponds roughly to the data size that will have the created
+ * sstable.
+ *
+ * The default is 128MiB, which should be reasonable for a 1GiB heap. If you experience
+ * OOM while using the writer, you should lower this value.
+ *
+ * @param size the size to use in MiB.
+ * @return this builder.
+ * @deprecated This method is deprecated in favor of the new withMaxSSTableSizeInMiB(int size)
+ */
+ @Deprecated(since = "5.0")
+ public Builder withBufferSizeInMiB(int size)
+ {
+ return withMaxSSTableSizeInMiB(size);
+ }
+
+ /**
+ * The size of the buffer to use.
+ *
+ * This defines how much data will be buffered before being written as
+ * a new SSTable. This corresponds roughly to the data size that will have the created
+ * sstable.
+ *
+ * The default is 128MiB, which should be reasonable for a 1GiB heap. If you experience
+ * OOM while using the writer, you should lower this value.
+ *
+ * @param size the size to use in MiB.
+ * @return this builder.
+ * @deprecated This method is deprecated in favor of the new withBufferSizeInMiB(int size). See CASSANDRA-17675
+ */
+ @Deprecated(since = "4.1")
+ public Builder withBufferSizeInMB(int size)
+ {
+ return withBufferSizeInMiB(size);
+ }
+
+ /**
+ * Creates a CQLSSTableWriter that expects sorted inputs.
+ *
+ * If this option is used, the resulting writer will expect rows to be
+ * added in SSTable sorted order (and an exception will be thrown if that
+ * is not the case during modification). The SSTable sorted order means that
+ * rows are added such that their partition key respect the partitioner
+ * order.
+ *
+ * You should thus only use this option is you know that you can provide
+ * the rows in order, which is rarely the case. If you can provide the
+ * rows in order however, using this sorted might be more efficient.
+ *
+ * Note that if used, some option like withBufferSizeInMiB will be ignored.
+ *
+ * @return this builder.
+ */
+ public Builder sorted()
+ {
+ this.sorted = true;
+ return this;
+ }
+
+ /**
+ * Whether indexes should be built and serialized to disk along data. Defaults to true.
+ *
+ * @param buildIndexes true if indexes should be built, false otherwise
+ * @return this builder
+ */
+ public Builder withBuildIndexes(boolean buildIndexes)
+ {
+ this.buildIndexes = buildIndexes;
+ return this;
+ }
+
+ /**
+ * Set the listener to receive notifications on sstable produced
+ *
+ * Note that if listener is registered, the sstables are opened into {@link SSTableReader}.
+ * The consumer is responsible for releasing the {@link SSTableReader}
+ *
+ * @param sstableProducedListener receives the produced sstables
+ * @return this builder
+ */
+ public Builder withSSTableProducedListener(Consumer> sstableProducedListener)
+ {
+ this.sstableProducedListener = sstableProducedListener;
+ return this;
+ }
+
+ /**
+ * Whether the produced sstable should be open or not.
+ * By default, the writer does not open the produced sstables
+ *
+ * @return this builder
+ */
+ public Builder openSSTableOnProduced()
+ {
+ this.openSSTableOnProduced = true;
+ return this;
+ }
+
+ public HarrySSTableWriter build()
+ {
+ if (directory == null)
+ throw new IllegalStateException("No ouptut directory specified, you should provide a directory with inDirectory()");
+ if (schemaStatement == null)
+ throw new IllegalStateException("Missing schema, you should provide the schema for the SSTable to create with forTable()");
+
+ Preconditions.checkState(Sets.difference(SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES, Schema.instance.getKeyspaces()).isEmpty(),
+ "Local keyspaces were not loaded. If this is running as a client, please make sure to add %s=true system property.",
+ CassandraRelevantProperties.FORCE_LOAD_LOCAL_KEYSPACES.getKey());
+
+ // Assign the default max SSTable size if not defined in builder
+ if (isMaxSSTableSizeUnset())
+ {
+ maxSSTableSizeInMiB = sorted ? -1L : DEFAULT_BUFFER_SIZE_IN_MIB_FOR_UNSORTED;
+ }
+
+ synchronized (HarrySSTableWriter.class)
+ {
+ String keyspaceName = schemaStatement.keyspace();
+ String tableName = schemaStatement.table();
+
+ Schema.instance.submit(SchemaTransformations.addKeyspace(KeyspaceMetadata.create(keyspaceName,
+ KeyspaceParams.simple(1),
+ Tables.none(),
+ Views.none(),
+ Types.none(),
+ UserFunctions.none()), true));
+
+ KeyspaceMetadata ksm = KeyspaceMetadata.create(keyspaceName,
+ KeyspaceParams.simple(1),
+ Tables.none(),
+ Views.none(),
+ Types.none(),
+ UserFunctions.none());
+
+ TableMetadata tableMetadata = Schema.instance.getTableMetadata(keyspaceName, tableName);
+ if (tableMetadata == null)
+ {
+ Types types = createTypes(keyspaceName);
+ Schema.instance.submit(SchemaTransformations.addTypes(types, true));
+ tableMetadata = createTable(types, ksm.userFunctions);
+ Schema.instance.submit(SchemaTransformations.addTable(tableMetadata, true));
+
+ if (buildIndexes && !indexStatements.isEmpty())
+ {
+ // we need to commit keyspace metadata first so applyIndexes sees that keyspace from TCM
+ commitKeyspaceMetadata(ksm.withSwapped(ksm.tables.with(tableMetadata)));
+ applyIndexes(keyspaceName);
+ }
+
+ KeyspaceMetadata keyspaceMetadata = ClusterMetadata.current().schema.getKeyspaceMetadata(keyspaceName);
+ tableMetadata = keyspaceMetadata.tables.getNullable(tableName);
+
+ Schema.instance.submit(SchemaTransformations.addTable(tableMetadata, true));
+ }
+
+ KeyspaceMetadata keyspaceMetadata = ClusterMetadata.current().schema.getKeyspaceMetadata(keyspaceName);
+ Keyspace keyspace = Keyspace.mockKS(keyspaceMetadata);
+ Directories directories = new Directories(tableMetadata, Collections.singleton(new Directories.DataDirectory(new File(directory.toPath()))));
+ ColumnFamilyStore cfs = ColumnFamilyStore.createColumnFamilyStore(keyspace,
+ tableName,
+ tableMetadata,
+ directories,
+ false,
+ false);
+
+ keyspace.initCfCustom(cfs);
+
+ // this is the empty directory / leftover from times we initialized ColumnFamilyStore
+ // it will automatically create directories for keyspace and table on disk after initialization
+ // we set that directory to the destination of generated SSTables so we just remove empty directories here
+ try
+ {
+ new File(directory, keyspaceName).deleteRecursive();
+ }
+ catch (UncheckedIOException ex)
+ {
+ if (!(ex.getCause() instanceof NoSuchFileException))
+ {
+ throw ex;
+ }
+ }
+
+ TableMetadataRef ref = tableMetadata.ref;
+ AbstractSSTableSimpleWriter writer = sorted
+ ? new SSTableSimpleWriter(directory, ref, cfs.metadata.get().regularAndStaticColumns(), maxSSTableSizeInMiB)
+ : new SSTableSimpleUnsortedWriter(directory, ref, cfs.metadata.get().regularAndStaticColumns(), maxSSTableSizeInMiB);
+
+ if (format != null)
+ writer.setSSTableFormatType(format);
+
+ if (buildIndexes && !indexStatements.isEmpty() && cfs != null)
+ {
+ StorageAttachedIndexGroup saiGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+ if (saiGroup != null)
+ writer.addIndexGroup(saiGroup);
+ }
+
+ if (sstableProducedListener != null)
+ writer.setSSTableProducedListener(sstableProducedListener);
+
+ writer.setShouldOpenProducedSSTable(openSSTableOnProduced);
+
+ return new HarrySSTableWriter(writer);
+ }
+ }
+
+ private boolean isMaxSSTableSizeUnset()
+ {
+ return maxSSTableSizeInMiB <= 0;
+ }
+
+ private Types createTypes(String keyspace)
+ {
+ Types.RawBuilder builder = Types.rawBuilder(keyspace);
+ for (CreateTypeStatement.Raw st : typeStatements)
+ st.addToRawBuilder(builder);
+ return builder.build();
+ }
+
+ /**
+ * Applies any provided index definitions to the target table
+ *
+ * @param keyspaceName name of the keyspace to apply indexes for
+ * @return table metadata reflecting applied indexes
+ */
+ private void applyIndexes(String keyspaceName)
+ {
+ ClientState state = ClientState.forInternalCalls();
+
+ for (CreateIndexStatement.Raw statement : indexStatements)
+ {
+ Keyspaces keyspaces = statement.prepare(state).apply(ClusterMetadata.current());
+ commitKeyspaceMetadata(keyspaces.getNullable(keyspaceName));
+ }
+ }
+
+ private void commitKeyspaceMetadata(KeyspaceMetadata keyspaceMetadata)
+ {
+ SchemaTransformation schemaTransformation = new SchemaTransformation()
+ {
+ @Override
+ public Keyspaces apply(ClusterMetadata metadata)
+ {
+ return metadata.schema.getKeyspaces().withAddedOrUpdated(keyspaceMetadata);
+ }
+
+ @Override
+ public boolean compatibleWith(ClusterMetadata metadata)
+ {
+ return true;
+ }
+ };
+ ClusterMetadataService.instance().commit(new AlterSchema(schemaTransformation));
+ }
+
+ /**
+ * Creates the table according to schema statement
+ *
+ * @param types types this table should be created with
+ */
+ private TableMetadata createTable(Types types, UserFunctions functions)
+ {
+ ClientState state = ClientState.forInternalCalls();
+ CreateTableStatement statement = schemaStatement.prepare(state);
+ statement.validate(ClientState.forInternalCalls());
+
+ TableMetadata.Builder builder = statement.builder(types, functions);
+ if (partitioner != null)
+ builder.partitioner(partitioner);
+
+ return builder.build();
+ }
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
index edbe249b2e..8205fd02b3 100644
--- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
+++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java
@@ -23,64 +23,87 @@ import java.io.IOException;
import java.util.List;
import java.util.concurrent.*;
+import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
import org.apache.cassandra.io.util.File;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.service.snapshot.SnapshotManager;
+import org.apache.cassandra.tcm.ClusterMetadataService;
import org.openjdk.jmh.annotations.*;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
@Fork(value = 1)
@Threads(1)
@State(Scope.Benchmark)
public class CompactionBench extends CQLTester
{
- static String keyspace;
- String table;
- String writeStatement;
- String readStatement;
- ColumnFamilyStore cfs;
- List snapshotFiles;
- List liveFiles;
+ protected static String keyspace;
+ protected String table;
+ protected String writeStatement;
+ protected ColumnFamilyStore cfs;
+ protected List snapshotFiles;
+
+ @Param("2")
+ protected int sstableCount = 2;
+
+ @Param("50000")
+ protected int rowCount = 50000;
+
+ @Param("NONE")
+ protected String overlap = "NONE";
+
+ @Param("true")
+ protected boolean isCursor = true;
@Setup(Level.Trial)
public void setup() throws Throwable
{
CQLTester.prepareServer();
+ DatabaseDescriptor.setCursorCompactionEnabled(isCursor);
+ DatabaseDescriptor.setCompactionThroughputMebibytesPerSec(10*1024); // no rate limiting
+ createSStables();
+ takeSnapshot();
+ }
+
+ protected void createSStables()
+ {
keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))");
execute("use "+keyspace+";");
writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)";
- readStatement = "SELECT * from "+table+" limit 100";
Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
cfs.disableAutoCompaction();
- //Warm up
- System.err.println("Writing 50k");
- for (long i = 0; i < 50000; i++)
- execute(writeStatement, i, i, i );
+ for (int j = 0; j < sstableCount; j++)
+ {
+ int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+ int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+ for (long i = 0; i < rowCount; i++)
+ {
+ execute(writeStatement, (pPrefix + i), (rPrefix + i), j * rowCount + i);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ }
+ }
- cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
-
- System.err.println("Writing 50k again...");
- for (long i = 0; i < 50000; i++)
- execute(writeStatement, i, i, i );
-
- cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
-
+ private void takeSnapshot()
+ {
SnapshotManager.instance.takeSnapshot("originals", cfs.getKeyspaceTableName());
snapshotFiles = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots("originals").listFiles();
+ long[] sum = new long[1];
+ snapshotFiles.forEach(f -> sum[0] += f.length());
+ System.out.println("Total input size: " + sum[0]);
}
@TearDown(Level.Trial)
@@ -95,28 +118,36 @@ public class CompactionBench extends CQLTester
System.err.println("Thread "+t.getName());
}
+ CommitLog.instance.shutdownBlocking();
+ ClusterMetadataService.instance().log().close();
+ CQLTester.tearDownClass();
CQLTester.cleanup();
}
@TearDown(Level.Invocation)
- public void resetSnapshot()
+ public void resetSnapshot() throws IOException, InterruptedException
{
cfs.truncateBlocking();
List directories = cfs.getDirectories().getCFDirectories();
-
- for (File file : directories)
+ // Sometimes deletes are unreliable...
+ int deleted = 0;
+ do
{
- for (File f : file.tryList())
+ deleted = 0;
+ for (File file : directories)
{
- if (f.isDirectory())
- continue;
-
- FileUtils.delete(f);
+ for (File f : file.tryList())
+ {
+ if (f.isDirectory())
+ continue;
+ f.tryDelete();
+ deleted++;
+ }
}
- }
-
+ Thread.sleep(10);
+ } while (deleted != 0);
for (File file : snapshotFiles)
FileUtils.createHardLink(file, new File(new File(file.toPath().getParent().getParent().getParent()), file.name()));
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionLargeCellBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionLargeCellBench.java
new file mode 100644
index 0000000000..0a17d7ae95
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionLargeCellBench.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.test.microbench.CompactionBench;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class CompactionLargeCellBench extends CompactionBench
+{
+ @Param("128")
+ int blobSize = 128;
+
+ protected void createSStables()
+ {
+ keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, b blob, PRIMARY KEY(userid, picid))");
+ execute("use "+keyspace+";");
+ writeStatement = "INSERT INTO "+table+"(userid,picid,b)VALUES(?,?,?)";
+
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ byte[] blob = new byte[blobSize];
+ for (int j = 0; j < sstableCount; j++)
+ {
+ int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+ int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+ for (long i = 0; i < rowCount; i++)
+ {
+ ThreadLocalRandom.current().nextBytes(blob);
+ execute(writeStatement, (pPrefix + i), (rPrefix + i), ByteBuffer.wrap(blob));
+ }
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ }
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionStringsBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionStringsBench.java
new file mode 100644
index 0000000000..ee1af71e4d
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionStringsBench.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.test.microbench.CompactionBench;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class CompactionStringsBench extends CompactionBench
+{
+ @Param("128")
+ int stringSizeMin = 128;
+ @Param("256")
+ int stringSizeMax = 256;
+
+ protected void createSStables()
+ {
+ keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ table = createTable(keyspace, "CREATE TABLE %s ( pk ascii, ck ascii, c ascii, PRIMARY KEY(pk, ck))");
+ execute("use "+keyspace+";");
+ writeStatement = "INSERT INTO "+table+"(pk,ck,c)VALUES(?,?,?)";
+
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ for (int j = 0; j < sstableCount; j++)
+ {
+ Random r = new Random(42L);
+ int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+ int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+ for (long i = 0; i < rowCount; i++)
+ {
+ execute(writeStatement, (pPrefix + i) + getNextString(r), (rPrefix + i) + getNextString(r), getNextString(r));
+ }
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ }
+ }
+
+ private String getNextString(Random r)
+ {
+ int rangeDelta = stringSizeMax - stringSizeMin;
+ int rangeRandom = rangeDelta > 0 ? r.nextInt(rangeDelta) : 0;
+ byte[] blob = new byte[stringSizeMin + rangeRandom];
+ r.nextBytes(blob);
+ for (int i = 0; i < blob.length; i++)
+ {
+ if (blob[i] < 0)
+ blob[i] = 0;
+ }
+ String nextString = new String(blob);
+ return nextString;
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionWideRowBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionWideRowBench.java
new file mode 100644
index 0000000000..3b0e0745e8
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionWideRowBench.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.test.microbench.CompactionBench;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class CompactionWideRowBench extends CompactionBench
+{
+ @Param("1")
+ int rowPerPkCount = 1;
+
+ @Param("1")
+ int ckCount = 1;
+
+ @Param("1")
+ int colCount = 1;
+
+ protected void createSStables()
+ {
+ keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String tableCreate = "CREATE TABLE %s ( userid bigint";
+ for (int i=0;i k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ int pkCount = rowCount/rowPerPkCount;
+ for (int j = 0; j < sstableCount; j++)
+ {
+ int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+ int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+ for (long pkIndex = 0; pkIndex < pkCount; pkIndex++)
+ {
+ for (long rowIndex = 0; rowIndex < rowPerPkCount; rowIndex++)
+ {
+ values[0] = (pPrefix + pkIndex);
+ Arrays.fill(values, 1, ckCount + 1, (rPrefix + rowIndex));
+ Arrays.fill(values, 1 + ckCount, values.length, j * rowCount + pkIndex);
+ execute(writeStatement, values);
+ }
+ }
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ }
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableAbstractBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableAbstractBench.java
new file mode 100644
index 0000000000..836394deb6
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableAbstractBench.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 1)
+@Measurement(iterations = 10, time = 1)
+@Fork(value = 1)
+@State(Scope.Benchmark)
+public class SSTableAbstractBench extends CQLTester
+{
+ ColumnFamilyStore cfs;
+ String keyspace;
+
+ @Param("50000")
+ int rowCount = 50000;
+ private String table;
+
+ // TODO: elaborate data setup with multiple schemas
+ @Setup(Level.Trial)
+ public void setup() throws Throwable
+ {
+ prepareServer();
+ beforeTest();
+
+ setupTable();
+ setupData();
+ }
+
+ protected void setupTable()
+ {
+ keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid1 bigint, picid2 bigint, commentid bigint, " +
+ "PRIMARY KEY(userid, picid1, picid2))");
+ execute("use "+keyspace+";");
+
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ }
+
+ protected void setupData()
+ {
+ String writeStatement = "INSERT INTO " + table + "(userid,picid1,picid2,commentid)VALUES(?,?,?,?)";
+ for (long i = 0; i < rowCount; i++)
+ insertForIndex(writeStatement, i);
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ }
+
+ protected UntypedResultSet insertForIndex(String writeStatement, long i)
+ {
+ return execute(writeStatement, i, i, i, i);
+ }
+
+ @TearDown(Level.Trial)
+ public void teardown() throws IOException, ExecutionException, InterruptedException
+ {
+ CommitLog.instance.shutdownBlocking();
+ ClusterMetadataService.instance().log().close();
+ CQLTester.tearDownClass();
+ CQLTester.cleanup();
+ }
+
+ public SSTableReader getReader() throws IOException
+ {
+ return cfs.getLiveSSTables().stream().filter(s -> s.getKeyspaceName().equals(keyspace)).findFirst().orElse(null);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableAbstractPipeBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableAbstractPipeBench.java
new file mode 100644
index 0000000000..98c6cbb19a
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableAbstractPipeBench.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.snapshot.SnapshotManager;
+import org.apache.cassandra.tools.Util;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+
+@State(Scope.Benchmark)
+public class SSTableAbstractPipeBench extends SSTableAbstractBench
+{
+ Descriptor desc;
+ TableMetadata metadata;
+ File tmpDir;
+ List snapshotFiles;
+
+ @Setup(Level.Trial)
+ public void setupSnapshots() throws Throwable
+ {
+ SnapshotManager.instance.takeSnapshot("originals", cfs.getKeyspaceTableName());
+
+ snapshotFiles = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots("originals").listFiles();
+
+ desc = Descriptor.fromFileWithComponent(snapshotFiles.get(0), false).left;
+ metadata = Util.metadataFromSSTable(desc);
+ tmpDir = Files.createTempDirectory("sstable-copy").toFile();
+ System.err.println("Writing to : " + tmpDir);
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableCursorPipeUtil.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableCursorPipeUtil.java
new file mode 100644
index 0000000000..a2b8b5661b
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableCursorPipeUtil.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.IOException;
+
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.io.sstable.PartitionDescriptor;
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.io.sstable.SSTableCursorReader;
+import org.apache.cassandra.io.sstable.SSTableCursorWriter;
+import org.apache.cassandra.io.sstable.UnfilteredDescriptor;
+
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.*;
+
+public class SSTableCursorPipeUtil
+{
+
+ public static void copySSTable(SSTableCursorReader reader, SSTableCursorWriter writer) throws Throwable
+ {
+ PartitionDescriptor pHeader = new PartitionDescriptor(reader.ssTableReader().getPartitioner().createReusableKey(0));
+ UnfilteredDescriptor unfilteredDescriptor = new UnfilteredDescriptor(reader.ssTableReader().header.clusteringTypes().toArray(AbstractType[]::new));
+ int readerState = PARTITION_START;
+ boolean first = true;
+
+ while (readerState != DONE)
+ {
+ readerState = reader.readPartitionHeader(pHeader);
+ if (first)
+ {
+ first = false;
+ writer.setFirst(pHeader.keyBuffer());
+ }
+ readerState = copyPartition(reader, writer, pHeader, unfilteredDescriptor, readerState);
+ }
+ writer.setLast(pHeader.keyBuffer());
+ }
+
+ public static int copyPartition(SSTableCursorReader reader, SSTableCursorWriter writer, PartitionDescriptor pHeader, UnfilteredDescriptor unfilteredDescriptor, int readerState) throws IOException
+ {
+ if (readerState != STATIC_ROW_START &&
+ readerState != ROW_START &&
+ readerState != TOMBSTONE_START &&
+ readerState != PARTITION_END)
+ throw new IllegalStateException();
+
+ final byte[] keyBytes = pHeader.keyBytes();
+ final int keyLength = pHeader.keyLength();
+ final DeletionTime pDeletionTime = pHeader.deletionTime();
+
+ int headerLength = writer.writePartitionStart(keyBytes, keyLength, pDeletionTime);
+ int unfilteredCounter = 0;
+ while (readerState != PARTITION_END)
+ {
+ switch (readerState)
+ {
+ case STATIC_ROW_START:
+ readerState = copyStaticRow(reader, writer, unfilteredDescriptor);
+ headerLength = (int) (writer.getPosition() - writer.getPartitionStart());
+ break;
+ case ROW_START:
+ readerState = copyRow(reader, writer, unfilteredDescriptor, unfilteredCounter++);
+ break;
+ case TOMBSTONE_START:
+ readerState = copyRangeTombstone(reader, writer, unfilteredDescriptor, unfilteredCounter++);
+ }
+ }
+ writer.writePartitionEnd(keyBytes, keyLength, pDeletionTime, headerLength);
+ if (unfilteredCounter > 1) {
+ writer.updateClusteringMetadata(unfilteredDescriptor);
+ }
+ return reader.continueReading();
+ }
+
+ public static int copyStaticRow(SSTableCursorReader reader, SSTableCursorWriter writer, UnfilteredDescriptor unfilteredDescriptor) throws IOException
+ {
+ int readerState = reader.readStaticRowHeader(unfilteredDescriptor);
+ return copyRowAfterDescriptor(reader, writer, unfilteredDescriptor, readerState, true, false);
+ }
+
+ public static int copyRow(SSTableCursorReader reader, SSTableCursorWriter writer, UnfilteredDescriptor unfilteredDescriptor, int unfilteredIndex) throws IOException
+ {
+ int readerState = reader.readRowHeader(unfilteredDescriptor);
+ return copyRowAfterDescriptor(reader, writer, unfilteredDescriptor, readerState, false, unfilteredIndex == 0);
+ }
+
+ public static int copyRangeTombstone(SSTableCursorReader reader, SSTableCursorWriter writer, UnfilteredDescriptor unfilteredDescriptor, int unfilteredIndex) throws IOException
+ {
+ int readerState = reader.readTombstoneMarker(unfilteredDescriptor);
+ writer.writeRangeTombstone(unfilteredDescriptor, unfilteredIndex == 0);
+ return readerState;
+ }
+
+ private final static byte[] copyColumnValueBuffer = new byte[4096]; // used to copy cell contents (maybe piecemeal if very large, since we don't have a direct read option)
+
+ public static int copyRowAfterDescriptor(SSTableCursorReader reader, SSTableCursorWriter writer, UnfilteredDescriptor unfilteredDescriptor, int readerState, boolean isStatic, boolean updateClusteringMetadata) throws IOException
+ {
+ writer.writeRowStart(unfilteredDescriptor.livenessInfo(), unfilteredDescriptor.deletionTime(), isStatic);
+
+ // Copy cells
+ while (readerState != UNFILTERED_END)
+ {
+ if (readerState != CELL_HEADER_START)
+ throw new IllegalStateException("Unexpected reader state: " + readerState);
+ readerState = reader.readCellHeader();
+ SSTableCursorReader.CellCursor cellCursor = reader.cellCursor();
+
+ /**
+ * {@link Cell.Serializer#serialize}
+ */
+ int cellFlags = cellCursor.cellFlags;
+ ReusableLivenessInfo cellLiveness = cellCursor.cellLiveness;
+ writer.writeCellHeader(cellFlags, cellLiveness, cellCursor.cellColumn);
+ if (readerState == CELL_VALUE_START)
+ {
+ readerState = writer.writeCellValue(reader, copyColumnValueBuffer);
+ }
+ else if (Cell.Serializer.hasValue(cellFlags))
+ {
+ throw new IllegalStateException("Flags and state contradict");
+ }
+ if (readerState != CELL_END)
+ throw new IllegalStateException("Expect CELL_END after cell read. State: " + readerState);
+
+ readerState = reader.continueReading();
+ }
+
+ writer.writeRowEnd(unfilteredDescriptor, updateClusteringMetadata);
+
+ return reader.continueReading();
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeBench.java
new file mode 100644
index 0000000000..e69900007e
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeBench.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.File;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.tools.Util;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+
+@State(Scope.Benchmark)
+public class SSTablePipeBench extends SSTableAbstractPipeBench
+{
+ @TearDown(Level.Invocation)
+ public void closeReaderAndDeleteOutput()
+ {
+ for (File file : tmpDir.listFiles())
+ {
+ file.delete();
+ }
+ }
+
+ @Benchmark
+ public void readAndWrite() throws Throwable
+ {
+ SSTableReader ssTableReader = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata));
+ try (SSTableWriter ssTableWriter = CompactionManager.createWriter(cfs, new org.apache.cassandra.io.util.File(tmpDir), -1, -1, null, false, ssTableReader, LifecycleTransaction.offline(OperationType.COMPACTION));)
+ {
+ final ISSTableScanner currentScanner = ssTableReader.getScanner();
+ Stream partitions = Util.iterToStream(currentScanner);
+ partitions.forEach(unfilteredRowIterator -> {
+ ssTableWriter.append(unfilteredRowIterator);
+ });
+ ssTableWriter.finish(false);
+ }
+ ssTableReader.ref().close();
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeCursorBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeCursorBench.java
new file mode 100644
index 0000000000..83fc957ac8
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTablePipeCursorBench.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.File;
+
+import org.apache.cassandra.io.sstable.SSTableCursorReader;
+import org.apache.cassandra.io.sstable.SSTableCursorWriter;
+import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SortedTableWriter;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+
+@State(Scope.Benchmark)
+public class SSTablePipeCursorBench extends SSTableAbstractPipeBench
+{
+ @TearDown(Level.Invocation)
+ public void closeReaderAndDeleteOutput() throws Exception
+ {
+ for (File file : tmpDir.listFiles())
+ {
+ file.delete();
+ }
+ }
+
+ @Benchmark
+ public void readAndWrite() throws Throwable
+ {
+ try(SSTableCursorReader cursorReader = SSTableCursorReader.fromDescriptor(desc);
+ SortedTableWriter ssTableWriter = (SortedTableWriter) CompactionManager.createWriter(cfs, new org.apache.cassandra.io.util.File(tmpDir), 0, 0, null, false, cursorReader.ssTableReader(), LifecycleTransaction.offline(OperationType.COMPACTION));
+ SSTableCursorWriter cursorWriter = new SSTableCursorWriter(ssTableWriter);){
+ SSTableCursorPipeUtil.copySSTable(cursorReader, cursorWriter);
+ }
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableRawVisitorBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableRawVisitorBench.java
new file mode 100644
index 0000000000..5d4db56cc5
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableRawVisitorBench.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.IOException;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.utils.vint.VIntCoding;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+@State(Scope.Benchmark)
+public class SSTableRawVisitorBench extends SSTableAbstractBench
+{
+ private Version version;
+ private TableMetadata metadata;
+ private ImmutableList clusteringColumns;
+ private int clusteringColumnCount;
+ private AbstractType[] clusteringColumnTypes;
+ private boolean hasUIntDeletionTime;
+
+ private SSTableReader ssTableReader;
+
+ @Setup(Level.Invocation)
+ public void prepareReader() throws IOException
+ {
+ ssTableReader = getReader();
+ TableMetadata metadata = ssTableReader.metadata();
+ version = ssTableReader.descriptor.version;
+ hasUIntDeletionTime = version.hasUIntDeletionTime();
+ clusteringColumns = metadata.clusteringColumns();
+ clusteringColumnCount = clusteringColumns.size();
+ clusteringColumnTypes = new AbstractType[clusteringColumnCount];
+ for (int i = 0; i< clusteringColumnTypes.length; i++) {
+ clusteringColumnTypes[i] = clusteringColumns.get(i).type;
+ }
+ }
+
+ @TearDown(Level.Invocation)
+ public void closeReader() {
+ ssTableReader.ref().close();
+ }
+
+ long[] counters = new long[4];
+ @Benchmark
+ public void countPartitionsAndUnfiltered() throws IOException
+ {
+ for (int i = 0; i < counters.length; i++)
+ {
+ counters[i] = 0;
+ }
+ try (RandomAccessReader randomAccessReader = ssTableReader.openDataReader()) {
+ long length = randomAccessReader.length();
+ long nextPartition = 0;
+ do
+ {
+ nextPartition = readPartition(randomAccessReader, nextPartition, counters);
+ counters[0]++;
+ } while (!randomAccessReader.isEOF() && nextPartition < length);
+ }
+ }
+
+
+ // struct partition {
+ // struct partition_header header
+ // optional row
+ // struct unfiltered unfiltereds[];
+ //};
+ private long readPartition(RandomAccessReader randomAccessReader, long nextPartition, long[] counters) throws IOException
+ {
+ int cursor = (int) nextPartition;
+ int headerPosition = cursor;
+ // struct partition_header header {
+ // be16 key_length; e.g. 8 if long
+ // byte key[key_length];
+ // struct deletion_time deletion_time {
+ // be32 local_deletion_time;
+ // be64 marked_for_delete_at;
+ // };
+ // };
+ int keyLength = randomAccessReader.readUnsignedShort();
+ // TODO: print key according to metadata (need the type for formatting)
+ int keyPosition = (cursor += 2);
+ randomAccessReader.skipBytes(keyLength);
+ cursor += keyLength;
+
+ // PARTITION DELETION TIME
+ int deletionTimePosition = cursor;
+ int deletionTimeSize = 12;
+ if (hasUIntDeletionTime) {
+ byte flags = randomAccessReader.readByte();
+ if ((IS_LIVE_DELETION & flags) != 0) {
+ deletionTimeSize = 1;
+ // no delete times
+ }
+ else {
+ long position = randomAccessReader.getPosition();
+ randomAccessReader.seek(position - 1);
+ long markedForDeleteAt = randomAccessReader.readLong();
+ int localDeletionTime = randomAccessReader.readInt();
+ }
+ }
+ else
+ {
+ int localDeletionTime = randomAccessReader.readInt();
+ long markedForDeleteAt = randomAccessReader.readLong();
+ }
+ // read the rows until END_OF_PARTITION
+ int nextUnfilteredPosition = (cursor += deletionTimeSize);
+ byte nextUnfilteredFlags = randomAccessReader.readByte();
+ while (!UnfilteredSerializer.isEndOfPartition(nextUnfilteredFlags)) {
+ nextUnfilteredPosition = readUnfiltered(randomAccessReader, nextUnfilteredFlags, nextUnfilteredPosition, counters);
+ nextUnfilteredFlags = randomAccessReader.readByte();
+ }
+ return nextUnfilteredPosition + 1;
+ }
+ // struct row {
+ // byte flags;
+ // optional extended_flags; // only present for static rows
+ // optional clustering_blocks {
+ // varint clustering_block_header;
+ // simple_cell[] clustering_cells;
+ // }; // only present for non-static rows
+ // varint row_body_size;
+ // varint prev_unfiltered_size; // for backward traversing
+ // optional liveness_info;
+ // optional deletion_time;
+ // optional missing_columns;
+ // cell[] cells;
+ // }; // Has IS_STATIC flag set
+ private int readUnfiltered(RandomAccessReader randomAccessReader, byte flags, final int unfilteredStartPosition, long[] counters) throws IOException
+ {
+ if (UnfilteredSerializer.isEndOfPartition(flags)) throw new IllegalStateException();
+
+ int cursor = unfilteredStartPosition + 1;
+ boolean isRow = UnfilteredSerializer.isRow(flags);
+ boolean isTombstoneMarker = UnfilteredSerializer.isTombstoneMarker(flags);
+ boolean isStatic = false;
+ boolean deletionIsShadowable = false;
+ if (UnfilteredSerializer.isExtended(flags)) {
+ byte extendedFlags = randomAccessReader.readByte(); cursor++;
+
+ isStatic = UnfilteredSerializer.isStatic(extendedFlags);
+ deletionIsShadowable = UnfilteredSerializer.deletionIsShadowable(extendedFlags);
+
+ }
+ if ((isStatic && !isRow) || (isStatic && isTombstoneMarker)) throw new IllegalStateException();
+
+ if (isStatic) { // this should only apply to first row read
+ // static row
+ long rowSize = randomAccessReader.readUnsignedVInt();
+ randomAccessReader.skipBytes((int)rowSize);
+
+ cursor += VIntCoding.computeUnsignedVIntSize(rowSize) + rowSize;
+ // TODO: handle row contents
+
+ counters[1]++;
+ }
+ else if (isRow)
+ {
+ final int rowClusteringStart = cursor;
+ // READ CLUSTERING, repeated for tombstone, will de-dup later
+ long clusteringBlockHeader = 0;
+ AbstractType[] types = clusteringColumnTypes;
+ for (int clusteringIndex = 0; clusteringIndex < types.length; clusteringIndex++)
+ {
+ // struct clustering_block {
+ // varint clustering_block_header;
+ // simple_cell[] clustering_cells;
+ // };
+ if (clusteringIndex % 32 == 0) {
+ // TODO: ideally we'd like to get the size while reading rather than have to compute it
+ clusteringBlockHeader = randomAccessReader.readUnsignedVInt();
+ cursor += VIntCoding.computeUnsignedVIntSize(clusteringBlockHeader);
+ }
+ AbstractType type = types[clusteringIndex];
+ if (isNull(clusteringBlockHeader, clusteringIndex)) {
+ // handle null
+ } else if (isEmpty(clusteringBlockHeader, clusteringIndex)) {
+ // handle empty
+ } else if (type.isValueLengthFixed()) {
+ // handle value (TODO: add some JSON sonversion without Strings)
+ int length = type.valueLengthIfFixed();
+ cursor += length;
+ randomAccessReader.skipBytes(length);
+ } else {
+ int length = randomAccessReader.readUnsignedVInt32();
+ cursor += VIntCoding.computeUnsignedVIntSize(length);
+ if (length < 0)
+ throw new IllegalStateException("Corrupt (negative) value length encountered");
+ // handle value (TODO: add some JSON sonversion without Strings)
+ cursor += length;
+ randomAccessReader.skipBytes(length);
+ }
+ }
+ // READ CLUSTERING DONE
+ final int rowBodyStart = cursor;
+
+ long rowSize = randomAccessReader.readUnsignedVInt();
+ randomAccessReader.skipBytes((int)rowSize);
+ cursor += VIntCoding.computeUnsignedVIntSize(rowSize) + rowSize;
+ // TODO: handle row contents
+ counters[2]++;
+ }
+ else if (isTombstoneMarker) {
+ // struct range_tombstone_marker {
+ // byte flags = IS_MARKER;
+ // byte kind_ordinal;
+ // be16 bound_values_count;
+ // struct clustering_block[] clustering_blocks;
+ // varint marker_body_size;
+ // varint prev_unfiltered_size;
+ // };
+ byte kind = randomAccessReader.readByte();
+ cursor++;
+
+ int clusteringColumnsBound = randomAccessReader.readUnsignedShort();
+ cursor+=2;
+
+ // READ CLUSTERING, repeated for row, will de-dup later
+ long clusteringBlockHeader = 0;
+ AbstractType[] types = clusteringColumnTypes;
+ for (int clusteringIndex = 0; clusteringIndex < clusteringColumnsBound; clusteringIndex++)
+ {
+ // struct clustering_block {
+ // varint clustering_block_header;
+ // simple_cell[] clustering_cells;
+ // };
+ if (clusteringIndex % 32 == 0) {
+ // TODO: ideally we'd like to get the size while reading rather than have to compute it
+ clusteringBlockHeader = randomAccessReader.readUnsignedVInt();
+ cursor += VIntCoding.computeUnsignedVIntSize(clusteringBlockHeader);
+ }
+ AbstractType type = types[clusteringIndex];
+ if (isNull(clusteringBlockHeader, clusteringIndex)) {
+ // handle null
+ } else if (isEmpty(clusteringBlockHeader, clusteringIndex)) {
+ // handle empty
+ } else if (type.isValueLengthFixed()) {
+ // handle value (TODO: add some JSON sonversion without Strings)
+ cursor += type.valueLengthIfFixed();
+ } else {
+ int length = randomAccessReader.readUnsignedVInt32();
+ //cursor += VIntCoding.computeUnsignedVIntSize(length);
+ if (length < 0)
+ throw new IllegalStateException("Corrupt (negative) value length encountered");
+ // handle value (TODO: add some JSON sonversion without Strings)
+ cursor += length;
+ }
+ }
+ // READ CLUSTERING DONE
+ long length = randomAccessReader.readUnsignedVInt();
+ cursor += VIntCoding.computeUnsignedVIntSize(length);
+ length = randomAccessReader.readUnsignedVInt();
+ cursor += VIntCoding.computeUnsignedVIntSize(length);
+ counters[3]++;
+ }
+
+ return cursor;
+ }
+
+ // TODO: C&P from Clustering
+ // ---Clustering
+ // no need to do modulo arithmetic for i, since the left-shift execute on the modulus of RH operand by definition
+ private static boolean isNull(long header, int i)
+ {
+ long mask = 1L << (i * 2) + 1;
+ return (header & mask) != 0;
+ }
+
+ // no need to do modulo arithmetic for i, since the left-shift execute on the modulus of RH operand by definition
+ private static boolean isEmpty(long header, int i)
+ {
+ long mask = 1L << (i * 2);
+ return (header & mask) != 0;
+ }
+ // ---Clustering
+
+ // TODO: C&P from DeletionTime
+ // We use the sign bit to signal LIVE DeletionTimes
+ private final static int IS_LIVE_DELETION = 0b1000_0000;
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingBench.java
new file mode 100644
index 0000000000..d6a820ee4d
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingBench.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.IOException;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.tools.Util;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+@State(Scope.Benchmark)
+public class SSTableReadingBench extends SSTableAbstractBench
+{
+ private SSTableReader ssTableReader;
+
+ @Setup(Level.Invocation)
+ public void prepareReader() throws IOException
+ {
+ ssTableReader = super.getReader();
+ }
+
+ @TearDown(Level.Invocation)
+ public void closeReader() {
+ ssTableReader.ref().close();
+ }
+
+ long[] counters = new long[4];
+ @Benchmark
+ public void countPartitionsAndUnfiltered()
+ {
+ for (int i = 0; i < counters.length; i++)
+ {
+ counters[i] = 0;
+ }
+ final ISSTableScanner currentScanner = ssTableReader.getScanner();
+ Stream partitions = Util.iterToStream(currentScanner);
+ partitions.forEach(unfilteredRowIterator -> {
+ counters[0]++;
+ Row staticRow = unfilteredRowIterator.staticRow();
+ if (staticRow != null) {
+ counters[1]++;
+ }
+ unfilteredRowIterator.forEachRemaining(unfiltered -> {
+ if (unfiltered.isRow()) {
+ counters[2]++;
+ }
+ else {
+ counters[3]++;
+ }
+ });
+ });
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingCursorBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingCursorBench.java
new file mode 100644
index 0000000000..e5c5d957cc
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingCursorBench.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.IOException;
+
+import org.apache.cassandra.io.sstable.SSTableCursorReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+@State(Scope.Benchmark)
+public class SSTableReadingCursorBench extends SSTableAbstractBench
+{
+ private SSTableReader ssTableReader;
+ private SSTableCursorReader cursor;
+
+ @Setup(Level.Invocation)
+ public void prepareReader() throws IOException
+ {
+ ssTableReader = super.getReader();
+ cursor = new SSTableCursorReader(ssTableReader);
+ }
+
+ @TearDown(Level.Invocation)
+ public void closeReader() throws Exception
+ {
+ cursor.close();
+ ssTableReader.ref().close();
+ }
+
+ long[] counters = new long[4];
+
+
+ @Benchmark
+ public void readPartitionAndUnfiltered() throws IOException
+ {
+ SSTableReadingFileCursorBench.readPartitionAndUnfiltered(counters, cursor);
+ }
+
+ @Benchmark
+ public void readPartitionSkipUnfiltered() throws IOException
+ {
+ SSTableReadingFileCursorBench.readPartitionSkipUnfiltered(counters, cursor);
+ }
+
+ @Benchmark
+ public void skipPartition() throws IOException
+ {
+ SSTableReadingFileCursorBench.skipPartition(counters, cursor);
+ }
+
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingFileBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingFileBench.java
new file mode 100644
index 0000000000..dab8faa36b
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingFileBench.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.tools.Util;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 1)
+@Measurement(iterations = 10, time = 1)
+@Fork(value = 1)
+@State(Scope.Benchmark)
+public class SSTableReadingFileBench
+{
+ static
+ {
+ DatabaseDescriptor.toolInitialization(!TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean());
+ }
+ @Param("test/data/compaction/oa-70-big-Data.db")
+ String sstableFileName;
+ private Descriptor desc;
+ private SSTableReader ssTableReader;
+
+ @Setup(Level.Trial)
+ public void loadDescriptor() throws FileNotFoundException
+ {
+ File ssTableFile = new File(sstableFileName);
+
+ if (!ssTableFile.exists())
+ {
+ throw new FileNotFoundException("Cannot find file " + ssTableFile.absolutePath());
+ }
+ desc = Descriptor.fromFileWithComponent(ssTableFile, false).left;
+ }
+
+ @Setup(Level.Invocation)
+ public void prepareReader() throws IOException
+ {
+ TableMetadata metadata = Util.metadataFromSSTable(desc);
+ ssTableReader = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata));
+ }
+
+ @TearDown(Level.Invocation)
+ public void closeReader() {
+ ssTableReader.ref().close();
+ }
+
+ long[] counters = new long[4];
+ @Benchmark
+ public void countPartitionsAndUnfiltered()
+ {
+ for (int i = 0; i < counters.length; i++)
+ {
+ counters[i] = 0;
+ }
+ final ISSTableScanner currentScanner = ssTableReader.getScanner();
+ Stream partitions = Util.iterToStream(currentScanner);
+ partitions.forEach(unfilteredRowIterator -> {
+ counters[0]++;
+ Row staticRow = unfilteredRowIterator.staticRow();
+ if (staticRow != null) {
+ counters[1]++;
+ }
+ unfilteredRowIterator.forEachRemaining(unfiltered -> {
+ if (unfiltered.isRow()) {
+ counters[2]++;
+ }
+ else
+ {
+ counters[3]++;
+ }
+ });
+ });
+ }
+}
diff --git a/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingFileCursorBench.java b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingFileCursorBench.java
new file mode 100644
index 0000000000..a7d2ebf444
--- /dev/null
+++ b/test/microbench/org/apache/cassandra/test/microbench/sstable/SSTableReadingFileCursorBench.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.io.sstable.PartitionDescriptor;
+import org.apache.cassandra.io.sstable.UnfilteredDescriptor;
+import org.apache.cassandra.io.sstable.SSTableCursorReader;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.util.File;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.*;
+import static org.apache.cassandra.config.CassandraRelevantProperties.TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 1)
+@Measurement(iterations = 10, time = 1)
+@Fork(value = 1)
+@State(Scope.Benchmark)
+public class SSTableReadingFileCursorBench
+{
+ static
+ {
+ if(!DatabaseDescriptor.isDaemonInitialized())
+ DatabaseDescriptor.toolInitialization(!TEST_UTIL_ALLOW_TOOL_REINIT_FOR_TEST.getBoolean());
+ }
+
+ @Param("test/data/compaction/oa-70-big-Data.db")
+ String sstableFileName;
+ private Descriptor desc;
+ private SSTableCursorReader cursor;
+
+ @Setup(Level.Trial)
+ public void loadDescriptor() throws FileNotFoundException
+ {
+ File ssTableFile = new File(sstableFileName);
+
+ if (!ssTableFile.exists())
+ {
+ throw new FileNotFoundException("Cannot find file " + ssTableFile.absolutePath());
+ }
+ desc = Descriptor.fromFileWithComponent(ssTableFile, false).left;
+ }
+
+ @Setup(Level.Invocation)
+ public void prepareReader() throws IOException
+ {
+ cursor = SSTableCursorReader.fromDescriptor(desc);
+ }
+
+ @TearDown(Level.Invocation)
+ public void closeReader() throws Exception
+ {
+ cursor.close();
+ }
+
+ long[] counters = new long[4];
+
+ @Benchmark
+ public void readPartitionAndUnfiltered() throws IOException
+ {
+ readPartitionAndUnfiltered(counters, cursor);
+ }
+
+ static void readPartitionAndUnfiltered(long[] counters, SSTableCursorReader cursor) throws IOException
+ {
+ Arrays.fill(counters, 0);
+ int state = PARTITION_START;
+ PartitionDescriptor pHeader = new PartitionDescriptor(cursor.ssTableReader().getPartitioner().createReusableKey(0));
+ UnfilteredDescriptor rHeader = new UnfilteredDescriptor(cursor.ssTableReader().header.clusteringTypes().toArray(AbstractType[]::new));
+ while (state != DONE) {
+ state = cursor.readPartitionHeader(pHeader);
+ counters[0]++;
+ state = readThroughPartition(counters, cursor, state, rHeader);
+ }
+ }
+
+ private static int readThroughPartition(long[] counters, SSTableCursorReader cursor, int state, UnfilteredDescriptor unfilteredDescriptor) throws IOException
+ {
+ while (state != PARTITION_END) {
+ switch (state) {
+ case STATIC_ROW_START:
+ counters[1]++;
+ state = readThroughStaticRow(cursor, unfilteredDescriptor);
+ break;
+ case ROW_START:
+ counters[2]++;
+ state = readThroughRow(cursor, unfilteredDescriptor);
+ break;
+ case TOMBSTONE_START:
+ counters[3]++;
+ state = cursor.readTombstoneMarker(unfilteredDescriptor);
+ break;
+ }
+ }
+ return cursor.continueReading();
+ }
+
+ static int readThroughStaticRow(SSTableCursorReader cursor, UnfilteredDescriptor unfilteredDescriptor) throws IOException
+ {
+ int state = cursor.readStaticRowHeader(unfilteredDescriptor);
+ while (state != UNFILTERED_END) {
+ state = readThroughCell(cursor);
+ }
+ return cursor.continueReading();
+ }
+
+ static int readThroughRow(SSTableCursorReader cursor, UnfilteredDescriptor unfilteredDescriptor) throws IOException
+ {
+ int state = cursor.readRowHeader(unfilteredDescriptor);
+ while (state != UNFILTERED_END) {
+ state = readThroughCell(cursor);
+ }
+ return cursor.continueReading();
+ }
+
+ private static int readThroughCell(SSTableCursorReader cursor) throws IOException
+ {
+ int state = cursor.readCellHeader();
+ if (state == CELL_VALUE_START)
+ {
+ state = cursor.skipCellValue();
+ }
+ if (state == CELL_END)
+ state = cursor.continueReading();
+ return state;
+ }
+
+ @Benchmark
+ public void readPartitionSkipUnfiltered() throws IOException
+ {
+ readPartitionSkipUnfiltered(counters, cursor);
+ }
+
+ static void readPartitionSkipUnfiltered(long[] counters, SSTableCursorReader cursor) throws IOException
+ {
+ for (int i = 0; i < counters.length; i++)
+ {
+ counters[i] = 0;
+ }
+ int state = PARTITION_START;
+ PartitionDescriptor pHeader = new PartitionDescriptor(cursor.ssTableReader().getPartitioner().createReusableKey(0));
+ while (state != DONE) {
+ state = cursor.readPartitionHeader(pHeader);
+ counters[0]++;
+ if (state == PARTITION_END) continue;
+ while (state != DONE && state != PARTITION_START) {
+ switch (state) {
+ case STATIC_ROW_START:
+ counters[1]++;
+ state = cursor.skipStaticRow(true);
+ break;
+ case ROW_START:
+ counters[2]++;
+ state = cursor.skipUnfiltered(true);
+ break;
+ case TOMBSTONE_START:
+ counters[3]++;
+ state = cursor.skipUnfiltered(true);
+ break;
+ }
+ }
+ }
+ }
+
+ @Benchmark
+ public void skipPartition() throws IOException
+ {
+ skipPartition(counters, cursor);
+ }
+
+ static void skipPartition(long[] counters, SSTableCursorReader cursor) throws IOException
+ {
+ for (int i = 0; i < counters.length; i++)
+ {
+ counters[i] = 0;
+ }
+ while (cursor.skipPartition() != DONE) {
+ counters[0]++;
+ }
+ }
+}
diff --git a/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java b/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java
index 6ae41ab74e..2f47f8cc1e 100644
--- a/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java
+++ b/test/simulator/main/org/apache/cassandra/simulator/cluster/KeyspaceActions.java
@@ -220,8 +220,8 @@ public class KeyspaceActions extends ClusterActions
{
int primaryKey = primaryKeys[i];
LongToken token = Murmur3Partitioner.instance.getToken(Int32Type.instance.decompose(primaryKey));
- List readReplicas = readPlacements.replicasFor(token.token);
- List writeReplicas = writePlacements.replicasFor(token.token);
+ List readReplicas = readPlacements.replicasFor(token.getLongValue());
+ List writeReplicas = writePlacements.replicasFor(token.getLongValue());
replicasForKey[i] = readReplicas.stream().mapToInt(r -> r.node().idx()).toArray();
Set pendingReplicas = new HashSet<>(writeReplicas);
diff --git a/test/unit/org/apache/cassandra/cql3/CQLTester.java b/test/unit/org/apache/cassandra/cql3/CQLTester.java
index 4ac41a608b..192b6a059e 100644
--- a/test/unit/org/apache/cassandra/cql3/CQLTester.java
+++ b/test/unit/org/apache/cassandra/cql3/CQLTester.java
@@ -2251,6 +2251,9 @@ public abstract class CQLTester
Assert.assertEquals(String.format("expected %d rows but received %d", expectedCount, actualRowCount), expectedCount, actualRowCount);
}
+ public static void assertRows(UntypedResultSet result, Object[]... rows) {
+ assertRows(result, List.of(rows));
+ }
public abstract static class CellValidator
{
public abstract ByteBuffer expected();
@@ -2378,21 +2381,21 @@ public abstract class CQLTester
};
}
- public static void assertRows(UntypedResultSet result, Object[]... rows)
+ public static void assertRows(UntypedResultSet result, List rows)
{
if (result == null)
{
- if (rows.length > 0)
- Assert.fail(String.format("No rows returned by query but %d expected", rows.length));
+ if (rows.size() > 0)
+ Assert.fail(String.format("No rows returned by query but %d expected", rows.size()));
return;
}
List meta = result.metadata();
Iterator iter = result.iterator();
int i = 0;
- while (iter.hasNext() && i < rows.length)
+ while (iter.hasNext() && i < rows.size())
{
- Object[] expected = rows[i];
+ Object[] expected = rows.get(i);
UntypedResultSet.Row actual = iter.next();
Assert.assertEquals(String.format("Invalid number of (expected) values provided for row %d", i), expected == null ? 1 : expected.length, meta.size());
@@ -2445,10 +2448,11 @@ public abstract class CQLTester
}
logger.info("Extra row num {}: {}", i, str);
}
- Assert.fail(String.format("Got more rows than expected. Expected %d but got %d.\nExpected: %s\nActual: %s", rows.length, i, toString(rows), result.toStringUnsafe()));
+ Assert.fail(String.format("Got more rows than expected. Expected %d but got %d.", rows.size(), i));
+ Assert.fail(String.format("Got more rows than expected. Expected %d but got %d.\nExpected: %s\nActual: %s", rows.size(), i, toString(rows), result.toStringUnsafe()));
}
- Assert.assertTrue(String.format("Got %s rows than expected. Expected %d but got %d", rows.length>i ? "less" : "more", rows.length, i), i == rows.length);
+ Assert.assertTrue(String.format("Got %s rows than expected. Expected %d but got %d", rows.size()>i ? "less" : "more", rows.size(), i), i == rows.size());
}
private static String toString(Object o)
diff --git a/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java b/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
index 075fbf8e91..84fd815378 100644
--- a/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
+++ b/test/unit/org/apache/cassandra/cql3/RandomSchemaTest.java
@@ -32,6 +32,8 @@ import org.junit.Test;
import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.CursorCompactor;
import org.apache.cassandra.io.sstable.format.SSTableFormat;
import org.apache.cassandra.io.util.DataInputBuffer;
import org.apache.cassandra.io.util.DataOutputBuffer;
@@ -56,6 +58,7 @@ import static org.junit.Assert.assertTrue;
public class RandomSchemaTest extends CQLTester.InMemory
{
+ static final boolean STRESS_CURSOR_COMPACTION = false;
static
{
// make sure blob is always the same
@@ -75,30 +78,34 @@ public class RandomSchemaTest extends CQLTester.InMemory
resetSchema();
// TODO : when table level override of sstable format is allowed, migrate to that
- DatabaseDescriptor.setSelectedSSTableFormat(sstableFormatGen.generate(random));
+ if (!STRESS_CURSOR_COMPACTION) DatabaseDescriptor.setSelectedSSTableFormat(sstableFormatGen.generate(random));
Gen udtName = Generators.unique(IDENTIFIER_GEN);
TypeGenBuilder withoutUnsafeEquality = AbstractTypeGenerators.withoutUnsafeEquality()
.withUDTNames(udtName);
- TableMetadata metadata = new TableMetadataBuilder()
- .withKeyspaceName(KEYSPACE)
- .withTableKinds(TableMetadata.Kind.REGULAR)
- .withKnownMemtables()
- .withDefaultTypeGen(AbstractTypeGenerators.builder()
- .withoutEmpty()
- .withMaxDepth(2)
- .withDefaultSetKey(withoutUnsafeEquality)
- .withoutTypeKinds(AbstractTypeGenerators.TypeKind.COUNTER)
- .withUDTNames(udtName))
- .withPartitionColumnsCount(1)
- .withPrimaryColumnTypeGen(new TypeGenBuilder(withoutUnsafeEquality)
- // map of vector of map crossed the size cut-off for one of the tests, so changed max depth from 2 to 1, so we can't have the second map
- .withMaxDepth(1))
- .withClusteringColumnsBetween(1, 2)
- .withRegularColumnsBetween(1, 5)
- .withStaticColumnsBetween(0, 2)
- .build(random);
+ TableMetadata metadata;
+ do
+ {
+ metadata = new TableMetadataBuilder()
+ .withKeyspaceName(KEYSPACE)
+ .withTableKinds(TableMetadata.Kind.REGULAR)
+ .withKnownMemtables()
+ .withDefaultTypeGen(AbstractTypeGenerators.builder()
+ .withoutEmpty()
+ .withMaxDepth(2)
+ .withDefaultSetKey(withoutUnsafeEquality)
+ .withoutTypeKinds(AbstractTypeGenerators.TypeKind.COUNTER)
+ .withUDTNames(udtName))
+ .withPartitionColumnsCount(1)
+ .withPrimaryColumnTypeGen(new TypeGenBuilder(withoutUnsafeEquality)
+ // map of vector of map crossed the size cut-off for one of the tests, so changed max depth from 2 to 1, so we can't have the second map
+ .withMaxDepth(1))
+ .withClusteringColumnsBetween(1, 2)
+ .withRegularColumnsBetween(1, 5)
+ .withStaticColumnsBetween(0, 2)
+ .build(random);
+ } while (STRESS_CURSOR_COMPACTION && CursorCompactor.unsupportedMetadata(metadata));
maybeCreateUDTs(metadata);
String createTable = metadata.toCqlString(true, false, false);
// just to make the CREATE TABLE stmt easier to read for CUSTOM types
@@ -129,6 +136,7 @@ public class RandomSchemaTest extends CQLTester.InMemory
// check sstable
flush(KEYSPACE, metadata.name);
compact(KEYSPACE, metadata.name);
+ ColumnFamilyStore cfs = getColumnFamilyStore(KEYSPACE, metadata.name);
assertRows(execute(selectStmt, (Object[]) rowKey), expected);
assertRows(execute(tokenStmt, (Object[]) partitionKeys), partitionKeys);
assertRowsNet(executeNet(selectStmt, (Object[]) rowKey), expected);
diff --git a/test/unit/org/apache/cassandra/db/ClusteringComparatorTest.java b/test/unit/org/apache/cassandra/db/ClusteringComparatorTest.java
new file mode 100644
index 0000000000..586d203c81
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/ClusteringComparatorTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.LongType;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+import static org.junit.Assert.assertEquals;
+
+public class ClusteringComparatorTest
+{
+ @Test
+ public void compareLong()
+ {
+ Iterable> types;
+ ClusteringComparator comparator = new ClusteringComparator(LongType.instance);
+ for (int i=0;i<1000; i++) {
+ long l1 = ThreadLocalRandom.current().nextLong();
+ long l2 = ThreadLocalRandom.current().nextLong();
+ int cmp = comparator.compare(
+ Clustering.make(ByteBufferUtil.bytes(l1)),
+ Clustering.make(ByteBufferUtil.bytes(l2)));
+ assertEquals(Long.compare(l1, l2), cmp == 0?0:cmp<0?-1:1);
+
+ }
+ }
+
+ @Test
+ public void compareRawLong() throws IOException
+ {
+ AbstractType[] types = {LongType.instance};
+ for (int i=0;i<1000; i++) {
+ long l1 = ThreadLocalRandom.current().nextLong();
+ long l2 = ThreadLocalRandom.current().nextLong();
+
+ int compare = Long.compare(l1, l2);
+ int compareCluster = ClusteringComparator.compare(types, clusteringOfLongAsBuffer(types, l1),
+ clusteringOfLongAsBuffer(types, l2));
+ assertEquals("FFS: l1=" + l1 + ", l2=" + l2,
+ compare,
+ compareCluster == 0?0:compareCluster<0?-1:1);
+ assertEquals("FFS: v1=" + l1 + ", v2=" + l2,
+ compare > 0,
+ compareCluster > 0);
+ assertEquals("FFS: v1=" + l1 + ", v2=" + l2,
+ compare < 0,
+ compareCluster < 0);
+ assertEquals("FFS: v1=" + l1 + ", v2=" + l2,
+ compare == 0,
+ compareCluster == 0);
+ }
+ }
+ @Test
+ public void compareRawInt() throws IOException
+ {
+ AbstractType[] types = { Int32Type.instance};
+ for (int i=0;i<1000; i++) {
+ int i1 = ThreadLocalRandom.current().nextInt();
+ int i2 = ThreadLocalRandom.current().nextInt();
+
+ int compare = Integer.compare(i1, i2);
+ int compareCluster = ClusteringComparator.compare(types, clusteringOfIntAsBuffer(types, i1),
+ clusteringOfIntAsBuffer(types, i2));
+ assertEquals("FFS: v1=" + i1 + ", v2=" + i2,
+ compare > 0,
+ compareCluster > 0);
+ assertEquals("FFS: v1=" + i1 + ", v2=" + i2,
+ compare < 0,
+ compareCluster < 0);
+ assertEquals("FFS: v1=" + i1 + ", v2=" + i2,
+ compare == 0,
+ compareCluster == 0);
+ }
+ }
+
+ private static ByteBuffer clusteringOfLongAsBuffer(AbstractType[] types, long v1) throws IOException
+ {
+ Clustering clustering = Clustering.make(ByteBufferUtil.bytes(v1));
+ DataOutputBuffer out = new DataOutputBuffer();
+ Clustering.serializer.serialize(clustering, out, 0, List.of(types));
+ return out.asNewBuffer();
+ }
+
+ private static ByteBuffer clusteringOfIntAsBuffer(AbstractType[] types, int v1) throws IOException
+ {
+ Clustering clustering = Clustering.make(ByteBufferUtil.bytes(v1));
+ DataOutputBuffer out = new DataOutputBuffer();
+ Clustering.serializer.serialize(clustering, out, 0, List.of(types));
+ return out.asNewBuffer();
+ }
+}
\ No newline at end of file
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
index d09c955173..5d09a91bfe 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionIteratorTest.java
@@ -482,6 +482,12 @@ public class CompactionIteratorTest extends CQLTester
{
return ImmutableSet.of();
}
+
+ @Override
+ public boolean isFullRange()
+ {
+ return false;
+ }
}
@Test
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
index b18a20ec98..756b51dd08 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsCQLTest.java
@@ -664,6 +664,9 @@ public class CompactionsCQLTest extends CQLTester
Throwable cause = t;
while (cause != null && !(cause instanceof MarshalException))
cause = cause.getCause();
+ if (cause == null) {
+ t.printStackTrace();
+ }
assertNotNull(cause);
MarshalException me = (MarshalException) cause;
assertTrue(me.getMessage().contains(cfs.metadata.keyspace+"."+cfs.metadata.name));
diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
index 79f01f9a59..8d9a8dcaa7 100644
--- a/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.compaction;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -331,6 +332,7 @@ public class CompactionsTest
{
keys.add(Util.dk(Integer.toString(i)));
}
+ Collections.sort(keys);
int[] dks = {0, 1, 3};
writeSSTableWithRangeTombstoneMaskingOneColumn(cfs, table, dks);
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionColumnDeleteAndPurgeTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionColumnDeleteAndPurgeTest.java
new file mode 100644
index 0000000000..27cedf7f77
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionColumnDeleteAndPurgeTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TestHelper;
+
+import static org.junit.Assert.assertTrue;
+
+@SuppressWarnings({ "UnnecessaryBoxing", "SingleCharacterStringConcatenation" })
+public class CompactionColumnDeleteAndPurgeTest extends SimpleCompactionTest
+{
+ @Test
+ public void testColumn1DeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', " +
+ "'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 " +
+ "bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with " +
+ "gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+
+ // Delete cell
+ execute("DELETE c1 FROM " + table + " using timestamp 1 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteAllColumnsCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', " +
+ "'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 " +
+ "bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with " +
+ "gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete cells
+ execute("DELETE c1, c2 FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ TestHelper.verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ assertTrue(!cells.hasNext());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteOneColumnCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', " +
+ "'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 " +
+ "bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with " +
+ "gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete cells
+ execute("DELETE c1 FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ TestHelper.verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ assertTrue(!cells.hasNext());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteOneColumnViaTTLCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', " +
+ "'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 " +
+ "bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with " +
+ "gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // set column TTL
+ execute("UPDATE " + table + " using TTL 1 SET c1 = ? WHERE pk = ? AND ck1 = ? AND ck2 = ?",
+ Long.valueOf(2), // c1
+ Long.valueOf(0), //pk
+ Long.valueOf(0), Integer.valueOf(0));//ck1,ck2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(2000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ TestHelper.verifyAndPrint(cfs, sstable);
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ assertTrue(!cells.hasNext());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteOneStaticColumnCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', " +
+ "'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 " +
+ "bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with " +
+ "gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete cells
+ execute("DELETE sc1 FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ TestHelper.verifyAndPrint(cfs, sstable);
+ // Expected:{"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":[{"type":"static_block",
+ // "position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2",
+ // "value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},{"type":"row","position":31,"clustering":[0,0],
+ // "liveness_info":{"tstamp":"1970-01-01T00:00:00.000001Z"},"cells":[{"name":"c1",
+ // "deletion_info":{"local_delete_time":"2025-01-25T08:48:55Z"},"tstamp":"1970-01-01T00:00:00.000002Z"},
+ // {"name":"c2","deletion_info":{"local_delete_time":"2025-01-25T08:48:55Z"},"tstamp":"1970-01-01T00:00:00
+ // .000002Z"}]}]}
+ // {"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":[{"type":"static_block",
+ // "position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},
+ // {"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},{"type":"row","position":31,
+ // "clustering":[0,0],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000001Z"},
+ // "cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-01-25T08:49:54Z"},
+ // "tstamp":"1970-01-01T00:00:00.000002Z"},{"name":"c2",
+ // "deletion_info":{"local_delete_time":"2025-01-25T08:49:54Z"},"tstamp":"1970-01-01T00:00:00
+ // .000002Z"}]}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ Row staticRow = partition.staticRow();
+ assertTrue(!staticRow.isEmpty());
+ Iterator> staticCells = staticRow.cells().iterator();
+ Cell cell = staticCells.next();
+ assertTrue(!cell.isTombstone());
+ assertTrue(!staticCells.hasNext());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionColumnTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionColumnTest.java
new file mode 100644
index 0000000000..fdeb5722d3
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionColumnTest.java
@@ -0,0 +1,555 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@SuppressWarnings({ "UnnecessaryBoxing", "SingleCharacterStringConcatenation" })
+public class CompactionColumnTest extends SimpleCompactionTest
+{
+ @Test
+ public void testColumn1DeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete cell
+ execute("DELETE c1 FROM " + table + " using timestamp 1 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":11},"rows":[{"type":"row","position":11,"clustering":[0,0],"cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-03-12T11:32:18Z"},"tstamp":"1970-01-01T00:00:00.000001Z"}]}]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive());
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ assertEquals(1, cell.timestamp());
+ assertTrue(cell.isTombstone());
+ }
+
+ @Test
+ public void testColumnCompactionIntoSingleRow() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, c3 bigint, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c3)VALUES(?, ?,?, ?,?, ?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(3));//c3
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c2)VALUES(?, ?,?, ?,?, ?) using timestamp 2",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Integer.valueOf(2));//c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1)VALUES(?, ?,?, ?,?, ?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1));//c1
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive());
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ cell = cells.next();
+ cell = cells.next();
+ assertTrue(!partition.hasNext());
+
+ }
+
+ @Test
+ public void testPartialColumnsCompaction64Columns() throws Throwable
+ {
+ int columnCount = 64;
+ testPartialColoumnsCompaction(columnCount);
+ }
+
+ @Test
+ public void testPartialColumnsCompactionOver64Columns() throws Throwable
+ {
+ int columnCount = 68;
+ testPartialColoumnsCompaction(columnCount);
+ }
+
+ private void testPartialColoumnsCompaction(int columnCount) throws IOException
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String createTable = "CREATE TABLE %s ( pk bigint, ck1 bigint";
+ for (int i = 0; i < columnCount; i++) createTable += ", c" + i + " bigint";
+ createTable += ", PRIMARY KEY(pk, ck1))";
+
+ String table = createTable(keyspace, createTable);
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+ // one row has all the columns
+ String insertAll = "INSERT INTO " + table + "(pk,ck1";
+ for (int i = 0; i < columnCount; i++) insertAll += ", c" + i;
+ insertAll += ")VALUES(?,?";
+ for (int i = 0; i < columnCount; i++) insertAll += ",?";
+ insertAll += ") using timestamp 1";
+ Long[] values = new Long[2 + columnCount];
+ Arrays.fill(values, Long.valueOf(0));
+ execute(insertAll, (Object[]) values);
+
+ String insertEven = "INSERT INTO " + table + "(pk,ck1";
+ for (int i = 0; i < columnCount; i+=2) insertEven += ", c" + i;
+ insertEven += ")VALUES(?,?";
+ for (int i = 0; i < columnCount; i+=2) insertEven += ",?";
+ insertEven += ") using timestamp 2";
+ values = new Long[2 + columnCount / 2];
+ Arrays.fill(values, Long.valueOf(1));
+ execute(insertEven, (Object[]) values);
+
+ String insertOdd = "INSERT INTO " + table + "(pk,ck1";
+ for (int i = 1; i < columnCount; i+=2) insertOdd += ", c" + i;
+ insertOdd += ")VALUES(?,?";
+ for (int i = 1; i < columnCount; i+=2) insertOdd += ",?";
+ insertOdd += ") using timestamp 3";
+ values = new Long[2 + columnCount / 2];
+ Arrays.fill(values, Long.valueOf(2));
+ execute(insertOdd, (Object[]) values);
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ ISSTableScanner partitions = sstable.getScanner();
+
+ for (int i=0;i<3;i++)
+ {
+ UnfilteredRowIterator partition = partitions.next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+ long timestamp = ((Row) row).primaryKeyLivenessInfo().timestamp();
+ Iterator> cells = ((Row) row).cells().iterator();
+ if (timestamp == 1)
+ {
+ for (int colIndex=0;colIndex cell = cells.next();
+ assertTrue(cell.valueSize()!=0);
+ }
+ }
+ else if (timestamp == 2)
+ {
+ for (int colIndex=0;colIndex cell = cells.next();
+ assertTrue(cell.valueSize()!=0);
+ String columnName = cell.column().name.toString();
+ int cellColIndex = Integer.parseInt(columnName.substring(1));
+ assertEquals("Unexpected position:" + cellColIndex, 0, cellColIndex % 2);
+ }
+ }
+ else if (timestamp == 3)
+ {
+ for (int colIndex = 0; colIndex cell = cells.next();
+ assertTrue(cell.valueSize()!=0);
+ String columnName = cell.column().name.toString();
+ int cellColIndex = Integer.parseInt(columnName.substring(1));
+ assertEquals("Unexpected position:" + cellColIndex, 1, cellColIndex % 2);
+ }
+ }
+ else {
+ fail();
+ }
+ assertTrue(!cells.hasNext());
+ assertTrue(!partition.hasNext());
+ }
+ assertTrue(!partitions.hasNext());
+ }
+
+ @Test
+ public void testPartialColumnsCompactionUnder64Columns() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, c3 bigint, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write 1,1,c1
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1)VALUES(?, ?,?, ?,?, ?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(1));//c1
+
+ // Write 2,2,c2
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c2)VALUES(?, ?,?, ?,?, ?) using timestamp 2",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Integer.valueOf(2));//c2
+
+ // Write 3,3,c3
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c3)VALUES(?, ?,?, ?,?, ?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(3));//c3
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write 1,1,c3
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c3)VALUES(?, ?,?, ?,?, ?) using timestamp 4",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(1));//c3
+
+ // Write 2,2,c1
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1)VALUES(?, ?,?, ?,?, ?) using timestamp 5",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(2));//c1
+
+ // Write 3,3,c2
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c2)VALUES(?, ?,?, ?,?, ?) using timestamp 6",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Integer.valueOf(3));//c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ // We expect:
+ // 1,1,c1=1,c3=1
+ // 2,2,c1=2,c2=2
+ // 3,3,c2=3,c3=3
+ // {"table kind":"REGULAR","partition":{"key":["0"],"position":31},
+ // "rows":[
+ // {"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000006Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000006Z"}]},
+ // {"type":"row","position":31,"clustering":[1,1],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000004Z"},
+ // "cells":[{"name":"c1","value":1},{"name":"c3","value":1}]},
+ // {"type":"row","position":67,"clustering":[2,2],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000005Z"},
+ // "cells":[{"name":"c1","value":2},{"name":"c2","value":2}]},
+ // {"type":"row","position":99,"clustering":[3,3],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000006Z"},
+ // "cells":[{"name":"c2","value":3},{"name":"c3","value":3}]}]}
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ for (int i=0;i<3;i++)
+ {
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ cell = cells.next();
+ assertTrue(!cells.hasNext());
+ }
+ assertTrue(!partition.hasNext());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteAllColumnsCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete cells
+ execute("DELETE c1, c2 FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected:{"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":[{"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},{"type":"row","position":31,"clustering":[0,0],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000001Z"},"cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-01-25T08:48:55Z"},"tstamp":"1970-01-01T00:00:00.000002Z"},{"name":"c2","deletion_info":{"local_delete_time":"2025-01-25T08:48:55Z"},"tstamp":"1970-01-01T00:00:00.000002Z"}]}]}
+ // {"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":[{"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},{"type":"row","position":31,"clustering":[0,0],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000001Z"},"cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-01-25T08:49:54Z"},"tstamp":"1970-01-01T00:00:00.000002Z"},{"name":"c2","deletion_info":{"local_delete_time":"2025-01-25T08:49:54Z"},"tstamp":"1970-01-01T00:00:00.000002Z"}]}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ assertEquals(2, cell.timestamp());
+ assertTrue(cell.isTombstone());
+ cell = cells.next();
+ assertEquals(2, cell.timestamp());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteOneColumnCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete cells
+ execute("DELETE c1 FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ assertEquals(2, cell.timestamp());
+ assertTrue(cell.isTombstone());
+
+ cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteOneColumnViaTTLCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', " +
+ "'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 " +
+ "bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // set column TTL
+ execute("UPDATE " + table + " using TTL 1 SET c1 = ? WHERE pk = ? AND ck1 = ? AND ck2 = ?",
+ Long.valueOf(2), // c1
+ Long.valueOf(0), //pk
+ Long.valueOf(0), Integer.valueOf(0));//ck1,ck2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(2000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row) row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ // The cell is converted to a tombstone, and the expiration time becomes both the TS and the LDT
+ assertEquals(cell.localDeletionTime(), cell.timestamp()/1000000);
+ assertTrue(cell.isTombstone());
+
+ assertTrue(cells.hasNext());
+ cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ assertTrue(!cells.hasNext());
+ }
+
+ @Test
+ public void testWriteRowAndDeleteOneStaticColumnCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(0), Integer.valueOf(0),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete cells
+ execute("DELETE sc1 FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected:{"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":[{"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},{"type":"row","position":31,"clustering":[0,0],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000001Z"},"cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-01-25T08:48:55Z"},"tstamp":"1970-01-01T00:00:00.000002Z"},{"name":"c2","deletion_info":{"local_delete_time":"2025-01-25T08:48:55Z"},"tstamp":"1970-01-01T00:00:00.000002Z"}]}]}
+ // {"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":[{"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},{"type":"row","position":31,"clustering":[0,0],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000001Z"},"cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-01-25T08:49:54Z"},"tstamp":"1970-01-01T00:00:00.000002Z"},{"name":"c2","deletion_info":{"local_delete_time":"2025-01-25T08:49:54Z"},"tstamp":"1970-01-01T00:00:00.000002Z"}]}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ Row staticRow = partition.staticRow();
+ assertTrue(!staticRow.isEmpty());
+ Iterator> staticCells = staticRow.cells().iterator();
+ Cell cell = staticCells.next();
+ assertTrue(cell.isTombstone());
+ cell = staticCells.next();
+ assertTrue(!cell.isTombstone());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ cell = cells.next();
+ assertTrue(!cell.isTombstone());
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteAndPurgePKTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteAndPurgePKTest.java
new file mode 100644
index 0000000000..74a15c6740
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteAndPurgePKTest.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import org.junit.Test;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CompactionDeleteAndPurgePKTest extends SimpleCompactionTest
+{
+ @Test
+ public void testPK1DeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ // even with GC period 0, needs some time
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testPK1WriteAndDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testPK2WriteAndDeleteCompactionTwice() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(112), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(12), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(12), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 4 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testPK3DeleteAndWriteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 1 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 2",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ assertTrue(!partition.next().isEmpty());
+ }
+
+ @Test
+ public void testPKDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete every other partition
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i) //pk
+ );
+ }
+
+ // delete a partition that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5 || pk % 2 == 0) {
+ fail("Expecting pk==5 to be purged");
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteAndPurgeRowTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteAndPurgeRowTest.java
new file mode 100644
index 0000000000..64e95633b8
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteAndPurgeRowTest.java
@@ -0,0 +1,637 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.junit.Test;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.KeyReader;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@SuppressWarnings({ "UnnecessaryBoxing", "SingleCharacterStringConcatenation" })
+public class CompactionDeleteAndPurgeRowTest extends SimpleCompactionTest
+{
+ @Test
+ public void testRow1DeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testRow1WriteAndDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11),Integer.valueOf(21) //ck1,ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+
+ cfs.forceMajorCompaction();
+
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ assertTrue(!partition.hasNext());
+ }
+
+ @Test
+ public void testRow1WriteAndDeleteViaTTLCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // set row TTL
+ execute("INSERT INTO " + table + "(pk, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using TTL 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(2000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ assertTrue(!partition.hasNext());
+ }
+
+ @Test
+ public void testRow1WriteAndRowDeleteAndPKDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testRow1WriteAndPKDeleteAndRowDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ assertTrue(cfs.getLiveSSTables().isEmpty());
+ }
+
+ @Test
+ public void testRow2WriteDeleteWriteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(112), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(12), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive());
+ assertTrue(!row.isEmpty());
+ }
+
+ @Test
+ public void testRowDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete rows [0,2,2] and [2,2,2]
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i), //pk
+ Long.valueOf(2), //ck1
+ Integer.valueOf(2) //ck2
+ );
+ }
+
+ // delete a partition + row that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5) {
+ fail();
+ }
+ else if (pk % 2 == 0)
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ // only have 2 live rows
+ Unfiltered row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,((Row)row).deletion().time().isLive());
+
+ row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,((Row)row).deletion().time().isLive());
+
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testLargeRowDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 blob, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ byte[] blob = new byte[DatabaseDescriptor.getColumnIndexCacheSize()*2];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(blob);
+ ThreadLocalRandom.current().nextBytes(blob);
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), byteBuffer,//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), byteBuffer,//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), byteBuffer,//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete rows [0,2,2] and [2,2,2]
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i), //pk
+ Long.valueOf(2), //ck1
+ Integer.valueOf(2) //ck2
+ );
+ }
+
+ // delete a partition + row that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5) {
+ fail();
+ }
+ else if (pk % 2 == 0)
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ // only have 2 live rows
+ Unfiltered row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,((Row)row).deletion().time().isLive());
+
+ row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,((Row)row).deletion().time().isLive());
+
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+ try (KeyReader it = sstable.keyReader())
+ {
+ ByteBuffer last = it.key();
+ while (it.advance()) last = it.key(); // no-op, just check if index is readable
+ if (!Objects.equals(last, sstable.getLast().getKey()))
+ throw new CorruptSSTableException(new IOException("Failed to read partition index"), it.toString());
+ }
+ }
+
+ @Test
+ public void testLargeCKRowDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 blob, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ byte[] blob = new byte[DatabaseDescriptor.getColumnIndexCacheSize()*2];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(blob);
+ ThreadLocalRandom.current().nextBytes(blob);
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), byteBuffer,//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(1),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), byteBuffer,//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(1),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), byteBuffer, //ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(1), //c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete rows [0,2,bb] and [2,2,bb]
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i), //pk
+ Long.valueOf(2), //ck1
+ byteBuffer //ck2
+ );
+ }
+
+ // delete a partition + row that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5), //pk
+ Long.valueOf(11), //ck1
+ byteBuffer //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5) {
+ fail();
+ }
+ else if (pk % 2 == 0)
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ // only have 2 live rows
+ Unfiltered row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,((Row)row).deletion().time().isLive());
+
+ row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,((Row)row).deletion().time().isLive());
+
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+ try (KeyReader it = sstable.keyReader())
+ {
+ ByteBuffer last = it.key();
+ while (it.advance()) last = it.key(); // no-op, just check if index is readable
+ if (!Objects.equals(last, sstable.getLast().getKey()))
+ throw new CorruptSSTableException(new IOException("Failed to read partition index"), it.toString());
+ }
+ }
+
+ @Test
+ public void testSingleLargeCKRow() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, ck1 bigint, ck2 blob, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2)) with gc_grace_seconds=0");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ byte[] blob = new byte[DatabaseDescriptor.getColumnIndexCacheSize()*2];
+ ByteBuffer byteBuffer = ByteBuffer.wrap(blob);
+ ThreadLocalRandom.current().nextBytes(blob);
+ for (int i = 0; i < 1; i++)
+ {
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), byteBuffer,//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(1),//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(1000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeletePKTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeletePKTest.java
new file mode 100644
index 0000000000..30b0972086
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeletePKTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class CompactionDeletePKTest extends SimpleCompactionTest
+{
+ @Test
+ public void testPK1DeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(!partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ assertEquals(2, partition.partitionLevelDeletion().markedForDeleteAt());
+ }
+
+ @Test
+ public void testPK1WriteAndDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(!partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ assertEquals(2, partition.partitionLevelDeletion().markedForDeleteAt());
+ }
+
+ @Test
+ public void testPK2WriteAndDeleteCompactionTwice() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(112), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(12), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(12), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 4 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(!partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ assertEquals(4, partition.partitionLevelDeletion().markedForDeleteAt());
+ }
+
+ @Test
+ public void testPK3DeleteAndWriteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 1 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 2",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ assertTrue(!partition.next().isEmpty());
+ assertEquals(1, partition.partitionLevelDeletion().markedForDeleteAt());
+ }
+
+ @Test
+ public void testPKDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete every other partition
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i) //pk
+ );
+ }
+
+ // delete a partition that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5) {
+ assertTrue(!partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ assertEquals(pk + timestamp, partition.partitionLevelDeletion().markedForDeleteAt());
+ }
+ else if (pk % 2 == 0)
+ {
+ assertTrue("pk="+pk, !partition.hasNext());
+ assertTrue("pk="+pk,!partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,partition.staticRow().isEmpty());
+ assertEquals(pk + timestamp, partition.partitionLevelDeletion().markedForDeleteAt());
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteRowRangeTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteRowRangeTest.java
new file mode 100644
index 0000000000..174fd9bba1
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteRowRangeTest.java
@@ -0,0 +1,771 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneBoundaryMarker;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@SuppressWarnings({ "UnnecessaryBoxing", "SingleCharacterStringConcatenation" })
+public class CompactionDeleteRowRangeTest extends SimpleCompactionTest
+{
+ @Test
+ public void testRow1DeleteRangeCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 < ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR",
+ // "partition":{"key":["0"],"position":11},
+ // "rows": [
+ // {"type":"range_tombstone_bound","start":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T10:29:02Z"}}},
+ // {"type":"range_tombstone_bound","end":{"type":"exclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T10:29:02Z"}}}]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void test2DeleteRangeWithExclusiveMatchingBoundCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 < ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ? AND ck1 = ? AND ck2 > ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR",
+ // "partition":{"key":["0"],"position":11},"rows":[
+ // {"type":"range_tombstone_bound","start":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:05:05Z"}}},
+ // {"type":"range_tombstone_bound","end":{"type":"exclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:05:05Z"}}},
+ // {"type":"range_tombstone_bound","start":{"type":"exclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:05:05Z"}}},
+ // {"type":"range_tombstone_bound","end":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:05:05Z"}}}]}
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(!((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(3, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(3, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void test2DeleteRangeWithInclusiveMatchingBoundCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 <= ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ? AND ck1 = ? AND ck2 >= ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":11},"rows":[
+ // {"type":"range_tombstone_bound",
+ // "start":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}},
+ // {"type":"range_tombstone_boundary",
+ // "start":{"type":"inclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000003Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}},
+ // "end":{"type":"exclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}},
+ // {"type":"range_tombstone_bound",
+ // "end":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000003Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}}]}
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).openIsInclusive(false));
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundaryMarker)tombstoneMarker).endDeletionTime().markedForDeleteAt());
+ assertEquals(3, ((RangeTombstoneBoundaryMarker)tombstoneMarker).startDeletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(3, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void test2DeleteRangeWithInclusiveOverlapingBoundCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 <= ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(3) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ? AND ck1 = ? AND ck2 >= ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":11},"rows":[
+ // {"type":"range_tombstone_bound",
+ // "start":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}},
+ // {"type":"range_tombstone_boundary",
+ // "start":{"type":"inclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000003Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}},
+ // "end":{"type":"exclusive","clustering":[0,0],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}},
+ // {"type":"range_tombstone_bound",
+ // "end":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000003Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}}]}
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).openIsInclusive(false));
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundaryMarker)tombstoneMarker).endDeletionTime().markedForDeleteAt());
+ assertEquals(3, ((RangeTombstoneBoundaryMarker)tombstoneMarker).startDeletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(3, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void test2DeleteRangeWithOverlapingBoundAndSameTimestampCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 <= ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(3) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 >= ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":11},"rows":[
+ // {"type":"range_tombstone_bound",
+ // "start":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}},
+ // {"type":"range_tombstone_bound",
+ // "end":{"type":"inclusive","clustering":[0,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-05-17T09:12:27Z"}}}]}
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void testWrite1RowAndRangeDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 < ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(22) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":31},"rows":
+ // [{"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},
+ // {"type":"range_tombstone_bound","start":
+ // {"type":"inclusive","clustering":[11,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T10:29:02Z"}}},
+ // {"type":"range_tombstone_bound","end":
+ // {"type":"exclusive","clustering":[11,22],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T10:29:02Z"}}}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void testWrite1RowAndRangeDeleteAndPKDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete RT
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 < ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(22) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete PK which should remove the RT and row
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(!partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertEquals(3, partition.partitionLevelDeletion().markedForDeleteAt());
+ assertTrue(partition.staticRow().isEmpty());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void testWrite1RowAndPKDeleteAndRangeDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete PK
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete RT, which is later than the PK delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ? AND ck1 = ? AND ck2 < ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(22) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ // {"table kind":"REGULAR","partition":
+ // {"key":["0"],"position":27,
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T12:22:21Z"}},
+ // "rows":[
+ // {"type":"range_tombstone_bound","start":
+ // {"type":"inclusive","clustering":[11,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000003Z","local_delete_time":"2025-04-18T12:22:21Z"}}},
+ // {"type":"range_tombstone_bound","end":
+ // {"type":"exclusive","clustering":[11,22],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000003Z","local_delete_time":"2025-04-18T12:22:21Z"}}}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertEquals(2, partition.partitionLevelDeletion().markedForDeleteAt());
+ assertTrue(partition.staticRow().isEmpty());
+
+ Unfiltered tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(((RangeTombstoneBoundMarker)tombstoneMarker).openIsInclusive(false));
+ assertEquals(3, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+
+ tombstoneMarker = partition.next();
+ assertTrue(tombstoneMarker.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).isBoundary());
+ assertTrue(!((RangeTombstoneMarker)tombstoneMarker).closeIsInclusive(false));
+ assertEquals(3, ((RangeTombstoneBoundMarker)tombstoneMarker).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void testRow2WriteDeleteWriteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 > ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(10) //ck1
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(112), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(12), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ // {"table kind":"REGULAR","partition":{"key":["0"],"position":31}, "rows":[
+ // {"type":"static_block","position":31,"cells":[{"name":"sc1","value":112,"tstamp":"1970-01-01T00:00:00.000003Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000003Z"}]},
+ // {"type":"range_tombstone_bound","start":
+ // {"type":"exclusive","clustering":[10,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T12:15:58Z"}}},
+ // {"type": "row",
+ // "position":48,"clustering":[11,21],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000003Z"},
+ // "cells":[{"name":"c1","value":12},{"name":"c2","value":2}]},
+ // {"type":"range_tombstone_bound","end":
+ // {"type":"inclusive",
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000002Z",
+ // "local_delete_time":"2025-04-18T12:15:58Z"}}}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered unfiltered = partition.next();
+ assertTrue(unfiltered.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)unfiltered).isBoundary());
+ assertTrue(!((RangeTombstoneBoundMarker)unfiltered).openIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)unfiltered).deletionTime().markedForDeleteAt());
+
+ unfiltered = partition.next();
+ assertTrue(unfiltered.isRow());
+ assertTrue(!((Row)unfiltered).isEmpty());
+ assertEquals(3, ((Row)unfiltered).primaryKeyLivenessInfo().timestamp());
+
+ unfiltered = partition.next();
+ assertTrue(unfiltered.isRangeTombstoneMarker());
+ assertTrue(!((RangeTombstoneMarker)unfiltered).isBoundary());
+ assertTrue(((RangeTombstoneMarker)unfiltered).closeIsInclusive(false));
+ assertEquals(2, ((RangeTombstoneBoundMarker)unfiltered).deletionTime().markedForDeleteAt());
+ assertFalse(partition.hasNext());
+ }
+
+ @Test
+ public void testRowDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete every other partition
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 < ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i), //pk
+ Long.valueOf(2), //ck1
+ Integer.valueOf(3) //ck2
+ );
+ }
+
+ // delete a partition + row that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // {"table kind":"REGULAR","partition":{"key":["2"],"position":31},
+ // "rows":[{"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000006Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000006Z"}]},
+ // {"type":"row","position":31,"clustering":[1,1],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000006Z"},"cells":[{"name":"c1","value":2},{"name":"c2","value":2}]},
+ // {"type":"range_tombstone_bound","start":
+ // {"type":"inclusive","clustering":[2,"*"],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000014Z","local_delete_time":"2025-04-18T12:42:52Z"}}},
+ // {"type":"range_tombstone_bound","end":
+ // {"type":"exclusive","clustering":[2,3],
+ // "deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000014Z","local_delete_time":"2025-04-18T12:42:52Z"}}},
+ // {"type":"row","position":100,"clustering":[3,3],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000008Z"},"cells":[{"name":"c1","value":2},{"name":"c2","value":2}]}]}
+ //{"table kind":"REGULAR","partition":{"key":["3"],"position":163},
+ // "rows":[{"type":"static_block","position":163,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000009Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000009Z"}]},{"type":"row","position":163,"clustering":[1,1],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000009Z"},"cells":[{"name":"c1","value":3},{"name":"c2","value":3}]},{"type":"row","position":194,"clustering":[2,2],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000010Z"},"cells":[{"name":"c1","value":3},{"name":"c2","value":3}]},{"type":"row","position":225,"clustering":[3,3],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000011Z"},"cells":[{"name":"c1","value":3},{"name":"c2","value":3}]}]}
+ //{"table kind":"REGULAR","partition":{"key":["0"],"position":288},"rows":[
+ // {"type":"static_block","position":288,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00Z"}]},{"type":"row","position":288,"clustering":[1,1],"liveness_info":{"tstamp":"1970-01-01T00:00:00Z"},"cells":[{"name":"c1","value":0},{"name":"c2","value":0}]},{"type":"range_tombstone_bound","start":{"type":"inclusive","clustering":[2,"*"],"deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000012Z","local_delete_time":"2025-04-18T12:42:52Z"}}},{"type":"range_tombstone_bound","end":{"type":"exclusive","clustering":[2,3],"deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000012Z","local_delete_time":"2025-04-18T12:42:52Z"}}},{"type":"row","position":357,"clustering":[3,3],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000002Z"},"cells":[{"name":"c1","value":0},{"name":"c2","value":0}]}]}
+ //{"table kind":"REGULAR","partition":{"key":["5"],"position":405},"rows":[
+ // {"type":"row","position":405,"clustering":[11,21],"deletion_info":{"marked_deleted":"1970-01-01T00:00:00.000017Z","local_delete_time":"2025-04-18T12:42:52Z"},"cells":[]}]}
+ //{"table kind":"REGULAR","partition":{"key":["1"],"position":456},"rows":[
+ // {"type":"static_block","position":456,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000003Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000003Z"}]},{"type":"row","position":456,"clustering":[1,1],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000003Z"},"cells":[{"name":"c1","value":1},{"name":"c2","value":1}]},{"type":"row","position":487,"clustering":[2,2],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000004Z"},"cells":[{"name":"c1","value":1},{"name":"c2","value":1}]},{"type":"row","position":518,"clustering":[3,3],"liveness_info":{"tstamp":"1970-01-01T00:00:00.000005Z"},"cells":[{"name":"c1","value":1},{"name":"c2","value":1}]}]}
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5) {
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+ assertTrue("pk="+pk,row.isRow());
+ assertTrue("pk="+pk,!((Row)row).deletion().time().isLive());
+ assertEquals("pk="+pk,pk + timestamp, ((Row)row).deletion().time().markedForDeleteAt());
+ }
+ else if (pk % 2 == 0)
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ assertTrue("pk="+pk,row.isRow());
+
+ row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+ assertTrue("pk="+pk,row.isRangeTombstoneMarker());
+
+ row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+ assertTrue("pk="+pk,row.isRangeTombstoneMarker());
+
+ row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteRowTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteRowTest.java
new file mode 100644
index 0000000000..ad745757cb
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionDeleteRowTest.java
@@ -0,0 +1,462 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@SuppressWarnings({ "UnnecessaryBoxing", "SingleCharacterStringConcatenation" })
+public class CompactionDeleteRowTest extends SimpleCompactionTest
+{
+ @Test
+ public void testRow1DeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(ColumnFamilyStore::disableAutoCompaction));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(0), //ck1
+ Integer.valueOf(0) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(!((Row)row).deletion().time().isLive());
+ assertEquals(2, ((Row)row).deletion().time().markedForDeleteAt());
+ }
+
+ @Test
+ public void testRow1WriteAndDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(!((Row)row).deletion().time().isLive());
+ assertEquals(2, ((Row)row).deletion().time().markedForDeleteAt());
+
+ Iterator> cells = ((Row) row).cells().iterator();
+ assertTrue(!cells.hasNext());
+ }
+
+ @Test
+ public void testRow1WriteAndDeleteViaTTLCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // set column TTL
+ execute("INSERT INTO " + table + "(pk, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using TTL 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ Thread.sleep(2000);
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ // {"table kind":"REGULAR",
+ // "partition":{"key":["0"],"position":31},
+ // "rows":[
+ // {"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},
+ // {"type":"row","position":31,"clustering":[11,21],
+ // "liveness_info":{"tstamp":"2025-03-12T09:01:59.127Z","ttl":1,"expires_at":"2025-03-12T09:02:00Z","expired":true},
+ // "cells":[{"name":"c1","deletion_info":{"local_delete_time":"2025-03-12T09:01:59Z"}},{"name":"c2","deletion_info":{"local_delete_time":"2025-03-12T09:01:59Z"}}]}]}
+
+ // {"table kind":"REGULAR","partition":{"key":["0"],"position":31},
+ // "rows":[
+ // {"type":"static_block","position":31,"cells":[{"name":"sc1","value":111,"tstamp":"1970-01-01T00:00:00.000001Z"},{"name":"sc2","value":222,"tstamp":"1970-01-01T00:00:00.000001Z"}]},
+ // {"type":"row","position":31,"clustering":[11,21],
+ // "liveness_info":{"tstamp":"2025-03-12T09:47:44.760Z","ttl":1,"expires_at":"2025-03-12T09:47:45Z","expired":true},
+ // "cells":[{"name":"c1","value":""},{"name":"c2","value":""}]}]}
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(((Row)row).deletion().time().isLive()); // expired rows are not transformed into tombstones
+ LivenessInfo livenessInfo = ((Row) row).primaryKeyLivenessInfo();
+ assertEquals(1, livenessInfo.ttl()); // TTL as set
+ assertEquals(livenessInfo.localExpirationTime()-1, livenessInfo.timestamp()/1000000);
+
+ // TTL expiry for the row turns the cells into tombstones
+ Iterator> cells = ((Row) row).cells().iterator();
+ Cell cell = cells.next();
+ assertEquals(cell.localDeletionTime(), cell.timestamp()/1000000);
+ assertTrue(cell.isTombstone());
+ assertTrue(cells.hasNext());
+ cell = cells.next();
+ assertEquals(cell.localDeletionTime(), cell.timestamp()/1000000);
+ assertTrue(cell.isTombstone());
+ assertTrue(!cells.hasNext());
+ }
+
+ @Test
+ public void testRow1WriteAndRowDeleteAndPKDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ // Expected: {"table kind":"REGULAR","partition":{"key":["0"],"position":27,"deletion_info":{"marked_deleted":"2025-01-14T11:20:37.220Z","local_delete_time":"2025-01-14T11:20:37Z"}},"rows":[]}
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(!partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertEquals(3, partition.partitionLevelDeletion().markedForDeleteAt());
+ assertTrue(partition.staticRow().isEmpty());
+ }
+
+ @Test
+ public void testRow1WriteAndPKDeleteAndRowDeleteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ?;",
+ Long.valueOf(0) //pk
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 3 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(!partition.partitionLevelDeletion().isLive());
+ assertEquals(2, partition.partitionLevelDeletion().markedForDeleteAt());
+ assertTrue(partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(!((Row)row).deletion().time().isLive());
+ assertEquals(3, ((Row)row).deletion().time().markedForDeleteAt());
+ assertTrue(((Row)row).columnData().isEmpty());
+ }
+
+ @Test
+ public void testRow2WriteDeleteWriteCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 1",
+ Long.valueOf(0), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(1), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Delete
+ execute("DELETE FROM " + table + " using timestamp 2 WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(0), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // Write
+ execute("INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp 3",
+ Long.valueOf(0), //pk
+ Long.valueOf(112), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(11), Integer.valueOf(21),//ck1,ck2
+ Long.valueOf(12), Integer.valueOf(2));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ UnfilteredRowIterator partition = sstable.getScanner().next();
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(!partition.staticRow().isEmpty());
+
+ Unfiltered row = partition.next();
+ assertTrue(row.isRow());
+ assertTrue(!((Row)row).deletion().time().isLive());
+ assertEquals(2, ((Row)row).deletion().time().markedForDeleteAt());
+ assertTrue(!row.isEmpty());
+ }
+
+ @Test
+ public void testRowDeleteCompactionInterleaving() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?) using timestamp ?";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?) using timestamp ?";
+ int prefix = 0;
+ long timestamp = 0;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i),//c1,c2
+ timestamp++);
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ // delete every other partition
+ for (int i = 0; i < 4; i+=2)
+ {
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + i), // timestamp
+ Long.valueOf(i), //pk
+ Long.valueOf(2), //ck1
+ Integer.valueOf(2) //ck2
+ );
+ }
+
+ // delete a partition + row that we don't have
+ execute("DELETE FROM " + table + " using timestamp ? WHERE pk = ? AND ck1 = ? AND ck2 = ?;",
+ Long.valueOf(timestamp + 5), // timestamp
+ Long.valueOf(5), //pk
+ Long.valueOf(11), //ck1
+ Integer.valueOf(21) //ck2
+ );
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ try(ISSTableScanner scanner = sstable.getScanner())
+ {
+ while (scanner.hasNext()) {
+ UnfilteredRowIterator partition = scanner.next();
+ long pk = partition.partitionKey().getKey().getLong();
+ if (pk == 5) {
+ assertTrue(partition.hasNext());
+ assertTrue(partition.partitionLevelDeletion().isLive());
+ assertTrue(partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+ assertTrue("pk="+pk,row.isRow());
+ assertTrue("pk="+pk,!((Row)row).deletion().time().isLive());
+ assertEquals("pk="+pk,pk + timestamp, ((Row)row).deletion().time().markedForDeleteAt());
+
+ }
+ else if (pk % 2 == 0)
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ Unfiltered row = partition.next();
+ assertTrue("pk=" + pk, !row.isEmpty());
+ row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+ assertTrue("pk="+pk,row.isRow());
+ assertTrue("pk="+pk,!((Row)row).deletion().time().isLive());
+ assertEquals("pk="+pk,pk + timestamp, ((Row)row).deletion().time().markedForDeleteAt());
+
+ row = partition.next();
+ assertTrue("pk="+pk,!row.isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ else
+ {
+ assertTrue("pk="+pk,partition.hasNext());
+ assertTrue("pk="+pk,partition.partitionLevelDeletion().isLive());
+ assertTrue("pk="+pk,!partition.staticRow().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.next().isEmpty());
+ assertTrue("pk="+pk,!partition.hasNext());
+ }
+ }
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/CompactionSimpleValueMergeTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionSimpleValueMergeTest.java
new file mode 100644
index 0000000000..2223a1b88a
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/CompactionSimpleValueMergeTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Test;
+
+import org.apache.cassandra.cql3.UntypedResultSet;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.utils.TestHelper.verifyAndPrint;
+
+public class CompactionSimpleValueMergeTest extends SimpleCompactionTest
+{
+ @Test
+ public void testStaticRowCompaction() throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement1 = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?)";
+ String writeStatement2 = "INSERT INTO " + table + "(pk,ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?)";
+ int prefix = 0;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(111), Integer.valueOf(222),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ prefix = 10;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ prefix = 20;
+ // Writes, 3 rows in each partition
+ for (int i = 0; i < 4; i++)
+ {
+ execute(writeStatement1,
+ Long.valueOf(i), //pk
+ Long.valueOf(311), Integer.valueOf(322),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ execute(writeStatement2,
+ Long.valueOf(i), //pk
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+i), Integer.valueOf(prefix+i));//c1,c2
+ }
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+ List rows = new ArrayList<>();
+ int[] pks = {2,3,0,1};
+ for (int pk : pks)
+ {
+ rows.add(new Object[]{
+ Long.valueOf(pk), //pk
+ Long.valueOf(311), Integer.valueOf(322),//sc1,sc2
+ Long.valueOf(1), Integer.valueOf(1),//ck1,ck2
+ Long.valueOf(prefix+pk), Integer.valueOf(prefix+pk)});//c1,c2
+ rows.add(new Object[]{
+ Long.valueOf(pk), //pk
+ Long.valueOf(311), Integer.valueOf(322),//sc1,sc2
+ Long.valueOf(2), Integer.valueOf(2),//ck1,ck2
+ Long.valueOf(prefix+pk), Integer.valueOf(prefix+pk)});//c1,c2
+ rows.add(new Object[]{
+ Long.valueOf(pk), //pk
+ Long.valueOf(311), Integer.valueOf(322),//sc1,sc2
+ Long.valueOf(3), Integer.valueOf(3),//ck1,ck2
+ Long.valueOf(prefix+pk), Integer.valueOf(prefix+pk)});//c1,c2
+ }
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result,
+ rows);
+ }
+
+ @Test
+ public void testCompaction1TableNoPartitionOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(1, 10, false, false, false);
+ }
+
+ @Test
+ public void testCompaction2TableNoPartitionOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(2, 10, false, false, false);
+ }
+
+ @Test
+ public void testCompaction3TableNoPartitionOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(3, 10, false, false, false);
+ }
+
+ @Test
+ public void testCompaction2TableWithPartitionOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(2, 10, true, false, false);
+ }
+
+ @Test
+ public void testCompaction3TableWithPartitionOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(3, 10, true, false, false);
+ }
+ @Test
+ public void testCompaction2TableWithRowOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(2, 10, true, true, false);
+ }
+
+ @Test
+ public void testCompaction3TableWithRowOverlap() throws Throwable
+ {
+ genetrateCompactAndVerify(3, 10, true, true, false);
+ }
+
+ protected void genetrateCompactAndVerify(int sstableCount, int partitionCount, boolean pOverlap, boolean rOverlap, boolean cOverlap) throws Throwable
+ {
+ String keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+ String table = createTable(keyspace, "CREATE TABLE %s ( pk bigint, sc1 bigint static, sc2 int static, ck1 bigint, ck2 int, c1 bigint, c2 int, PRIMARY KEY(pk, ck1, ck2))");
+ execute("use " + keyspace + ";");
+ Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+ ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+ cfs.disableAutoCompaction();
+
+ String writeStatement = "INSERT INTO " + table + "(pk,sc1,sc2, ck1,ck2, c1,c2)VALUES(?, ?,?, ?,?, ?,?)";
+ for (int j = 0; j < sstableCount; j++)
+ {
+ for (int i = 0; i < partitionCount; i++)
+ execute(writeStatement,
+ (Long.valueOf(pOverlap ? 0 : j * partitionCount) + i), //pk
+ Long.valueOf(j * partitionCount + i), Integer.valueOf(j * partitionCount + i),//sc1,sc2
+ Long.valueOf((rOverlap ? 0 : j * partitionCount) + i), Integer.valueOf((rOverlap ? 0 : j * partitionCount) + i),//ck1,ck2
+ Long.valueOf(j * partitionCount + i), Integer.valueOf(j * partitionCount + i));//c1,c2
+
+ cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+ }
+
+ cfs.forceMajorCompaction();
+ SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
+ verifyAndPrint(cfs, sstable);
+
+ List rows = new ArrayList<>();
+
+ if (pOverlap && rOverlap)
+ {
+ int prefix = (sstableCount - 1) * partitionCount;
+ int[] pks = {2,3,7,9,4,0,8,5,6,1};
+ for (int pk : pks)
+ {
+ rows.add(new Object[]{ Long.valueOf(pk), //pk
+ Long.valueOf(prefix + pk), Integer.valueOf(prefix + pk),//sc1,sc2
+ Long.valueOf(pk), Integer.valueOf(pk),//ck1,ck2
+ Long.valueOf(prefix + pk), Integer.valueOf(prefix + pk) });//c1,c2
+ }
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result,
+ rows);
+ }
+
+ if (pOverlap && !rOverlap)
+ {
+ int prefix = (sstableCount - 1) * partitionCount;
+ int[] pks = {2,3,7,9,4,0,8,5,6,1};
+ for (int pk : pks)
+ {
+ for (int j = 0; j < sstableCount; j++)
+ {
+ rows.add(new Object[]{ Long.valueOf(pk), //pk
+ Long.valueOf(prefix + pk), Integer.valueOf(prefix + pk),//sc1,sc2
+ Long.valueOf(j * partitionCount + pk), Integer.valueOf(j * partitionCount + pk),//ck1,ck2
+ Long.valueOf(j * partitionCount + pk), Integer.valueOf(j * partitionCount + pk) });//c1,c2
+ }
+ }
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result,
+ rows);
+ }
+
+ if (!pOverlap && !rOverlap)
+ {
+ int[][] pks = {{2,3,7,9,4,0,8,5,6,1},
+ {19,2,3,16,12,13,7,15,9,4,10,0,11,14,8,5,6,1,18,17},
+ {19,2,24,3,16,25,12,20,13,7,26,15,23,9,27,21,4,10,28,0,11,14,8,5,22,6,1,18,17,29}};
+ for (int pk : pks[sstableCount - 1])
+ {
+ rows.add(new Object[]{ Long.valueOf(pk), //pk
+ Long.valueOf(pk), Integer.valueOf(pk),//sc1,sc2
+ Long.valueOf(pk), Integer.valueOf(pk),//ck1,ck2
+ Long.valueOf(pk), Integer.valueOf(pk) });//c1,c2
+
+ }
+ UntypedResultSet result = execute("SELECT pk,sc1,sc2, ck1,ck2, c1,c2 FROM " + table);
+ assertRows(result,
+ rows);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/simple/SimpleCompactionTest.java b/test/unit/org/apache/cassandra/db/compaction/simple/SimpleCompactionTest.java
new file mode 100644
index 0000000000..27c7ba7e17
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/compaction/simple/SimpleCompactionTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.simple;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.AfterClass;
+import org.junit.Ignore;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.utils.TestHelper;
+
+
+@Ignore
+public abstract class SimpleCompactionTest extends CQLTester
+{
+ @AfterClass
+ public static void teardown() throws IOException, InterruptedException, ExecutionException
+ {
+ TestHelper.teardown();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java
index 212182e2b8..33e615ce75 100644
--- a/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/unified/BackgroundCompactionTrackingTest.java
@@ -151,7 +151,7 @@ public class BackgroundCompactionTrackingTest extends CQLTester
if (i == 0)
Assert.assertEquals(uncompressedSize * 1.0 / tasks, op.getTotal(), uncompressedSize * 0.03);
- assertTrue(op.getCompleted() <= op.getTotal() * 1.03);
+ assertTrue("Expected:" + op.getCompleted() +"<=" + (long)(op.getTotal() * 1.03), op.getCompleted() <= op.getTotal() * 1.03);
if (op.getCompleted() >= op.getTotal() * 0.97)
++finished;
}
diff --git a/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java b/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java
index 9a02fb086c..9a00ce49a4 100644
--- a/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java
+++ b/test/unit/org/apache/cassandra/dht/Murmur3PartitionerTest.java
@@ -74,7 +74,7 @@ public class Murmur3PartitionerTest extends PartitionerTestCase
qt().forAll(longs().between(Long.MIN_VALUE + 1, Long.MAX_VALUE))
.check(token -> {
ByteBuffer key = Murmur3Partitioner.LongToken.keyForToken(new Murmur3Partitioner.LongToken(token));
- return Murmur3Partitioner.instance.getToken(key).token == token;
+ return Murmur3Partitioner.instance.getToken(key).getLongValue() == token;
});
}
}
diff --git a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
index 865d1b799e..92b825ba35 100644
--- a/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
+++ b/test/unit/org/apache/cassandra/index/accord/RouteIndexTest.java
@@ -178,8 +178,8 @@ public class RouteIndexTest extends CQLTester
TokenRange range = selectExistingRange(rs, ranges);
// have a key, so find a key within the range
- long start = range.start().isMin() ? Long.MIN_VALUE : ((LongToken) range.start().token()).token;
- long end = range.end().isMax() ? Long.MAX_VALUE : ((LongToken) range.end().token()).token;
+ long start = range.start().isMin() ? Long.MIN_VALUE : ((LongToken) range.start().token()).getLongValue();
+ long end = range.end().isMax() ? Long.MAX_VALUE : ((LongToken) range.end().token()).getLongValue();
long token = 1 + rs.nextLong(start, end);
@Nullable DecidedRX decidedRX = state.nextDecidedRX(rs);
return new KeySearch(storeId, new TokenKey(tableId, new LongToken(token)), state.nextTxnRange(rs), decidedRX);
diff --git a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
index 3b651fc36b..1d79c1d29d 100644
--- a/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/VerifyTest.java
@@ -142,6 +142,10 @@ public class VerifyTest
standardCFMD(KEYSPACE, BF_ALWAYS_PRESENT).bloomFilterFpChance(1.0));
}
+ protected IVerifier getVerifier(SSTableReader sstable, ColumnFamilyStore cfs, IVerifier.Options.Builder verifierOptions)
+ {
+ return sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, verifierOptions.build());
+ }
@Test
public void testVerifyCorrect()
@@ -154,7 +158,7 @@ public class VerifyTest
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
@@ -174,7 +178,7 @@ public class VerifyTest
fillCounterCF(cfs, 2);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
@@ -194,7 +198,7 @@ public class VerifyTest
fillCF(cfs, 2);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
@@ -215,7 +219,7 @@ public class VerifyTest
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true)))
{
verifier.verify();
}
@@ -236,7 +240,7 @@ public class VerifyTest
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
@@ -257,7 +261,7 @@ public class VerifyTest
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
@@ -278,7 +282,7 @@ public class VerifyTest
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
@@ -299,12 +303,13 @@ public class VerifyTest
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().extendedVerification(true).invokeDiskFailurePolicy(true)))
{
verifier.verify();
}
catch (CorruptSSTableException err)
{
+ err.printStackTrace();
fail("Unexpected CorruptSSTableException");
}
}
@@ -331,7 +336,7 @@ public class VerifyTest
writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("Expected a CorruptSSTableException to be thrown");
@@ -340,7 +345,7 @@ public class VerifyTest
{
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(false).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(false)))
{
verifier.verify();
fail("Expected a RuntimeException to be thrown");
@@ -380,7 +385,7 @@ public class VerifyTest
// Update the Digest to have the right Checksum
writeChecksum(simpleFullChecksum(sstable.getFilename()), sstable.descriptor.fileFor(Components.DIGEST));
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
// First a simple verify checking digest, which should succeed
try
@@ -393,7 +398,7 @@ public class VerifyTest
fail("Simple verify should have succeeded as digest matched");
}
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true)))
{
// Now try extended verify
try
@@ -425,7 +430,7 @@ public class VerifyTest
file.position(0);
file.write(ByteBufferUtil.bytes(StringUtils.repeat('z', 2)));
file.close();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("Expected a CorruptSSTableException to be thrown");
@@ -433,7 +438,7 @@ public class VerifyTest
catch (CorruptSSTableException expected)
{
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(false).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(false)))
{
verifier.verify();
fail("Expected a RuntimeException to be thrown");
@@ -470,7 +475,7 @@ public class VerifyTest
correctChecksum = file.readLong();
}
writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().mutateRepairStatus(false).invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("Expected a CorruptSSTableException to be thrown");
@@ -482,7 +487,7 @@ public class VerifyTest
assertTrue(sstable.isRepaired());
// now the repair status should be changed:
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().mutateRepairStatus(true).invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("Expected a CorruptSSTableException to be thrown");
@@ -507,7 +512,7 @@ public class VerifyTest
.update();
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkOwnsTokens(true).extendedVerification(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().checkOwnsTokens(true).extendedVerification(true)))
{
verifier.verify();
}
@@ -536,7 +541,7 @@ public class VerifyTest
correctChecksum = file.readLong();
}
writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true).mutateRepairStatus(true)))
{
verifier.verify();
fail("should be corrupt");
@@ -581,7 +586,7 @@ public class VerifyTest
fillCF(cfs, 2);
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options()))
{
verifier.verify(); //still not corrupt, should pass
}
@@ -590,7 +595,7 @@ public class VerifyTest
fileChannel.truncate(3);
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("should throw exception");
@@ -622,7 +627,7 @@ public class VerifyTest
writeChecksum(++correctChecksum, sstable.descriptor.fileFor(Components.DIGEST));
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("Expected a CorruptSSTableException to be thrown");
@@ -631,12 +636,12 @@ public class VerifyTest
{
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).quick(true).build())) // with quick = true we don't verify the digest
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true).quick(true))) // with quick = true we don't verify the digest
{
verifier.verify();
}
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().invokeDiskFailurePolicy(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().invokeDiskFailurePolicy(true)))
{
verifier.verify();
fail("Expected a RuntimeException to be thrown");
@@ -740,7 +745,7 @@ public class VerifyTest
for (SSTableReader sstable : cfs.getLiveSSTables())
{
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().checkOwnsTokens(true).build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options().checkOwnsTokens(true)))
{
verifier.verify();
}
@@ -759,7 +764,7 @@ public class VerifyTest
{
File f = sstable.descriptor.fileFor(Components.FILTER);
assertFalse(f.exists());
- try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false, IVerifier.options().build()))
+ try (IVerifier verifier = getVerifier(sstable, cfs, IVerifier.options()))
{
verifier.verify();
}
diff --git a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
index 625e2e65ef..3527fd4bc8 100644
--- a/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
+++ b/test/unit/org/apache/cassandra/locator/SimpleStrategyTest.java
@@ -286,7 +286,7 @@ public class SimpleStrategyTest extends CassandraTestBase
configOptions.put(ReplicationParams.CLASS, SimpleStrategy.class.getName());
configOptions.put("replication_factor", "3/1");
SchemaLoader.createKeyspace("ks", KeyspaceParams.create(false, configOptions));
- Range range1 = range(Murmur3Partitioner.MINIMUM.token, 100);
+ Range range1 = range(Murmur3Partitioner.MINIMUM.getLongValue(), 100);
Util.assertRCEquals(EndpointsForToken.of(range1.right,
Replica.fullReplica(endpoints.get(0), range1),
diff --git a/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java
index 9846895a12..65b9f332bb 100644
--- a/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java
+++ b/test/unit/org/apache/cassandra/service/paxos/PaxosRepairHistoryTest.java
@@ -481,7 +481,7 @@ public class PaxosRepairHistoryTest
Ballot ballotForToken(LongToken token)
{
return canonical
- .floorEntry(token.token == Long.MIN_VALUE ? token : token.decreaseSlightly())
+ .floorEntry(token.getLongValue() == Long.MIN_VALUE ? token : token.decreaseSlightly())
.getValue();
}
diff --git a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
index bfc3a7f5b8..1285186767 100644
--- a/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
+++ b/test/unit/org/apache/cassandra/tools/OfflineToolUtils.java
@@ -79,6 +79,8 @@ public abstract class OfflineToolUtils
"Attach Listener", // spawned in intellij IDEA
"JNA Cleaner", // spawned by JNA
"ThreadLocalMetrics-Cleaner", // spawned by org.apache.cassandra.metrics.ThreadLocalMetrics
+ "Native reference cleanup thread",
+ "^ForkJoinPool\\.commonPool-worker-\\d+$"
};
static final String[] NON_DEFAULT_MEMTABLE_THREADS =
diff --git a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
index edfbe88163..e179eb2f52 100644
--- a/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
+++ b/test/unit/org/apache/cassandra/utils/CassandraGenerators.java
@@ -1408,16 +1408,16 @@ public final class CassandraGenerators
List> unwrap = range.unwrap();
return rs -> {
Range subRange = unwrap.get(Math.toIntExact(rs.next(Constraint.between(0, unwrap.size() - 1))));
- long end = ((Murmur3Partitioner.LongToken) subRange.right).token;
+ long end = ((Murmur3Partitioner.LongToken) subRange.right).getLongValue();
if (end == Long.MIN_VALUE)
end = Long.MAX_VALUE;
- Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) subRange.left).token + 1, end);
+ Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) subRange.left).getLongValue() + 1, end);
return new Murmur3Partitioner.LongToken(rs.next(token));
};
}
else
{
- Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) range.left).token + 1, ((Murmur3Partitioner.LongToken) range.right).token);
+ Constraint token = Constraint.between(((Murmur3Partitioner.LongToken) range.left).getLongValue() + 1, ((Murmur3Partitioner.LongToken) range.right).getLongValue());
return rs -> new Murmur3Partitioner.LongToken(rs.next(token));
}
}
diff --git a/test/unit/org/apache/cassandra/utils/PreSortedBubbleInsertTest.java b/test/unit/org/apache/cassandra/utils/PreSortedBubbleInsertTest.java
new file mode 100644
index 0000000000..4f60c16e7c
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/PreSortedBubbleInsertTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.Comparator;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.cassandra.db.compaction.CursorCompactor;
+
+public class PreSortedBubbleInsertTest
+{
+ @Test
+ public void testSortArray123() {
+ Integer[] array = new Integer[]{1,2,3};
+ boolean[] equalsNext = new boolean[array.length];
+ sort(array, equalsNext, Integer::compareTo);
+ Assert.assertArrayEquals(new Integer[]{1,2,3}, array);
+ Assert.assertArrayEquals(new boolean[array.length], equalsNext);
+ }
+
+ @Test
+ public void testSortArray321() {
+ Integer[] array = new Integer[]{3,2,1};
+ boolean[] equalsNext = new boolean[array.length];
+ sort(array, equalsNext, Integer::compareTo);
+ Assert.assertArrayEquals(new Integer[]{1,2,3}, array);
+ Assert.assertArrayEquals(new boolean[array.length], equalsNext);
+ }
+
+ @Test
+ public void testSortArray213() {
+ Integer[] array = new Integer[]{2,1,3};
+ boolean[] equalsNext = new boolean[array.length];
+ sort(array, equalsNext, Integer::compareTo);
+ Assert.assertArrayEquals(new Integer[]{1,2,3}, array);
+ Assert.assertArrayEquals(new boolean[array.length], equalsNext);
+ }
+
+ @Test
+ public void testSortArray231() {
+ Integer[] array = new Integer[]{2,3,1};
+ boolean[] equalsNext = new boolean[array.length];
+ sort(array, equalsNext, Integer::compareTo);
+ Assert.assertArrayEquals(new Integer[]{1,2,3}, array);
+ Assert.assertArrayEquals(new boolean[array.length], equalsNext);
+ }
+
+ @Test
+ public void testSortArray111() {
+ Integer[] array = new Integer[]{1,1,1};
+ boolean[] equalsNext = new boolean[array.length];
+ sort(array, equalsNext, Integer::compareTo);
+ Assert.assertArrayEquals(new Integer[]{1,1,1}, array);
+ Assert.assertArrayEquals(new boolean[]{true,true,false}, equalsNext);
+ }
+
+ @Test
+ public void testSortFrom2Array113Presorted() {
+ Integer[] array = new Integer[]{1,1,3};
+ boolean[] equalsNext = new boolean[]{true,true,false};
+ bubbleSortFrom(array, equalsNext, Integer::compareTo, 2);
+ Assert.assertArrayEquals(new Integer[]{1,1,3}, array);
+ Assert.assertArrayEquals(new boolean[]{true,false,false}, equalsNext);
+ }
+
+ @Test
+ public void testSortFrom1Array113Presorted() {
+ Integer[] array = new Integer[]{1,1,3};
+ boolean[] equalsNext = new boolean[]{false,false,false};
+ bubbleSortFrom(array, equalsNext, Integer::compareTo, 1);
+ Assert.assertArrayEquals(new Integer[]{1,1,3}, array);
+ Assert.assertArrayEquals(new boolean[]{true,false,false}, equalsNext);
+ }
+
+ @Test
+ public void testSortFrom1Array2113Presorted() {
+ Integer[] array = new Integer[]{2,1,1,3};
+ boolean[] equalsNext = new boolean[]{false,true,false,false};
+ bubbleSortFrom(array, equalsNext, Integer::compareTo, 1);
+ Assert.assertArrayEquals(new Integer[]{1,1,2,3}, array);
+ Assert.assertArrayEquals(new boolean[]{true,false,false,false}, equalsNext);
+ }
+
+ @Test
+ public void testSortFrom2Array2411113Presorted() {
+ Integer[] array = new Integer[]{2,4,1,1,1,1,3};
+ boolean[] equalsNext = new boolean[]{false,true,true,true,true,false,false};
+ bubbleSortFrom(array, equalsNext, Integer::compareTo, 2);
+ Assert.assertArrayEquals(new Integer[]{1,1,1,1,2,3,4}, array);
+ Assert.assertArrayEquals(new boolean[]{true,true,true,false,false,false,false}, equalsNext);
+ }
+
+ @Test
+ public void testSortFrom2Array4411113Presorted() {
+ Integer[] array = new Integer[]{4,4,1,1,1,1,3};
+ boolean[] equalsNext = new boolean[]{false,true,true,true,true,false,false};
+ bubbleSortFrom(array, equalsNext, Integer::compareTo, 2);
+ Assert.assertArrayEquals(new Integer[]{1,1,1,1,3,4,4}, array);
+ Assert.assertArrayEquals(new boolean[]{true,true,true,false,false,true,false}, equalsNext);
+ }
+
+ static void sort(T[] array, boolean[] equalsNext, Comparator comparator) {
+ int perturbedLimit = array.length;
+ bubbleSortFrom(array, equalsNext, comparator, perturbedLimit);
+ }
+
+ private static void bubbleSortFrom(T[] array, boolean[] equalsNext, Comparator comparator, int perturbedLimit)
+ {
+ for (; perturbedLimit > 0; perturbedLimit--) {
+ CursorCompactor.bubbleInsertElementToPreSorted(array, equalsNext, perturbedLimit, array.length, comparator);
+ }
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/TestHelper.java b/test/unit/org/apache/cassandra/utils/TestHelper.java
new file mode 100644
index 0000000000..11772f81f6
--- /dev/null
+++ b/test/unit/org/apache/cassandra/utils/TestHelper.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import org.junit.AfterClass;
+
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.commitlog.CommitLog;
+import org.apache.cassandra.io.sstable.ISSTableScanner;
+import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tools.JsonTransformer;
+import org.apache.cassandra.tools.Util;
+
+public class TestHelper
+{
+ public static void verifyAndPrint(ColumnFamilyStore cfs, SSTableReader sstable) throws IOException
+ {
+ try (IVerifier verifier = sstable.getVerifier(cfs, new OutputHandler.LogOutput(), false,
+ IVerifier.options().invokeDiskFailurePolicy(true).extendedVerification(true).build()))
+ {
+ verifier.verify();
+ }
+ try (ISSTableScanner scanner = sstable.getScanner())
+ {
+ JsonTransformer.toJsonLines(scanner, Util.iterToStream(scanner), false, false, sstable.metadata(),
+ Clock.Global.currentTimeMillis() / 1000, System.out);
+ }
+ }
+
+ @AfterClass
+ public static void teardown() throws IOException, ExecutionException, InterruptedException
+ {
+ CommitLog.instance.shutdownBlocking();
+ ClusterMetadataService.instance().log().close();
+ CQLTester.tearDownClass();
+ CQLTester.cleanup();
+ }
+}
diff --git a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
index e1f11ac15a..df4b661915 100644
--- a/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
+++ b/test/unit/org/apache/cassandra/utils/bytecomparable/ByteSourceComparisonTest.java
@@ -812,9 +812,14 @@ public class ByteSourceComparisonTest extends ByteSourceTestBase
ByteBuffer collision = Util.generateMurmurCollision(original, append.getBytes(StandardCharsets.UTF_8));
long[] hash = new long[2];
+ long[] hash2 = new long[2];
MurmurHash.hash3_x64_128(original, 0, original.limit(), 0, hash);
+ MurmurHash.hash3_x64_128(original.array(), original.arrayOffset(), original.limit(), 0, hash2);
+ Assert.assertArrayEquals(hash, hash2);
logger.info(String.format("Original hash %016x,%016x", hash[0], hash[1]));
MurmurHash.hash3_x64_128(collision, 0, collision.limit(), 0, hash);
+ MurmurHash.hash3_x64_128(collision.array(), collision.arrayOffset(), original.limit(), 0, hash2);
+ Assert.assertArrayEquals(hash, hash2);
logger.info(String.format("Collision hash %016x,%016x", hash[0], hash[1]));
DecoratedKey kk1 = partitioner.decorateKey(original);