Add cursor based optimized compaction path

Adds a compaction implementation utilizing new fixed allocation SSTable reader/writer implementations, and other purpose built code, leading to improved efficiencies.

patch by Nitsan Wakart; reviewed by Branimir Lambov, Dmitry Konstantinov for CASSANDRA-20918

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Add cursor based optimized compaction path (CASSANDRA-20918)
  * Ensure peers with LEFT status are expired from gossip state (CASSANDRA-21035)
  * Optimize UTF8Validator.validate for ASCII prefixed Strings (CASSANDRA-21075)
  * Switch LatencyMetrics to use ThreadLocalTimer/ThreadLocalCounter (CASSANDRA-21080)

build.xml

@@ -1414,6 +1414,7 @@
         <jvmarg value="-Dcassandra.config=file:///${scm_none_yaml}"/>
         <jvmarg value="-Dcassandra.test.storage_compatibility_mode=NONE"/>
         <jvmarg value="-Dcassandra.skip_sync=true" />
+        <jvmarg value="-Dcassandra.cursor_compaction_enabled=false" />
       </testmacrohelper>
     </sequential>
   </macrodef>

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -193,6 +193,7 @@ public enum CassandraRelevantProperties
     CONSISTENT_RANGE_MOVEMENT("cassandra.consistent.rangemovement", "true"),
     CONSISTENT_SIMULTANEOUS_MOVES_ALLOW("cassandra.consistent.simultaneousmoves.allow"),
     CRYPTO_PROVIDER_CLASS_NAME("cassandra.crypto_provider_class_name"),
+    CURSOR_COMPACTION_ENABLED("cassandra.cursor_compaction_enabled", "true"),
     CUSTOM_DISK_ERROR_HANDLER("cassandra.custom_disk_error_handler"),
     CUSTOM_GUARDRAILS_CONFIG_PROVIDER_CLASS("cassandra.custom_guardrails_config_provider_class"),
     CUSTOM_QUERY_HANDLER_CLASS("cassandra.custom_query_handler_class"),

src/java/org/apache/cassandra/config/Config.java

@@ -49,6 +49,7 @@ import org.apache.cassandra.utils.StorageCompatibilityMode;
 
 import static org.apache.cassandra.config.CassandraRelevantProperties.AUTOCOMPACTION_ON_STARTUP_ENABLED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_AVAILABLE_PROCESSORS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.CURSOR_COMPACTION_ENABLED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.FILE_CACHE_ENABLED;
 import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.SKIP_PAXOS_REPAIR_ON_TOPOLOGY_CHANGE_KEYSPACES;
@@ -661,6 +662,8 @@ public class Config
     @Replaces(oldName = "enable_drop_compact_storage", converter = Converters.IDENTITY, deprecated = true)
     public volatile boolean drop_compact_storage_enabled = false;
 
+    public boolean cursor_compaction_enabled = CURSOR_COMPACTION_ENABLED.getBoolean();
+
     public volatile boolean use_statements_enabled = true;
 
     /**

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -4725,6 +4725,17 @@ public class DatabaseDescriptor
         conf.transient_replication_enabled = enabled;
     }
 
+    public static boolean cursorCompactionEnabled()
+    {
+        return conf.cursor_compaction_enabled;
+    }
+
+    @VisibleForTesting
+    public static void setCursorCompactionEnabled(boolean cursor_compaction_enabled)
+    {
+        conf.cursor_compaction_enabled = cursor_compaction_enabled;
+    }
+
     public static boolean enableDropCompactStorage()
     {
         return conf.drop_compact_storage_enabled;

src/java/org/apache/cassandra/db/BufferDecoratedKey.java

@@ -25,7 +25,7 @@ import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 
 public class BufferDecoratedKey extends DecoratedKey
 {
-    private final ByteBuffer key;
+    protected ByteBuffer key;
 
     public BufferDecoratedKey(Token token, ByteBuffer key)
     {

src/java/org/apache/cassandra/db/Clustering.java

@@ -133,7 +133,7 @@ public interface Clustering<V> extends ClusteringPrefix<V>, IMeasurableMemory
     /**
      * Serializer for Clustering object.
      * <p>
-     * Because every clustering in a given table must have the same size (ant that size cannot actually change once the table
+     * Because every clustering in a given table must have the same size (and that size cannot actually change once the table
      * has been defined), we don't record that size.
      */
     public static class Serializer

src/java/org/apache/cassandra/db/ClusteringComparator.java

@@ -26,14 +26,17 @@ import java.util.Objects;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 
+import org.apache.cassandra.io.sstable.ClusteringDescriptor;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteBufferAccessor;
 import org.apache.cassandra.db.marshal.ValueAccessor;
 import org.apache.cassandra.db.rows.Row;
-import org.apache.cassandra.db.marshal.AbstractType;
-import org.apache.cassandra.serializers.MarshalException;
-
 import org.apache.cassandra.io.sstable.IndexInfo;
+import org.apache.cassandra.serializers.MarshalException;
+import org.apache.cassandra.utils.ByteBufferUtil;
 import org.apache.cassandra.utils.bytecomparable.ByteComparable;
 import org.apache.cassandra.utils.bytecomparable.ByteSource;
+import org.apache.cassandra.utils.vint.VIntCoding;
 
 import static org.apache.cassandra.utils.bytecomparable.ByteSource.EXCLUDED;
 import static org.apache.cassandra.utils.bytecomparable.ByteSource.NEXT_COMPONENT;
@@ -156,6 +159,107 @@ public class ClusteringComparator implements Comparator<Clusterable>
         return s1 < s2 ? c1.kind().comparedToClustering : -c2.kind().comparedToClustering;
     }
 
+    public static int compare(ClusteringDescriptor c1, ClusteringDescriptor c2)
+    {
+        final int c1Size = c1.clusteringColumnsBound();
+        final int c2Size = c2.clusteringColumnsBound();
+        final int minColumns = Math.min(c1Size, c2Size);
+
+        final int cmp = compare(c1.clusteringTypes(), c1.clusteringBuffer(), c2.clusteringBuffer(), minColumns);
+        if (cmp != 0)
+            return cmp;
+
+        final ClusteringPrefix.Kind c1Kind = c1.clusteringKind();
+        final ClusteringPrefix.Kind c2Kind = c2.clusteringKind();
+        if (c1Size == c2Size)
+        {
+            return ClusteringPrefix.Kind.compare(c1Kind, c2Kind);
+        }
+
+        return c1Size < c2Size ? c1Kind.comparedToClustering : -c2Kind.comparedToClustering;
+    }
+
+    public static int compare(AbstractType<?>[] types, ByteBuffer c1, ByteBuffer c2) {
+        return compare(types, c1, c2, types.length);
+    }
+
+    private static int compare(AbstractType<?>[] types, ByteBuffer c1, ByteBuffer c2, int size)
+    {
+        long clusteringBlock1 = 0;
+        long clusteringBlock2 = 0;
+        final int position1 = c1.position();
+        final int position2 = c2.position();
+        final int limit1 = c1.limit();
+        final int limit2 = c2.limit();
+        try
+        {
+            for (int clusteringIndex = 0; clusteringIndex < size; clusteringIndex++)
+            {
+                if (clusteringIndex % 32 == 0)
+                {
+                    clusteringBlock1 = VIntCoding.readUnsignedVInt(c1);
+                    clusteringBlock2 = VIntCoding.readUnsignedVInt(c2);
+                }
+
+                AbstractType<?> type = types[clusteringIndex];
+
+                byte v1Flags = (byte) (clusteringBlock1 & 0b11);
+                byte v2Flags = (byte) (clusteringBlock2 & 0b11);
+
+                // both values are present
+                if ((v1Flags|v2Flags) == 0)
+                {
+                    boolean isByteOrderComparable = type.isByteOrderComparable;
+                    int vlen1,vlen2;
+                    if (type.isValueLengthFixed())
+                    {
+                        vlen1 = vlen2 = type.valueLengthIfFixed();
+                    }
+                    else
+                    {
+                        vlen1 = VIntCoding.readUnsignedVInt32(c1);
+                        vlen2 = VIntCoding.readUnsignedVInt32(c2);
+                    }
+                    int v1Limit = c1.position() + vlen1;
+                    if (v1Limit > limit1)
+                        throw new IllegalArgumentException("Value limit exceeds buffer limit.");
+                    c1.limit(v1Limit);
+                    int v2Limit = c2.position() + vlen2;
+                    if (v2Limit > limit2)
+                        throw new IllegalArgumentException("Value limit exceeds buffer limit.");
+                    c2.limit(v2Limit);
+                    int cmp = isByteOrderComparable ?
+                                ByteBufferUtil.compareUnsigned(c1, c2) :
+                                type.compareCustom(c1, ByteBufferAccessor.instance, c2, ByteBufferAccessor.instance);
+                    if (cmp != 0)
+                        return cmp;
+                    c1.position(v1Limit);
+                    c2.position(v2Limit);
+                    c1.limit(limit1);
+                    c2.limit(limit2);
+                }
+                // present > not present
+                else
+                {
+                    // null (0b10) is smaller than empty (0b01) which is smaller than valued (0b00);
+                    // compare swapped arguments to reverse the order
+                    int cmp = Long.compare(v2Flags, v1Flags);
+                    if (cmp != 0)
+                        return cmp;
+                    // null/empty == null/empty, continue...
+                }
+                clusteringBlock1 = clusteringBlock1 >>> 2;
+                clusteringBlock2 = clusteringBlock2 >>> 2;
+            }
+        }
+        finally
+        {
+            c1.position(position1).limit(limit1);
+            c2.position(position2).limit(limit2);
+        }
+        return 0;
+    }
+
     public <V1, V2> int compare(Clustering<V1> c1, Clustering<V2> c2)
     {
         return compare(c1, c2, size());

src/java/org/apache/cassandra/db/ClusteringPrefix.java

@@ -519,7 +519,7 @@ public interface ClusteringPrefix<V> extends IMeasurableMemory, Clusterable<V>
             return result;
         }
 
-        byte[][] deserializeValuesWithoutSize(DataInputPlus in, int size, int version, List<AbstractType<?>> types) throws IOException
+        public byte[][] deserializeValuesWithoutSize(DataInputPlus in, int size, int version, List<AbstractType<?>> types) throws IOException
         {
             // Callers of this method should handle the case where size = 0 (in all case we want to return a special value anyway).
             assert size > 0;

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -2330,6 +2330,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         return partitionKeySetIgnoreGcGrace.contains(dk);
     }
 
+    public boolean shouldIgnoreGcGraceForAnyKey()
+    {
+        return !partitionKeySetIgnoreGcGrace.isEmpty();
+    }
+
     public static Iterable<ColumnFamilyStore> all()
     {
         List<Iterable<ColumnFamilyStore>> stores = new ArrayList<>(Schema.instance.getKeyspaces().size());

src/java/org/apache/cassandra/db/Columns.java

@@ -529,6 +529,7 @@ public class Columns extends AbstractCollection<ColumnMetadata> implements Colle
             int supersetCount = superset.size();
             if (columnCount == supersetCount)
             {
+                /** This is prevented by caller for row serialization: {@link org.apache.cassandra.db.rows.UnfilteredSerializer#serializeRowBody}*/
                 out.writeUnsignedVInt32(0);
             }
             else if (supersetCount < 64)

src/java/org/apache/cassandra/db/DecoratedKey.java

@@ -114,7 +114,7 @@ public abstract class DecoratedKey implements PartitionPosition, FilterKey
         // The OSS50 version avoids this by adding a terminator.
         return ByteSource.withTerminatorMaybeLegacy(version,
                                                     ByteSource.END_OF_STREAM,
-                                                    token.asComparableBytes(version),
+                                                    getToken().asComparableBytes(version),
                                                     keyComparableBytes(version));
     }
 
@@ -127,7 +127,7 @@ public abstract class DecoratedKey implements PartitionPosition, FilterKey
 
             return ByteSource.withTerminator(
                     before ? ByteSource.LT_NEXT_COMPONENT : ByteSource.GT_NEXT_COMPONENT,
-                    token.asComparableBytes(version),
+                    getToken().asComparableBytes(version),
                     keyComparableBytes(version));
         };
     }

src/java/org/apache/cassandra/db/DeletionPurger.java

@@ -19,16 +19,16 @@ package org.apache.cassandra.db;
 
 public interface DeletionPurger
 {
-    public static final DeletionPurger PURGE_ALL = (ts, ldt) -> true;
+    DeletionPurger PURGE_ALL = (ts, ldt) -> true;
 
-    public boolean shouldPurge(long timestamp, long localDeletionTime);
+    boolean shouldPurge(long timestamp, long localDeletionTime);
 
-    public default boolean shouldPurge(DeletionTime dt)
+    default boolean shouldPurge(DeletionTime dt)
     {
         return !dt.isLive() && shouldPurge(dt.markedForDeleteAt(), dt.localDeletionTime());
     }
 
-    public default boolean shouldPurge(LivenessInfo liveness, long nowInSec)
+    default boolean shouldPurge(LivenessInfo liveness, long nowInSec)
     {
         return !liveness.isLive(nowInSec) && shouldPurge(liveness.timestamp(), liveness.localExpirationTime());
     }

src/java/org/apache/cassandra/db/DeletionTime.java

@@ -36,27 +36,29 @@ import static java.lang.Math.min;
 /**
  * Information on deletion of a storage engine object.
  */
-public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
+public abstract class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
 {
-    public static final long EMPTY_SIZE = ObjectSizes.measure(new DeletionTime(0, 0));
+    private static final int LOCAL_DELETION_TIME_LIVE = Cell.deletionTimeLongToUnsignedInteger(Long.MAX_VALUE);
+    private static final long MARKED_FOR_DELETE_AT_LIVE = Long.MIN_VALUE;
+    public static final long EMPTY_SIZE = ObjectSizes.measure(new ImmutableDeletionTime(0, 0));
 
     /**
      * A special DeletionTime that signifies that there is no top-level (row) tombstone.
      */
-    public static final DeletionTime LIVE = new DeletionTime(Long.MIN_VALUE, Long.MAX_VALUE);
+    public static final DeletionTime LIVE = new ImmutableDeletionTime(MARKED_FOR_DELETE_AT_LIVE, LOCAL_DELETION_TIME_LIVE);
 
     private static final Serializer serializer = new Serializer();
     private static final Serializer legacySerializer = new LegacySerializer();
 
-    private final long markedForDeleteAt;
-    final int localDeletionTimeUnsignedInteger;
+    protected long markedForDeleteAt;
+    protected int localDeletionTimeUnsignedInteger;
 
     public static DeletionTime build(long markedForDeleteAt, long localDeletionTime)
     {
         // Negative ldts can only be a result of a corruption or when scrubbing legacy sstables with overflown int ldts
         return localDeletionTime < 0 || localDeletionTime > Cell.MAX_DELETION_TIME
                     ? new InvalidDeletionTime(markedForDeleteAt)
-                    : new DeletionTime(markedForDeleteAt, localDeletionTime);
+                    : new ImmutableDeletionTime(markedForDeleteAt, localDeletionTime);
     }
 
     // Do not use. This is a perf optimization where some data structures known to hold valid uints are allowed to use it.
@@ -65,7 +67,7 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
     {
         return CassandraUInt.compare(Cell.MAX_DELETION_TIME_UNSIGNED_INTEGER, localDeletionTimeUnsignedInteger) < 0
                 ? new InvalidDeletionTime(markedForDeleteAt)
-                : new DeletionTime(markedForDeleteAt, localDeletionTimeUnsignedInteger);
+                : new ImmutableDeletionTime(markedForDeleteAt, localDeletionTimeUnsignedInteger);
     }
 
     private DeletionTime(long markedForDeleteAt, long localDeletionTime)
@@ -98,6 +100,11 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
         return Cell.deletionTimeUnsignedIntegerToLong(localDeletionTimeUnsignedInteger);
     }
 
+    public int localDeletionTimeUnsignedInteger()
+    {
+        return localDeletionTimeUnsignedInteger;
+    }
+
     /**
      * Returns whether this DeletionTime is live, that is deletes no columns.
      */
@@ -143,7 +150,7 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
     @Override
     public String toString()
     {
-        return String.format("deletedAt=%d, localDeletion=%d", markedForDeleteAt(), localDeletionTime());
+        return isLive() ? "LIVE" : String.format("deletedAt=%d, localDeletion=%d", markedForDeleteAt(), localDeletionTime());
     }
 
     public int compareTo(DeletionTime dt)
@@ -155,6 +162,10 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
         else return CassandraUInt.compare(localDeletionTimeUnsignedInteger, dt.localDeletionTimeUnsignedInteger);
     }
 
+    /**
+     * supersedes: supplants, replaces, in this case: "is more recent"
+     * @return true if dt is deleted BEFORE this (markedForDeleteAt > dt.markedForDeleteAt || (markedForDeleteAt == dt.markedForDeleteAt && localDeletionTime > dt.localDeletionTime))
+     */
     public boolean supersedes(DeletionTime dt)
     {
         return markedForDeleteAt() > dt.markedForDeleteAt() || (markedForDeleteAt() == dt.markedForDeleteAt() && localDeletionTime() > dt.localDeletionTime());
@@ -209,7 +220,7 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
 
         public void serialize(DeletionTime delTime, DataOutputPlus out) throws IOException
         {
-            if (delTime == LIVE)
+            if (delTime == LIVE || delTime.isLive())
                 out.writeByte(IS_LIVE_DELETION);
             else
             {
@@ -238,7 +249,30 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
                 long mfda = readBytesToMFDA(flags, bytes1, bytes2, bytes4);
                 int localDeletionTimeUnsignedInteger = in.readInt();
 
-                return new DeletionTime(mfda, localDeletionTimeUnsignedInteger);
+                return new ImmutableDeletionTime(mfda, localDeletionTimeUnsignedInteger);
+            }
+        }
+
+        public void deserialize(DataInputPlus in, ReusableDeletionTime reuse) throws IOException
+        {
+            int flags = in.readByte();
+            if ((flags & IS_LIVE_DELETION) != 0)
+            {
+                if ((flags & 0xFF) != IS_LIVE_DELETION)
+                    throw new IOException("Corrupted sstable. Invalid flags found deserializing DeletionTime: " + Integer.toBinaryString(flags & 0xFF));
+                reuse.resetLive();
+            }
+            else
+            {
+                // Read the remaining 7 bytes
+                int bytes1 = in.readByte();
+                int bytes2 = in.readShort();
+                int bytes4 = in.readInt();
+
+                long mfda = readBytesToMFDA(flags, bytes1, bytes2, bytes4);
+                int localDeletionTimeUnsignedInteger = in.readInt();
+
+                reuse.reset(mfda, localDeletionTimeUnsignedInteger);
             }
         }
 
@@ -256,7 +290,7 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
                 long mfda = buf.getLong(offset);
                 int localDeletionTimeUnsignedInteger = buf.getInt(offset + TypeSizes.LONG_SIZE);
 
-                return new DeletionTime(mfda, localDeletionTimeUnsignedInteger);
+                return new ImmutableDeletionTime(mfda, localDeletionTimeUnsignedInteger);
             }
         }
 
@@ -315,13 +349,26 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
                  : DeletionTime.build(mfda, ldt);
         }
 
+        public void deserialize(DataInputPlus in, ReusableDeletionTime reuse) throws IOException
+        {
+            int ldt = in.readInt();
+            long mfda = in.readLong();
+            if (mfda == Long.MIN_VALUE && ldt == Integer.MAX_VALUE) {
+                reuse.resetLive();
+            }
+            else
+            {
+                reuse.reset(mfda, ldt);
+            }
+        }
+
         public DeletionTime deserialize(ByteBuffer buf, int offset)
         {
             int ldt = buf.getInt(offset);
             long mfda = buf.getLong(offset + 4);
             return mfda == Long.MIN_VALUE && ldt == Integer.MAX_VALUE
                    ? LIVE
-                   : new DeletionTime(mfda, ldt);
+                   : new ImmutableDeletionTime(mfda, ldt);
         }
 
         public void skip(DataInputPlus in) throws IOException
@@ -336,8 +383,22 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
         }
     }
 
+
+    private static class ImmutableDeletionTime extends DeletionTime
+    {
+        private ImmutableDeletionTime(long markedForDeleteAt, long localDeletionTime)
+        {
+            super(markedForDeleteAt, localDeletionTime);
+        }
+
+        private ImmutableDeletionTime(long markedForDeleteAt, int localDeletionTimeUnsignedInteger)
+        {
+            super(markedForDeleteAt, localDeletionTimeUnsignedInteger);
+        }
+    }
+
     // When scrubbing legacy sstables (overflown) or upon sstable corruption we could have negative ldts
-    public static class InvalidDeletionTime extends DeletionTime
+    public static class InvalidDeletionTime extends ImmutableDeletionTime
     {
         private InvalidDeletionTime(long markedForDeleteAt)
         {
@@ -358,4 +419,61 @@ public class DeletionTime implements Comparable<DeletionTime>, IMeasurableMemory
             return false;
         }
     }
+
+    public static class ReusableDeletionTime extends DeletionTime
+    {
+        private ReusableDeletionTime(long markedForDeleteAt, int localDeletionTimeUnsignedInteger)
+        {
+            super(markedForDeleteAt, localDeletionTimeUnsignedInteger);
+        }
+
+        public void resetLive()
+        {
+            markedForDeleteAt = MARKED_FOR_DELETE_AT_LIVE;
+            localDeletionTimeUnsignedInteger = LOCAL_DELETION_TIME_LIVE;
+        }
+
+        void reset(long markedForDeleteAt, int localDeletionTimeUnsignedInteger)
+        {
+            this.markedForDeleteAt = markedForDeleteAt;
+            this.localDeletionTimeUnsignedInteger = localDeletionTimeUnsignedInteger;
+        }
+
+        public void reset(long markedForDeleteAt, long localDeletionTime)
+        {
+            this.markedForDeleteAt = markedForDeleteAt;
+            if (localDeletionTime < 0 || localDeletionTime > Cell.MAX_DELETION_TIME) // invalid
+                this.localDeletionTimeUnsignedInteger = Cell.MAX_DELETION_TIME_UNSIGNED_INTEGER + 1;
+            else
+                this.localDeletionTimeUnsignedInteger = Cell.deletionTimeLongToUnsignedInteger(localDeletionTime);
+        }
+
+        public void reset(DeletionTime deletionTime)
+        {
+            if (deletionTime == null)
+                resetLive();
+            else
+                reset(deletionTime.markedForDeleteAt, deletionTime.localDeletionTimeUnsignedInteger);
+        }
+
+        @Override
+        public boolean validate()
+        {
+            return localDeletionTimeUnsignedInteger == LOCAL_DELETION_TIME_LIVE || CassandraUInt.compare(Cell.MAX_DELETION_TIME_UNSIGNED_INTEGER, localDeletionTimeUnsignedInteger) >= 0;
+        }
+
+        public static ReusableDeletionTime copy(DeletionTime original)
+        {
+            if (original == null)
+                return live();
+            // Negative ldts can only be a result of a corruption or when scrubbing legacy sstables with overflown int ldts
+            return new ReusableDeletionTime(original.markedForDeleteAt, original.localDeletionTimeUnsignedInteger);
+        }
+
+        public static ReusableDeletionTime live()
+        {
+            // Negative ldts can only be a result of a corruption or when scrubbing legacy sstables with overflown int ldts
+            return new ReusableDeletionTime(LIVE.markedForDeleteAt, LIVE.localDeletionTimeUnsignedInteger);
+        }
+    }
 }

src/java/org/apache/cassandra/db/LivenessInfo.java

@@ -37,35 +37,28 @@ import org.apache.cassandra.utils.ObjectSizes;
  * unaffected (of course, the rest of said row data might be ttl'ed on its own but this is
  * separate).
  */
-public class LivenessInfo implements IMeasurableMemory
+public interface LivenessInfo extends IMeasurableMemory
 {
-    public static final long NO_TIMESTAMP = Long.MIN_VALUE;
-    public static final int NO_TTL = Cell.NO_TTL;
+    long NO_TIMESTAMP = Long.MIN_VALUE;
+    int NO_TTL = Cell.NO_TTL;
     /**
      * Used as flag for representing an expired liveness.
      *
      * TTL per request is at most 20 yrs, so this shouldn't conflict
      * (See {@link org.apache.cassandra.cql3.Attributes#MAX_TTL})
      */
-    public static final int EXPIRED_LIVENESS_TTL = Integer.MAX_VALUE;
-    public static final long NO_EXPIRATION_TIME = Cell.NO_DELETION_TIME;
+    int EXPIRED_LIVENESS_TTL = Integer.MAX_VALUE;
+    long NO_EXPIRATION_TIME = Cell.NO_DELETION_TIME;
 
-    public static final LivenessInfo EMPTY = new LivenessInfo(NO_TIMESTAMP);
-    private static final long UNSHARED_HEAP_SIZE = ObjectSizes.measure(EMPTY);
+    LivenessInfo EMPTY = new ImmutableLivenessInfo(NO_TIMESTAMP);
+    long UNSHARED_HEAP_SIZE = ObjectSizes.measure(EMPTY);
 
-    protected final long timestamp;
-
-    protected LivenessInfo(long timestamp)
+    static LivenessInfo create(long timestamp, long nowInSec)
     {
-        this.timestamp = timestamp;
+        return new ImmutableLivenessInfo(timestamp);
     }
 
-    public static LivenessInfo create(long timestamp, long nowInSec)
-    {
-        return new LivenessInfo(timestamp);
-    }
-
-    public static LivenessInfo expiring(long timestamp, int ttl, long nowInSec)
+    static LivenessInfo expiring(long timestamp, int ttl, long nowInSec)
     {
         assert ttl != EXPIRED_LIVENESS_TTL;
         return new ExpiringLivenessInfo(timestamp, ttl, ExpirationDateOverflowHandling.computeLocalExpirationTime(nowInSec, ttl));
@@ -86,7 +79,7 @@ public class LivenessInfo implements IMeasurableMemory
              : expiring(timestamp, ttl, nowInSec, applyOverflowPolicy);
     }
 
-    public static LivenessInfo create(long timestamp, int ttl, long nowInSec)
+    static LivenessInfo create(long timestamp, int ttl, long nowInSec)
     {
         return ttl == NO_TTL
              ? create(timestamp, nowInSec)
@@ -95,11 +88,11 @@ public class LivenessInfo implements IMeasurableMemory
 
     // Note that this ctor takes the expiration time, not the current time.
     // Use when you know that's what you want.
-    public static LivenessInfo withExpirationTime(long timestamp, int ttl, long localExpirationTime)
+    static LivenessInfo withExpirationTime(long timestamp, int ttl, long localExpirationTime)
     {
         if (ttl == EXPIRED_LIVENESS_TTL)
             return new ExpiredLivenessInfo(timestamp, ttl, localExpirationTime);
-        return ttl == NO_TTL ? new LivenessInfo(timestamp) : new ExpiringLivenessInfo(timestamp, ttl, localExpirationTime);
+        return ttl == NO_TTL ? new ImmutableLivenessInfo(timestamp) : new ExpiringLivenessInfo(timestamp, ttl, localExpirationTime);
     }
 
     /**
@@ -107,9 +100,9 @@ public class LivenessInfo implements IMeasurableMemory
      *
      * @return whether this liveness info is empty or not.
      */
-    public boolean isEmpty()
+    default boolean isEmpty()
     {
-        return timestamp == NO_TIMESTAMP;
+        return timestamp() == NO_TIMESTAMP;
     }
 
     /**
@@ -117,18 +110,11 @@ public class LivenessInfo implements IMeasurableMemory
      *
      * @return the liveness info timestamp (or {@link #NO_TIMESTAMP} if the info is empty).
      */
-    public long timestamp()
-    {
-        return timestamp;
-    }
-
+    long timestamp();
     /**
      * Whether the info has a ttl.
      */
-    public boolean isExpiring()
-    {
-        return false;
-    }
+    boolean isExpiring();
 
     /**
      * The ttl (if any) on the row primary key columns or {@link #NO_TTL} if it is not
@@ -137,19 +123,13 @@ public class LivenessInfo implements IMeasurableMemory
      * Please note that this value is the TTL that was set originally and is thus not
      * changing.
      */
-    public int ttl()
-    {
-        return NO_TTL;
-    }
+    int ttl();
 
     /**
      * The expiration time (in seconds) if the info is expiring ({@link #NO_EXPIRATION_TIME} otherwise).
      *
      */
-    public long localExpirationTime()
-    {
-        return NO_EXPIRATION_TIME;
-    }
+    long localExpirationTime();
 
     /**
      * Whether that info is still live.
@@ -160,17 +140,15 @@ public class LivenessInfo implements IMeasurableMemory
      * @param nowInSec the current time in seconds.
      * @return whether this liveness info is live or not.
      */
-    public boolean isLive(long nowInSec)
-    {
-        return !isEmpty();
-    }
+    boolean isLive(long nowInSec);
+
 
     /**
      * Adds this liveness information to the provided digest.
      *
      * @param digest the digest to add this liveness information to.
      */
-    public void digest(Digest digest)
+    default void digest(Digest digest)
     {
         digest.updateWithLong(timestamp());
     }
@@ -180,7 +158,7 @@ public class LivenessInfo implements IMeasurableMemory
      *
      * @throws MarshalException if some of the data is corrupted.
      */
-    public void validate()
+    default void validate()
     {
     }
 
@@ -189,18 +167,18 @@ public class LivenessInfo implements IMeasurableMemory
      *
      * @return the size of the data this liveness information contains.
      */
-    public int dataSize()
+    default int dataSize()
     {
         return TypeSizes.sizeof(timestamp());
     }
 
     /**
      * Whether this liveness information supersedes another one (that is
-     * whether is has a greater timestamp than the other or not).
+     * whether it has a greater timestamp than the other or not).
      *
      * If timestamps are the same and none of them are expired livenessInfo,
      * livenessInfo with greater TTL supersedes another. It also means, if timestamps are the same,
-     * ttl superseders no-ttl. This is the same rule as {@link Conflicts#resolveRegular}
+     * ttl superseders no-ttl. This is the same rule as {@link org.apache.cassandra.db.rows.Cells#resolveRegular}
      *
      * If timestamps are the same and one of them is expired livenessInfo. Expired livenessInfo
      * supersedes, ie. tombstone supersedes.
@@ -216,10 +194,12 @@ public class LivenessInfo implements IMeasurableMemory
      *
      * @return whether this {@code LivenessInfo} supersedes {@code other}.
      */
-    public boolean supersedes(LivenessInfo other)
+    default boolean supersedes(LivenessInfo other)
     {
-        if (timestamp != other.timestamp)
-            return timestamp > other.timestamp;
+        long tTimestamp = timestamp();
+        long oTimestamp = other.timestamp();
+        if (tTimestamp != oTimestamp)
+            return tTimestamp > oTimestamp;
         if (isExpired() ^ other.isExpired())
             return isExpired();
         if (isExpiring() == other.isExpiring())
@@ -231,7 +211,7 @@ public class LivenessInfo implements IMeasurableMemory
         return isExpiring();
     }
 
-    protected boolean isExpired()
+    default boolean isExpired()
     {
         return false;
     }
@@ -244,47 +224,24 @@ public class LivenessInfo implements IMeasurableMemory
      * as timestamp. If it has no timestamp however, this liveness info is returned
      * unchanged.
      */
-    public LivenessInfo withUpdatedTimestamp(long newTimestamp)
+    default LivenessInfo withUpdatedTimestamp(long newTimestamp)
     {
-        return new LivenessInfo(newTimestamp);
+        return new ImmutableLivenessInfo(newTimestamp);
     }
 
-    public LivenessInfo withUpdatedTimestampAndLocalDeletionTime(long newTimestamp, long newLocalDeletionTime)
+    default LivenessInfo withUpdatedTimestampAndLocalDeletionTime(long newTimestamp, long newLocalDeletionTime)
     {
         return LivenessInfo.create(newTimestamp, ttl(), newLocalDeletionTime, true);
     }
 
     // C14227 To prevent row resurrection and be backwards compatible sometimes we need to force an overflowed ldt
-    public LivenessInfo withUpdatedTimestampAndLocalDeletionTime(long newTimestamp, long newLocalDeletionTime, boolean applyOverflowPolicy)
+    default LivenessInfo withUpdatedTimestampAndLocalDeletionTime(long newTimestamp, long newLocalDeletionTime, boolean applyOverflowPolicy)
     {
         return LivenessInfo.create(newTimestamp, ttl(), newLocalDeletionTime, applyOverflowPolicy);
     }
 
     @Override
-    public String toString()
-    {
-        return String.format("[ts=%d]", timestamp);
-    }
-
-    @Override
-    public boolean equals(Object other)
-    {
-        if(!(other instanceof LivenessInfo))
-            return false;
-
-        LivenessInfo that = (LivenessInfo)other;
-        return this.timestamp() == that.timestamp()
-            && this.ttl() == that.ttl()
-            && this.localExpirationTime() == that.localExpirationTime();
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(timestamp(), ttl(), localExpirationTime());
-    }
-
-    public long unsharedHeapSize()
+    default long unsharedHeapSize()
     {
         return this == EMPTY ? 0 : UNSHARED_HEAP_SIZE;
     }
@@ -295,7 +252,7 @@ public class LivenessInfo implements IMeasurableMemory
      *
      * See {@link org.apache.cassandra.db.view.ViewUpdateGenerator#deleteOldEntryInternal}.
      */
-    private static class ExpiredLivenessInfo extends ExpiringLivenessInfo
+    class ExpiredLivenessInfo extends ExpiringLivenessInfo
     {
         private ExpiredLivenessInfo(long timestamp, int ttl, long localExpirationTime)
         {
@@ -324,7 +281,7 @@ public class LivenessInfo implements IMeasurableMemory
         }
     }
 
-    private static class ExpiringLivenessInfo extends LivenessInfo
+    class ExpiringLivenessInfo extends ImmutableLivenessInfo
     {
         private final int ttl;
         private final long localExpirationTime;
@@ -401,7 +358,7 @@ public class LivenessInfo implements IMeasurableMemory
         @Override
         public String toString()
         {
-            return String.format("[ts=%d ttl=%d, let=%d]", timestamp, ttl, localExpirationTime);
+            return String.format("[ts=%d ttl=%d, let=%d]", timestamp(), ttl, localExpirationTime);
         }
 
         public long unsharedHeapSize()
@@ -409,4 +366,66 @@ public class LivenessInfo implements IMeasurableMemory
             return UNSHARED_HEAP_SIZE;
         }
     }
+
+    class ImmutableLivenessInfo implements LivenessInfo {
+        private final long timestamp;
+
+        private ImmutableLivenessInfo(long timestamp)
+        {
+            this.timestamp = timestamp;
+        }
+
+        @Override
+        public final long timestamp()
+        {
+            return timestamp;
+        }
+
+        public int ttl()
+        {
+            return NO_TTL;
+        }
+
+        @Override
+        public boolean isExpiring()
+        {
+            return false;
+        }
+
+        @Override
+        public long localExpirationTime()
+        {
+            return NO_EXPIRATION_TIME;
+        }
+
+        @Override
+        public boolean isLive(long nowInSec)
+        {
+            return !isEmpty();
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("[ts=%d]", timestamp);
+        }
+
+        @Override
+        public boolean equals(Object other)
+        {
+            if(!(other instanceof LivenessInfo))
+                return false;
+
+            LivenessInfo that = (LivenessInfo)other;
+            return this.timestamp() == that.timestamp()
+                   && this.ttl() == that.ttl()
+                   && this.localExpirationTime() == that.localExpirationTime();
+        }
+
+        @Override
+        public int hashCode()
+        {
+            return Objects.hash(timestamp(), ttl(), localExpirationTime());
+        }
+    }
 }

src/java/org/apache/cassandra/db/RangeTombstoneList.java

@@ -145,7 +145,7 @@ public class RangeTombstoneList implements Iterable<RangeTombstone>, IMeasurable
         add(tombstone.deletedSlice().start(),
             tombstone.deletedSlice().end(),
             tombstone.deletionTime().markedForDeleteAt(),
-            tombstone.deletionTime().localDeletionTimeUnsignedInteger);
+            tombstone.deletionTime().localDeletionTimeUnsignedInteger());
     }
 
     /**

src/java/org/apache/cassandra/db/ReusableLivenessInfo.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db;
+
+// TODO: maybe flatten into descriptor classes?
+public class ReusableLivenessInfo implements LivenessInfo
+{
+    private int ttl = NO_TTL;
+    private long localExpirationTime = NO_EXPIRATION_TIME;
+    private long timestamp = NO_TIMESTAMP;
+
+    @Override
+    public long timestamp()
+    {
+        return timestamp;
+    }
+
+    @Override
+    public int ttl()
+    {
+        return ttl;
+    }
+
+    @Override
+    public long localExpirationTime()
+    {
+        return localExpirationTime;
+    }
+
+    @Override
+    public boolean isExpiring()
+    {
+        return localExpirationTime != NO_EXPIRATION_TIME;
+    }
+
+    /**
+     * {@link org.apache.cassandra.db.rows.AbstractCell#isTombstone()}
+     */
+    public boolean isTombstone()
+    {
+        return localExpirationTime != NO_EXPIRATION_TIME && ttl == NO_TTL;
+    }
+
+    @Override
+    public boolean isLive(long nowInSec)
+    {
+        return localExpirationTime == NO_EXPIRATION_TIME || ttl != NO_TTL && !isExpired(nowInSec);
+    }
+
+    public boolean isExpired(long nowInSec)
+    {
+        return nowInSec >= localExpirationTime;
+    }
+
+    public void ttlToTombstone()
+    {
+        // LET/LDT is now the time the TTL would have expired
+        localExpirationTime = localExpirationTime - ttl;
+        ttl = NO_TTL;
+    }
+
+    public void reset(long timestamp, int ttl, long localExpirationTime)
+    {
+        this.timestamp = timestamp;
+        this.ttl = ttl;
+        this.localExpirationTime = localExpirationTime;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "ReusableLivenessInfo{" + ((timestamp == NO_TIMESTAMP && ttl == NO_TTL && localExpirationTime == NO_EXPIRATION_TIME) ? "NONE }" :
+               "timestamp=" + (timestamp == NO_TIMESTAMP ? "NO_TIMESTAMP" : timestamp) +
+               ", ttl=" + (ttl == NO_TTL ? "NO_TTL" : ttl)  +
+               ", localExpirationTime=" + (localExpirationTime == NO_EXPIRATION_TIME ? "NO_EXPIRATION_TIME" : localExpirationTime) +
+               '}');
+    }
+}

src/java/org/apache/cassandra/db/SerializationHeader.java

@@ -207,6 +207,14 @@ public class SerializationHeader
         return DeletionTime.build(markedAt, localDeletionTime);
     }
 
+    public void readDeletionTime(DataInputPlus in, DeletionTime.ReusableDeletionTime reuse) throws IOException
+    {
+        long markedAt = readTimestamp(in);
+        long localDeletionTime = readLocalDeletionTime(in);
+        reuse.reset(markedAt, localDeletionTime);
+    }
+
+
     public long timestampSerializedSize(long timestamp)
     {
         return TypeSizes.sizeofUnsignedVInt(timestamp - stats.minTimestamp);

src/java/org/apache/cassandra/db/compaction/AbstractCompactionPipeline.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.AbstractCompactionController;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+abstract class AbstractCompactionPipeline extends CompactionInfo.Holder implements AutoCloseable {
+    static AbstractCompactionPipeline create(
+        CompactionTask task,
+        OperationType type,
+        AbstractCompactionStrategy.ScannerList scanners,
+        AbstractCompactionController controller,
+        long nowInSec,
+        TimeUUID compactionId)
+    {
+        if (DatabaseDescriptor.cursorCompactionEnabled()) {
+            if (CursorCompactor.isSupported(scanners, controller))
+            {
+                return new CursorCompactionPipeline(task, type, scanners, controller, nowInSec, compactionId);
+            }
+        }
+        return new IteratorCompactionPipeline(task, type, scanners, controller, nowInSec, compactionId);
+    }
+
+    abstract boolean processNextPartitionKey() throws IOException;
+
+    public abstract long[] getMergedRowCounts();
+
+    public abstract long getTotalSourceCQLRows();
+
+    public abstract long getTotalKeysWritten();
+
+    public abstract long getTotalBytesScanned();
+
+    public abstract AutoCloseable openWriterResource(ColumnFamilyStore cfs,
+                                                     Directories directories,
+                                                     ILifecycleTransaction transaction,
+                                                     Set<SSTableReader> nonExpiredSSTables);
+
+    @Override
+    public abstract void close() throws IOException;
+
+    public abstract Collection<SSTableReader> finishWriting();
+
+    public abstract long estimatedKeys();
+
+    public abstract void stop();
+}

src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java

@@ -820,7 +820,7 @@ public class CompactionStrategyManager implements INotificationConsumer
      *
      * lives in matches the list index of the holder that's responsible for it
      */
-    public List<GroupedSSTableContainer> groupSSTables(Iterable<SSTableReader> sstables)
+    public final List<GroupedSSTableContainer> groupSSTables(Iterable<SSTableReader> sstables)
     {
         List<GroupedSSTableContainer> classified = new ArrayList<>(holders.size());
         for (AbstractStrategyHolder holder : holders)
@@ -970,7 +970,7 @@ public class CompactionStrategyManager implements INotificationConsumer
      * @param ranges
      * @return
      */
-    public AbstractCompactionStrategy.ScannerList maybeGetScanners(Collection<SSTableReader> sstables,  Collection<Range<Token>> ranges)
+    public final AbstractCompactionStrategy.ScannerList maybeGetScanners(Collection<SSTableReader> sstables,  Collection<Range<Token>> ranges)
     {
         maybeReloadDiskBoundaries();
         List<ISSTableScanner> scanners = new ArrayList<>(sstables.size());

src/java/org/apache/cassandra/db/compaction/CompactionTask.java

@@ -70,7 +70,9 @@ import static org.apache.cassandra.utils.Clock.Global.nanoTime;
 
 public class CompactionTask extends AbstractCompactionTask
 {
+    private static final int MEGABYTE = 1024 * 1024;
     protected static final Logger logger = LoggerFactory.getLogger(CompactionTask.class);
+
     protected final long gcBefore;
     protected final boolean keepOriginals;
     protected static long totalBytesCompacted = 0;
@@ -151,9 +153,11 @@ public class CompactionTask extends AbstractCompactionTask
      * For internal use and testing only.  The rest of the system should go through the submit* methods,
      * which are properly serialized.
      * Caller is in charge of marking/unmarking the sstables as compacting.
+     *
+     * NOTE: this method is a Byteman hook location
      */
     @Override
-    protected void runMayThrow() throws Exception
+    protected final void runMayThrow() throws Exception
     {
         // The collection of sstables passed may be empty (but not null); even if
         // it is not empty, it may compact down to nothing if all rows are deleted.
@@ -245,7 +249,7 @@ public class CompactionTask extends AbstractCompactionTask
             long nowInSec = FBUtilities.nowInSeconds();
             try (Refs<SSTableReader> refs = Refs.ref(actuallyCompact);
                  AbstractCompactionStrategy.ScannerList scanners = strategy.getScanners(actuallyCompact, rangeList);
-                 CompactionIterator ci = new CompactionIterator(compactionType, scanners.scanners, controller, nowInSec, taskId))
+                 AbstractCompactionPipeline ci = AbstractCompactionPipeline.create(this, compactionType, scanners, controller, nowInSec, taskId))
             {
                 long lastCheckObsoletion = start;
                 inputSizeBytes = scanners.getTotalCompressedSize();
@@ -253,30 +257,29 @@ public class CompactionTask extends AbstractCompactionTask
                 if (compressionRatio == MetadataCollector.NO_COMPRESSION_RATIO)
                     compressionRatio = 1.0;
 
-                long lastBytesScanned = 0;
-
                 activeCompactions.beginCompaction(ci);
-                try (CompactionAwareWriter writer = getCompactionAwareWriter(cfs, getDirectories(), transaction, actuallyCompact))
+                try (AutoCloseable resource = getCompactionAwareWriter(actuallyCompact, ci))
                 {
+                    long lastBytesScanned = 0;
                     // Note that we need to re-check this flag after calling beginCompaction above to avoid a window
                     // where the compaction does not exist in activeCompactions but the CSM gets paused.
                     // We already have the sstables marked compacting here so CompactionManager#waitForCessation will
                     // block until the below exception is thrown and the transaction is cancelled.
                     if (!controller.cfs.getCompactionStrategyManager().isActive())
                         throw new CompactionInterruptedException(ci.getCompactionInfo());
-                    estimatedKeys = writer.estimatedKeys();
-                    while (ci.hasNext())
+                    estimatedKeys = ci.estimatedKeys();
+                    while (ci.processNextPartitionKey())
                     {
-                        if (writer.append(ci.next()))
-                            totalKeysWritten++;
-
-                        ci.setTargetDirectory(writer.getSStableDirectory().path());
-                        long bytesScanned = scanners.getTotalBytesScanned();
+                        long bytesScanned = ci.getTotalBytesScanned();
 
+                        // If we ingested less than a MB, keep going
+                        if (bytesScanned - lastBytesScanned > MEGABYTE)
+                        {
                             // Rate limit the scanners, and account for compression
                             CompactionManager.instance.compactionRateLimiterAcquire(limiter, bytesScanned, lastBytesScanned, compressionRatio);
 
                             lastBytesScanned = bytesScanned;
+                        }
 
                         if (nanoTime() - lastCheckObsoletion > TimeUnit.MINUTES.toNanos(1L))
                         {
@@ -284,19 +287,27 @@ public class CompactionTask extends AbstractCompactionTask
                             lastCheckObsoletion = nanoTime();
                         }
                     }
+                    if (ci.getTotalBytesScanned() != lastBytesScanned)
+                    {
+                        // Report any leftover bytes
+                        CompactionManager.instance.compactionRateLimiterAcquire(limiter, ci.getTotalBytesScanned(), lastBytesScanned, compressionRatio);
+                    }
                     timeSpentWritingKeys = TimeUnit.NANOSECONDS.toMillis(nanoTime() - start);
 
                     // point of no return
-                    newSStables = writer.finish();
+                    newSStables = finish(ci);
                 }
                 finally
                 {
                     activeCompactions.finishCompaction(ci);
                     mergedRowCounts = ci.getMergedRowCounts();
                     totalSourceCQLRows = ci.getTotalSourceCQLRows();
+
+                    totalKeysWritten = ci.getTotalKeysWritten();
                 }
             }
 
+
             if (transaction.isOffline())
                 return;
 
@@ -343,6 +354,28 @@ public class CompactionTask extends AbstractCompactionTask
             // update the metrics
             cfs.metric.compactionBytesWritten.inc(endsize);
         }
+        catch (Throwable e)
+        {
+            /** This should not be needed (because {@link org.apache.cassandra.utils.WrappedRunnable}) but some exceptions seem to slip by in tests */
+            logger.debug("Unexpected exception in compaction", e);
+            throw e;
+        }
+    }
+
+    /**
+     * NOTE: a Byteman hook
+     */
+    protected Collection<SSTableReader> finish(AbstractCompactionPipeline pipeline)
+    {
+        return pipeline.finishWriting();
+    }
+
+    /**
+     * NOTE: a Byteman hook
+     */
+    protected AutoCloseable getCompactionAwareWriter(Set<SSTableReader> actuallyCompact, AbstractCompactionPipeline pipeline)
+    {
+        return pipeline.openWriterResource(cfs, getDirectories(), transaction, actuallyCompact);
     }
 
     public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,

src/java/org/apache/cassandra/db/compaction/CursorCompactionPipeline.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.db.AbstractCompactionController;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+class CursorCompactionPipeline extends AbstractCompactionPipeline {
+    final CursorCompactor cursorCompactor;
+    final CompactionTask task;
+    long totalKeysWritten;
+    CompactionAwareWriter writer;
+
+    CursorCompactionPipeline(CompactionTask task, OperationType type, AbstractCompactionStrategy.ScannerList scanners, AbstractCompactionController controller, long nowInSec, TimeUUID compactionId) {
+        this.task = task;
+        cursorCompactor = new CursorCompactor(type, scanners.scanners, controller, nowInSec, compactionId);
+    }
+
+    public AutoCloseable openWriterResource(ColumnFamilyStore cfs,
+                                            Directories directories,
+                                            ILifecycleTransaction transaction,
+                                            Set<SSTableReader> nonExpiredSSTables) {
+        this.writer = task.getCompactionAwareWriter(cfs, directories, transaction, nonExpiredSSTables);
+        return writer;
+    }
+
+
+    @Override
+    public Collection<SSTableReader> finishWriting() {
+        return writer.finish();
+    }
+
+    @Override
+    public long estimatedKeys() {
+        return writer.estimatedKeys();
+    }
+
+    @Override
+    public CompactionInfo getCompactionInfo() {
+        return cursorCompactor.getCompactionInfo();
+    }
+
+    @Override
+    public boolean isGlobal() {
+        return cursorCompactor.isGlobal();
+    }
+
+    @Override
+    boolean processNextPartitionKey() throws IOException {
+        if (cursorCompactor.writeNextPartition(writer)) {
+            totalKeysWritten++;
+            cursorCompactor.setTargetDirectory(writer.getSStableDirectoryPath());
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public long[] getMergedRowCounts() {
+        return cursorCompactor.getMergedRowsCounts();
+    }
+
+    @Override
+    public long getTotalSourceCQLRows() {
+        return cursorCompactor.getTotalSourceCQLRows();
+    }
+
+    @Override
+    public long getTotalKeysWritten() {
+        return totalKeysWritten;
+    }
+
+    @Override
+    public long getTotalBytesScanned() {
+        return cursorCompactor.getTotalBytesScanned();
+    }
+
+    @Override
+    public void close() throws IOException {
+        cursorCompactor.close();
+    }
+
+    @Override
+    public void stop() {
+        cursorCompactor.stop();
+    }
+}

src/java/org/apache/cassandra/db/compaction/IteratorCompactionPipeline.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.db.AbstractCompactionController;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.utils.TimeUUID;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+
+class IteratorCompactionPipeline extends AbstractCompactionPipeline {
+    final CompactionIterator ci;
+    final AbstractCompactionStrategy.ScannerList scanners;
+    final CompactionTask task;
+    long totalKeysWritten;
+    CompactionAwareWriter writer;
+
+    IteratorCompactionPipeline(CompactionTask task, OperationType type, AbstractCompactionStrategy.ScannerList scanners, AbstractCompactionController controller, long nowInSec, TimeUUID compactionId) {
+        this.task = task;
+        this.scanners = scanners;
+        ci = new CompactionIterator(type, this.scanners.scanners, controller, nowInSec, compactionId);
+    }
+
+    public AutoCloseable openWriterResource(ColumnFamilyStore cfs,
+                                            Directories directories,
+                                            ILifecycleTransaction transaction,
+                                            Set<SSTableReader> nonExpiredSSTables) {
+        this.writer = task.getCompactionAwareWriter(cfs, directories, transaction, nonExpiredSSTables);
+        return writer;
+    }
+
+
+    @Override
+    public Collection<SSTableReader> finishWriting() {
+        return writer.finish();
+    }
+
+    @Override
+    public long estimatedKeys() {
+        return writer.estimatedKeys();
+    }
+
+    @Override
+    public CompactionInfo getCompactionInfo() {
+        return ci.getCompactionInfo();
+    }
+
+    @Override
+    public boolean isGlobal() {
+        return ci.isGlobal();
+    }
+
+    @Override
+    boolean processNextPartitionKey() throws IOException {
+        if (ci.hasNext()) {
+            if (writer.append(ci.next()))
+                totalKeysWritten++;
+            ci.setTargetDirectory(writer.getSStableDirectoryPath());
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public long[] getMergedRowCounts() {
+        return ci.getMergedRowCounts();
+    }
+
+    @Override
+    public long getTotalSourceCQLRows() {
+        return ci.getTotalSourceCQLRows();
+    }
+
+    @Override
+    public long getTotalKeysWritten() {
+        return totalKeysWritten;
+    }
+
+    @Override
+    public long getTotalBytesScanned() {
+        return scanners.getTotalBytesScanned();
+    }
+
+    @Override
+    public void close() throws IOException {
+        ci.close();
+    }
+
+    @Override
+    public void stop() {
+        ci.stop();
+    }
+}

src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java

@@ -297,6 +297,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
         return levelFanoutSize;
     }
 
+    @Override
     public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)
     {
         Set<SSTableReader>[] sstablesPerLevel = manifest.getSStablesPerLevelSnapshot();
@@ -432,7 +433,12 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
             assert sstableIterator.hasNext(); // caller should check intersecting first
             SSTableReader currentSSTable = sstableIterator.next();
             currentScanner = currentSSTable.getScanner(ranges);
+        }
 
+        @Override
+        public boolean isFullRange()
+        {
+            return ranges == null;
         }
 
         public static Collection<SSTableReader> intersecting(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges)

src/java/org/apache/cassandra/db/compaction/StatefulCursor.java

@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import org.apache.cassandra.config.Config;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.UnfilteredValidation;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.io.sstable.PartitionDescriptor;
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.io.sstable.SSTableCursorReader;
+import org.apache.cassandra.io.sstable.UnfilteredDescriptor;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+import static org.apache.cassandra.db.rows.Cell.INVALID_DELETION_TIME;
+import static org.apache.cassandra.db.rows.Cell.NO_DELETION_TIME;
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.CELL_END;
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.CELL_HEADER_START;
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.CELL_VALUE_START;
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.isState;
+
+// Cursor state
+class StatefulCursor extends SSTableCursorReader
+{
+    private final Config.CorruptedTombstoneStrategy corruptedTombstoneStrategy = DatabaseDescriptor.getCorruptedTombstoneStrategy();
+    private final boolean corruptedTombstoneValidationEnabled = corruptedTombstoneStrategy != Config.CorruptedTombstoneStrategy.disabled;
+
+    private PartitionDescriptor currPartition;
+    /**
+     * used for tracking reader and writer order, as well as last pk
+     */
+    private PartitionDescriptor prevPartition;
+
+    private final UnfilteredDescriptor unfiltered;
+
+    private boolean resetAfterDone = false;
+    private long bytesReadPositionSnapshot = 0;
+
+    private boolean isOpenRangeTombstonePresent = false;
+
+    public StatefulCursor(SSTableReader reader)
+    {
+        super(reader);
+        currPartition = new PartitionDescriptor(reader.getPartitioner().createReusableKey(0));
+        prevPartition = new PartitionDescriptor(reader.getPartitioner().createReusableKey(0));
+        unfiltered = new UnfilteredDescriptor(reader.header.clusteringTypes().toArray(AbstractType[]::new));
+    }
+
+    public int readPartitionHeader()
+    {
+        swapCurrAndPrevPartition();
+        int state = readPartitionHeader(currPartition);
+
+        if (prevPartition.keyLength() != 0 && prevPartition.key().compareTo(currPartition.key()) >= 0)
+            corruptSSTableKeysOOO();
+        if (corruptedTombstoneValidationEnabled)
+            validateInvalidPartitionDeletion();
+        return state;
+    }
+
+    private int corruptSSTableKeysOOO()
+    {
+        return corruptSSTable("Keys out of order. Current key: " + keyToString(currentKey()) + " <= "  + keyToString(prevKey()));
+    }
+
+    private void swapCurrAndPrevPartition()
+    {
+        PartitionDescriptor temp = currPartition;
+        currPartition = prevPartition;
+        prevPartition = temp;
+    }
+
+    public int skipUnfiltered()
+    {
+        if (isState(state(), CELL_HEADER_START | CELL_VALUE_START | CELL_END))
+            return super.skipRowCells(unfiltered().dataStart(), unfiltered().size(), false);
+
+        return super.skipUnfiltered(false);
+    }
+
+    public int skipStaticRow()
+    {
+        if (isState(state(), CELL_HEADER_START | CELL_VALUE_START | CELL_END))
+            return super.skipRowCells(unfiltered().dataStart(), unfiltered().size(), false);
+
+        return super.skipStaticRow(false);
+    }
+
+    @Override
+    public String toString()
+    {
+        return "StatefulCursor{" +
+               "pHeader=" + currPartition() +
+               ", rHeader=" + unfiltered() +
+               ", state=" + state() +
+               '}';
+    }
+
+    /**
+     * @return true if reset, false if already been reset
+     */
+    public boolean resetAfterDone()
+    {
+        if (resetAfterDone)
+            return false;
+        resetAfterDone = true;
+        swapCurrAndPrevPartition();
+        // only current is reset, prev is still needed.
+        currPartition().resetPartition();
+        unfiltered().resetUnfiltered();
+        return true;
+    }
+
+    DecoratedKey currentKey()
+    {
+        return currPartition.key();
+    }
+
+    DecoratedKey prevKey()
+    {
+        return prevPartition.key();
+    }
+
+    public PartitionDescriptor currPartition()
+    {
+        return currPartition;
+    }
+
+    public UnfilteredDescriptor unfiltered()
+    {
+        return unfiltered;
+    }
+
+    public long bytesReadSinceSnapshot()
+    {
+        long latestByteReadPosition = isEOF() ? uncompressedLength() : position();
+        long cursorBytesRead = latestByteReadPosition - bytesReadPositionSnapshot;
+        bytesReadPositionSnapshot = latestByteReadPosition;
+        return cursorBytesRead;
+    }
+
+    private String keyToString(DecoratedKey key)
+    {
+        String keyString;
+        try
+        {
+            keyString = ssTableReader().metadata().partitionKeyType.getString(key.getKey());
+        }
+        catch (Throwable t)
+        {
+            keyString = "[corrupt token="+key.getToken()+"]";
+        }
+        return keyString;
+    }
+
+    public void readRowHeader()
+    {
+        super.readRowHeader(unfiltered);
+        if (corruptedTombstoneValidationEnabled)
+            validateInvalidRowDeletion();
+    }
+
+    public void readTombstoneMarker()
+    {
+        super.readTombstoneMarker(unfiltered);
+
+        if (corruptedTombstoneValidationEnabled)
+            validateInvalidTombstoneDeletion();
+
+        boolean isStartBound = unfiltered.isStartBound();
+        if (isOpenRangeTombstonePresent && isStartBound)
+            corruptSSTable("Encountered an open range tombstone marker before the prev was closed: " + unfiltered);
+        if (!isOpenRangeTombstonePresent && !isStartBound)
+            corruptSSTable("Encountered an close/boundary range tombstone marker before an open one: " + unfiltered);
+        isOpenRangeTombstonePresent = isStartBound || unfiltered.isBoundary();
+        // TODO: can also add verification of open/close timestamp match
+    }
+
+    public void readStaticRowHeader()
+    {
+        super.readStaticRowHeader(unfiltered);
+        if (corruptedTombstoneValidationEnabled)
+            validateInvalidRowDeletion();
+    }
+
+    @Override
+    public int readCellHeader()
+    {
+        int state = super.readCellHeader();
+        if (corruptedTombstoneValidationEnabled)
+            validateInvalidCellDeletion();
+        return state;
+    }
+
+    private void validateInvalidTombstoneDeletion()
+    {
+        if (!unfiltered.deletionTime().validate()) {
+            UnfilteredValidation.handleInvalid(
+                ssTableReader().metadata(),
+                currPartition.key(),
+                ssTableReader(),
+                "rowDeletion="+currPartition.deletionTime().toString());
+        }
+        if (unfiltered.isBoundary() && !unfiltered.deletionTime2().validate()) {
+            UnfilteredValidation.handleInvalid(
+                ssTableReader().metadata(),
+                currPartition.key(),
+                ssTableReader(),
+                "rowDeletion2="+currPartition.deletionTime().toString());
+        }
+    }
+
+    private void validateInvalidCellDeletion()
+    {
+        ReusableLivenessInfo cellLiveness = cellCursor().cellLiveness;
+        long ldt = cellLiveness.localExpirationTime();
+        if (cellLiveness.ttl() < 0 || ldt == INVALID_DELETION_TIME || ldt < 0 || (cellLiveness.isExpiring() && ldt == NO_DELETION_TIME)) {
+            UnfilteredValidation.handleInvalid(
+            ssTableReader().metadata(),
+            currPartition.key(),
+            ssTableReader(),
+            "cellLiveness="+cellLiveness);
+        }
+    }
+
+    private void validateInvalidRowDeletion()
+    {
+        if (!unfiltered.deletionTime().validate()) {
+            UnfilteredValidation.handleInvalid(
+                ssTableReader().metadata(),
+                currPartition.key(),
+                ssTableReader(),
+                "rowDeletion="+currPartition.deletionTime().toString());
+        }
+        ReusableLivenessInfo livenessInfo = unfiltered.livenessInfo();
+        if (livenessInfo.isExpiring() && (livenessInfo.ttl() < 0 || livenessInfo.localExpirationTime() < 0)) {
+            UnfilteredValidation.handleInvalid(
+                ssTableReader().metadata(),
+                currPartition.key(),
+                ssTableReader(),
+                "rowLiveness="+livenessInfo.toString());
+        }
+
+    }
+
+    private void validateInvalidPartitionDeletion()
+    {
+        if (!currPartition.deletionTime().validate()) {
+            UnfilteredValidation.handleInvalid(
+                ssTableReader().metadata(),
+                currPartition.key(),
+                ssTableReader(),
+                "partitionLevelDeletion="+currPartition.deletionTime().toString());
+        }
+    }
+}

src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java

@@ -40,7 +40,6 @@ import org.apache.cassandra.io.sstable.SSTableRewriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SSTableWriter;
 import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
@@ -69,7 +68,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
     private final List<PartitionPosition> diskBoundaries;
     private int locationIndex;
     protected Directories.DataDirectory currentDirectory;
-
+    protected String sstableDirectoryPath;
     public CompactionAwareWriter(ColumnFamilyStore cfs,
                                  Directories directories,
                                  ILifecycleTransaction txn,
@@ -151,9 +150,10 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
         return realAppend(partition);
     }
 
-    public final File getSStableDirectory() throws IOException
+    // hot path, called per partition
+    public final String getSStableDirectoryPath() throws IOException
     {
-        return getDirectories().getLocationForDisk(currentDirectory);
+        return sstableDirectoryPath;
     }
 
     @Override
@@ -173,35 +173,36 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
      * specific strategy has decided a new sstable is needed.
      * Guaranteed to be called before the first call to realAppend.
      */
-    protected void maybeSwitchWriter(DecoratedKey key)
+    public final SSTableWriter maybeSwitchWriter(DecoratedKey key)
     {
-        if (maybeSwitchLocation(key))
-            return;
-
-        if (shouldSwitchWriterInCurrentLocation(key))
-            switchCompactionWriter(currentDirectory, key);
+        SSTableWriter newWriter = maybeSwitchLocation(key);
+        if (newWriter == null && shouldSwitchWriterInCurrentLocation(key))
+        {
+            newWriter = switchCompactionWriter(currentDirectory, key);
+        }
+        return newWriter;
     }
 
     /**
      * Switches the file location and writer and returns true if the new key should be placed in a different data
      * directory.
      */
-    protected boolean maybeSwitchLocation(DecoratedKey key)
+    private SSTableWriter maybeSwitchLocation(DecoratedKey key)
     {
         if (diskBoundaries == null)
         {
             if (locationIndex < 0)
             {
                 Directories.DataDirectory defaultLocation = getWriteDirectory(nonExpiredSSTables, getExpectedWriteSize());
-                switchCompactionWriter(defaultLocation, key);
+                SSTableWriter writer = switchCompactionWriter(defaultLocation, key);
                 locationIndex = 0;
-                return true;
+                return writer;
             }
-            return false;
+            return null;
         }
 
         if (locationIndex > -1 && key.compareTo(diskBoundaries.get(locationIndex)) < 0)
-            return false;
+            return null;
 
         int prevIdx = locationIndex;
         while (locationIndex == -1 || key.compareTo(diskBoundaries.get(locationIndex)) > 0)
@@ -209,8 +210,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
         Directories.DataDirectory newLocation = locations.get(locationIndex);
         if (prevIdx >= 0)
             logger.debug("Switching write location from {} to {}", locations.get(prevIdx), newLocation);
-        switchCompactionWriter(newLocation, key);
-        return true;
+        return switchCompactionWriter(newLocation, key);
     }
 
     /**
@@ -223,14 +223,14 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
      * Implementations of this method should finish the current sstable writer and start writing to this directory.
      * <p>
      * Called once before starting to append and then whenever we see a need to start writing to another directory.
-     *
-     * @param directory
-     * @param nextKey
      */
-    protected void switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+    protected SSTableWriter switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
     {
         currentDirectory = directory;
-        sstableWriter.switchWriter(sstableWriter(directory, nextKey));
+        sstableDirectoryPath = getDirectories().getLocationForDisk(currentDirectory).path();
+        SSTableWriter newWriter = sstableWriter(directory, nextKey);
+        sstableWriter.switchWriter(newWriter);
+        return newWriter;
     }
 
     protected SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)

src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java

@@ -26,6 +26,7 @@ import org.apache.cassandra.db.compaction.LeveledManifest;
 import org.apache.cassandra.db.lifecycle.ILifecycleTransaction;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
 
 public class MajorLeveledCompactionWriter extends CompactionAwareWriter
 {
@@ -87,12 +88,12 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     }
 
     @Override
-    public void switchCompactionWriter(Directories.DataDirectory location, DecoratedKey nextKey)
+    public SSTableWriter switchCompactionWriter(Directories.DataDirectory location, DecoratedKey nextKey)
     {
         averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1));
         partitionsWritten = 0;
         sstablesWritten = 0;
-        super.switchCompactionWriter(location, nextKey);
+        return super.switchCompactionWriter(location, nextKey);
     }
 
     protected int sstableLevel()

src/java/org/apache/cassandra/db/guardrails/MaxThreshold.java

@@ -53,14 +53,14 @@ public class MaxThreshold extends Threshold
     }
 
     @Override
-    protected long failValue(ClientState state)
+    public long failValue(ClientState state)
     {
         long failValue = failThreshold.applyAsLong(state);
         return failValue <= 0 ? Long.MAX_VALUE : failValue;
     }
 
     @Override
-    protected long warnValue(ClientState state)
+    public long warnValue(ClientState state)
     {
         long warnValue = warnThreshold.applyAsLong(state);
         return warnValue <= 0 ? Long.MAX_VALUE : warnValue;

src/java/org/apache/cassandra/db/partitions/UnfilteredPartitionIterators.java

@@ -288,7 +288,7 @@ public abstract class UnfilteredPartitionIterators
     }
 
     /**
-     * Digests the the provided iterator.
+     * Digests the provided iterator.
      *
      * Caller must close the provided iterator.
      *

src/java/org/apache/cassandra/db/rows/BTreeRow.java

@@ -80,13 +80,15 @@ public class BTreeRow extends AbstractRow
     private static final Comparator<ColumnData> COLUMN_COMPARATOR = (cd1, cd2) -> cd1.column.compareTo(cd2.column);
 
 
-    // We need to filter the tombstones of a row on every read (twice in fact: first to remove purgeable tombstone, and then after reconciliation to remove
-    // all tombstone since we don't return them to the client) as well as on compaction. But it's likely that many rows won't have any tombstone at all, so
-    // we want to speed up that case by not having to iterate/copy the row in this case. We could keep a single boolean telling us if we have tombstones,
-    // but that doesn't work for expiring columns. So instead we keep the deletion time for the first thing in the row to be deleted. This allow at any given
-    // time to know if we have any deleted information or not. If we any "true" tombstone (i.e. not an expiring cell), this value will be forced to
-    // Long.MIN_VALUE, but if we don't and have expiring cells, this will the time at which the first expiring cell expires. If we have no tombstones and
-    // no expiring cells, this will be Cell.MAX_DELETION_TIME;
+    // We need to filter the tombstones of a row on every read (twice in fact: first to remove purgeable tombstone,
+    // and then after reconciliation to remove all tombstone since we don't return them to the client) as well as on
+    // compaction. But it's likely that many rows won't have any tombstone at all, so we want to speed up that case
+    // by not having to iterate/copy the row in this case. We could keep a single boolean telling us if we have
+    // tombstones, but that doesn't work for expiring columns. So instead we keep the deletion time for the first
+    // thing in the row to be deleted. This allows at any given time to know if we have any deleted information or not.
+    // If we have any "true" tombstone (i.e. not an expiring cell), this value will be forced to Long.MIN_VALUE,
+    // but if we don't and have expiring cells, this will the time at which the first expiring cell expires. If we
+    // have no tombstones and no expiring cells, this will be Cell.MAX_DELETION_TIME;
     private final long minLocalDeletionTime;
 
     private BTreeRow(Clustering clustering,

src/java/org/apache/cassandra/db/rows/Cell.java

@@ -258,7 +258,7 @@ public abstract class Cell<V> extends ColumnData
      * where not all field are always present (in fact, only the [ flags ] are guaranteed to be present). The fields have the following
      * meaning:
      *   - [ flags ] is the cell flags. It is a byte for which each bit represents a flag whose meaning is explained below (*_MASK constants)
-     *   - [ timestamp ] is the cell timestamp. Present unless the cell has the USE_TIMESTAMP_MASK.
+     *   - [ timestamp ] is the cell timestamp. Present unless the cell has the USE_ROW_TIMESTAMP_MASK.
      *   - [ deletion time]: the local deletion time for the cell. Present if either the cell is deleted (IS_DELETED_MASK)
      *       or it is expiring (IS_EXPIRING_MASK) but doesn't have the USE_ROW_TTL_MASK.
      *   - [ ttl ]: the ttl for the cell. Present if the row is expiring (IS_EXPIRING_MASK) but doesn't have the
@@ -269,13 +269,13 @@ public abstract class Cell<V> extends ColumnData
      *   - [ value ]: the cell value, unless it has the HAS_EMPTY_VALUE_MASK.
      *   - [ path ]: the cell path if the column this is a cell of is complex.
      */
-    static class Serializer
+    public static class Serializer
     {
-        private final static int IS_DELETED_MASK             = 0x01; // Whether the cell is a tombstone or not.
-        private final static int IS_EXPIRING_MASK            = 0x02; // Whether the cell is expiring.
-        private final static int HAS_EMPTY_VALUE_MASK        = 0x04; // Wether the cell has an empty value. This will be the case for tombstone in particular.
-        private final static int USE_ROW_TIMESTAMP_MASK      = 0x08; // Wether the cell has the same timestamp than the row this is a cell of.
-        private final static int USE_ROW_TTL_MASK            = 0x10; // Wether the cell has the same ttl than the row this is a cell of.
+        public final static int IS_DELETED_MASK             = 0x01; // Whether the cell is a tombstone or not.
+        public final static int IS_EXPIRING_MASK            = 0x02; // Whether the cell is expiring.
+        public final static int HAS_EMPTY_VALUE_MASK        = 0x04; // Wether the cell has an empty value. This will be the case for tombstone in particular.
+        public final static int USE_ROW_TIMESTAMP_MASK      = 0x08; // Wether the cell has the same timestamp than the row this is a cell of.
+        public final static int USE_ROW_TTL_MASK            = 0x10; // Wether the cell has the same ttl than the row this is a cell of.
 
         public <T> void serialize(Cell<T> cell, ColumnMetadata column, DataOutputPlus out, LivenessInfo rowLiveness, SerializationHeader header) throws IOException
         {
@@ -319,11 +319,11 @@ public abstract class Cell<V> extends ColumnData
         public <V> Cell<V> deserialize(DataInputPlus in, LivenessInfo rowLiveness, ColumnMetadata column, SerializationHeader header, DeserializationHelper helper, ValueAccessor<V> accessor) throws IOException
         {
             int flags = in.readUnsignedByte();
-            boolean hasValue = (flags & HAS_EMPTY_VALUE_MASK) == 0;
-            boolean isDeleted = (flags & IS_DELETED_MASK) != 0;
-            boolean isExpiring = (flags & IS_EXPIRING_MASK) != 0;
-            boolean useRowTimestamp = (flags & USE_ROW_TIMESTAMP_MASK) != 0;
-            boolean useRowTTL = (flags & USE_ROW_TTL_MASK) != 0;
+            boolean hasValue = hasValue(flags);
+            boolean isDeleted = isDeleted(flags);
+            boolean isExpiring = isExpiring(flags);
+            boolean useRowTimestamp = useRowTimestamp(flags);
+            boolean useRowTTL = useRowTTL(flags);
 
             long timestamp = useRowTimestamp ? rowLiveness.timestamp() : header.readTimestamp(in);
 
@@ -390,11 +390,11 @@ public abstract class Cell<V> extends ColumnData
         public boolean skip(DataInputPlus in, ColumnMetadata column, SerializationHeader header) throws IOException
         {
             int flags = in.readUnsignedByte();
-            boolean hasValue = (flags & HAS_EMPTY_VALUE_MASK) == 0;
-            boolean isDeleted = (flags & IS_DELETED_MASK) != 0;
-            boolean isExpiring = (flags & IS_EXPIRING_MASK) != 0;
-            boolean useRowTimestamp = (flags & USE_ROW_TIMESTAMP_MASK) != 0;
-            boolean useRowTTL = (flags & USE_ROW_TTL_MASK) != 0;
+            boolean hasValue = hasValue(flags);
+            boolean isDeleted = isDeleted(flags);
+            boolean isExpiring = isExpiring(flags);
+            boolean useRowTimestamp = useRowTimestamp(flags);
+            boolean useRowTTL = useRowTTL(flags);
 
             if (!useRowTimestamp)
                 header.skipTimestamp(in);
@@ -413,5 +413,30 @@ public abstract class Cell<V> extends ColumnData
 
             return true;
         }
+
+        public static boolean useRowTTL(int cellFlags)
+        {
+            return (cellFlags & USE_ROW_TTL_MASK) != 0;
+        }
+
+        public static boolean useRowTimestamp(int cellFlags)
+        {
+            return (cellFlags & USE_ROW_TIMESTAMP_MASK) != 0;
+        }
+
+        public static boolean isExpiring(int cellFlags)
+        {
+            return (cellFlags & IS_EXPIRING_MASK) != 0;
+        }
+
+        public static boolean isDeleted(int cellFlags)
+        {
+            return (cellFlags & IS_DELETED_MASK) != 0;
+        }
+
+        public static boolean hasValue(int cellFlags)
+        {
+            return (cellFlags & HAS_EMPTY_VALUE_MASK) == 0;
+        }
     }
 }

src/java/org/apache/cassandra/db/rows/Cells.java

@@ -36,7 +36,7 @@ public abstract class Cells
     private Cells() {}
 
     /**
-     * Collect statistics ont a given cell.
+     * Collect statistics on a given cell.
      *
      * @param cell the cell for which to collect stats.
      * @param collector the stats collector.

src/java/org/apache/cassandra/db/rows/RangeTombstoneMarker.java

@@ -159,7 +159,7 @@ public interface RangeTombstoneMarker extends Unfiltered, IMeasurableMemory
                 return DeletionTime.LIVE;
 
             DeletionTime biggestDeletionTime = openMarkers[biggestOpenMarker];
-            // it's only open in the merged iterator if it doesn't supersedes the partition level deletion
+            // it's only open in the merged iterator if it doesn't supersede the partition level deletion
             return !biggestDeletionTime.supersedes(partitionDeletion) ? DeletionTime.LIVE : biggestDeletionTime;
         }
 
@@ -172,7 +172,7 @@ public interface RangeTombstoneMarker extends Unfiltered, IMeasurableMemory
                     continue;
 
                 // Note that we can have boundaries that are both open and close, but in that case all we care about
-                // is what it the open deletion after the marker, so we favor the opening part in this case.
+                // is what is the open deletion after the marker, so we favor the opening part in this case.
                 if (marker.isOpen(reversed))
                     openMarkers[i] = marker.openDeletionTime(reversed);
                 else
@@ -192,7 +192,7 @@ public interface RangeTombstoneMarker extends Unfiltered, IMeasurableMemory
         {
             DeletionTime openMarker = currentOpenDeletionTimeInMerged();
             // We only have an open marker in the merged stream if it's not shadowed by the partition deletion (which can be LIVE itself), so
-            // if have an open marker, we know it's the "active" deletion for the merged stream.
+            // if we have an open marker, we know it's the "active" deletion for the merged stream.
             return openMarker.isLive() ? partitionDeletion : openMarker;
         }
     }

src/java/org/apache/cassandra/db/rows/UnfilteredSerializer.java

@@ -99,19 +99,19 @@ public class UnfilteredSerializer
     /*
      * Unfiltered flags constants.
      */
-    private final static int END_OF_PARTITION     = 0x01; // Signal the end of the partition. Nothing follows a <flags> field with that flag.
-    private final static int IS_MARKER            = 0x02; // Whether the encoded unfiltered is a marker or a row. All following markers applies only to rows.
-    private final static int HAS_TIMESTAMP        = 0x04; // Whether the encoded row has a timestamp (i.e. if row.partitionKeyLivenessInfo().hasTimestamp() == true).
-    private final static int HAS_TTL              = 0x08; // Whether the encoded row has some expiration info (i.e. if row.partitionKeyLivenessInfo().hasTTL() == true).
-    private final static int HAS_DELETION         = 0x10; // Whether the encoded row has some deletion info.
-    private final static int HAS_ALL_COLUMNS      = 0x20; // Whether the encoded row has all of the columns from the header present.
-    private final static int HAS_COMPLEX_DELETION = 0x40; // Whether the encoded row has some complex deletion for at least one of its columns.
-    private final static int EXTENSION_FLAG       = 0x80; // If present, another byte is read containing the "extended flags" above.
+    public final static int END_OF_PARTITION     = 0x01; // Signal the end of the partition. Nothing follows a <flags> field with that flag.
+    public final static int IS_MARKER            = 0x02; // Whether the encoded unfiltered is a marker or a row. All following markers applies only to rows.
+    public final static int HAS_TIMESTAMP        = 0x04; // Whether the encoded row has a timestamp (i.e. if row.partitionKeyLivenessInfo().hasTimestamp() == true).
+    public final static int HAS_TTL              = 0x08; // Whether the encoded row has some expiration info (i.e. if row.partitionKeyLivenessInfo().hasTTL() == true).
+    public final static int HAS_DELETION         = 0x10; // Whether the encoded row has some deletion info.
+    public final static int HAS_ALL_COLUMNS      = 0x20; // Whether the encoded row has all of the columns from the header present.
+    public final static int HAS_COMPLEX_DELETION = 0x40; // Whether the encoded row has some complex deletion for at least one of its columns.
+    public final static int EXTENSION_FLAG       = 0x80; // If present, another byte is read containing the "extended flags" above.
 
     /*
      * Extended flags
      */
-    private final static int IS_STATIC               = 0x01; // Whether the encoded row is a static. If there is no extended flag, the row is assumed not static.
+    public final static int IS_STATIC               = 0x01; // Whether the encoded row is a static. If there is no extended flag, the row is assumed not static.
     /**
      * A shadowable tombstone cannot replace a previous row deletion otherwise it could resurrect a
      * previously deleted cell not updated by a subsequent update, SEE CASSANDRA-11500
@@ -119,7 +119,7 @@ public class UnfilteredSerializer
      * @deprecated See CASSANDRA-11500
      */
     @Deprecated(since = "4.0")
-    private final static int HAS_SHADOWABLE_DELETION = 0x02; // Whether the row deletion is shadowable. If there is no extended flag (or no row deletion), the deletion is assumed not shadowable.
+    public final static int HAS_SHADOWABLE_DELETION = 0x02; // Whether the row deletion is shadowable. If there is no extended flag (or no row deletion), the deletion is assumed not shadowable.
 
     public void serialize(Unfiltered unfiltered, SerializationHelper helper, DataOutputPlus out, int version)
     throws IOException
@@ -220,14 +220,14 @@ public class UnfilteredSerializer
         LivenessInfo pkLiveness = row.primaryKeyLivenessInfo();
         Row.Deletion deletion = row.deletion();
 
-        if ((flags & HAS_TIMESTAMP) != 0)
+        if (hasTimestamp(flags))
             header.writeTimestamp(pkLiveness.timestamp(), out);
-        if ((flags & HAS_TTL) != 0)
+        if (hasTTL(flags))
         {
             header.writeTTL(pkLiveness.ttl(), out);
             header.writeLocalDeletionTime(pkLiveness.localExpirationTime(), out);
         }
-        if ((flags & HAS_DELETION) != 0)
+        if (hasDeletion(flags))
             header.writeDeletionTime(deletion.time(), out);
 
         if ((flags & HAS_ALL_COLUMNS) == 0)
@@ -251,7 +251,7 @@ public class UnfilteredSerializer
                     if (cd.column.isSimple())
                         Cell.serializer.serialize((Cell<?>) cd, column, out, pkLiveness, header);
                     else
-                        writeComplexColumn((ComplexColumnData) cd, column, (flags & HAS_COMPLEX_DELETION) != 0, pkLiveness, header, out);
+                        writeComplexColumn((ComplexColumnData) cd, column, hasComplexDeletion(flags), pkLiveness, header, out);
                 }
                 catch (IOException e)
                 {
@@ -412,7 +412,7 @@ public class UnfilteredSerializer
 
     public void writeEndOfPartition(DataOutputPlus out) throws IOException
     {
-        out.writeByte((byte)1);
+        out.writeByte((byte)END_OF_PARTITION);
     }
 
     public long serializedSizeEndOfPartition()
@@ -502,12 +502,12 @@ public class UnfilteredSerializer
             else
             {
                 assert !isStatic(extendedFlags); // deserializeStaticRow should be used for that.
-                if ((flags & HAS_DELETION) != 0)
+                if (hasDeletion(flags))
                 {
                     assert header.isForSSTable();
-                    boolean hasTimestamp = (flags & HAS_TIMESTAMP) != 0;
-                    boolean hasTTL = (flags & HAS_TTL) != 0;
-                    boolean deletionIsShadowable = (extendedFlags & HAS_SHADOWABLE_DELETION) != 0;
+                    boolean hasTimestamp = hasTimestamp(flags);
+                    boolean hasTTL = hasTTL(flags);
+                    boolean deletionIsShadowable = deletionIsShadowable(extendedFlags);
                     Clustering<byte[]> clustering = Clustering.serializer.deserialize(in, helper.version, header.clusteringTypes());
                     long nextPosition = in.readUnsignedVInt() + in.getFilePointer();
                     in.readUnsignedVInt(); // skip previous unfiltered size
@@ -572,12 +572,12 @@ public class UnfilteredSerializer
         try
         {
             boolean isStatic = isStatic(extendedFlags);
-            boolean hasTimestamp = (flags & HAS_TIMESTAMP) != 0;
-            boolean hasTTL = (flags & HAS_TTL) != 0;
-            boolean hasDeletion = (flags & HAS_DELETION) != 0;
-            boolean deletionIsShadowable = (extendedFlags & HAS_SHADOWABLE_DELETION) != 0;
-            boolean hasComplexDeletion = (flags & HAS_COMPLEX_DELETION) != 0;
-            boolean hasAllColumns = (flags & HAS_ALL_COLUMNS) != 0;
+            boolean hasTimestamp = hasTimestamp(flags);
+            boolean hasTTL = hasTTL(flags);
+            boolean hasDeletion = hasDeletion(flags);
+            boolean deletionIsShadowable = deletionIsShadowable(extendedFlags);
+            boolean hasComplexDeletion = hasComplexDeletion(flags);
+            boolean hasAllColumns = hasAllColumns(flags);
             Columns headerColumns = header.columns(isStatic);
 
             if (header.isForSSTable())
@@ -734,7 +734,17 @@ public class UnfilteredSerializer
 
     public static Unfiltered.Kind kind(int flags)
     {
-        return (flags & IS_MARKER) != 0 ? Unfiltered.Kind.RANGE_TOMBSTONE_MARKER : Unfiltered.Kind.ROW;
+        return isTombstoneMarker(flags) ? Unfiltered.Kind.RANGE_TOMBSTONE_MARKER : Unfiltered.Kind.ROW;
+    }
+
+    public static boolean isTombstoneMarker(int flags)
+    {
+        return (flags & IS_MARKER) != 0;
+    }
+
+    public static boolean isRow(int flags)
+    {
+        return (flags & IS_MARKER) == 0;
     }
 
     public static boolean isStatic(int extendedFlags)
@@ -742,7 +752,12 @@ public class UnfilteredSerializer
         return (extendedFlags & IS_STATIC) != 0;
     }
 
-    private static boolean isExtended(int flags)
+    public static boolean deletionIsShadowable(int extendedFlags)
+    {
+        return (extendedFlags & HAS_SHADOWABLE_DELETION) != 0;
+    }
+
+    public static boolean isExtended(int flags)
     {
         return (flags & EXTENSION_FLAG) != 0;
     }
@@ -756,4 +771,29 @@ public class UnfilteredSerializer
     {
         return row.isStatic() || row.deletion().isShadowable();
     }
+
+    public static boolean hasTTL(int flags)
+    {
+        return (flags & HAS_TTL) != 0;
+    }
+
+    public static boolean hasTimestamp(int flags)
+    {
+        return (flags & HAS_TIMESTAMP) != 0;
+    }
+
+    public static boolean hasAllColumns(int flags)
+    {
+        return (flags & HAS_ALL_COLUMNS) != 0;
+    }
+
+    public static boolean hasComplexDeletion(int flags)
+    {
+        return (flags & HAS_COMPLEX_DELETION) != 0;
+    }
+
+    public static boolean hasDeletion(int flags)
+    {
+        return (flags & HAS_DELETION) != 0;
+    }
 }

src/java/org/apache/cassandra/dht/ComparableObjectToken.java

@@ -21,7 +21,7 @@ abstract class ComparableObjectToken<C extends Comparable<C>> extends Token
 {
     private static final long serialVersionUID = 1L;
 
-    final C token;   // Package-private to allow access from subtypes, which should all reside in the dht package.
+    C token;   // Package-private to allow access from subtypes, which should all reside in the dht package.
 
     protected ComparableObjectToken(C token)
     {

src/java/org/apache/cassandra/dht/IPartitioner.java

@@ -41,6 +41,19 @@ public interface IPartitioner
         return DatabaseDescriptor.getPartitioner();
     }
 
+    /**
+     * @return a new instance of a reusable key
+     */
+    default ReusableDecoratedKey createReusableKey(int initialSize)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    default boolean supportsReusableKeys()
+    {
+        return false;
+    }
+
     /**
      * Transform key to object representation of the on-disk format.
      *

src/java/org/apache/cassandra/dht/LocalPartitioner.java

@@ -243,4 +243,38 @@ public class LocalPartitioner implements IPartitioner
     {
         return AccordBytesSplitter::new;
     }
+
+    private class ReusableLocalToken extends LocalToken
+    {
+        void setToken(ByteBuffer token)
+        {
+            this.token = token;
+        }
+    }
+
+    private class ReusableLocalKey extends ReusableDecoratedKey
+    {
+        public ReusableLocalKey(int initialSize)
+        {
+            super(new ReusableLocalToken(), initialSize);
+        }
+
+        @Override
+        protected void recalculateToken()
+        {
+            ((ReusableLocalToken)getToken()).setToken(key);
+        }
+    }
+
+    @Override
+    public ReusableDecoratedKey createReusableKey(int initialSize)
+    {
+        return new ReusableLocalKey(initialSize);
+    }
+
+    @Override
+    public boolean supportsReusableKeys()
+    {
+        return true;
+    }
 }

src/java/org/apache/cassandra/dht/Murmur3Partitioner.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -35,6 +36,7 @@ import com.google.common.primitives.Longs;
 
 import accord.primitives.Ranges;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.PartitionPosition;
 import org.apache.cassandra.db.PreHashedDecoratedKey;
 import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.marshal.AbstractType;
@@ -177,7 +179,7 @@ public class Murmur3Partitioner implements IPartitioner
     {
         static final long serialVersionUID = -5833580143318243006L;
 
-        public final long token;
+         long token;
 
         public LongToken(long token)
         {
@@ -320,6 +322,18 @@ public class Murmur3Partitioner implements IPartitioner
         return new LongToken(normalize(hash[0]));
     }
 
+    private long calculateTokenValue(ByteBuffer key, long[] hash)
+    {
+        if (key.remaining() == 0)
+        {
+            hash[0] = MINIMUM.token;
+            hash[1] = 0;
+            return MINIMUM.token;
+        }
+        populateHash(key, hash);
+        return normalize(hash[0]);
+    }
+
     @Override
     public boolean isFixedLength()
     {
@@ -386,10 +400,15 @@ public class Murmur3Partitioner implements IPartitioner
     private long[] getHash(ByteBuffer key)
     {
         long[] hash = new long[2];
-        MurmurHash.hash3_x64_128(key, key.position(), key.remaining(), 0, hash);
+        populateHash(key, hash);
         return hash;
     }
 
+    private void populateHash(ByteBuffer key, long[] hash)
+    {
+        MurmurHash.hash3_x64_128(key, key.position(), key.remaining(), 0, hash);
+    }
+
     public LongToken getRandomToken()
     {
         return getRandomToken(ThreadLocalRandom.current());
@@ -565,4 +584,79 @@ public class Murmur3Partitioner implements IPartitioner
     {
         return ignore -> splitter;
     }
+
+    private static class ReusableLongToken extends LongToken
+    {
+        public ReusableLongToken()
+        {
+            super(MINIMUM.token);
+        }
+
+        void setToken(long token)
+        {
+            this.token = token;
+        }
+    }
+
+    private static class ReusableMurmurKey extends ReusableDecoratedKey
+    {
+        protected final long[] hash = new long[2];
+        private long tokenValue = MINIMUM.token;
+        ReusableMurmurKey(int initialSize)
+        {
+            super(new ReusableLongToken(), initialSize);
+        }
+
+        protected void recalculateToken()
+        {
+            Token token = getToken();
+            ((ReusableLongToken) token).setToken(Murmur3Partitioner.instance.calculateTokenValue(key, hash));
+            tokenValue = token.getLongValue();
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            return (obj instanceof ReusableMurmurKey) ? equals((ReusableMurmurKey) obj) : super.equals(obj);
+        }
+
+        public boolean equals(ReusableMurmurKey obj)
+        {
+            if (this == obj)
+                return true;
+            if (obj == null)
+                return false;
+
+            if (tokenValue != obj.tokenValue)
+                return false;
+            return Arrays.equals(keyBytes, 0, keyLength, obj.keyBytes, 0, obj.getKeyLength()); // we compare faster than BB.equals for array backed BB
+        }
+
+        @Override
+        public int compareTo(PartitionPosition pos)
+        {
+            return (pos instanceof ReusableMurmurKey) ? compareTo((ReusableMurmurKey) pos) : super.compareTo(pos);
+        }
+
+        public int compareTo(ReusableMurmurKey obj)
+        {
+            if (this == obj)
+                return 0;
+
+            int cmp = Long.compare(tokenValue, obj.tokenValue);
+            return cmp == 0 ? Arrays.compareUnsigned(keyBytes, 0, keyLength, obj.keyBytes, 0, obj.keyLength) : cmp;
+        }
+    }
+
+    @Override
+    public ReusableDecoratedKey createReusableKey(int initialSize)
+    {
+        return new ReusableMurmurKey(initialSize);
+    }
+
+    @Override
+    public boolean supportsReusableKeys()
+    {
+        return true;
+    }
 }

src/java/org/apache/cassandra/dht/ReusableDecoratedKey.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.dht;
+
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.BufferDecoratedKey;
+import org.apache.cassandra.utils.ByteBufferUtil;
+
+public abstract class ReusableDecoratedKey extends BufferDecoratedKey
+{
+    protected int keyLength = 0;
+    protected byte[] keyBytes;
+
+    public ReusableDecoratedKey(Token token, int initialSize)
+    {
+        super(token, ByteBuffer.wrap(new byte[initialSize]).limit(0));
+        keyBytes = key.array();
+    }
+
+    abstract void recalculateToken();
+
+    public void copyKey(ByteBuffer newKey)
+    {
+        int length = newKey.remaining();
+        maybeResizeKey(length);
+        ByteBufferUtil.copyBytes(newKey, newKey.position(), key, 0, length);
+        keyLength = length;
+        key.limit(length);
+        recalculateToken();
+    }
+
+    /** WARNING: retains ref to external buffer */
+    public void shadowKey(ByteBuffer newKey, byte[] newKeyBytes, int newKeyLength)
+    {
+        key = newKey;
+        keyBytes = newKeyBytes;
+        keyLength = newKeyLength;
+        recalculateToken();
+    }
+
+    /** WARNING: retains ref to external buffer */
+    public void shadowKey(int newKeyLength)
+    {
+        keyLength = newKeyLength;
+        recalculateToken();
+    }
+
+    @Override
+    public int getKeyLength()
+    {
+        return keyLength;
+    }
+
+    public byte[] keyBytes()
+    {
+        return keyBytes;
+    }
+
+    public void reset()
+    {
+        keyLength = 0;
+        key.limit(0);
+        recalculateToken();
+    }
+
+    private void maybeResizeKey(int length)
+    {
+        int capacity = keyBytes.length;
+        if (capacity > length)
+            return;
+        keyBytes = new byte[Math.max(length, capacity * 2)];
+        key = ByteBuffer.wrap(keyBytes);
+    }
+}

src/java/org/apache/cassandra/io/sstable/AbstractSSTableIterator.java

@@ -500,7 +500,7 @@ public abstract class AbstractSSTableIterator<RIE extends AbstractRowIndexEntry>
             {
                 // We use a same reasoning as in handlePreSliceData regarding the strictness of the inequality below.
                 // We want to exclude deserialized unfiltered equal to end, because 1) we won't miss any rows since those
-                // woudn't be equal to a slice bound and 2) a end bound can be equal to a start bound
+                // wouldn't be equal to a slice bound and 2) a end bound can be equal to a start bound
                 // (EXCL_END(x) == INCL_START(x) for instance) and in that case we don't want to return start bound because
                 // it's fundamentally excluded. And if the bound is a  end (for a range tombstone), it means it's exactly
                 // our slice end, but in that  case we will properly close the range tombstone anyway as part of our "close

src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java

@@ -47,7 +47,7 @@ import org.apache.cassandra.service.ActiveRepairService;
 /**
  * Base class for the sstable writers used by CQLSSTableWriter.
  */
-abstract class AbstractSSTableSimpleWriter implements Closeable
+public abstract class AbstractSSTableSimpleWriter implements Closeable
 {
     protected final File directory;
     protected final TableMetadataRef metadata;
@@ -172,7 +172,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
         }
     }
 
-    PartitionUpdate.Builder getUpdateFor(ByteBuffer key) throws IOException
+    public PartitionUpdate.Builder getUpdateFor(ByteBuffer key) throws IOException
     {
         return getUpdateFor(metadata.get().partitioner.decorateKey(key));
     }

src/java/org/apache/cassandra/io/sstable/ClusteringDescriptor.java

@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ClusteringBound;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.ByteArrayAccessor;
+import org.apache.cassandra.io.util.DataInputBuffer;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.readUnfilteredClustering;
+
+public class ClusteringDescriptor extends ResizableByteBuffer
+{
+    private static final byte MAX_START_KIND = (byte) ClusteringBound.MAX_START.kind().ordinal();
+    private static final byte MIN_END_KIND = (byte) ClusteringBound.MIN_END.kind().ordinal();
+
+    static final byte EXCL_END_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.EXCL_END_BOUND.ordinal();
+    static final byte INCL_START_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.INCL_START_BOUND.ordinal();
+    static final byte INCL_END_EXCL_START_BOUNDARY_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.INCL_END_EXCL_START_BOUNDARY.ordinal();
+
+    static final byte STATIC_CLUSTERING_KIND = (byte)ClusteringPrefix.Kind.STATIC_CLUSTERING.ordinal();
+    static final byte ROW_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.CLUSTERING.ordinal();
+
+    static final byte EXCL_END_INCL_START_BOUNDARY_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.EXCL_END_INCL_START_BOUNDARY.ordinal();
+    static final byte INCL_END_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.INCL_END_BOUND.ordinal();
+    static final byte EXCL_START_BOUND_CLUSTERING_KIND = (byte) ClusteringPrefix.Kind.EXCL_START_BOUND.ordinal();
+
+    protected final AbstractType<?>[] clusteringTypes;
+    protected ClusteringPrefix.Kind clusteringKind;
+    protected byte clusteringKindEncoded;
+    protected int clusteringColumnsBound;
+
+    public ClusteringDescriptor(AbstractType<?>[] clusteringTypes)
+    {
+        this.clusteringTypes = clusteringTypes;
+    }
+
+    protected void loadClustering(RandomAccessReader dataReader, byte clusteringKind, int clusteringColumnsBound) throws IOException
+    {
+        set(ClusteringPrefix.Kind.values()[clusteringKind], clusteringKind, clusteringColumnsBound);
+        if (clusteringKind != STATIC_CLUSTERING_KIND)
+            readUnfilteredClustering(dataReader, clusteringTypes, this.clusteringColumnsBound, this);
+        else
+            resetBuffer();
+    }
+
+    public final ClusteringDescriptor resetMinEnd() {
+        set(MIN_END_KIND, 0);
+        resetBuffer();
+        return this;
+    }
+
+    public final ClusteringDescriptor resetMaxStart() {
+        set(MAX_START_KIND, 0);
+        resetBuffer();
+        return this;
+    }
+
+    public final boolean isMaxStart()
+    {
+        return clusteringKindEncoded == MAX_START_KIND && clusteringColumnsBound == 0;
+    }
+
+    public final void resetClustering()
+    {
+        set(ClusteringPrefix.Kind.CLUSTERING, 0);
+
+        resetBuffer();
+    }
+
+    public final void copy(ClusteringDescriptor newClustering)
+    {
+        set(newClustering.clusteringKind, newClustering.clusteringColumnsBound());
+        overwrite(newClustering.clusteringBytes(), newClustering.clusteringLength());
+    }
+
+    private void set(ClusteringPrefix.Kind clusteringKind, int clusteringColumnsBound) {
+        set(clusteringKind, (byte) clusteringKind.ordinal(), clusteringColumnsBound);
+    }
+
+    private void set(byte clusteringKindEncoded, int clusteringColumnsBound) {
+        set(ClusteringPrefix.Kind.values()[clusteringKindEncoded], clusteringKindEncoded, clusteringColumnsBound);
+    }
+
+    private void set(ClusteringPrefix.Kind clusteringKind, byte clusteringKindEncoded, int clusteringColumnsBound)
+    {
+        this.clusteringKindEncoded = clusteringKindEncoded;
+        this.clusteringKind = clusteringKind;
+        this.clusteringColumnsBound = clusteringColumnsBound;
+    }
+
+    // Expose and rename parent data
+    public final ByteBuffer clusteringBuffer() {
+        return buffer();
+    }
+
+    public final int clusteringLength() {
+        return length();
+    }
+
+    public final byte[] clusteringBytes() {
+        return bytes();
+    }
+
+    public final AbstractType<?>[] clusteringTypes()
+    {
+        return clusteringTypes;
+    }
+
+    public final byte clusteringKindEncoded() {
+        return clusteringKindEncoded;
+    }
+
+    public final ClusteringPrefix.Kind clusteringKind() {
+        return clusteringKind;
+    }
+
+    public final void clusteringKind(ClusteringPrefix.Kind kind)
+    {
+        clusteringKind = kind;
+        clusteringKindEncoded = (byte)kind.ordinal();
+    }
+
+    public final int clusteringColumnsBound() {
+        return clusteringColumnsBound;
+    }
+
+    public final boolean isStartBound()
+    {
+        return (clusteringKindEncoded == INCL_START_BOUND_CLUSTERING_KIND || clusteringKindEncoded == EXCL_START_BOUND_CLUSTERING_KIND);
+    }
+
+    public final boolean isEndBound()
+    {
+        return (clusteringKindEncoded == INCL_END_BOUND_CLUSTERING_KIND || clusteringKindEncoded == EXCL_END_BOUND_CLUSTERING_KIND);
+    }
+
+    public final boolean isBoundary()
+    {
+        return (clusteringKindEncoded == EXCL_END_INCL_START_BOUNDARY_CLUSTERING_KIND || clusteringKindEncoded == INCL_END_EXCL_START_BOUNDARY_CLUSTERING_KIND);
+    }
+
+    public final ClusteringPrefix<?> toClusteringPrefix(List<AbstractType<?>> clusteringTypesList) {
+        if (clusteringKindEncoded == ROW_CLUSTERING_KIND) {
+            return Clustering.serializer.deserialize(clusteringBuffer(), 0, clusteringTypesList);
+        }
+        else if (clusteringColumnsBound == 0) {
+            return ByteArrayAccessor.factory.bound(clusteringKind);
+        }
+        else {
+            byte[][] values;
+            try (DataInputBuffer buffer = new DataInputBuffer(clusteringBuffer(), true))
+            {
+                values = ClusteringPrefix.serializer.deserializeValuesWithoutSize(buffer, clusteringColumnsBound, 0, clusteringTypesList);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException("Reading from an in-memory buffer shouldn't trigger an IOException", e);
+            }
+            return ByteArrayAccessor.factory.boundOrBoundary(clusteringKind, values);
+        }
+    }
+}

src/java/org/apache/cassandra/io/sstable/EmptySSTableScanner.java

@@ -56,6 +56,12 @@ public class EmptySSTableScanner extends AbstractUnfilteredPartitionIterator imp
         return ImmutableSet.of(sstable);
     }
 
+    @Override
+    public boolean isFullRange()
+    {
+        return false;
+    }
+
     public long getCurrentPosition()
     {
         return 0;

src/java/org/apache/cassandra/io/sstable/ISSTableScanner.java

@@ -39,6 +39,7 @@ public interface ISSTableScanner extends UnfilteredPartitionIterator
     public long getCurrentPosition();
     public long getBytesScanned();
     public Set<SSTableReader> getBackingSSTables();
+    public boolean isFullRange();
 
     public static void closeAllAndPropagate(Collection<ISSTableScanner> scanners, Throwable throwable)
     {

src/java/org/apache/cassandra/io/sstable/IndexInfo.java

@@ -31,6 +31,7 @@ import org.apache.cassandra.io.sstable.format.Version;
 import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
 import org.apache.cassandra.io.util.DataInputPlus;
 import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.schema.TableMetadata;
 import org.apache.cassandra.utils.ObjectSizes;
 
 /**
@@ -87,6 +88,17 @@ public class IndexInfo
         return new IndexInfo.Serializer(version, header.clusteringTypes());
     }
 
+    public String toString(TableMetadata metadata)
+    {
+        return "IndexInfo{" +
+               "offset=" + offset +
+               ", width=" + width +
+               ", firstName=" + firstName.toString(metadata) +
+               ", lastName=" + lastName.toString(metadata) +
+               ", endOpenMarker=" + endOpenMarker +
+               '}';
+    }
+
     public static class Serializer implements ISerializer<IndexInfo>
     {
         // This is the default index size that we use to delta-encode width when serializing so we get better vint-encoding.

src/java/org/apache/cassandra/io/sstable/PartitionDescriptor.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime.ReusableDeletionTime;
+import org.apache.cassandra.dht.ReusableDecoratedKey;
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+public final class PartitionDescriptor extends ResizableByteBuffer
+{
+    private long position;
+    private final ReusableDeletionTime deletionTime = ReusableDeletionTime.live();
+    private final ReusableDecoratedKey decoratedKey;
+
+    public PartitionDescriptor(ReusableDecoratedKey decoratedKey)
+    {
+        this.decoratedKey = decoratedKey;
+        this.decoratedKey.shadowKey(buffer(), bytes(), length());
+    }
+
+    /**
+     *  Loads the following structure:
+     *  <pre>
+     *   struct partition_header header {
+     *     be16 key_length; // e.g. 8 if long
+     *     byte key[key_length];
+     *     struct deletion_time deletion_time {
+     *       be32 local_deletion_time;
+     *       be64 marked_for_delete_at;
+     *     };
+     *   };
+     *   </pre>
+     */
+    void load(RandomAccessReader dataReader, DeletionTime.Serializer serializer) throws IOException
+    {
+        position = dataReader.getPosition();
+
+        boolean resize = loadShortLength(dataReader);
+        if (!resize)
+        {
+            decoratedKey.shadowKey(length());
+        }
+        else
+        {
+            decoratedKey.shadowKey(buffer(), bytes(), length());
+        }
+        serializer.deserialize(dataReader, deletionTime);
+    }
+
+    public long position()
+    {
+        return position;
+    }
+
+    public DeletionTime deletionTime()
+    {
+        return deletionTime;
+    }
+
+    public DecoratedKey key()
+    {
+        return decoratedKey;
+    }
+
+    public ByteBuffer keyBuffer() {
+        return super.buffer();
+    }
+
+    public int keyLength() {
+        return super.length();
+    }
+
+    public byte[] keyBytes() {
+        return super.bytes();
+    }
+
+    public void resetPartition()
+    {
+        resetBuffer();
+        decoratedKey.reset();
+        deletionTime.resetLive();
+        position = 0;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "PartitionDescriptor{ key=" + decoratedKey +
+               ", position=" + position +
+               ", deletionTime=" + deletionTime +
+               '}';
+    }
+}

src/java/org/apache/cassandra/io/sstable/SSTableCursorKeyReader.java

@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import javax.annotation.concurrent.NotThreadSafe;
+
+
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.io.util.FileHandle;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.utils.Throwables;
+
+@NotThreadSafe
+public class SSTableCursorKeyReader implements AutoCloseable
+{
+    private final FileHandle indexFile;
+    private final RandomAccessReader indexFileReader;
+    private final long initialPosition;
+
+    public static class Entry extends ResizableByteBuffer
+    {
+        private long dataPosition = -1;
+        private long keyPosition = -1;
+
+        public void load(RandomAccessReader indexReader) throws IOException
+        {
+            keyPosition = indexReader.getFilePointer();
+            super.loadShortLength(indexReader);
+            if (length() != 0)
+            {
+                dataPosition = indexReader.readUnsignedVInt();
+                // skip row index entries
+                int size = indexReader.readUnsignedVInt32();
+                if (size > 0)
+                    indexReader.skipBytesFully(size);
+            }
+            else
+            {
+                dataPosition = -1;
+            }
+        }
+
+        public long dataPosition()
+        {
+            return dataPosition;
+        }
+
+        public long keyPosition()
+        {
+            return keyPosition;
+        }
+
+        public ByteBuffer getKey()
+        {
+            return buffer();
+        }
+    }
+
+    private SSTableCursorKeyReader(FileHandle indexFile,
+                                   RandomAccessReader indexFileReader)
+    {
+        this.indexFile = indexFile;
+        this.indexFileReader = indexFileReader;
+        this.initialPosition = indexFileReader.getFilePointer();
+    }
+
+    public static SSTableCursorKeyReader create(RandomAccessReader indexFileReader) throws IOException
+    {
+        return new SSTableCursorKeyReader(null, indexFileReader);
+    }
+
+    @SuppressWarnings({ "resource", "RedundantSuppression" }) // iFile and reader are closed in the BigTableKeyReader#close method
+    public static SSTableCursorKeyReader create(FileHandle indexFile) throws IOException
+    {
+        FileHandle iFile = null;
+        RandomAccessReader reader = null;
+        try
+        {
+            iFile = indexFile.sharedCopy();
+            reader = iFile.createReader();
+            return new SSTableCursorKeyReader(iFile, reader);
+        }
+        catch (RuntimeException ex)
+        {
+            Throwables.closeNonNullAndAddSuppressed(ex, reader, iFile);
+            throw ex;
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        FileUtils.closeQuietly(indexFileReader);
+        FileUtils.closeQuietly(indexFile);
+    }
+
+    public boolean advance(Entry entry) throws IOException
+    {
+        if (indexFileReader.isEOF())
+        {
+            return false;
+        }
+        entry.load(indexFileReader);
+        return true;
+    }
+
+    public boolean isExhausted()
+    {
+        return indexFileReader.isEOF();
+    }
+
+    public long indexPosition()
+    {
+        return indexFileReader.getFilePointer();
+    }
+
+    public void seek(long position) throws IOException
+    {
+        if (position > indexLength())
+            throw new IndexOutOfBoundsException("The requested position exceeds the index length");
+        indexFileReader.seek(position);
+    }
+
+    public long indexLength()
+    {
+        return indexFileReader.length();
+    }
+
+    public void reset() throws IOException
+    {
+        indexFileReader.seek(initialPosition);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("BigTable-SSTableCursorKeyReader(%s), indexPosition=%d", indexFile.path(), indexFileReader.getFilePointer());
+    }
+}

src/java/org/apache/cassandra/io/sstable/SSTableCursorReader.java

@@ -0,0 +1,891 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.io.util.ResizableByteBuffer;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.CellPath;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.RangeTombstoneMarker;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.Version;
+import org.apache.cassandra.io.util.DataInputPlus;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.tools.Util;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.io.sstable.SSTableCursorReader.State.*;
+
+public class SSTableCursorReader implements AutoCloseable
+{
+    private static final ColumnMetadata[] COLUMN_METADATA_TYPE = new ColumnMetadata[0];
+    private final boolean hasStaticColumns;
+
+    public interface State
+    {
+        /** start of file, after partition end but before EOF */
+        int PARTITION_START = 1;
+        int STATIC_ROW_START = 1 << 1;
+        int ROW_START = 1 << 2;
+        /** common to row/static row cells */
+        int CELL_HEADER_START = 1 << 3;
+        int CELL_VALUE_START = 1 << 4;
+        int CELL_END = 1 << 5;
+        int TOMBSTONE_START = 1 << 6;
+
+        /** common to rows/tombstones. Call continue(); for next unfiltered, or maybe partition end */
+        int UNFILTERED_END = 1 << 7;
+        /** at {@link UnfilteredSerializer#isEndOfPartition(int)} */
+        int PARTITION_END = 1 << 8;
+        /** EOF */
+        int DONE = 1 << 9;
+
+        /* Special case for seeking in file */
+        int SEEK = 1 << 10;
+        static boolean isState(int state, int mask) {
+            return (state & mask) != 0;
+        }
+    }
+
+    public class CellCursor {
+        public ReusableLivenessInfo rowLiveness;
+        public Columns columns;
+
+        public int columnsSize;
+        public int columnsIndex;
+        public int cellFlags;
+        public final ReusableLivenessInfo cellLiveness = new ReusableLivenessInfo();
+        public CellPath cellPath;
+        public AbstractType<?> cellType;
+        public ColumnMetadata cellColumn;
+        private ColumnMetadata[] columnsArray;
+        private AbstractType<?>[] cellTypeArray;
+
+        void init (Columns columns, ReusableLivenessInfo rowLiveness)
+        {
+            if (this.columns != columns)
+            {
+                // This will be a problem with changing columns
+                this.columns = columns;
+                columnsArray = columns.toArray(COLUMN_METADATA_TYPE);
+                cellTypeArray = new AbstractType<?>[columnsArray.length];
+                for (int i = 0; i < columnsArray.length; i++)
+                {
+                    ColumnMetadata cellColumn = columnsArray[i];
+                    cellTypeArray[i]  = serializationHeader.getType(cellColumn);
+                }
+                columnsSize = columns.size();
+            }
+            this.rowLiveness = rowLiveness;
+            columnsIndex = 0;
+            cellFlags = 0;
+            cellPath = null;
+            cellType = null;
+        }
+
+        public boolean hasNext()
+        {
+            return columnsIndex < columnsSize;
+        }
+
+        /**
+         * For Cell deserialization see {@link Cell.Serializer#deserialize}
+         *
+         * @return true if the cell has a value, false otherwise
+         */
+        boolean readCellHeader() throws IOException
+        {
+            if (!(columnsIndex < columnsSize)) throw new IllegalStateException();
+
+            // HOTSPOT: suprisingly expensive
+            int currIndex = columnsIndex++;
+            cellColumn = columnsArray[currIndex];
+            cellType = cellTypeArray[currIndex];
+            cellFlags = dataReader.readUnsignedByte();
+            // TODO: specialize common case where flags == HAS_VALUE | USE_ROW_TS?
+            boolean hasValue = Cell.Serializer.hasValue(cellFlags);
+            boolean isDeleted = Cell.Serializer.isDeleted(cellFlags);
+            boolean isExpiring = Cell.Serializer.isExpiring(cellFlags);
+            boolean useRowTimestamp = Cell.Serializer.useRowTimestamp(cellFlags);
+            boolean useRowTTL = Cell.Serializer.useRowTTL(cellFlags);
+
+            long timestamp = useRowTimestamp ? rowLiveness.timestamp() : serializationHeader.readTimestamp(dataReader);
+
+            long localDeletionTime = useRowTTL
+                                     ? rowLiveness.localExpirationTime()
+                                     : (isDeleted || isExpiring ? serializationHeader.readLocalDeletionTime(dataReader) : Cell.NO_DELETION_TIME);
+
+            int ttl = useRowTTL ? rowLiveness.ttl() : (isExpiring ? serializationHeader.readTTL(dataReader) : Cell.NO_TTL);
+            localDeletionTime = Cell.decodeLocalDeletionTime(localDeletionTime, ttl, deserializationHelper);
+
+            cellLiveness.reset(timestamp, ttl, localDeletionTime);
+            cellPath = cellColumn.isComplex()
+                            ? cellColumn.cellPathSerializer().deserialize(dataReader)
+                            : null;
+            return hasValue;
+        }
+    }
+
+    private final Ref<SSTableReader> ssTableReaderRef;
+    private final AbstractType<?>[] clusteringColumnTypes;
+    private final DeserializationHelper deserializationHelper;
+    private final SerializationHeader serializationHeader;
+
+    // need to be closed
+    private final SSTableReader ssTableReader;
+    private final RandomAccessReader dataReader;
+    private final DeletionTime.Serializer deletionTimeSerializer;
+
+    private final CellCursor staticRowCellCursor = new CellCursor();
+    private final CellCursor rowCellCursor = new CellCursor();
+
+
+    private CellCursor cellCursor;
+
+    // SHARED STATIC_ROW/ROW/TOMB
+    private int basicUnfilteredFlags = 0;
+    private int extendedFlags = 0;
+
+    private int state = PARTITION_START;
+
+    public static SSTableCursorReader fromDescriptor(Descriptor desc) throws IOException
+    {
+        TableMetadata metadata = Util.metadataFromSSTable(desc);
+        SSTableReader reader = SSTableReader.openNoValidation(null, desc, TableMetadataRef.forOfflineTools(metadata));
+        return new SSTableCursorReader(reader, metadata, reader.ref());
+    }
+
+    public SSTableCursorReader(SSTableReader reader)
+    {
+        this(reader, reader.metadata(), null);
+    }
+
+    private SSTableCursorReader(SSTableReader reader, TableMetadata metadata, Ref<SSTableReader> readerRef)
+    {
+        ssTableReader = reader;
+        ssTableReaderRef = readerRef;
+        Version version = reader.descriptor.version;
+        deletionTimeSerializer = DeletionTime.getSerializer(version);
+        ImmutableList<ColumnMetadata> clusteringColumns = metadata.clusteringColumns();
+        int clusteringColumnCount = clusteringColumns.size();
+        clusteringColumnTypes = new AbstractType<?>[clusteringColumnCount];
+        for (int i = 0; i < clusteringColumnTypes.length; i++)
+        {
+            clusteringColumnTypes[i] = clusteringColumns.get(i).type;
+        }
+        deserializationHelper = new DeserializationHelper(metadata, version.correspondingMessagingVersion(), DeserializationHelper.Flag.LOCAL, null);
+        serializationHeader = reader.header;
+
+        dataReader = reader.openDataReader();
+        hasStaticColumns = metadata.hasStaticColumns();
+    }
+
+    @Override
+    public void close()
+    {
+        dataReader.close();
+        if (ssTableReaderRef != null)
+            ssTableReaderRef.close();
+    }
+
+    private void resetOnPartitionStart()
+    {
+        basicUnfilteredFlags = 0;
+        extendedFlags = 0;
+    }
+
+    public int seekPartition(long position)
+    {
+        state = SEEK;
+        if (position == 0)
+        {
+            dataReader.seek(position);
+            state = PARTITION_START;
+        }
+        else {
+            // verify partition start is after a partition end marker
+            dataReader.seek(position - 1);
+            try
+            {
+                basicUnfilteredFlags = dataReader.readUnsignedByte();
+            }
+            catch (Exception e)
+            {
+                return corruptSSTable(e);
+            }
+            // end of partition
+            if (!UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags)) {
+                throw new IllegalArgumentException("Seeking to a partition at: " + position + " did not result in a valid state");
+            }
+            state = dataReader.isEOF() ? DONE : PARTITION_START;
+        }
+        resetOnPartitionStart();
+        return state;
+    }
+
+    public int seekUnfiltered(long position)
+    {
+        state = SEEK;
+        // partition elements (Unfiltered) have flags
+        dataReader.seek(position);
+        int state = 0;
+        try
+        {
+            state = checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+        }
+        catch (IOException e)
+        {
+            return corruptSSTable(e);
+        }
+        if (!isState(state , ROW_START | TOMBSTONE_START | DONE)) throw new IllegalStateException();
+        return state;
+    }
+
+    // struct partition {
+    //   struct partition_header header
+    //   optional<struct row> row
+    //   struct unfiltered unfiltereds[];
+    //};
+    public int readPartitionHeader(PartitionDescriptor header)
+    {
+        if (state != PARTITION_START) throw new IllegalStateException();
+        resetOnPartitionStart();
+        try
+        {
+            header.load(dataReader, deletionTimeSerializer);
+            return checkNextFlagsAfterPartitionStart(false);
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    // struct static_row {
+    //    byte flags;          // preloaded
+    //    byte extended_flags; // preloaded
+    //    varint row_body_size;
+    //    varint prev_unfiltered_size; // for backward traversing, ignored
+    //    optional<struct liveness_info> liveness_info;
+    //    optional<struct delta_deletion_time> deletion_time;
+    // ***  We read the columns in a separate method ***
+    //      optional<varint[]> missing_columns;
+    //      cell[] cells; // potentially only some
+    //};
+    public int readStaticRowHeader(UnfilteredDescriptor unfilteredDescriptor)
+    {
+        if (state != STATIC_ROW_START) throw new IllegalStateException();
+        try
+        {
+            unfilteredDescriptor.loadStaticRow(dataReader, serializationHeader, deserializationHelper, basicUnfilteredFlags, extendedFlags);
+        }
+        catch (IOException e)
+        {
+            return corruptSSTable(e);
+        }
+
+        staticRowCellCursor.init(unfilteredDescriptor.rowColumns(), unfilteredDescriptor.livenessInfo());
+        cellCursor = staticRowCellCursor;
+        if (!staticRowCellCursor.hasNext())
+        {
+            try
+            {
+                return checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+            }
+            catch (Exception e)
+            {
+                return corruptSSTable(e);
+            }
+        }
+        else
+        {
+            return state = State.CELL_HEADER_START;
+        }
+    }
+
+    public int copyCellValue(DataOutputPlus writer, byte[] buffer) throws IOException
+    {
+        if (state != CELL_VALUE_START) throw new IllegalStateException();
+        if (cellCursor.cellType == null) throw new IllegalStateException();
+        int length = cellCursor.cellType.valueLengthIfFixed();
+        copyCellContents(writer, buffer, length);
+
+        try
+        {
+            if (!cellCursor.hasNext())
+            {
+                try
+                {
+                    return checkNextFlagsAfterCellValuesEnd();
+                }
+                catch (Exception e)
+                {
+                    return corruptSSTable(e);
+                }
+            }
+            return state = State.CELL_END;
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    // TODO: move to cell cursor? maybe avoid copy through buffer?
+    private void copyCellContents(DataOutputPlus writer, byte[] transferBuffer, int length) throws IOException
+    {
+        if (length >= 0)
+        {
+            try
+            {
+                dataReader.readFully(transferBuffer, 0, length);
+            }
+            catch (Exception e)
+            {
+                corruptSSTable(e);
+            }
+            writer.write(transferBuffer, 0, length);
+        }
+        else
+        {
+            try
+            {
+                length = dataReader.readUnsignedVInt32();
+            }
+            catch (IOException e)
+            {
+                corruptSSTable(e);
+            }
+            if (length < 0)
+                corruptSSTable("Corrupt (negative) value length encountered");
+            writer.writeUnsignedVInt32(length);
+            int remaining = length;
+            while (remaining > 0)
+            {
+                int readLength = Math.min(remaining, transferBuffer.length);
+                try
+                {
+                    dataReader.readFully(transferBuffer, 0, readLength);
+                }
+                catch (Exception e)
+                {
+                    corruptSSTable(e);
+                }
+                writer.write(transferBuffer, 0, readLength);
+                remaining -= readLength;
+            }
+        }
+    }
+
+    // struct row {
+    //    byte flags;
+    //    optional<struct clustering_block[]> clustering_blocks;
+    //    varint row_body_size;
+    //    varint prev_unfiltered_size; // for backward traversing, ignored
+    //    optional<struct liveness_info> liveness_info;
+    //    optional<struct delta_deletion_time> deletion_time;
+    // ***  We read the columns in a separate step ***
+    //    optional<varint[]> missing_columns;
+    //    cell[] cells; // potentially only some
+    //};
+    public int readRowHeader(UnfilteredDescriptor unfilteredDescriptor)
+    {
+        if (state != State.ROW_START) throw new IllegalStateException();
+        if (!UnfilteredSerializer.isRow(basicUnfilteredFlags)) throw new IllegalStateException();
+        try
+        {
+            unfilteredDescriptor.loadRow(dataReader, serializationHeader, deserializationHelper, basicUnfilteredFlags);
+
+            rowCellCursor.init(unfilteredDescriptor.rowColumns(), unfilteredDescriptor.livenessInfo());
+            cellCursor = rowCellCursor;
+            if (!rowCellCursor.hasNext())
+            {
+                return checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+            }
+            else
+            {
+                return state = State.CELL_HEADER_START;
+            }
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    // TODO: introduce cell header class
+    public int readCellHeader()
+    {
+        if (state != State.CELL_HEADER_START) throw new IllegalStateException();
+        try
+        {
+            if (cellCursor.readCellHeader())
+            {
+                return state = State.CELL_VALUE_START;
+            }
+            if (!cellCursor.hasNext())
+                return checkNextFlagsAfterCellValuesEnd();
+            return state = State.CELL_END;
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    public int skipCellValue()
+    {
+        if (state != State.CELL_VALUE_START) throw new IllegalStateException();
+        try
+        {
+            cellCursor.cellType.skipValue(dataReader);
+            return !cellCursor.hasNext() ? checkNextFlagsAfterCellValuesEnd() : (state = State.CELL_HEADER_START);
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    /**
+     * See: {@link org.apache.cassandra.db.rows.UnfilteredSerializer#serialize(RangeTombstoneMarker, SerializationHelper, DataOutputPlus, long, int)}
+     * <pre>
+     * struct range_tombstone_marker {
+     *   byte flags = IS_MARKER;
+     *   byte kind_ordinal;
+     *   be16 bound_values_count;
+     *   struct clustering_block[] clustering_blocks;
+     *   varint marker_body_size;
+     *   varint prev_unfiltered_size;
+     * };
+     * struct range_tombstone_bound_marker : range_tombstone_marker {
+     *   struct delta_deletion_time deletion_time;
+     * };
+     * struct range_tombstone_boundary_marker : range_tombstone_marker {
+     *   struct delta_deletion_time end_deletion_time;
+ *       struct delta_deletion_time start_deletion_time;
+     * };
+     * </pre>
+     *
+     */
+    public int readTombstoneMarker(UnfilteredDescriptor unfilteredDescriptor)
+    {
+        try
+        {
+            if (state != TOMBSTONE_START) throw new IllegalStateException();
+            if (!UnfilteredSerializer.isTombstoneMarker(basicUnfilteredFlags)) throw new IllegalStateException();
+            unfilteredDescriptor.loadTombstone(dataReader, serializationHeader, basicUnfilteredFlags);
+            return checkNextFlagsAfterStaticRowOrUnfilteredStart(false);
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+
+    /**
+     * {@link ClusteringPrefix.Serializer#deserializeValuesWithoutSize}
+     */
+    static void readUnfilteredClustering(RandomAccessReader dataReader, AbstractType<?>[] types, int clusteringColumnsBound, ResizableByteBuffer clustering) throws IOException
+    {
+        clustering.resetBuffer();
+        if (clusteringColumnsBound == 0) {
+            return;
+        }
+        long clusteringBlockHeader = 0;
+        int fixedLengthClusteringLength = 0;
+        for (int clusteringIndex = 0; clusteringIndex < clusteringColumnsBound; clusteringIndex++)
+        {
+            // struct clustering_block {
+            //    varint clustering_block_header;
+            //    simple_cell[] clustering_cells;
+            // };
+            if (clusteringIndex % 32 == 0)
+            {
+                if (fixedLengthClusteringLength != 0) {
+                    clustering.loadPart(dataReader, fixedLengthClusteringLength);
+                    fixedLengthClusteringLength = 0;
+                }
+                clusteringBlockHeader = dataReader.readUnsignedVInt();
+                clustering.writeUnsignedVInt(clusteringBlockHeader);
+            }
+
+            // load value if present
+            if ((clusteringBlockHeader & 0b11) == 0)
+            {
+                AbstractType<?> type = types[clusteringIndex];
+                if (type.isValueLengthFixed())
+                {
+                    fixedLengthClusteringLength += type.valueLengthIfFixed();
+                }
+                else
+                {
+                    if (fixedLengthClusteringLength != 0) {
+                        clustering.loadPart(dataReader, fixedLengthClusteringLength);
+                        fixedLengthClusteringLength = 0;
+                    }
+                    int varLength = dataReader.readUnsignedVInt32();
+                    clustering.writeUnsignedVInt(varLength);
+                    clustering.loadPart(dataReader, varLength);
+                }
+            }
+            clusteringBlockHeader = clusteringBlockHeader >>> 2;
+        }
+        if (fixedLengthClusteringLength != 0) clustering.loadPart(dataReader, fixedLengthClusteringLength);
+        if (clusteringBlockHeader != 0) {
+            throw new IOException("Clustering block upper bits (those not associated with keys) expected to be 0:" + clusteringBlockHeader);
+        }
+    }
+
+    private static void skipClustering(RandomAccessReader dataReader, AbstractType<?>[] types, int clusteringColumnsBound) throws IOException
+    {
+        long clusteringBlockHeader = 0;
+        for (int clusteringIndex = 0; clusteringIndex < clusteringColumnsBound; clusteringIndex++)
+        {
+            // struct clustering_block {
+            //    varint clustering_block_header;
+            //    simple_cell[] clustering_cells;
+            // };
+            if (clusteringIndex % 32 == 0)
+            {
+                clusteringBlockHeader = dataReader.readUnsignedVInt();
+            }
+            // skip value if present
+            if ((clusteringBlockHeader & 0b11) == 0)
+            {
+                AbstractType<?> type = types[clusteringIndex];
+                int len = type.isValueLengthFixed() ? type.valueLengthIfFixed() : dataReader.readUnsignedVInt32();
+                dataReader.skipBytes(len);
+            }
+            clusteringBlockHeader = clusteringBlockHeader >>> 2;
+        }
+        if (clusteringBlockHeader != 0) {
+            throw new IOException("Clustering block upper bits (those not associated with keys) expected to be 0:" + clusteringBlockHeader);
+        }
+    }
+
+    /**
+     * {@link UnfilteredSerializer#deserializeRowBody(DataInputPlus, SerializationHeader, DeserializationHelper, int, int, Row.Builder)}
+     */
+    static void readLivenessInfo(RandomAccessReader dataReader, SerializationHeader serializationHeader, DeserializationHelper deserializationHelper, int flags, ReusableLivenessInfo livenessInfo) throws IOException
+    {
+        long timestamp = LivenessInfo.NO_TIMESTAMP;
+        int ttl = LivenessInfo.NO_TTL;
+        long localExpirationTime = LivenessInfo.NO_EXPIRATION_TIME;
+        if (UnfilteredSerializer.hasTimestamp(flags))
+        {
+            // struct liveness_info {
+            //    varint64 delta_timestamp;
+            //    optional<varint32> delta_ttl;
+            //    optional<varint64> delta_local_deletion_time;
+            //};
+            timestamp = serializationHeader.readTimestamp(dataReader);
+            if (UnfilteredSerializer.hasTTL(flags))
+            {
+                ttl = serializationHeader.readTTL(dataReader);
+                localExpirationTime = Cell.decodeLocalDeletionTime(serializationHeader.readLocalDeletionTime(dataReader), ttl, deserializationHelper);
+            }
+        }
+        livenessInfo.reset(timestamp, ttl, localExpirationTime);
+    }
+
+    // SKIPPING
+    public int skipPartition()
+    {
+        if (state == PARTITION_END)
+            return continueReading();
+
+        if (state == PARTITION_START)
+        {
+            try
+            {
+                int partitionKeyLength = dataReader.readUnsignedShort();
+                dataReader.skipBytes(partitionKeyLength);
+
+                // PARTITION DELETION TIME
+                deletionTimeSerializer.skip(dataReader);
+                checkNextFlagsAfterPartitionStart(true);
+            }
+            catch (Exception e)
+            {
+                return corruptSSTable(e);
+            }
+        }
+        else if (!isState(state, STATIC_ROW_START | ROW_START | TOMBSTONE_START | PARTITION_END))
+        {
+            throw new IllegalStateException("Unexpected state: " + state);
+        }
+
+        while (!isState(state,PARTITION_START | DONE))
+        {
+            switch (state)
+            {
+                case STATIC_ROW_START:
+                    state = skipStaticRow(true);
+                    break;
+                case ROW_START:
+                case TOMBSTONE_START:
+                    state = skipUnfiltered(true);
+                    break;
+            }
+        }
+        return state;
+    }
+
+    public int skipStaticRow(boolean autoContinue)
+    {
+        if (state != State.STATIC_ROW_START) throw new IllegalStateException();
+
+        try
+        {
+            long rowSize = dataReader.readUnsignedVInt();
+            dataReader.seek(dataReader.getPosition() + rowSize);
+            return checkNextFlagsAfterStaticRowOrUnfilteredStart(autoContinue);
+        }
+        catch (IOException e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    public int skipUnfiltered(boolean autoContinue)
+    {
+        if (!isState(state, ROW_START | TOMBSTONE_START))
+            throw new IllegalStateException();
+
+        AbstractType<?>[] types = clusteringColumnTypes;
+        int clusteringColumnsBound = types.length;
+        // tombstone markers have `kind` & `clusteringColumnsBound`
+        try
+        {
+            if (!UnfilteredSerializer.isRow(basicUnfilteredFlags))
+            {
+                dataReader.readByte();// byte kind =
+                clusteringColumnsBound = dataReader.readUnsignedShort();
+            }
+            /**
+             * {@link org.apache.cassandra.db.ClusteringPrefix.Deserializer}
+             */
+            skipClustering(dataReader, types, clusteringColumnsBound);
+            // same for row/tombstone
+            long rowSize = dataReader.readUnsignedVInt();
+            dataReader.seek(dataReader.getPosition() + rowSize);
+
+            return checkNextFlagsAfterStaticRowOrUnfilteredStart(autoContinue);
+        }
+        catch (Exception e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    public int skipRowCells(long unfilteredDataStart, long unfilteredSize, boolean autoContinue)
+    {
+        if (!(isState(state,CELL_HEADER_START | CELL_VALUE_START | CELL_END))) throw new IllegalStateException();
+
+        try
+        {
+            dataReader.seek(unfilteredDataStart + unfilteredSize);
+            return checkNextFlagsAfterStaticRowOrUnfilteredStart(autoContinue);
+        }
+        catch (IOException e)
+        {
+            return corruptSSTable(e);
+        }
+    }
+
+    public int continueReading() {
+        // TODO: can be optimized by pre-calculating next state when the flags are read
+        switch (state)
+        {
+            case PARTITION_END:
+                state = dataReader.isEOF() ? DONE : PARTITION_START;
+                break;
+            case UNFILTERED_END:
+                if (UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags))
+                {
+                    state = PARTITION_END;
+                }
+                else
+                {
+                    state = UnfilteredSerializer.isRow(basicUnfilteredFlags) ? ROW_START : TOMBSTONE_START;
+                }
+                break;
+            case CELL_END:
+                if (cellCursor.hasNext())
+                {
+                    state = CELL_HEADER_START;
+                }
+                else
+                {
+                    state = UNFILTERED_END;
+                }
+                break;
+            default:
+                throw new IllegalStateException("Cannot continue reading in current state: " + state);
+        }
+        return state;
+    }
+
+    private int checkNextFlagsAfterPartitionStart(boolean autoContinue) throws IOException
+    {
+        long preFlagsPosition = dataReader.getPosition();
+        basicUnfilteredFlags = dataReader.readUnsignedByte();
+        if (UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags))
+        {
+            state = !autoContinue ? PARTITION_END :
+                                    dataReader.isEOF() ? DONE : PARTITION_START;
+        }
+        else if (UnfilteredSerializer.isExtended(basicUnfilteredFlags))
+        {
+            state = STATIC_ROW_START;
+            extendedFlags = dataReader.readUnsignedByte();
+            validateStaticRowFlags(preFlagsPosition);
+        }
+        else
+        {
+            state = UnfilteredSerializer.isRow(basicUnfilteredFlags) ? ROW_START : TOMBSTONE_START;
+        }
+        return state;
+    }
+
+    private void validateStaticRowFlags(long preFlagsPosition)
+    {
+        if (!UnfilteredSerializer.isStatic(extendedFlags))
+        {
+            corruptSSTable("Row at: " + preFlagsPosition + " has extended flags but is not static, extendedFlags: " + extendedFlags);
+        }
+        if (!UnfilteredSerializer.isRow(basicUnfilteredFlags))
+        {
+            corruptSSTable("Static row at: " + preFlagsPosition + " is not a row, flags: " + basicUnfilteredFlags);
+        }
+        if (!hasStaticColumns)
+        {
+            corruptSSTable("Row at: " + preFlagsPosition + " is static, but table has no static columns " + ssTableReader.metadata());
+        }
+        if (UnfilteredSerializer.deletionIsShadowable(extendedFlags))
+        {
+            throw new UnsupportedOperationException("Static row at: " + preFlagsPosition + " has deletionIsShadowable, which is deprecated since 4.0");
+        }
+    }
+
+    private int checkNextFlagsAfterStaticRowOrUnfilteredStart(boolean autoContinue) throws IOException
+    {
+        int flags = this.basicUnfilteredFlags = dataReader.readUnsignedByte();
+        if (UnfilteredSerializer.isExtended(flags))
+        {
+            corruptSSTable("Unexpected static row (flags=" + flags + ") mid-partition, at position: " + (dataReader.getPosition() - 1));
+        }
+
+        if (!autoContinue) {
+            return this.state = UNFILTERED_END;
+        }
+        else
+        {
+            return this.state = nextStateMidPartition(flags);
+        }
+    }
+
+    private int checkNextFlagsAfterCellValuesEnd() throws IOException
+    {
+        int flags = this.basicUnfilteredFlags = dataReader.readUnsignedByte();
+        if (UnfilteredSerializer.isExtended(flags))
+        {
+            corruptSSTable("Unexpected static row (flags=" + flags + ") mid-partition, at position: " + (dataReader.getPosition() - 1));
+        }
+        return this.state = CELL_END;
+    }
+
+    private int corruptSSTable(Exception e)
+    {
+        ssTableReader.markSuspect();
+        if (e instanceof CorruptSSTableException)
+            throw (CorruptSSTableException) e;
+
+        throw new CorruptSSTableException(e, ssTableReader.getFilename());
+    }
+
+    protected int corruptSSTable(String message)
+    {
+        return corruptSSTable(new IllegalStateException(message));
+    }
+
+    private int nextStateMidPartition(int basicUnfilteredFlags)
+    {
+        if (UnfilteredSerializer.isEndOfPartition(basicUnfilteredFlags))
+        {
+            return dataReader.isEOF() ? DONE : PARTITION_START;
+        }
+        else if (UnfilteredSerializer.isRow(basicUnfilteredFlags))
+        {
+            return ROW_START;
+        }
+        else
+        {
+            return TOMBSTONE_START;
+        }
+    }
+
+    public boolean isEOF() {
+        return state == DONE || dataReader.isEOF();
+    }
+
+    public int state()
+    {
+        return state;
+    }
+
+    public long position() {
+        return dataReader.getFilePointer();
+    }
+
+    public long uncompressedLength()
+    {
+        return ssTableReader.uncompressedLength();
+    }
+
+    public SSTableReader ssTableReader()
+    {
+        return ssTableReader;
+    }
+
+    public CellCursor cellCursor()
+    {
+        return cellCursor;
+    }
+}

src/java/org/apache/cassandra/io/sstable/SSTableCursorWriter.java

@@ -0,0 +1,689 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import com.google.common.primitives.Ints;
+
+import org.agrona.collections.IntArrayList;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ClusteringPrefix;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.DeletionTime;
+import org.apache.cassandra.db.DeletionTime.ReusableDeletionTime;
+import org.apache.cassandra.db.LivenessInfo;
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.guardrails.Guardrails;
+import org.apache.cassandra.db.guardrails.Threshold;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.Row;
+import org.apache.cassandra.db.rows.SerializationHelper;
+import org.apache.cassandra.db.rows.Unfiltered;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.io.FSWriteError;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SortedTableWriter;
+import org.apache.cassandra.io.sstable.format.big.BigFormatPartitionWriter;
+import org.apache.cassandra.io.sstable.format.big.BigTableWriter;
+import org.apache.cassandra.io.sstable.format.big.RowIndexEntry;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.util.DataOutputBuffer;
+import org.apache.cassandra.io.util.DataOutputPlus;
+import org.apache.cassandra.io.util.SequentialWriter;
+import org.apache.cassandra.schema.ColumnMetadata;
+import org.apache.cassandra.utils.BloomFilter;
+import org.apache.cassandra.utils.ByteArrayUtil;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.concurrent.Ref;
+
+import static org.apache.cassandra.db.rows.UnfilteredSerializer.*;
+
+public class SSTableCursorWriter implements AutoCloseable
+{
+    private static final UnfilteredSerializer SERIALIZER = UnfilteredSerializer.serializer;
+    private static final ColumnMetadata[] EMPTY_COL_META = new ColumnMetadata[0];
+    private final SortedTableWriter<?,?> ssTableWriter;
+    private final SequentialWriter dataWriter;
+    private final SortedTableWriter.AbstractIndexWriter indexWriter;
+    private final DeletionTime.Serializer deletionTimeSerializer;
+    private final MetadataCollector metadataCollector;
+    private final SerializationHeader serializationHeader;
+    /**
+     * See: {@link BloomFilter#reusableIndexes}
+     */
+    private final long[] reusableIndexes = new long[21];
+    private final boolean hasStaticColumns;
+
+    private long partitionStart;
+    // ROW contents, needed because of the order of writing and the var int fields
+    private int rowFlags; // discovered as we go along
+    private int rowExtendedFlags;
+    private final DataOutputBuffer rowHeaderBuffer = new DataOutputBuffer(); // holds the contents between FLAGS and SIZE
+    private final DataOutputBuffer rowBuffer = new DataOutputBuffer();
+    private final ReusableDeletionTime openMarker = ReusableDeletionTime.live();
+
+    private final ColumnMetadata[] staticColumns;
+    private final ColumnMetadata[] regularColumns;
+    private final IntArrayList missingColumns = new IntArrayList();
+    private ColumnMetadata[] columns; // points to static/regular
+    private int columnsWrittenCount = 0;
+    private int nextCellIndex = 0;
+    // Index info
+    private final DataOutputBuffer rowIndexEntries = new DataOutputBuffer();
+    private final IntArrayList rowIndexEntriesOffsets = new IntArrayList();
+    private final ClusteringDescriptor rowIndexEntryLastClustering;
+    private int indexBlockStartOffset;
+    private int rowIndexEntryOffset;
+    private final int indexBlockThreshold;
+
+    private SSTableCursorWriter(
+        Descriptor desc,
+        SortedTableWriter<?,?> ssTableWriter,
+        SequentialWriter dataWriter,
+        SortedTableWriter.AbstractIndexWriter indexWriter,
+        MetadataCollector metadataCollector,
+        SerializationHeader serializationHeader)
+    {
+        this.ssTableWriter = ssTableWriter;
+        this.dataWriter = dataWriter;
+        this.indexWriter = indexWriter;
+        this.deletionTimeSerializer = DeletionTime.getSerializer(desc.version);
+        this.metadataCollector = metadataCollector;
+        this.serializationHeader = serializationHeader;
+        hasStaticColumns = serializationHeader.hasStatic();
+        staticColumns = hasStaticColumns ? serializationHeader.columns(true).toArray(EMPTY_COL_META) : EMPTY_COL_META;
+        regularColumns = serializationHeader.columns(false).toArray(EMPTY_COL_META);
+        this.indexBlockThreshold = DatabaseDescriptor.getColumnIndexSize(BigFormatPartitionWriter.DEFAULT_GRANULARITY);
+        rowIndexEntryLastClustering = new ClusteringDescriptor(serializationHeader.clusteringTypes().toArray(AbstractType[]::new));
+    }
+
+    public SSTableCursorWriter(SortedTableWriter<?,?> ssTableWriter)
+    {
+        this(ssTableWriter.descriptor,
+             ssTableWriter,
+             ssTableWriter.dataWriter,
+             ssTableWriter.indexWriter,
+             ssTableWriter.metadataCollector,
+             ssTableWriter.partitionWriter.getHeader());
+    }
+
+    @Override
+    public void close()
+    {
+        SSTableReader finish = ssTableWriter.finish(false);
+        if (finish != null) {
+            Ref<SSTableReader> ref = finish.ref();
+            if (ref != null) ref.close();
+        }
+        ssTableWriter.close();
+    }
+
+    public long getPartitionStart()
+    {
+        return partitionStart;
+    }
+
+    public long getPosition()
+    {
+        return dataWriter.position();
+    }
+
+    public int writePartitionStart(byte[] partitionKey, int partitionKeyLength, DeletionTime partitionDeletionTime) throws IOException
+    {
+        rowIndexEntries.clear();
+        rowIndexEntriesOffsets.clear();
+        rowIndexEntryOffset = 0;
+        openMarker.resetLive();
+
+        partitionStart = dataWriter.position();
+        writePartitionHeader(partitionKey, partitionKeyLength, partitionDeletionTime);
+        updateIndexBlockStartOffset(dataWriter.position());
+        return indexBlockStartOffset;
+    }
+
+    public void writePartitionEnd(byte[] partitionKey, int partitionKeyLength, DeletionTime partitionDeletionTime, int headerLength) throws IOException
+    {
+        SERIALIZER.writeEndOfPartition(dataWriter);
+        long partitionEnd = dataWriter.position();
+        long partitionSize = partitionEnd - partitionStart;
+        addPartitionMetadata(partitionKey, partitionKeyLength, partitionSize, partitionDeletionTime);
+
+        /** {@link SortedTableWriter#endPartition(DecoratedKey, DeletionTime)}
+         lastWrittenKey = key; // tracked for verification, see {@link SortedTableWriter#verifyPartition(DecoratedKey)}, checking the key size and sorting
+         // first/last are retained for metadata {@link org.apache.cassandra.io.sstable.format.SSTableWriter#finalizeMetadata()}. They are also exposed via
+         // getters from the writer, but usage is unclear.
+         last = lastWrittenKey;
+         if (first == null)
+         first = lastWrittenKey;
+         // this is implemented differently for BIG/BTI
+         createRowIndexEntry(key, partitionLevelDeletion, partitionEnd - 1);
+         */
+        appendBIGIndex(partitionKey, partitionKeyLength, partitionStart, headerLength, partitionDeletionTime, partitionEnd);
+    }
+
+    private void appendBIGIndex(byte[] key, int keyLength, long partitionStart, int headerLength, DeletionTime partitionDeletionTime, long partitionEnd) throws IOException
+    {
+        /**
+         * {@link BigTableWriter#createRowIndexEntry(DecoratedKey, DeletionTime, long)}
+         * {@link BigTableWriter.IndexWriter#append(DecoratedKey, RowIndexEntry, long, ByteBuffer)}
+         *
+         */
+        BigTableWriter.IndexWriter indexWriter = (BigTableWriter.IndexWriter) this.indexWriter;
+        SequentialWriter indexFileWriter = indexWriter.writer;
+        ((BloomFilter)indexWriter.bf).add(key, 0, keyLength, reusableIndexes);
+        long indexStart = indexFileWriter.position();
+        try
+        {
+            ByteArrayUtil.writeWithShortLength(key, 0, keyLength, indexFileWriter);
+
+            indexFileWriter.writeUnsignedVInt(partitionStart);
+
+            // if the list of entries has one or fewer entries, no point in index entries.
+            /** See: {@link org.apache.cassandra.io.sstable.format.big.RowIndexEntry#create} */
+            if (rowIndexEntriesOffsets.size() <= 1)
+            {
+                /**
+                 * {@link RowIndexEntry#serialize(DataOutputPlus, ByteBuffer)}
+                 */
+                indexFileWriter.writeUnsignedVInt32(0); // size
+            }
+            else {
+                // add last block
+                long indexBlockSize = (partitionEnd - partitionStart - 1) - indexBlockStartOffset;
+                if (indexBlockSize != 0) {
+                    addIndexBlock(partitionEnd - 1, indexBlockSize);
+                }
+                // if we have intermeddiate index info elements we also need to serialize the partitionDeletionTime
+                /** {@link RowIndexEntry.IndexedEntry#serialize(DataOutputPlus, ByteBuffer) */
+                // size up to the offsets?
+                int endOfEntries = rowIndexEntries.getLength();
+                // Write the headerLength, partitionDeletionTime and rowIndexEntriesOffsets.size() after the entries,
+                // just to calculate size.
+                rowIndexEntries.writeUnsignedVInt((long)headerLength);
+                deletionTimeSerializer.serialize(partitionDeletionTime, rowIndexEntries);
+
+                rowIndexEntries.writeUnsignedVInt32(rowIndexEntriesOffsets.size()); // number of entries
+
+                // bytes until offsets
+                int entriesAndOffsetsSize = rowIndexEntries.getLength() + rowIndexEntriesOffsets.size() * 4;
+                assert entriesAndOffsetsSize > 0;
+                indexFileWriter.writeUnsignedVInt32(entriesAndOffsetsSize); // size != 0
+                // copy the header elements
+                indexFileWriter.write(rowIndexEntries.getData(), endOfEntries, rowIndexEntries.getLength() - endOfEntries);
+                indexFileWriter.write(rowIndexEntries.getData(), 0, endOfEntries);
+                for (int i = 0; i < rowIndexEntriesOffsets.size(); i++)
+                {
+                    int offset = rowIndexEntriesOffsets.get(i);
+                    indexFileWriter.writeInt(offset);
+                }
+            }
+        }
+        catch (IOException e)
+        {
+            throw new FSWriteError(e, indexFileWriter.getPath());
+        }
+        indexWriter.summary.maybeAddEntry(key, 0, keyLength, indexStart);
+    }
+
+    final long guardrailsPartitionSizeWarning = Guardrails.partitionSize.warnValue(null);
+    final long guardrailsPartitionTombstonesWarning = Guardrails.partitionTombstones.warnValue(null);
+
+    /**
+     *  update metadata like {@link SortedTableWriter#endPartition} and {@link SortedTableWriter#startPartition}
+     */
+    private void addPartitionMetadata(byte[] partitionKey, int partitionKeyLength, long partitionSize, DeletionTime partitionDeletionTime)
+    {
+        if (partitionSize > guardrailsPartitionSizeWarning)
+            guardPartitionThreshold(Guardrails.partitionSize, partitionKey, partitionKeyLength, partitionSize);
+
+        if (metadataCollector.totalTombstones > guardrailsPartitionTombstonesWarning)
+            guardPartitionThreshold(Guardrails.partitionTombstones, partitionKey, partitionKeyLength, metadataCollector.totalTombstones);
+
+        metadataCollector.updatePartitionDeletion(partitionDeletionTime);
+        metadataCollector.addPartitionSizeInBytes(partitionSize);
+        metadataCollector.addKey(partitionKey, 0, partitionKeyLength);
+        metadataCollector.addCellPerPartitionCount();
+    }
+
+    private void guardPartitionThreshold(Threshold guardrail, byte[] partitionKey, int partitionKeyLength, long size)
+    {
+        if (guardrail.triggersOn(size, null))
+        {
+            String message = String.format("%s.%s:%s on sstable %s",
+                    ssTableWriter.metadata().keyspace,
+                    ssTableWriter.metadata().name,
+                    ssTableWriter.metadata().partitionKeyType.getString(ByteBuffer.wrap(partitionKey, 0, partitionKeyLength)),
+                    ssTableWriter.getFilename());
+            guardrail.guard(size, message, true, null);
+        }
+    }
+
+    private void writePartitionHeader(byte[] partitionKey, int partitionKeyLength, DeletionTime partitionDeletionTime) throws IOException
+    {
+        dataWriter.writeShort(partitionKeyLength);
+        dataWriter.write(partitionKey, 0, partitionKeyLength);
+        deletionTimeSerializer.serialize(partitionDeletionTime, dataWriter);
+    }
+
+    public boolean writeEmptyStaticRow() throws IOException
+    {
+        if (!hasStaticColumns)
+            return false;
+        rowFlags = UnfilteredSerializer.EXTENSION_FLAG;
+        rowExtendedFlags = UnfilteredSerializer.IS_STATIC;
+        columns = staticColumns;
+        // TOD: we should be able to skip the use of the row buffers in this special case, maybe it doesn't matter
+        rowHeaderBuffer.clear();
+        // NOTE: if we are to write this value (which is not used), this is where we should compute it.
+        rowHeaderBuffer.writeUnsignedVInt32(0);
+        rowBuffer.clear();
+        columnsWrittenCount = 0;
+        missingColumns.clear();
+        writeRowEnd(null, false);
+
+        updateIndexBlockStartOffset(dataWriter.position());
+        return true;
+    }
+
+    public void writeRowStart(LivenessInfo livenessInfo, DeletionTime deletionTime, boolean isStatic) throws IOException
+    {
+        if (isStatic) {
+            rowFlags = UnfilteredSerializer.EXTENSION_FLAG;
+            rowExtendedFlags = UnfilteredSerializer.IS_STATIC;
+            columns = staticColumns;
+        }
+        else {
+            rowFlags = 0;
+            rowExtendedFlags = 0;
+            columns = regularColumns;
+        }
+        // NOTE: Data after this point needs a computed ahead of write size. This, combined with the cost of rewriting
+        // the size after the writing completes, means we have to buffer the row timestamps (most likely to differ in length)
+        // and the row columns data (will differ if they use their own timestamps, probably). Unfortunate.
+        // rest of header
+        rowHeaderBuffer.clear();
+        missingColumns.clear();
+        rowBuffer.clear();
+        columnsWrittenCount = 0;
+        nextCellIndex = 0;
+
+        // NOTE: if we are to write this value (which is not used), this is where we should compute it.
+        rowHeaderBuffer.writeUnsignedVInt32(0);
+
+        // copy TS/TTL/deletion data
+        rowFlags |= writeRowTimeData(livenessInfo, deletionTime, rowHeaderBuffer);
+    }
+
+    /**
+     * See {@link UnfilteredSerializer#serialize(Row, SerializationHelper, DataOutputPlus, long, int)}
+     */
+    private int writeRowTimeData(LivenessInfo livenessInfo, DeletionTime deletionTime, DataOutputPlus writer) throws IOException
+    {
+        int flags = 0;
+        boolean writtenLivenessMetadata = false;
+
+        if (!livenessInfo.isEmpty())
+        {
+            flags |= HAS_TIMESTAMP;
+            serializationHeader.writeTimestamp(livenessInfo.timestamp(), writer);
+            metadataCollector.update(livenessInfo);
+            writtenLivenessMetadata = true;
+        }
+        if (livenessInfo.isExpiring())
+        {
+            flags |= HAS_TTL;
+            serializationHeader.writeTTL(livenessInfo.ttl(), writer);
+            serializationHeader.writeLocalDeletionTime(livenessInfo.localExpirationTime(), writer);
+            if (!writtenLivenessMetadata) metadataCollector.update(livenessInfo);
+        }
+        if (!deletionTime.isLive())
+        {
+            flags |= HAS_DELETION;
+            writeDeletionTime(deletionTime, writer);
+        }
+
+        /**
+         * Metadata calls matching: {@link org.apache.cassandra.db.rows.Rows#collectStats}
+         * But the collection of data is conditional and the cell metadata is collected elsewhere.
+         */
+        return flags;
+    }
+
+    private void writeDeletionTime(DeletionTime deletionTime, DataOutputPlus writer) throws IOException
+    {
+        serializationHeader.writeDeletionTime(deletionTime, writer);
+        metadataCollector.update(deletionTime);
+    }
+
+    public void writeCellHeader(int cellFlags, ReusableLivenessInfo cellLiveness, ColumnMetadata cellColumn) throws IOException
+    {
+        for (; nextCellIndex < columns.length; nextCellIndex++) {
+            if (columns[nextCellIndex].compareTo(cellColumn) == 0)
+                break;
+            missingColumns.addInt(nextCellIndex);
+        }
+        if (nextCellIndex == columns.length)
+            throw new IllegalStateException("Column not found: " + cellColumn +" or cell writes out of order, or bug.");
+        nextCellIndex++;
+        writeCellHeader(cellFlags, cellLiveness, rowBuffer);
+    }
+
+    private void writeCellHeader(int cellFlags, ReusableLivenessInfo cellLiveness, DataOutputPlus writer) throws IOException
+    {
+        columnsWrittenCount++;
+        writer.writeByte(cellFlags);
+        if (!Cell.Serializer.useRowTimestamp(cellFlags)) {
+            long timestamp = cellLiveness.timestamp();
+            serializationHeader.writeTimestamp(timestamp, writer);
+        }
+        if (!Cell.Serializer.useRowTTL(cellFlags)) {
+            boolean isDeleted = Cell.Serializer.isDeleted(cellFlags);
+            boolean isExpiring = Cell.Serializer.isExpiring(cellFlags);
+            if (isDeleted || isExpiring) {
+                // TODO: is this conversion from LET to LDT correct?
+                serializationHeader.writeLocalDeletionTime(cellLiveness.localExpirationTime(), writer);
+            }
+            if (isExpiring) {
+                serializationHeader.writeTTL(cellLiveness.ttl(), writer);
+            }
+        }
+        /**
+         * matching {@link org.apache.cassandra.db.rows.Cells#collectStats};
+         */
+        metadataCollector.updateCellLiveness(cellLiveness);
+    }
+
+    public int writeCellValue(SSTableCursorReader cursor, byte[] copyColumnValueBuffer) throws IOException
+    {
+        return cursor.copyCellValue(rowBuffer, copyColumnValueBuffer);
+    }
+
+    public void writeCellValue(DataOutputBuffer tempCellBuffer) throws IOException
+    {
+        rowBuffer.write(tempCellBuffer.getData(), 0, tempCellBuffer.getLength());
+    }
+
+    public void writeRowEnd(UnfilteredDescriptor rHeader, boolean updateClusteringMetadata) throws IOException
+    {
+        boolean isExtended = isExtended(rowFlags);
+        boolean isStatic = isExtended && UnfilteredSerializer.isStatic(rowExtendedFlags);
+        int columnsLength = columns.length;
+        if (columnsWrittenCount == columnsLength)
+        {
+            rowFlags |= HAS_ALL_COLUMNS;
+        }
+        else if (columnsWrittenCount == 0) {
+            // Same as Columns.serializer.serializeSubset(Columns.NONE, serializationHeader.columns(isStatic), rowHeaderBuffer)
+            if (columnsLength < 64) {
+                // all the bits are set, because all the columns are missing, value is always positive
+                rowHeaderBuffer.writeUnsignedVInt(-1L >>> (64 - columnsLength));
+            }
+            else {
+                // no columns are present, nothing to write
+                rowHeaderBuffer.writeUnsignedVInt32(columnsLength);
+            }
+        }
+        else if (columnsWrittenCount < columnsLength)
+        {
+            for (; nextCellIndex < columnsLength; nextCellIndex++)
+                missingColumns.addInt(nextCellIndex);
+
+            if (columnsLength < 64) {
+                // set a bit for every missing column
+                long mask = 0;
+                for (int missingIndex : missingColumns) {
+                    mask |= (1L << missingIndex);
+                }
+                rowHeaderBuffer.writeUnsignedVInt(mask);
+            }
+            else {
+                encodeLargeColumnsSubset();
+            }
+        }
+        long unfilteredStartPosition = dataWriter.position();
+        /** See: {@link UnfilteredSerializer#serialize} */
+        dataWriter.writeByte(rowFlags);
+        if (isExtended)
+        {
+            dataWriter.writeByte(rowExtendedFlags);
+        }
+
+        if (!isStatic)
+        {
+            byte[] clustering = rHeader.clusteringBytes();
+            int clusteringLength = rHeader.clusteringLength();
+            dataWriter.write(clustering, 0, clusteringLength);
+        }
+
+        // Now that we know the size, write it + the rest of the data
+        dataWriter.writeUnsignedVInt32(rowHeaderBuffer.getLength() + rowBuffer.getLength());
+
+        dataWriter.write(rowHeaderBuffer.getData(), 0, rowHeaderBuffer.getLength());
+        dataWriter.write(rowBuffer.getData(), 0, rowBuffer.getLength());
+
+        long unfilteredEndPosition = getPosition();
+
+        /**
+         * Matching the: {@link org.apache.cassandra.db.rows.Rows#collectStats} along with above cell level metadata updates
+         */
+        metadataCollector.updateColumnSetPerRow(columnsWrittenCount);
+
+        if (isStatic)
+        {
+            updateIndexBlockStartOffset(dataWriter.position());
+        }
+        else
+        {
+            updateMetadataAndIndexBlock(rHeader, unfilteredStartPosition, unfilteredEndPosition, updateClusteringMetadata);
+        }
+    }
+
+    /**
+     * See: {@link org.apache.cassandra.io.sstable.format.SortedTableWriter#addRangeTomstoneMarker}
+     */
+    public void writeRangeTombstone(UnfilteredDescriptor rangeTombstone, boolean updateClusteringMetadata) throws IOException
+    {
+        int tombstoneKind = rangeTombstone.clusteringKindEncoded();
+        ClusteringPrefix.Kind kind = ClusteringPrefix.Kind.values()[tombstoneKind];
+        long unfilteredStartPosition = getPosition();
+        /** See: {@link org.apache.cassandra.db.rows.UnfilteredSerializer#serialize */
+        dataWriter.writeByte((byte)IS_MARKER);
+        /** See: {@link org.apache.cassandra.db.ClusteringBoundOrBoundary.Serializer#serialize} */
+        dataWriter.writeByte(tombstoneKind);
+        dataWriter.writeShort(rangeTombstone.clusteringColumnsBound());
+
+        int clusteringLength = rangeTombstone.clusteringLength();
+        if (clusteringLength != 0)
+        {
+            byte[] clustering = rangeTombstone.clusteringBytes();
+            dataWriter.write(clustering, 0, clusteringLength);
+        }
+        rowHeaderBuffer.clear();
+        // TODO: previousUnfilteredSize
+        rowHeaderBuffer.writeUnsignedVInt32(0);
+
+        if (kind.isBoundary())
+        {
+            writeDeletionTime(rangeTombstone.deletionTime(), rowHeaderBuffer);
+            writeDeletionTime(rangeTombstone.deletionTime2(), rowHeaderBuffer);
+            openMarker.reset(rangeTombstone.deletionTime2());
+        }
+        else
+        {
+            writeDeletionTime(rangeTombstone.deletionTime(), rowHeaderBuffer);
+            if (kind.isOpen(false))
+                openMarker.reset(rangeTombstone.deletionTime());
+            else
+                openMarker.resetLive();
+        }
+
+        dataWriter.writeUnsignedVInt32(rowHeaderBuffer.getLength());
+        dataWriter.write(rowHeaderBuffer.getData(), 0, rowHeaderBuffer.getLength());
+
+        long unfilteredEndPosition = getPosition();
+
+        /** {@link org.apache.cassandra.io.sstable.format.big.BigFormatPartitionWriter#addUnfiltered(Unfiltered)} */
+        // if we hit the index block size that we have to index after, go ahead and index it.
+        updateMetadataAndIndexBlock(rangeTombstone, unfilteredStartPosition, unfilteredEndPosition, updateClusteringMetadata);
+    }
+
+    private void updateMetadataAndIndexBlock(
+        UnfilteredDescriptor unfilteredDescriptor,
+        long unfilteredStartPosition,
+        long unfilteredEndPosition,
+        boolean updateClusteringMetadata) throws IOException
+    {
+        if (updateClusteringMetadata) updateClusteringMetadata(unfilteredDescriptor);
+        // write the first clustering into rowIndexEntries buffer (we will need it unless we never write the first entry)
+        if (unfilteredStartPosition == indexBlockStartOffset || (rowIndexEntryOffset == rowIndexEntries.position())) {
+            writeClusteringToRowIndexEntries(unfilteredDescriptor);
+        }
+        else
+        {
+            rowIndexEntryLastClustering.copy(unfilteredDescriptor);
+        }
+        /** {@link BigFormatPartitionWriter#addUnfiltered(Unfiltered)} */
+        // if we hit the index block size that we have to index after, go ahead and index it.
+        long indexBlockSize = currentOffsetInPartition(unfilteredEndPosition) - indexBlockStartOffset;
+        if (indexBlockSize >= this.indexBlockThreshold)
+            addIndexBlock(unfilteredEndPosition, indexBlockSize);
+    }
+
+    public void updateClusteringMetadata(UnfilteredDescriptor unfilteredDescriptor)
+    {
+        metadataCollector.updateClusteringValues(unfilteredDescriptor);
+    }
+
+    /**
+     *  See:
+     *  {@link BigFormatPartitionWriter#addIndexBlock()}
+     *  - {@link org.apache.cassandra.io.sstable.IndexInfo.Serializer#serialize(org.apache.cassandra.io.sstable.IndexInfo, org.apache.cassandra.io.util.DataOutputPlus)}
+     */
+    private void addIndexBlock(long endOfRowPosition, long indexBlockSize) throws IOException
+    {
+        if (rowIndexEntriesOffsets.isEmpty() && rowIndexEntryOffset != 0) {
+            throw new IllegalStateException();
+        }
+
+        // serialize the index info
+        /** {@link org.apache.cassandra.io.sstable.IndexInfo.Serializer#serialize(org.apache.cassandra.io.sstable.IndexInfo, org.apache.cassandra.io.util.DataOutputPlus)}*/
+        rowIndexEntriesOffsets.addInt(rowIndexEntryOffset);
+
+        // first clustering is already in, write last entry
+        if (rowIndexEntryLastClustering.clusteringLength() == 0) {
+            // first entry is the last entry, copy it
+            byte[] entriesData = rowIndexEntries.getData();
+            long endOfFirstEntry = rowIndexEntries.position();
+            rowIndexEntries.write(entriesData, rowIndexEntryOffset, (int) (endOfFirstEntry - rowIndexEntryOffset));
+        }
+        else
+        {
+            writeClusteringToRowIndexEntries(rowIndexEntryLastClustering);
+            rowIndexEntryLastClustering.resetClustering();
+        }
+        rowIndexEntries.writeUnsignedVInt((long)indexBlockStartOffset);
+        rowIndexEntries.writeVInt(indexBlockSize - IndexInfo.Serializer.WIDTH_BASE);
+
+        boolean isDeleteTimePresent = !openMarker.isLive();
+        rowIndexEntries.writeBoolean(isDeleteTimePresent);
+        if (isDeleteTimePresent)
+            deletionTimeSerializer.serialize(openMarker, rowIndexEntries);
+        // next block starts
+        rowIndexEntryOffset = Ints.checkedCast(rowIndexEntries.position());
+        updateIndexBlockStartOffset(endOfRowPosition);
+    }
+
+    private void updateIndexBlockStartOffset(long endOfRowPosition)
+    {
+        indexBlockStartOffset = (int) (endOfRowPosition - partitionStart);
+    }
+
+    private void writeClusteringToRowIndexEntries(ClusteringDescriptor clustering) throws IOException
+    {
+        ClusteringPrefix.Kind kind = clustering.clusteringKind();
+        rowIndexEntries.writeByte(kind.ordinal());
+        if (kind != ClusteringPrefix.Kind.CLUSTERING)
+            rowIndexEntries.writeShort(clustering.clusteringColumnsBound());
+        rowIndexEntries.write(clustering.clusteringBytes(), 0, clustering.clusteringLength());
+    }
+
+    private long currentOffsetInPartition(long position)
+    {
+        return position - partitionStart;
+    }
+
+    private void encodeLargeColumnsSubset() throws IOException
+    {
+        // no columns are present, nothing to write
+        rowHeaderBuffer.writeUnsignedVInt32(missingColumns.size());
+        if (missingColumns.size() > columns.length / 2)
+        {
+            // write present columns
+            int presentIndex = 0;
+            int missingIndex = 0;
+            for (int i = 0; i < missingColumns.size(); i++)
+            {
+                missingIndex = missingColumns.get(i);
+                for (; presentIndex < missingIndex; presentIndex++)
+                    rowHeaderBuffer.writeUnsignedVInt32(presentIndex);
+                presentIndex = missingIndex + 1;
+            }
+            if (missingIndex < columns.length-1) {
+                for (; presentIndex < missingIndex; presentIndex++)
+                    rowHeaderBuffer.writeUnsignedVInt32(presentIndex);
+            }
+        }
+        else
+        {
+            // write missing columns
+            for (int missingIndex : missingColumns) {
+                rowHeaderBuffer.writeUnsignedVInt32(missingIndex);
+            }
+        }
+    }
+
+    public void setLast(ByteBuffer key)
+    {
+        IPartitioner partitioner = ssTableWriter.getPartitioner();
+        DecoratedKey last = partitioner.decorateKey(ByteBufferUtil.clone(key));
+        ssTableWriter.setLast(last);
+    }
+
+    public void setFirst(ByteBuffer key)
+    {
+        IPartitioner partitioner = ssTableWriter.getPartitioner();
+        DecoratedKey first = partitioner.decorateKey(ByteBufferUtil.clone(key));
+        ssTableWriter.setFirst(first);
+        ssTableWriter.setLast(first);
+    }
+
+    public IPartitioner partitioner()
+    {
+        return ssTableWriter.getPartitioner();
+    }
+
+    public DeletionTime openMarker() {
+        return openMarker;
+    }
+}

src/java/org/apache/cassandra/io/sstable/SSTableSimpleUnsortedWriter.java

@@ -68,7 +68,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
     private final BlockingQueue<Buffer> writeQueue = newBlockingQueue(0);
     private final DiskWriter diskWriter = new DiskWriter();
 
-    SSTableSimpleUnsortedWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
+    public SSTableSimpleUnsortedWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
     {
         this(null, directory, metadata, columns, maxSSTableSizeInMiB);
     }
@@ -198,7 +198,7 @@ class SSTableSimpleUnsortedWriter extends AbstractSSTableSimpleWriter
         }
     }
 
-    static class SyncException extends RuntimeException
+    public static class SyncException extends RuntimeException
     {
         SyncException(IOException ioe)
         {

src/java/org/apache/cassandra/io/sstable/SSTableSimpleWriter.java

@@ -64,7 +64,7 @@ class SSTableSimpleWriter extends AbstractSSTableSimpleWriter
      * @param maxSSTableSizeInMiB defines the max SSTable size if the value is positive.
      *                            Any non-positive value indicates the sstable size is unlimited.
      */
-    protected SSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
+    public SSTableSimpleWriter(File directory, TableMetadataRef metadata, RegularAndStaticColumns columns, long maxSSTableSizeInMiB)
     {
         this(null, directory, metadata, columns, maxSSTableSizeInMiB);
     }

src/java/org/apache/cassandra/io/sstable/UnfilteredDescriptor.java

@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.cassandra.db.Columns;
+import org.apache.cassandra.db.DeletionTime.ReusableDeletionTime;
+import org.apache.cassandra.db.ReusableLivenessInfo;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.rows.DeserializationHelper;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.util.RandomAccessReader;
+
+public class UnfilteredDescriptor extends ClusteringDescriptor
+{
+    private final ReusableLivenessInfo rowLivenessInfo = new ReusableLivenessInfo();
+    private final ReusableDeletionTime deletionTime = ReusableDeletionTime.live();
+    private final ReusableDeletionTime deletionTime2 = ReusableDeletionTime.live();
+
+    private long position;
+    private int flags;
+    private int extendedFlags;
+
+    private long unfilteredSize;
+    private long unfilteredDataStart;
+    private long prevUnfilteredSize;
+    Columns rowColumns;
+
+    public UnfilteredDescriptor(AbstractType<?>[] clusteringTypes)
+    {
+        super(clusteringTypes);
+    }
+
+    void loadTombstone(RandomAccessReader dataReader,
+                       SerializationHeader serializationHeader,
+                       int flags) throws IOException
+    {
+        this.flags = flags;
+        this.extendedFlags = 0;
+        rowColumns = null;
+        byte clusteringKind = dataReader.readByte();
+        if (clusteringKind == STATIC_CLUSTERING_KIND || clusteringKind == ROW_CLUSTERING_KIND) {
+            // STATIC_CLUSTERING or CLUSTERING -> no deletion info, should not happen
+            throw new IllegalStateException();
+        }
+
+        int columnsBound = dataReader.readUnsignedShort();
+        loadClustering(dataReader, clusteringKind, columnsBound);
+        unfilteredSize = dataReader.readUnsignedVInt();
+        prevUnfilteredSize = dataReader.readUnsignedVInt(); // debug only, unused otherwise
+        if (clusteringKind == EXCL_END_INCL_START_BOUNDARY_CLUSTERING_KIND ||
+            clusteringKind == INCL_END_EXCL_START_BOUNDARY_CLUSTERING_KIND)
+        {
+            // boundary
+            // CLOSE
+            serializationHeader.readDeletionTime(dataReader, deletionTime);
+            // OPEN
+            serializationHeader.readDeletionTime(dataReader, deletionTime2);
+        }
+        else
+        {
+            // bound
+            // CLOSE|OPEN
+            serializationHeader.readDeletionTime(dataReader, deletionTime);
+        }
+    }
+
+    void loadRow(RandomAccessReader dataReader,
+                 SerializationHeader serializationHeader,
+                 DeserializationHelper deserializationHelper,
+                 int flags) throws IOException {
+        // body = whatever is covered by size, so inclusive of the prev_row_size inclusive of flags
+        position = dataReader.getPosition() - 1;
+        this.flags = flags;
+        this.extendedFlags = 0;
+
+        loadClustering(dataReader, ROW_CLUSTERING_KIND, this.clusteringTypes.length);
+
+        rowColumns = serializationHeader.columns(false);
+
+        loadCommonRowFields(dataReader, serializationHeader, deserializationHelper, flags);
+    }
+
+    void loadStaticRow(RandomAccessReader dataReader,
+                       SerializationHeader serializationHeader,
+                       DeserializationHelper deserializationHelper,
+                       int flags,
+                       int extendedFlags) throws IOException {
+        // body = whatever is covered by size, so inclusive of the prev_row_size inclusive of flags
+        position = dataReader.getPosition() - 2;
+        this.flags = flags;
+        this.extendedFlags = extendedFlags;
+        // no clustering
+        loadClustering(dataReader, STATIC_CLUSTERING_KIND, 0);
+        rowColumns = serializationHeader.columns(true);
+
+        loadCommonRowFields(dataReader, serializationHeader, deserializationHelper, flags);
+    }
+
+    private void loadCommonRowFields(RandomAccessReader dataReader,
+                                     SerializationHeader serializationHeader,
+                                     DeserializationHelper deserializationHelper,
+                                     int flags) throws IOException
+    {
+        unfilteredSize = dataReader.readUnsignedVInt();
+        unfilteredDataStart = dataReader.getPosition();
+        prevUnfilteredSize = dataReader.readUnsignedVInt(); // debug only, unused otherwise
+
+        SSTableCursorReader.readLivenessInfo(dataReader,
+                                             serializationHeader,
+                                             deserializationHelper,
+                                             flags,
+                                             rowLivenessInfo);
+
+        if (UnfilteredSerializer.hasDeletion(flags))
+        {
+            // struct delta_deletion_time {
+            //    varint delta_marked_for_delete_at;
+            //    varint delta_local_deletion_time;
+            //};
+            serializationHeader.readDeletionTime(dataReader, deletionTime);
+        }
+        else
+        {
+            deletionTime.resetLive();
+        }
+        if (!UnfilteredSerializer.hasAllColumns(flags))
+        {
+            // TODO: re-implement GC free
+            rowColumns = Columns.serializer.deserializeSubset(rowColumns, dataReader);
+        }
+    }
+
+    public void resetUnfiltered()
+    {
+        resetClustering();
+        position = 0;
+        flags = 0;
+        extendedFlags = 0;
+        unfilteredSize = 0;
+        unfilteredDataStart = 0;
+        prevUnfilteredSize = 0;
+        rowColumns = null;
+    }
+
+    public long position()
+    {
+        return position;
+    }
+
+    public ReusableLivenessInfo livenessInfo()
+    {
+        return rowLivenessInfo;
+    }
+
+    public ReusableDeletionTime deletionTime()
+    {
+        return deletionTime;
+    }
+
+    public ReusableDeletionTime openDeletionTime()
+    {
+        return isBoundary() ? deletionTime2 : isEndBound() ? null : deletionTime;
+    }
+
+    public ReusableDeletionTime deletionTime2()
+    {
+        return deletionTime2;
+    }
+
+    public int flags()
+    {
+        return flags;
+    }
+
+    public long size()
+    {
+        return unfilteredSize;
+    }
+
+    public long dataStart()
+    {
+        return unfilteredDataStart;
+    }
+
+    public Columns rowColumns()
+    {
+        return rowColumns;
+    }
+
+    @Override
+    public String toString()
+    {
+        return "UnfilteredDescriptor{" +
+               "rowLivenessInfo=" + rowLivenessInfo +
+               ", deletionTime=" + deletionTime +
+               ", position=" + position +
+               ", flags=" + flags +
+               ", extFlags=" + extendedFlags +
+               ", unfilteredSize=" + unfilteredSize +
+               ", prevUnfilteredSize=" + prevUnfilteredSize +
+               ", unfilteredDataStart=" + unfilteredDataStart +
+               ", rowColumns=" + rowColumns +
+               ", clusteringTypes=" + Arrays.toString(clusteringTypes()) +
+               '}';
+    }
+}

src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java

@@ -926,7 +926,16 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
     /**
      * Returns a {@link KeyReader} over all keys in the sstable.
      */
-    public abstract KeyReader keyReader() throws IOException;
+    public final KeyReader keyReader() throws IOException {
+        return keyReader(false);
+    }
+
+    /**
+     * Returns a {@link KeyReader} over all keys in the sstable.
+     *
+     * @param detailed should the iterator also provide details per partition entry(e.g. row entry details)
+     */
+    public abstract KeyReader keyReader(boolean detailed) throws IOException;
 
     /**
      * Returns a {@link KeyReader} over all keys in the sstable after a given key.

src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java

@@ -280,4 +280,12 @@ implements ISSTableScanner
             }
         }
     }
+
+    @Override
+    public boolean isFullRange()
+    {
+        return dataRange == null || (dataRange.startKey().equals(sstable.getFirst()) &&
+               dataRange.stopKey().equals(sstable.getLast()) &&
+               dataRange.isUnrestricted(sstable.metadata()));
+    }
 }

src/java/org/apache/cassandra/io/sstable/format/SSTableSimpleScanner.java

@@ -129,6 +129,13 @@ implements ISSTableScanner
         return ImmutableSet.of(sstable);
     }
 
+    @Override
+    public boolean isFullRange()
+    {
+        // hasNext will init start and end
+        return hasNext() && currentStartPosition == 0 && currentEndPosition == sstable.uncompressedLength();
+    }
+
     public TableMetadata metadata()
     {
         return sstable.metadata();

src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java

@@ -77,8 +77,8 @@ public abstract class SSTableWriter extends SSTable implements Transactional
     protected boolean isTransient;
     protected long maxDataAge = -1;
     protected final long keyCount;
-    protected final MetadataCollector metadataCollector;
-    protected final SerializationHeader header;
+    public final MetadataCollector metadataCollector;
+    public final SerializationHeader header;
     protected final List<SSTableFlushObserver> observers;
     protected final MmappedRegionsCache mmappedRegionsCache;
     protected final TransactionalProxy txnProxy = txnProxy();
@@ -334,7 +334,7 @@ public abstract class SSTableWriter extends SSTable implements Transactional
         }
     }
 
-    protected Map<MetadataType, MetadataComponent> finalizeMetadata()
+    protected final Map<MetadataType, MetadataComponent> finalizeMetadata()
     {
         return metadataCollector.finalizeMetadata(getPartitioner().getClass().getCanonicalName(),
                                                   metadata().params.bloomFilterFpChance,
@@ -595,4 +595,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional
             return new SSTableZeroCopyWriter(this, txn, owner);
         }
     }
+
+    public void setFirst(DecoratedKey key) {
+        first = key;
+    }
+
+    public void setLast(DecoratedKey key) {
+        last = key;
+    }
 }

src/java/org/apache/cassandra/io/sstable/format/SortedTablePartitionWriter.java

@@ -43,11 +43,11 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
     private final SerializationHelper helper;
     private final Version version;
 
-    private long previousRowStart;
-    private long initialPosition;
+    private long previousRowStartOffset;
+    private long partitionStartPosition;
     private long headerLength;
 
-    protected long startPosition;
+    protected long indexBlockStartOffset;
     protected int written;
 
     protected ClusteringPrefix<?> firstClustering;
@@ -78,9 +78,9 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
 
     protected void reset()
     {
-        this.initialPosition = writer.position();
-        this.startPosition = -1;
-        this.previousRowStart = 0;
+        this.partitionStartPosition = writer.position();
+        this.indexBlockStartOffset = -1;
+        this.previousRowStartOffset = 0;
         this.written = 0;
         this.firstClustering = null;
         this.lastClustering = null;
@@ -106,7 +106,7 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
 
         if (!header.hasStatic())
         {
-            this.headerLength = writer.position() - initialPosition;
+            this.headerLength = writer.position() - partitionStartPosition;
             state = State.AWAITING_ROWS;
             return;
         }
@@ -121,7 +121,7 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
 
         UnfilteredSerializer.serializer.serializeStaticRow(staticRow, helper, writer, version.correspondingMessagingVersion());
 
-        this.headerLength = writer.position() - initialPosition;
+        this.headerLength = writer.position() - partitionStartPosition;
         state = State.AWAITING_ROWS;
     }
 
@@ -129,21 +129,21 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
     {
         checkState(state == State.AWAITING_ROWS);
 
-        long pos = currentPosition();
+        long offset = currentOffsetInPartition();
 
         if (firstClustering == null)
         {
-            // Beginning of an index block. Remember the start and position
+            // Beginning of an index block. Remember the start clustering and position
             firstClustering = unfiltered.clustering();
-            startOpenMarker = openMarker;
-            startPosition = pos;
+            startOpenMarker = openMarker; // first entry is always LIVE (for BTI format)
+            indexBlockStartOffset = offset;
         }
 
         long unfilteredPosition = writer.position();
-        unfilteredSerializer.serialize(unfiltered, helper, writer, pos - previousRowStart, version.correspondingMessagingVersion());
+        unfilteredSerializer.serialize(unfiltered, helper, writer, offset - previousRowStartOffset, version.correspondingMessagingVersion());
 
         lastClustering = unfiltered.clustering();
-        previousRowStart = pos;
+        previousRowStartOffset = offset;
         ++written;
 
         if (unfiltered.kind() == Unfiltered.Kind.RANGE_TOMBSTONE_MARKER)
@@ -159,19 +159,30 @@ public abstract class SortedTablePartitionWriter implements AutoCloseable
 
         state = State.COMPLETED;
 
-        long endPosition = currentPosition();
+        long partitionLength = currentOffsetInPartition();
         unfilteredSerializer.writeEndOfPartition(writer);
 
-        return endPosition;
+        return partitionLength;
     }
 
-    protected long currentPosition()
+    protected long currentOffsetInPartition()
     {
-        return writer.position() - initialPosition;
+        return writer.position() - partitionStartPosition;
     }
 
-    public long getInitialPosition()
+    public long getPartitionStartPosition()
     {
-        return initialPosition;
+        return partitionStartPosition;
+    }
+
+    /** Some bullshit access for now */
+    public SerializationHeader getHeader()
+    {
+        return header;
+    }
+
+    public SerializationHelper getHelper()
+    {
+        return helper;
     }
 }

src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java

@@ -175,7 +175,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
     {
         try
         {
-            outputHandler.debug("Deserializing bloom filter for %s", sstable);
+            if (outputHandler.isDebugEnabled()) outputHandler.debug("Deserializing bloom filter for %s", sstable);
             deserializeBloomFilter(sstable);
         }
         catch (Throwable t)
@@ -217,7 +217,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
     protected int verifyOwnedRanges()
     {
         List<Range<Token>> ownedRanges = Collections.emptyList();
-        outputHandler.debug("Checking that all tokens are owned by the current node");
+        if (outputHandler.isDebugEnabled()) outputHandler.debug("Checking that all tokens are owned by the current node");
         try (KeyIterator iter = sstable.keyIterator())
         {
             ownedRanges = Range.normalize(tokenLookup.apply(cfs.metadata.keyspace));
@@ -287,7 +287,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
                     throw new CompactionInterruptedException(verifyInfo.getCompactionInfo());
 
                 long rowStart = dataFile.getFilePointer();
-                outputHandler.debug("Reading row at %d", rowStart);
+                if (outputHandler.isDebugEnabled()) outputHandler.debug("Reading row at %d", rowStart);
 
                 DecoratedKey key = null;
                 try
@@ -332,8 +332,11 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
 
                 long dataSize = nextRowPositionFromIndex - dataStartFromIndex;
                 // avoid an NPE if key is null
+                if (outputHandler.isDebugEnabled())
+                {
                     String keyName = key == null ? "(unreadable key)" : ByteBufferUtil.bytesToHex(key.getKey());
                     outputHandler.debug("row %s is %s", keyName, FBUtilities.prettyPrintMemory(dataSize));
+                }
 
                 try
                 {
@@ -352,7 +355,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
                     prevKey = key;
 
 
-                    outputHandler.debug("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex);
+                    if (outputHandler.isDebugEnabled()) outputHandler.debug("Row %s at %s valid, moving to next row at %s ", goodRows, rowStart, nextRowPositionFromIndex);
                     dataFile.seek(nextRowPositionFromIndex);
                 }
                 catch (Throwable th)
@@ -374,7 +377,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
     {
         try
         {
-            outputHandler.debug("Deserializing index for %s", sstable);
+            if (outputHandler.isDebugEnabled()) outputHandler.debug("Deserializing index for %s", sstable);
             deserializeIndex(sstable);
         }
         catch (Throwable t)
@@ -384,7 +387,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
         }
     }
 
-    private void deserializeIndex(SSTableReader sstable) throws IOException
+    protected void deserializeIndex(SSTableReader sstable) throws IOException
     {
         try (KeyReader it = sstable.keyReader())
         {

src/java/org/apache/cassandra/io/sstable/format/SortedTableWriter.java

@@ -79,9 +79,9 @@ public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I
 
     // TODO dataWriter is not needed to be directly accessible - we can access everything we need for the dataWriter
     //   from a partition writer
-    protected final SequentialWriter dataWriter;
-    protected final I indexWriter;
-    protected final P partitionWriter;
+    public final SequentialWriter dataWriter;
+    public final I indexWriter;
+    public final P partitionWriter;
     private final FileHandle.Builder dataFileBuilder = new FileHandle.Builder(descriptor.fileFor(Components.DATA));
     private DecoratedKey lastWrittenKey;
     private DataPosition dataMark;
@@ -237,13 +237,15 @@ public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I
 
     private AbstractRowIndexEntry endPartition(DecoratedKey key, DeletionTime partitionLevelDeletion) throws IOException
     {
+        // partitionLength not inclusive of last byte for BIG/default impl, but something different for BTI (and maybe other impls)
         long finishResult = partitionWriter.finish();
 
         long endPosition = dataWriter.position();
-        long rowSize = endPosition - partitionWriter.getInitialPosition();
-        guardPartitionThreshold(Guardrails.partitionSize, key, rowSize);
+        // inclusive of last byte
+        long partitionSize = endPosition - partitionWriter.getPartitionStartPosition();
+        guardPartitionThreshold(Guardrails.partitionSize, key, partitionSize);
         guardPartitionThreshold(Guardrails.partitionTombstones, key, metadataCollector.totalTombstones);
-        metadataCollector.addPartitionSizeInBytes(rowSize);
+        metadataCollector.addPartitionSizeInBytes(partitionSize);
         metadataCollector.addKey(key.getKey());
         metadataCollector.addCellPerPartitionCount();
 
@@ -260,7 +262,7 @@ public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I
     protected void onStartPartition(DecoratedKey key)
     {
         if (hasObservers())
-            notifyObservers(o -> o.startPartition(key, partitionWriter.getInitialPosition(), partitionWriter.getInitialPosition()));
+            notifyObservers(o -> o.startPartition(key, partitionWriter.getPartitionStartPosition(), partitionWriter.getPartitionStartPosition()));
     }
 
     protected void onStaticRow(Row row)
@@ -331,7 +333,7 @@ public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I
     }
 
     @Override
-    public long getFilePointer()
+    public final long getFilePointer()
     {
         return dataWriter.position();
     }
@@ -435,13 +437,13 @@ public abstract class SortedTableWriter<P extends SortedTablePartitionWriter, I
         }
     }
 
-    protected static abstract class AbstractIndexWriter extends AbstractTransactional implements Transactional
+    public static abstract class AbstractIndexWriter extends AbstractTransactional implements Transactional
     {
         protected final Descriptor descriptor;
         protected final TableMetadataRef metadata;
         protected final Set<Component> components;
 
-        protected final IFilter bf;
+        public final IFilter bf;
 
         protected AbstractIndexWriter(Builder<?, ?, ?, ?> b)
         {

src/java/org/apache/cassandra/io/sstable/format/big/BigFormatPartitionWriter.java

@@ -48,11 +48,11 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
     @VisibleForTesting
     public static final int DEFAULT_GRANULARITY = 64 * 1024;
 
-    // used, if the row-index-entry reaches config column_index_cache_size
-    private DataOutputBuffer buffer;
-    // used to track the size of the serialized size of row-index-entry (unused for buffer)
+    // used, if the row-index-entry reaches config switchIndexInfoToBufferThreshold
+    private DataOutputBuffer rowIndexEntryBuffer;
+    // used to track the total serialized size of indexSamples (unused for buffer)
     private int indexSamplesSerializedSize;
-    // used, until the row-index-entry reaches config column_index_cache_size
+    // used, until the row-index-entry reaches switchIndexInfoToBufferThreshold (from config column_index_cache_size, or default 64k)
     private final List<IndexInfo> indexSamples = new ArrayList<>();
 
     private DataOutputBuffer reusableBuffer;
@@ -62,8 +62,10 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
 
     private final ISerializer<IndexInfo> idxSerializer;
 
-    private final int cacheSizeThreshold;
-    private final int indexSize;
+    /** Beyond this limit we switch from storing IndexInfo in the list to directly serializing them into a buffer */
+    private final int switchIndexInfoToBufferThreshold;
+    /** If a partition grows beyond this size we store inter-partition index data in IndexInfo */
+    private final int indexBlockThreshold;
 
     BigFormatPartitionWriter(SerializationHeader header,
                              SequentialWriter writer,
@@ -82,8 +84,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
     {
         super(header, writer, version);
         this.idxSerializer = indexInfoSerializer;
-        this.cacheSizeThreshold = cacheSizeThreshold;
-        this.indexSize = indexSize;
+        this.switchIndexInfoToBufferThreshold = cacheSizeThreshold;
+        this.indexBlockThreshold = indexSize;
     }
 
     public void reset()
@@ -93,9 +95,9 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
         this.indexSamplesSerializedSize = 0;
         this.indexSamples.clear();
 
-        if (this.buffer != null)
-            this.reusableBuffer = this.buffer;
-        this.buffer = null;
+        if (this.rowIndexEntryBuffer != null)
+            this.reusableBuffer = this.rowIndexEntryBuffer;
+        this.rowIndexEntryBuffer = null;
     }
 
     public int getColumnIndexCount()
@@ -105,12 +107,12 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
 
     public ByteBuffer buffer()
     {
-        return buffer != null ? buffer.buffer() : null;
+        return rowIndexEntryBuffer != null ? rowIndexEntryBuffer.buffer() : null;
     }
 
     public List<IndexInfo> indexSamples()
     {
-        if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= cacheSizeThreshold)
+        if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) <= switchIndexInfoToBufferThreshold)
         {
             return indexSamples;
         }
@@ -129,8 +131,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
     {
         IndexInfo cIndexInfo = new IndexInfo(firstClustering,
                                              lastClustering,
-                                             startPosition,
-                                             currentPosition() - startPosition,
+                                             indexBlockStartOffset,
+                                             currentOffsetInPartition() - indexBlockStartOffset,
                                              !openMarker.isLive() ? openMarker : null);
 
         // indexOffsets is used for both shallow (ShallowIndexedEntry) and non-shallow IndexedEntry.
@@ -156,8 +158,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
             else
             {
                 indexOffsets[columnIndexCount] =
-                buffer != null
-                ? Ints.checkedCast(buffer.position())
+                rowIndexEntryBuffer != null
+                ? Ints.checkedCast(rowIndexEntryBuffer.position())
                 : indexSamplesSerializedSize;
             }
         }
@@ -165,16 +167,20 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
 
         // First, we collect the IndexInfo objects until we reach Config.column_index_cache_size in an ArrayList.
         // When column_index_cache_size is reached, we switch to byte-buffer mode.
-        if (buffer == null)
+        if (rowIndexEntryBuffer == null)
         {
             indexSamplesSerializedSize += idxSerializer.serializedSize(cIndexInfo);
-            if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0) > cacheSizeThreshold)
+            if (indexSamplesSerializedSize + columnIndexCount * TypeSizes.INT_SIZE > switchIndexInfoToBufferThreshold)
             {
-                buffer = reuseOrAllocateBuffer();
+                rowIndexEntryBuffer = reuseOrAllocateBuffer();
+                // serialize pre-existing samples
                 for (IndexInfo indexSample : indexSamples)
                 {
-                    idxSerializer.serialize(indexSample, buffer);
+                    /** {@link IndexInfo.Serializer#serialize} */
+                    idxSerializer.serialize(indexSample, rowIndexEntryBuffer);
                 }
+                // release pre-existing samples
+                indexSamples.clear();
             }
             else
             {
@@ -182,9 +188,10 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
             }
         }
         // don't put an else here since buffer may be allocated in preceding if block
-        if (buffer != null)
+        if (rowIndexEntryBuffer != null)
         {
-            idxSerializer.serialize(cIndexInfo, buffer);
+            /** {@link IndexInfo.Serializer#serialize} */
+            idxSerializer.serialize(cIndexInfo, rowIndexEntryBuffer);
         }
 
         firstClustering = null;
@@ -201,7 +208,7 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
             return buffer;
         }
         // don't use the standard RECYCLER as that only recycles up to 1MB and requires proper cleanup
-        return new DataOutputBuffer(cacheSizeThreshold * 2);
+        return new DataOutputBuffer(switchIndexInfoToBufferThreshold * 2);
     }
 
     @Override
@@ -210,7 +217,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
         super.addUnfiltered(unfiltered);
 
         // if we hit the column index size that we have to index after, go ahead and index it.
-        if (currentPosition() - startPosition >= indexSize)
+        long sizeSinceLastIndexBlock = currentOffsetInPartition() - indexBlockStartOffset;
+        if (sizeSinceLastIndexBlock >= this.indexBlockThreshold)
             addIndexBlock();
     }
 
@@ -231,10 +239,10 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
         // we have to write the offsts to these here. The offsets have already been collected
         // in indexOffsets[]. buffer is != null, if it exceeds Config.column_index_cache_size.
         // In the other case, when buffer==null, the offsets are serialized in RowIndexEntry.IndexedEntry.serialize().
-        if (buffer != null)
+        if (rowIndexEntryBuffer != null)
         {
             for (int i = 0; i < columnIndexCount; i++)
-                buffer.writeInt(indexOffsets[i]);
+                rowIndexEntryBuffer.writeInt(indexOffsets[i]);
         }
 
         // we should always have at least one computed index block, but we only write it out if there is more than that.
@@ -245,8 +253,8 @@ public class BigFormatPartitionWriter extends SortedTablePartitionWriter
 
     public int indexInfoSerializedSize()
     {
-        return buffer != null
-               ? buffer.buffer().limit()
+        return rowIndexEntryBuffer != null
+               ? rowIndexEntryBuffer.buffer().limit()
                : indexSamplesSerializedSize + columnIndexCount * TypeSizes.sizeof(0);
     }
 

src/java/org/apache/cassandra/io/sstable/format/big/BigSSTableReaderLoadingBuilder.java

@@ -178,7 +178,7 @@ public class BigSSTableReaderLoadingBuilder extends SortedTableReaderLoadingBuil
         checkNotNull(serializationHeader);
 
         RowIndexEntry.IndexSerializer serializer = new RowIndexEntry.Serializer(descriptor.version, serializationHeader, tableMetrics);
-        return BigTableKeyReader.create(indexFile, serializer);
+        return BigTableKeyReader.create(indexFile, serializer, false);
     }
 
     /**

src/java/org/apache/cassandra/io/sstable/format/big/BigTableKeyReader.java

@@ -36,19 +36,23 @@ public class BigTableKeyReader implements KeyReader
     private final RandomAccessReader indexFileReader;
     private final IndexSerializer rowIndexEntrySerializer;
     private final long initialPosition;
-
+    private final boolean detailed;
     private ByteBuffer key;
     private long dataPosition;
     private long keyPosition;
+    /** only if detailed */
+    private RowIndexEntry rowIndexEntry;
 
     private BigTableKeyReader(FileHandle indexFile,
                               RandomAccessReader indexFileReader,
-                              IndexSerializer rowIndexEntrySerializer)
+                              IndexSerializer rowIndexEntrySerializer,
+                              boolean detailed)
     {
         this.indexFile = indexFile;
         this.indexFileReader = indexFileReader;
         this.rowIndexEntrySerializer = rowIndexEntrySerializer;
         this.initialPosition = indexFileReader.getFilePointer();
+        this.detailed = detailed;
     }
 
     public static BigTableKeyReader create(RandomAccessReader indexFileReader, IndexSerializer serializer) throws IOException
@@ -58,7 +62,7 @@ public class BigTableKeyReader implements KeyReader
 
     public static BigTableKeyReader create(FileHandle indexFile, RandomAccessReader indexFileReader, IndexSerializer serializer) throws IOException
     {
-        BigTableKeyReader iterator = new BigTableKeyReader(indexFile, indexFileReader, serializer);
+        BigTableKeyReader iterator = new BigTableKeyReader(indexFile, indexFileReader, serializer, false);
         try
         {
             iterator.advance();
@@ -72,7 +76,7 @@ public class BigTableKeyReader implements KeyReader
     }
 
     @SuppressWarnings({ "resource", "RedundantSuppression" }) // iFile and reader are closed in the BigTableKeyReader#close method
-    public static BigTableKeyReader create(FileHandle indexFile, IndexSerializer serializer) throws IOException
+    public static BigTableKeyReader create(FileHandle indexFile, IndexSerializer serializer, boolean detailed) throws IOException
     {
         FileHandle iFile = null;
         RandomAccessReader reader = null;
@@ -81,7 +85,7 @@ public class BigTableKeyReader implements KeyReader
         {
             iFile = indexFile.sharedCopy();
             reader = iFile.createReader();
-            iterator = new BigTableKeyReader(iFile, reader, serializer);
+            iterator = new BigTableKeyReader(iFile, reader, serializer, detailed);
             iterator.advance();
             return iterator;
         }
@@ -116,7 +120,14 @@ public class BigTableKeyReader implements KeyReader
         {
             keyPosition = indexFileReader.getFilePointer();
             key = ByteBufferUtil.readWithShortLength(indexFileReader);
+            if (detailed) {
+                rowIndexEntry = rowIndexEntrySerializer.deserialize(indexFileReader);
+                dataPosition = rowIndexEntry.getPosition();
+            }
+            else
+            {
                 dataPosition = rowIndexEntrySerializer.deserializePositionAndSkip(indexFileReader);
+            }
             return true;
         }
         else
@@ -152,6 +163,15 @@ public class BigTableKeyReader implements KeyReader
         return dataPosition;
     }
 
+    public RowIndexEntry rowIndexEntry() {
+        assert detailed;
+        return rowIndexEntry;
+    }
+
+    public FileHandle indexFile() {
+        return indexFile;
+    }
+
     public long indexPosition()
     {
         return indexFileReader.getFilePointer();

src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java

@@ -141,9 +141,9 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
     }
 
     @Override
-    public KeyReader keyReader() throws IOException
+    public KeyReader keyReader(boolean detailed) throws IOException
     {
-        return BigTableKeyReader.create(ifile, rowIndexEntrySerializer);
+        return BigTableKeyReader.create(ifile, rowIndexEntrySerializer, detailed);
     }
 
     @Override

src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java

@@ -188,6 +188,7 @@ public class BigTableScanner extends SSTableScanner<BigTableReader, RowIndexEntr
                "dfile=" + dfile +
                " ifile=" + ifile +
                " sstable=" + sstable +
+               " rangeIterator=" + rangeIterator +
                ")";
     }
 }

src/java/org/apache/cassandra/io/sstable/format/big/BigTableVerifier.java

@@ -18,17 +18,25 @@
 package org.apache.cassandra.io.sstable.format.big;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.file.NoSuchFileException;
 import java.time.Instant;
+import java.util.Objects;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.cassandra.db.ClusteringComparator;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.rows.BTreeRow;
 import org.apache.cassandra.db.rows.Cell;
+import org.apache.cassandra.db.rows.DeserializationHelper;
 import org.apache.cassandra.db.rows.Row;
 import org.apache.cassandra.db.rows.Unfiltered;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.db.rows.UnfilteredSerializer;
+import org.apache.cassandra.io.sstable.CorruptSSTableException;
 import org.apache.cassandra.io.sstable.IVerifier;
+import org.apache.cassandra.io.sstable.IndexInfo;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.SortedTableVerifier;
 import org.apache.cassandra.io.sstable.format.big.BigFormat.Components;
@@ -82,7 +90,7 @@ public class BigTableVerifier extends SortedTableVerifier<BigTableReader> implem
     {
         try
         {
-            outputHandler.debug("Deserializing index summary for %s", sstable);
+            if (outputHandler.isDebugEnabled()) outputHandler.debug("Deserializing index summary for %s", sstable);
             deserializeIndexSummary(sstable);
         }
         catch (Throwable t)
@@ -99,6 +107,74 @@ public class BigTableVerifier extends SortedTableVerifier<BigTableReader> implem
         super.verifyIndex();
     }
 
+    @Override
+    protected void deserializeIndex(SSTableReader sstable) throws IOException
+    {
+        DeserializationHelper deserializationHelper = new DeserializationHelper(sstable.metadata(),
+                                                                                sstable.descriptor.version.correspondingMessagingVersion(),
+                                                                                DeserializationHelper.Flag.LOCAL);
+        ClusteringComparator comparator = sstable.metadata().comparator;
+        try (BigTableKeyReader it = (BigTableKeyReader)sstable.keyReader(true))
+        {
+            if (it.isExhausted())
+                return;
+            ByteBuffer key;
+            boolean isFirst = true;
+            do
+            {
+                key = it.key();
+                if (outputHandler.isDebugEnabled()) outputHandler.debug("Key %s", sstable.metadata().partitionKeyType.getString(key));
+
+                if (isFirst && !Objects.equals(key, sstable.getFirst().getKey()))
+                    throw new CorruptSSTableException(new IOException("First partition does not match index"), it.toString());
+                else
+                    isFirst = false;
+                RowIndexEntry rowIndexEntry = it.rowIndexEntry();
+                if (outputHandler.isDebugEnabled()) outputHandler.debug("rowIndexEntry %s", rowIndexEntry.toString());
+
+                long partitionBase = it.dataPosition();
+                int blockCount = rowIndexEntry.blockCount();
+                if (blockCount > 0)
+                {
+                    long expectedNextOffset = 0;
+                    RowIndexEntry.IndexInfoRetriever indexInfoRetriever = rowIndexEntry.openWithIndex(it.indexFile());
+                    for (int blockIndex = 0; blockIndex < blockCount; blockIndex++)
+                    {
+                        IndexInfo indexInfo = indexInfoRetriever.columnsIndex(blockIndex);
+                        if (outputHandler.isDebugEnabled()) outputHandler.debug("indexInfo %s", indexInfo.toString(sstable.metadata()));
+
+                        long dataFileOffset = partitionBase + indexInfo.offset;
+                        if (expectedNextOffset != 0)
+                        {
+                            if (expectedNextOffset != indexInfo.offset)
+                                throw new CorruptSSTableException(new IOException("Row entry indexInfo offset + width should match next block offset:" + indexInfo.offset + " expected: " + expectedNextOffset), it.toString());
+                        }
+                        expectedNextOffset = indexInfo.offset + indexInfo.width;
+                        dataFile.seek(dataFileOffset);
+
+                        Unfiltered unfiltered = UnfilteredSerializer.serializer.deserialize(dataFile,
+                                                                                            sstable.header,
+                                                                                            deserializationHelper,
+                                                                                            BTreeRow.sortedBuilder());
+                        if (!Objects.equals(indexInfo.firstName, unfiltered.clustering()))
+                            throw new CorruptSSTableException(new IOException("Unfiltered clustering in index does not match data:{info=" + indexInfo.toString(sstable.metadata()) + ", row:" + unfiltered.clustering().toString(sstable.metadata()) + "}"), it.toString());
+
+                        if (comparator.compare(indexInfo.firstName, indexInfo.lastName) > 0)
+                            throw new CorruptSSTableException(new IOException("First name is > Last name:{info=" + indexInfo.toString(sstable.metadata()) + "}"), it.toString());
+
+                    }
+                }
+            }
+            // The verifier checks that the partition key/position in the index and data match, here we waant to verify
+            // the entries clustering keys match.
+            while (it.advance()); // no-op, just check if index is readable
+
+            if (!Objects.equals(key, sstable.getLast().getKey()))
+                throw new CorruptSSTableException(new IOException("Last partition does not match index"), it.toString());
+        }
+        dataFile.reset();
+    }
+
     private void logDuplicates(DecoratedKey key, Row first, int duplicateRows, long minTimestamp, long maxTimestamp)
     {
         String keyString = sstable.metadata().partitionKeyType.getString(key.getKey());

src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java

@@ -87,7 +87,7 @@ public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter,
     @Override
     protected void onStartPartition(DecoratedKey key)
     {
-        notifyObservers(o -> o.startPartition(key, partitionWriter.getInitialPosition(), indexWriter.writer.position()));
+        notifyObservers(o -> o.startPartition(key, partitionWriter.getPartitionStartPosition(), indexWriter.writer.position()));
     }
 
     @Override
@@ -97,7 +97,7 @@ public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter,
         // serialized size to the index-writer position
         long indexFilePosition = ByteBufferUtil.serializedSizeWithShortLength(key.getKey()) + indexWriter.writer.position();
 
-        RowIndexEntry entry = RowIndexEntry.create(partitionWriter.getInitialPosition(),
+        RowIndexEntry entry = RowIndexEntry.create(partitionWriter.getPartitionStartPosition(),
                                                    indexFilePosition,
                                                    partitionLevelDeletion,
                                                    partitionWriter.getHeaderLength(),
@@ -229,16 +229,30 @@ public class BigTableWriter extends SortedTableWriter<BigFormatPartitionWriter,
         return openInternal(null, openReason);
     }
 
+    @Override
+    public void setFirst(DecoratedKey key)
+    {
+        super.setFirst(key);
+        indexWriter.first = key;
+    }
+
+    @Override
+    public void setLast(DecoratedKey key)
+    {
+        super.setLast(key);
+        indexWriter.last = key;
+    }
+
     /**
      * Encapsulates writing the index and filter for an SSTable. The state of this object is not valid until it has been closed.
      */
-    protected static class IndexWriter extends SortedTableWriter.AbstractIndexWriter
+    public static class IndexWriter extends SortedTableWriter.AbstractIndexWriter
     {
         private final RowIndexEntry.IndexSerializer rowIndexEntrySerializer;
 
-        final SequentialWriter writer;
+        public final SequentialWriter writer;
         final FileHandle.Builder builder;
-        final IndexSummaryBuilder summary;
+        public final IndexSummaryBuilder summary;
         private DataPosition mark;
         private DecoratedKey first;
         private DecoratedKey last;

src/java/org/apache/cassandra/io/sstable/format/big/RowIndexEntry.java

@@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable.format.big;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 import java.util.List;
 
 import com.codahale.metrics.Histogram;
@@ -150,6 +151,7 @@ public class RowIndexEntry extends AbstractRowIndexEntry
     static final Histogram indexInfoCountHistogram;
     static final Histogram indexInfoGetsHistogram;
     static final Histogram indexInfoReadsHistogram;
+
     static
     {
         MetricNameFactory factory = new DefaultNameFactory(TYPE_NAME, "RowIndexEntry");
@@ -259,7 +261,6 @@ public class RowIndexEntry extends AbstractRowIndexEntry
         default RowIndexEntry deserialize(FileDataInput input) throws IOException
         {
             return deserialize(input, input.getFilePointer());
-
         }
 
         void serializeForCache(RowIndexEntry rie, DataOutputPlus out) throws IOException;
@@ -358,6 +359,9 @@ public class RowIndexEntry extends AbstractRowIndexEntry
 
                 int indexedPartSize = size - serializedSize(deletionTime, headerLength, columnsIndexCount, version);
 
+                // We should never get here with 0/1 entries
+                /** See: {@link org.apache.cassandra.io.sstable.format.big.RowIndexEntry#create} */
+                assert columnsIndexCount > 1;
                 if (size <= DatabaseDescriptor.getColumnIndexCacheSize())
                 {
                     return new IndexedEntry(position, in, deletionTime, headerLength, columnsIndexCount,
@@ -663,6 +667,21 @@ public class RowIndexEntry extends AbstractRowIndexEntry
 
             in.readUnsignedVInt();
         }
+
+        @Override
+        public String toString()
+        {
+            return "IndexedEntry{" +
+                   "position=" + position +
+                   ", deletionTime=" + deletionTime +
+                   ", headerLength=" + headerLength +
+                   ", columnsIndex=" + Arrays.toString(columnsIndex) +
+                   ", offsets=" + Arrays.toString(offsets) +
+                   ", indexedPartSize=" + indexedPartSize +
+                   ", idxInfoSerializer=" + idxInfoSerializer +
+                   ", version=" + version +
+                   '}';
+        }
     }
 
     /**
@@ -678,6 +697,7 @@ public class RowIndexEntry extends AbstractRowIndexEntry
             BASE_SIZE = ObjectSizes.measure(new ShallowIndexedEntry(0, 0, DeletionTime.LIVE, 0, 10, 0, null, BigFormat.getInstance().getLatestVersion()));
         }
 
+        // only for cache serialization
         private final long indexFilePosition;
 
         private final DeletionTime deletionTime;
@@ -701,8 +721,6 @@ public class RowIndexEntry extends AbstractRowIndexEntry
         {
             super(dataFilePosition);
 
-            assert columnIndexCount > 1;
-
             this.indexFilePosition = indexFilePosition;
             this.headerLength = headerLength;
             this.deletionTime = deletionTime;
@@ -810,6 +828,23 @@ public class RowIndexEntry extends AbstractRowIndexEntry
 
             in.readUnsignedVInt();
         }
+
+        @Override
+        public String toString()
+        {
+            return "ShallowIndexedEntry{" +
+                   "position=" + position +
+                   ", indexFilePosition=" + indexFilePosition +
+                   ", deletionTime=" + deletionTime +
+                   ", headerLength=" + headerLength +
+                   ", columnsIndexCount=" + columnsIndexCount +
+                   ", indexedPartSize=" + indexedPartSize +
+                   ", offsetsOffset=" + offsetsOffset +
+                   ", idxInfoSerializer=" + idxInfoSerializer +
+                   ", fieldsSerializedSize=" + fieldsSerializedSize +
+                   ", version=" + version +
+                   '}';
+        }
     }
 
     private static final class ShallowInfoRetriever extends FileIndexInfoRetriever
@@ -900,4 +935,12 @@ public class RowIndexEntry extends AbstractRowIndexEntry
             super(message);
         }
     }
+
+    @Override
+    public String toString()
+    {
+        return "RowIndexEntry{" +
+               "position=" + position +
+               '}';
+    }
 }

src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormatPartitionWriter.java

@@ -79,7 +79,7 @@ class BtiFormatPartitionWriter extends SortedTablePartitionWriter
         super.addUnfiltered(unfiltered);
 
         // if we hit the column index size that we have to index after, go ahead and index it.
-        if (currentPosition() - startPosition >= rowIndexBlockSize)
+        if (currentOffsetInPartition() - indexBlockStartOffset >= rowIndexBlockSize)
             addIndexBlock();
     }
 
@@ -112,7 +112,7 @@ class BtiFormatPartitionWriter extends SortedTablePartitionWriter
 
     protected void addIndexBlock() throws IOException
     {
-        IndexInfo cIndexInfo = new IndexInfo(startPosition, startOpenMarker);
+        IndexInfo cIndexInfo = new IndexInfo(indexBlockStartOffset, startOpenMarker);
         rowTrie.add(firstClustering, lastClustering, cIndexInfo);
         firstClustering = null;
         ++rowIndexBlockCount;

src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java

@@ -317,7 +317,7 @@ public class BtiTableReader extends SSTableReaderWithFilter
     }
 
     @Override
-    public PartitionIterator keyReader() throws IOException
+    public PartitionIterator keyReader(boolean detailed) throws IOException
     {
         return PartitionIterator.create(partitionIndex, metadata().partitioner, rowIndexFile, dfile, descriptor.version);
     }

src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableWriter.java

@@ -73,7 +73,7 @@ public class BtiTableWriter extends SortedTableWriter<BtiFormatPartitionWriter,
     @Override
     protected TrieIndexEntry createRowIndexEntry(DecoratedKey key, DeletionTime partitionLevelDeletion, long finishResult) throws IOException
     {
-        TrieIndexEntry entry = TrieIndexEntry.create(partitionWriter.getInitialPosition(),
+        TrieIndexEntry entry = TrieIndexEntry.create(partitionWriter.getPartitionStartPosition(),
                                                      finishResult,
                                                      partitionLevelDeletion,
                                                      partitionWriter.getRowIndexBlockCount());

src/java/org/apache/cassandra/io/sstable/indexsummary/IndexSummaryBuilder.java

@@ -145,7 +145,7 @@ public class IndexSummaryBuilder implements AutoCloseable
      */
     private static long getEntrySize(long keySize)
     {
-        return keySize + TypeSizes.sizeof(0L);
+        return keySize + TypeSizes.LONG_SIZE;
     }
 
     // the index file has been flushed to the provided position; stash it and use that to recalculate our max readable boundary
@@ -188,6 +188,34 @@ public class IndexSummaryBuilder implements AutoCloseable
     {
         return maybeAddEntry(decoratedKey, indexStart, 0, 0);
     }
+    /**
+     * @param keyBytes the key data for this record
+     * @param offset key data offset in the keyBytes array
+     * @param length key data length
+     * @param indexStart the position in the index file this record begins
+     */
+    public IndexSummaryBuilder maybeAddEntry(byte[] keyBytes, int offset, int length, long indexStart) throws IOException
+    {
+        if (keysWritten == nextSamplePosition)
+        {
+            if ((entries.length() + getEntrySize(length)) <= Integer.MAX_VALUE)
+            {
+                offsets.writeInt((int) entries.length());
+                entries.write(keyBytes, offset, length);
+                entries.writeLong(indexStart);
+                setNextSamplePosition(keysWritten);
+            }
+            else
+            {
+                // we cannot fully sample this sstable due to too much memory in the index summary, so let's tell the user
+                logger.error("Memory capacity of index summary exceeded (2GiB), index summary will not cover full sstable, " +
+                             "you should increase min_sampling_level");
+            }
+        }
+
+        keysWritten++;
+        return this;
+    }
 
     /**
      *

src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java

@@ -23,6 +23,8 @@ import java.util.EnumMap;
 import java.util.Map;
 import java.util.UUID;
 
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.io.sstable.ClusteringDescriptor;
 import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
 import com.clearspring.analytics.stream.cardinality.ICardinality;
 import org.apache.cassandra.db.Clustering;
@@ -105,7 +107,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
     }
 
     protected EstimatedHistogram estimatedPartitionSize = defaultPartitionSizeHistogram();
-    // TODO: cound the number of row per partition (either with the number of cells, or instead)
+    // TODO: count the number of row per partition (either with the number of cells, or instead)
     protected EstimatedHistogram estimatedCellPerPartitionCount = defaultCellPerPartitionCountHistogram();
     protected IntervalSet<CommitLogPosition> commitLogIntervals = IntervalSet.empty();
     protected final MinMaxLongTracker timestampTracker = new MinMaxLongTracker();
@@ -122,6 +124,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
      * be a corresponding start bound that is smaller).
      */
     private ClusteringPrefix<?> minClustering = ClusteringBound.MAX_START;
+    private final ClusteringDescriptor minClusteringDescriptor;
+
     /**
      * The largest clustering prefix for any {@link Unfiltered} in the sstable.
      *
@@ -129,6 +133,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
      * be a corresponding end bound that is bigger).
      */
     private ClusteringPrefix<?> maxClustering = ClusteringBound.MIN_END;
+    private final ClusteringDescriptor maxClusteringDescriptor;
 
     protected boolean hasLegacyCounterShards = false;
     private boolean hasPartitionLevelDeletions = false;
@@ -159,6 +164,9 @@ public class MetadataCollector implements PartitionStatisticsCollector
     {
         this.comparator = comparator;
         this.originatingHostId = originatingHostId;
+        AbstractType<?>[] clusteringTypes = comparator.subtypes().toArray(AbstractType[]::new);
+        this.minClusteringDescriptor = new ClusteringDescriptor(clusteringTypes).resetMaxStart();
+        this.maxClusteringDescriptor = new ClusteringDescriptor(clusteringTypes).resetMinEnd();
     }
 
     public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator)
@@ -185,6 +193,14 @@ public class MetadataCollector implements PartitionStatisticsCollector
         return this;
     }
 
+    public MetadataCollector addKey(byte[] key, int offset, int length)
+    {
+        long hashed = MurmurHash.hash2_64(key, offset, length, 0);
+        cardinality.offerHashed(hashed);
+        totalTombstones = 0;
+        return this;
+    }
+
     public MetadataCollector addPartitionSizeInBytes(long partitionSize)
     {
         estimatedPartitionSize.add(partitionSize);
@@ -236,6 +252,15 @@ public class MetadataCollector implements PartitionStatisticsCollector
             updateTombstoneCount();
     }
 
+    /**
+     * Cell level stats, if we accept that LDT and LET are the same...
+     */
+    public void updateCellLiveness(LivenessInfo newInfo)
+    {
+        ++currentPartitionCells;
+        update(newInfo);
+    }
+
     public void updatePartitionDeletion(DeletionTime dt)
     {
         if (!dt.isLive())
@@ -299,6 +324,32 @@ public class MetadataCollector implements PartitionStatisticsCollector
         return this;
     }
 
+    public void updateClusteringValues(ClusteringDescriptor newClustering) {
+        // In a SSTable, every opening marker will be closed, so the start of a range tombstone marker will never be
+        // the maxClustering (the corresponding close might though) and there is no point in doing the comparison
+        // (and vice-versa for the close). By the same reasoning, a boundary will never be either the min or max
+        // clustering, and we can save on comparisons.
+        if (newClustering == null || newClustering.clusteringKind().isBoundary())
+            return;
+
+        // In case of monotonically growing stream of clusterings, we will usually require only one comparison
+        // because if we detected X is greater than the current MAX, then it cannot be lower than the current MIN
+        // at the same time. The only case when we need to update MIN when the current MAX was detected to be updated
+        // is the case when MIN was not yet initialized and still point the ClusteringBound.MAX_START
+        if (ClusteringComparator.compare(newClustering, maxClusteringDescriptor) > 0)
+        {
+            maxClusteringDescriptor.copy(newClustering);
+            if (minClusteringDescriptor.isMaxStart()) // min is unset
+            {
+                minClusteringDescriptor.copy(newClustering);
+            }
+        }
+        else if (ClusteringComparator.compare(newClustering, minClusteringDescriptor) < 0)
+        {
+            minClusteringDescriptor.copy(newClustering);
+        }
+    }
+
     public void updateClusteringValues(Clustering<?> clustering)
     {
         if (clustering == Clustering.STATIC_CLUSTERING)
@@ -361,6 +412,13 @@ public class MetadataCollector implements PartitionStatisticsCollector
 
         Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class);
         components.put(MetadataType.VALIDATION, new ValidationMetadata(partitioner, bloomFilterFPChance));
+        Slice coveredClustering;
+        if (!minClusteringDescriptor.isMaxStart()) // min is end only if the descriptors are unused
+        {
+            minClustering = minClusteringDescriptor.toClusteringPrefix(comparator.subtypes());
+            maxClustering = maxClusteringDescriptor.toClusteringPrefix(comparator.subtypes());
+        }
+        coveredClustering = Slice.make(minClustering.retainable().asStartBound(), maxClustering.retainable().asEndBound());
         components.put(MetadataType.STATS, new StatsMetadata(estimatedPartitionSize,
                                                              estimatedCellPerPartitionCount,
                                                              commitLogIntervals,
@@ -374,7 +432,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
                                                              estimatedTombstoneDropTime.build(),
                                                              sstableLevel,
                                                              comparator.subtypes(),
-                                                             Slice.make(minClustering.retainable().asStartBound(), maxClustering.retainable().asEndBound()),
+                                                             coveredClustering,
                                                              hasLegacyCounterShards,
                                                              repairedAt,
                                                              totalColumnsSet,

src/java/org/apache/cassandra/io/util/ResizableByteBuffer.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.cassandra.utils.vint.VIntCoding;
+import org.jctools.util.Pow2;
+
+public class ResizableByteBuffer
+{
+    private int length = 0;
+    private byte[] bytes = new byte[64];
+    private ByteBuffer buffer = ByteBuffer.wrap(bytes);
+
+    /**
+     * Loads the length (UnsignedShort) from the data reader, and then loads the new data into the buffer.
+     *
+     * @param dataReader data source
+     * @return true if the underlying array/buffer had to be resized to accomodate the new data, false otherwise
+     * @throws IOException
+     */
+    public final boolean loadShortLength(RandomAccessReader dataReader) throws IOException
+    {
+        int newLength = dataReader.readUnsignedShort();
+        return load(dataReader, newLength);
+    }
+
+    /**
+     * Loads the new data (of the given length) into the buffer. Buffer is rewritten.
+     *
+     * @param dataReader data source
+     * @param newLength new data length
+     * @return true if the underlying array/buffer had to be resized to accomodate the new data, false otherwise
+     * @throws IOException
+     */
+    public final boolean load(RandomAccessReader dataReader, int newLength) throws IOException
+    {
+        boolean resize = maybeResizeForOverwrite(newLength);
+        dataReader.readFully(bytes, 0, newLength);
+        setLength(newLength);
+        return resize;
+    }
+
+    public final void resetBuffer() {
+        setLength(0);
+    }
+
+    protected final byte[] bytes()
+    {
+        return bytes;
+    }
+
+    protected final ByteBuffer buffer()
+    {
+        return buffer;
+    }
+
+    protected final int length()
+    {
+        return length;
+    }
+
+    protected final void overwrite(byte[] newBytes, int newLength)
+    {
+        maybeResizeForOverwrite(newLength);
+        System.arraycopy(newBytes, 0, bytes, 0, newLength);
+        setLength(newLength);
+    }
+
+    private void setLength(int newLength)
+    {
+        int currentLength = length;
+        if (newLength != currentLength)
+        {
+            length = newLength;
+            buffer.limit(newLength);
+        }
+    }
+
+    /**
+     * Loads new partial data (of the given length) into the buffer. New data is appended.
+     *
+     * @param dataReader data source
+     * @param partLength new data length
+     * @return true if the underlying array/buffer had to be resized to accomodate the new data, false otherwise
+     * @throws IOException
+     */
+    public final boolean loadPart(RandomAccessReader dataReader, int partLength) throws IOException
+    {
+        int newLength = length + partLength;
+        boolean resize = maybeResizeForAppend(newLength);
+        dataReader.readFully(bytes, length, partLength);
+        setLength(newLength);
+        return resize;
+    }
+
+    public final boolean writeUnsignedVInt(long value)
+    {
+        int currentPosition = length;
+        int newLength = currentPosition + 9;
+        boolean resize = maybeResizeForAppend(newLength);
+        VIntCoding.writeUnsignedVInt(value, buffer.position(currentPosition).limit(newLength));
+        setLength(buffer.position());
+        buffer.position(0);
+        return resize;
+    }
+
+    private boolean maybeResizeForOverwrite(int newLength)
+    {
+        if (newLength > bytes.length)
+        {
+            bytes = new byte[Pow2.roundToPowerOfTwo(newLength)];
+            buffer = ByteBuffer.wrap(bytes);
+            return true;
+        }
+        return false;
+    }
+
+    private boolean maybeResizeForAppend(int newLength)
+    {
+        if (newLength > bytes.length)
+        {
+            int currLength = length;
+            byte[] currBytes = bytes;
+            bytes = new byte[Pow2.roundToPowerOfTwo(newLength)];
+            if (currLength != 0)
+                System.arraycopy(currBytes,0 , bytes, 0, currLength);
+            buffer = ByteBuffer.wrap(bytes);
+            return true;
+        }
+        return false;
+    }
+}

src/java/org/apache/cassandra/tools/JsonTransformer.java

@@ -436,8 +436,15 @@ public final class JsonTransformer
                 else
                 {
                     AbstractType<?> type = column.cellValueType();
+                    try
+                    {
                         json.writeRawValue(type.toJSONString(clustering.get(i), clustering.accessor(), ProtocolVersion.CURRENT));
                     }
+                    catch (Exception e)
+                    {
+                        json.writeString("unsupported conversion to JSON");
+                    }
+                }
             }
             json.writeEndArray();
             objectIndenter.setCompact(false);
@@ -552,8 +559,14 @@ public final class JsonTransformer
             else
             {
                 json.writeFieldName("value");
+                try
+                {
                     json.writeRawValue(cellType.toJSONString(cell.value(), cell.accessor(), ProtocolVersion.CURRENT));
                 }
+                catch (Exception e) {
+                    json.writeString("unsupported conversion to JSON");
+                }
+            }
             if (liveInfo.isEmpty() || cell.timestamp() != liveInfo.timestamp())
             {
                 json.writeFieldName("tstamp");

src/java/org/apache/cassandra/utils/BloomFilter.java

@@ -101,6 +101,22 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
         return indexes;
     }
 
+    private long[] indexes(byte[] key, int offset, int length)
+    {
+        // we use the same array both for storing the hash result, and for storing the indexes we return,
+        // so that we do not need to allocate two arrays.
+        long[] indexes = reusableIndexes.get();
+        return indexes(key, offset, length, indexes);
+    }
+
+    private long[] indexes(byte[] key, int offset, int length, long[] indexes)
+    {
+        MurmurHash.hash3_x64_128(key, offset, length, 0, indexes);
+
+        setIndexes(indexes[1], indexes[0], hashCount, bitset.capacity(), indexes);
+        return indexes;
+    }
+
     @Inline
     private void setIndexes(long base, long inc, int count, long max, long[] results)
     {
@@ -121,6 +137,24 @@ public class BloomFilter extends WrappedSharedCloseable implements IFilter
         }
     }
 
+    public void add(byte[] key, int offset, int length)
+    {
+        long[] indexes = indexes(key, offset, length);
+        for (int i = 0; i < hashCount; i++)
+        {
+            bitset.set(indexes[i]);
+        }
+    }
+
+    public void add(byte[] key, int offset, int length, long[] indexes)
+    {
+        indexes(key, offset, length, indexes);
+        for (int i = 0; i < hashCount; i++)
+        {
+            bitset.set(indexes[i]);
+        }
+    }
+
     @Override
     public final boolean isPresent(FilterKey key)
     {

src/java/org/apache/cassandra/utils/ByteArrayUtil.java

@@ -46,6 +46,11 @@ public class ByteArrayUtil
         return FastByteOperations.compareUnsigned(o1, off1, len, o2, off2, len);
     }
 
+    public static int compareUnsigned(byte[] o1, int off1, int len1, byte[] o2, int off2, int len2)
+    {
+        return FastByteOperations.compareUnsigned(o1, off1, len1, o2, off2, len2);
+    }
+
     public static byte[] bytes(byte b)
     {
         return new byte[] {b};
@@ -234,6 +239,16 @@ public class ByteArrayUtil
         out.write(buffer);
     }
 
+    public static void writeWithShortLength(byte[] buffer, int offset, int length, DataOutput out) throws IOException
+    {
+        assert length <= buffer.length;
+        assert offset <= length;
+        assert length <= FBUtilities.MAX_UNSIGNED_SHORT
+            : String.format("Attempted serializing to buffer exceeded maximum of %s bytes: %s", FBUtilities.MAX_UNSIGNED_SHORT, length);
+        out.writeShort(length);
+        out.write(buffer, offset, length);
+    }
+
     public static void writeWithVIntLength(byte[] bytes, DataOutputPlus out) throws IOException
     {
         out.writeUnsignedVInt32(bytes.length);

src/java/org/apache/cassandra/utils/MurmurHash.java

@@ -93,6 +93,66 @@ public class MurmurHash
         return h;
     }
 
+    /**
+     * We can go all in and do a unified unsafe implementation to serve arrays/direct memory and layer the byte buffer
+     * types on top. Left for later.
+     */
+    public static long hash2_64(byte[] key, int offset, int length, long seed)
+    {
+        long m64 = 0xc6a4a7935bd1e995L;
+        int r64 = 47;
+
+        long h64 = (seed & 0xffffffffL) ^ (m64 * length);
+
+        int lenLongs = length >> 3;
+
+        for (int i = 0; i < lenLongs; ++i)
+        {
+            int i_8 = i << 3;
+
+            long k64 =  ((long)  key[offset+i_8+0] & 0xff)      + (((long) key[offset+i_8+1] & 0xff)<<8)  +
+                        (((long) key[offset+i_8+2] & 0xff)<<16) + (((long) key[offset+i_8+3] & 0xff)<<24) +
+                        (((long) key[offset+i_8+4] & 0xff)<<32) + (((long) key[offset+i_8+5] & 0xff)<<40) +
+                        (((long) key[offset+i_8+6] & 0xff)<<48) + (((long) key[offset+i_8+7] & 0xff)<<56);
+
+            k64 *= m64;
+            k64 ^= k64 >>> r64;
+            k64 *= m64;
+
+            h64 ^= k64;
+            h64 *= m64;
+        }
+
+        int rem = length & 0x7;
+
+        switch (rem)
+        {
+            case 0:
+                break;
+            case 7:
+                h64 ^= (long) key[offset + length - rem + 6] << 48;
+            case 6:
+                h64 ^= (long) key[offset + length - rem + 5] << 40;
+            case 5:
+                h64 ^= (long) key[offset + length - rem + 4] << 32;
+            case 4:
+                h64 ^= (long) key[offset + length - rem + 3] << 24;
+            case 3:
+                h64 ^= (long) key[offset + length - rem + 2] << 16;
+            case 2:
+                h64 ^= (long) key[offset + length - rem + 1] << 8;
+            case 1:
+                h64 ^= (long) key[offset + length - rem];
+                h64 *= m64;
+        }
+
+        h64 ^= h64 >>> r64;
+        h64 *= m64;
+        h64 ^= h64 >>> r64;
+
+        return h64;
+    }
+
     public static long hash2_64(ByteBuffer key, int offset, int length, long seed)
     {
         long m64 = 0xc6a4a7935bd1e995L;
@@ -159,6 +219,16 @@ public class MurmurHash
                (((long) key.get(blockOffset + 6) & 0xff) << 48) + (((long) key.get(blockOffset + 7) & 0xff) << 56);
     }
 
+    protected static long getBlock(byte[] key, int offset, int index)
+    {
+        int i_8 = index << 3;
+        int blockOffset = offset + i_8;
+        return ((long)  key[blockOffset + 0] & 0xff) +        (((long) key[blockOffset + 1] & 0xff) << 8) +
+               (((long) key[blockOffset + 2] & 0xff) << 16) + (((long) key[blockOffset + 3] & 0xff) << 24) +
+               (((long) key[blockOffset + 4] & 0xff) << 32) + (((long) key[blockOffset + 5] & 0xff) << 40) +
+               (((long) key[blockOffset + 6] & 0xff) << 48) + (((long) key[blockOffset + 7] & 0xff) << 56);
+    }
+
     protected static long rotl64(long v, int n)
     {
         return ((v << n) | (v >>> (64 - n)));
@@ -251,6 +321,82 @@ public class MurmurHash
         result[1] = h2;
     }
 
+    public static void hash3_x64_128(byte[] key, int offset, int length, long seed, long[] result)
+    {
+        final int nblocks = length >> 4; // Process as 128-bit blocks.
+
+        long h1 = seed;
+        long h2 = seed;
+
+        long c1 = 0x87c37b91114253d5L;
+        long c2 = 0x4cf5ad432745937fL;
+
+        //----------
+        // body
+
+        for(int i = 0; i < nblocks; i++)
+        {
+            long k1 = getBlock(key, offset, i * 2 + 0);
+            long k2 = getBlock(key, offset, i * 2 + 1);
+
+            k1 *= c1; k1 = rotl64(k1,31); k1 *= c2; h1 ^= k1;
+
+            h1 = rotl64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
+
+            k2 *= c2; k2  = rotl64(k2,33); k2 *= c1; h2 ^= k2;
+
+            h2 = rotl64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
+        }
+
+        //----------
+        // tail
+
+        // Advance offset to the unprocessed tail of the data.
+        offset += nblocks * 16;
+
+        long k1 = 0;
+        long k2 = 0;
+
+        switch(length & 15)
+        {
+            case 15: k2 ^= ((long) key[offset+14]) << 48;
+            case 14: k2 ^= ((long) key[offset+13]) << 40;
+            case 13: k2 ^= ((long) key[offset+12]) << 32;
+            case 12: k2 ^= ((long) key[offset+11]) << 24;
+            case 11: k2 ^= ((long) key[offset+10]) << 16;
+            case 10: k2 ^= ((long) key[offset+9]) << 8;
+            case  9: k2 ^= ((long) key[offset+8]) << 0;
+                k2 *= c2; k2  = rotl64(k2,33); k2 *= c1; h2 ^= k2;
+
+            case  8: k1 ^= ((long) key[offset+7]) << 56;
+            case  7: k1 ^= ((long) key[offset+6]) << 48;
+            case  6: k1 ^= ((long) key[offset+5]) << 40;
+            case  5: k1 ^= ((long) key[offset+4]) << 32;
+            case  4: k1 ^= ((long) key[offset+3]) << 24;
+            case  3: k1 ^= ((long) key[offset+2]) << 16;
+            case  2: k1 ^= ((long) key[offset+1]) << 8;
+            case  1: k1 ^= ((long) key[offset]);
+                k1 *= c1; k1  = rotl64(k1,31); k1 *= c2; h1 ^= k1;
+        };
+
+        //----------
+        // finalization
+
+        h1 ^= length; h2 ^= length;
+
+        h1 += h2;
+        h2 += h1;
+
+        h1 = fmix(h1);
+        h2 = fmix(h2);
+
+        h1 += h2;
+        h2 += h1;
+
+        result[0] = h1;
+        result[1] = h2;
+    }
+
     protected static long invRotl64(long v, int n)
     {
         return ((v >>> n) | (v << (64 - n)));

src/java/org/apache/cassandra/utils/OutputHandler.java

@@ -36,7 +36,7 @@ public interface OutputHandler
     void debug(String msg);
     default void debug(String msg, Object ... args)
     {
-        debug(String.format(msg, args));
+        if (isDebugEnabled()) debug(String.format(msg, args));
     }
 
     // called when the user needs to be warn
@@ -55,6 +55,8 @@ public interface OutputHandler
         warn(String.format(msg, args));
     }
 
+    boolean isDebugEnabled();
+
     class LogOutput implements OutputHandler
     {
         private static Logger logger = LoggerFactory.getLogger(LogOutput.class);
@@ -66,7 +68,7 @@ public interface OutputHandler
 
         public void debug(String msg)
         {
-            logger.trace(msg);
+            logger.debug(msg);
         }
 
         public void warn(String msg)
@@ -78,6 +80,18 @@ public interface OutputHandler
         {
             logger.warn(msg, th);
         }
+
+        public boolean isDebugEnabled() { return logger.isDebugEnabled();}
+    }
+
+    class NullOutput implements OutputHandler
+    {
+        public void output(String msg) {}
+        public void debug(String msg, Object ... args) {}
+        public void debug(String msg) {}
+        public void warn(String msg) {}
+        public void warn(Throwable th, String msg) {}
+        public boolean isDebugEnabled() { return false;}
     }
 
     class SystemOutput implements OutputHandler
@@ -120,5 +134,10 @@ public interface OutputHandler
             if (printStack && th != null)
                 th.printStackTrace(warnOut);
         }
+
+        public boolean isDebugEnabled()
+        {
+            return debug;
+        }
     }
 }

src/java/org/apache/cassandra/utils/vint/VIntCoding.java

@@ -230,6 +230,7 @@ public class VIntCoding
     {
         return getUnsignedVInt(input, readerIndex, input.limit());
     }
+
     public static long getUnsignedVInt(ByteBuffer input, int readerIndex, int readerLimit)
     {
         if (readerIndex < 0)

test/bin/jmh

@@ -136,6 +136,6 @@ CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/test/conf/"
 CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/build/test/classes/"
 CLASSPATH="$CLASSPATH:$CASSANDRA_HOME/build/test/lib/jars/*"
 
-exec $NUMACTL "$JAVA" -cp "$CLASSPATH" org.openjdk.jmh.Main -jvmArgs="$cassandra_parms $JVM_OPTS" "$@"
+exec $NUMACTL "$JAVA" -Xmx8g -XX:+UseParallelGC -cp "$CLASSPATH" org.openjdk.jmh.Main -jvmArgs="-Djmh.shutdownTimeout=120 $cassandra_parms $JVM_OPTS" "$@"
 
 # vi:ai sw=4 ts=4 tw=0 et

test/distributed/org/apache/cassandra/distributed/test/FailingRepairTest.java

@@ -338,6 +338,12 @@ public class FailingRepairTest extends TestBaseImpl implements Serializable
             return Collections.emptySet();
         }
 
+        @Override
+        public boolean isFullRange()
+        {
+            return false;
+        }
+
         public TableMetadata metadata()
         {
             return null;

test/distributed/org/apache/cassandra/distributed/test/accord/AccordMigrationReadRaceTestBase.java

@@ -299,7 +299,7 @@ public abstract class AccordMigrationReadRaceTestBase extends AccordTestBase
     @Test
     public void testRangeRouting() throws Throwable
     {
-        String cql = "SELECT * FROM " + qualifiedAccordTableName + " WHERE token(pk) > " + Murmur3Partitioner.MINIMUM.token;
+        String cql = "SELECT * FROM " + qualifiedAccordTableName + " WHERE token(pk) > " + Murmur3Partitioner.MINIMUM.getLongValue();
         testSplitAndRetry(cql, this::loadData, result -> {
             assertThat(result).isDeepEqualTo(dataFlat);
         });

test/distributed/org/apache/cassandra/distributed/test/accord/InteropTokenRangeTest.java

@@ -120,7 +120,7 @@ public class InteropTokenRangeTest extends TestBaseImpl
     {
         NavigableSet<Long> set = new TreeSet<>();
         while (result.hasNext())
-            set.add(Murmur3Partitioner.instance.getToken(result.next().get("pk")).token);
+            set.add(Murmur3Partitioner.instance.getToken(result.next().get("pk")).getLongValue());
         return set;
     }
 

test/distributed/org/apache/cassandra/distributed/test/cql3/SingleNodeTokenConflictTest.java

@@ -405,10 +405,10 @@ public class SingleNodeTokenConflictTest extends StatefulASTBase
             for (ByteBuffer bb : values)
             {
                 var token = Murmur3Partitioner.instance.getToken(bb);
-                if (token.token > Long.MIN_VALUE + 1)
-                    neighbors.add(keyForToken(token.token - 1));
-                if (token.token < Long.MAX_VALUE)
-                    neighbors.add(keyForToken(token.token + 1));
+                if (token.getLongValue() > Long.MIN_VALUE + 1)
+                    neighbors.add(keyForToken(token.getLongValue() - 1));
+                if (token.getLongValue() < Long.MAX_VALUE)
+                    neighbors.add(keyForToken(token.getLongValue() + 1));
             }
             return neighbors.build();
         }

test/distributed/org/apache/cassandra/distributed/test/log/CoordinatorPathTestBase.java

@@ -1005,9 +1005,9 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
             SinglePartitionReadCommand sprc = (SinglePartitionReadCommand) command;
 
             Murmur3Partitioner.LongToken requestToken = (Murmur3Partitioner.LongToken) sprc.partitionKey().getToken();
-            assert node.cluster.state.get().isReadReplicaFor(requestToken.token, node.matcher) :
+            assert node.cluster.state.get().isReadReplicaFor(requestToken.getLongValue(), node.matcher) :
             String.format("Node %s is not a read replica for %s. Replicas: %s",
-                          node, requestToken.token, node.cluster.state.get().readReplicasFor(requestToken.token));
+                          node, requestToken.getLongValue(), node.cluster.state.get().readReplicasFor(requestToken.getLongValue()));
         }
 
         @Override
@@ -1048,7 +1048,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
 
             Murmur3Partitioner.LongToken leftToken = (Murmur3Partitioner.LongToken) prrc.dataRange().keyRange().left.getToken();
             Murmur3Partitioner.LongToken rightToken = (Murmur3Partitioner.LongToken) prrc.dataRange().keyRange().right.getToken();
-            assert node.cluster.state.get().isReadReplicaFor(leftToken.token, rightToken.token, node.matcher);
+            assert node.cluster.state.get().isReadReplicaFor(leftToken.getLongValue(), rightToken.getLongValue(), node.matcher);
         }
 
         @Override
@@ -1085,8 +1085,8 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
         {
             Mutation command = request.payload;
             Murmur3Partitioner.LongToken requestToken = (Murmur3Partitioner.LongToken) command.key().getToken();
-            assert node.cluster.state.get().isWriteTargetFor(requestToken.token, node.matcher) : String.format("Node %s is not a write target for %s. Write placements: %s",
-                                                                                                               node.node.idx(), requestToken, node.cluster.state.get().writePlacementsFor(requestToken.token));
+            assert node.cluster.state.get().isWriteTargetFor(requestToken.getLongValue(), node.matcher) : String.format("Node %s is not a write target for %s. Write placements: %s",
+                                                                                                               node.node.idx(), requestToken, node.cluster.state.get().writePlacementsFor(requestToken.getLongValue()));
         }
 
         public Message<NoPayload> respondTo(Message<Mutation> request)
@@ -1145,7 +1145,7 @@ public abstract class CoordinatorPathTestBase extends FuzzTestBase
 
     public static long token(Object... pk)
     {
-        return ((Murmur3Partitioner.LongToken) DatabaseDescriptor.getPartitioner().decorateKey(toBytes(pk)).getToken()).token;
+        return ((Murmur3Partitioner.LongToken) DatabaseDescriptor.getPartitioner().decorateKey(toBytes(pk)).getToken()).getLongValue();
     }
 
     public static ByteBuffer toBytes(Object... pk)

test/distributed/org/apache/cassandra/distributed/test/log/GossipDeadlockTest.java

@@ -71,7 +71,7 @@ public class GossipDeadlockTest extends TestBaseImpl
             ExecutorPlus e = ExecutorFactory.Global.executorFactory().pooled("BounceMove", 2);
             long startToken = cluster.get(2).callOnInstance(() -> {
                 NodeId nodeId = ClusterMetadata.current().myNodeId();
-                return ((Murmur3Partitioner.LongToken)ClusterMetadata.current().tokenMap.tokens(nodeId).get(0)).token;
+                return ((Murmur3Partitioner.LongToken)ClusterMetadata.current().tokenMap.tokens(nodeId).get(0)).getLongValue();
             });
             AtomicBoolean stop = new AtomicBoolean(false);
             Future<Integer> moves = e.submit(() -> {

test/distributed/org/apache/cassandra/distributed/test/log/MetadataChangeSimulationTest.java

@@ -555,7 +555,7 @@ public class MetadataChangeSimulationTest extends CMSTestBase
         Assert.assertEquals(replication, sut.rf.asKeyspaceParams().replication);
 
         Assert.assertEquals(modelState.simulatedPlacements.nodes.stream().map(Node::token).collect(Collectors.toSet()),
-                            actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).token).collect(Collectors.toSet()));
+                            actualMetadata.tokenMap.tokens().stream().map(t -> ((LongToken) t).getLongValue()).collect(Collectors.toSet()));
 
         for (Map.Entry<ReplicationParams, DataPlacement> e : actualMetadata.placements.asMap().entrySet())
         {

test/distributed/org/apache/cassandra/distributed/test/streaming/LCSStreamingKeepLevelTest.java

@@ -66,8 +66,8 @@ public class LCSStreamingKeepLevelTest extends TestBaseImpl
         {
             populate(cluster);
 
-            long tokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(3).getToken()).token;
-            long prevTokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(2).getToken()).token;
+            long tokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(3).getToken()).getLongValue();
+            long prevTokenVal = ((Murmur3Partitioner.LongToken)cluster.tokens().get(2).getToken()).getLongValue();
             // move node 4 to the middle point between its current position and the previous node
             long newToken = (tokenVal + prevTokenVal) / 2;
             cluster.get(4).nodetoolResult("move", String.valueOf(newToken)).asserts().success();

test/distributed/org/apache/cassandra/distributed/test/tcm/CMSPlacementAfterMoveTest.java

@@ -46,7 +46,7 @@ public class CMSPlacementAfterMoveTest extends TestBaseImpl
             long node1Token = cluster.get(1).callOnInstance(() -> {
                 ClusterMetadata metadata = ClusterMetadata.current();
                 ImmutableList<Token> tokens = ClusterMetadata.current().tokenMap.tokens(metadata.myNodeId());
-                return ((Murmur3Partitioner.LongToken) tokens.get(0)).token;
+                return ((Murmur3Partitioner.LongToken) tokens.get(0)).getLongValue();
             });
             long newNode4Token = node1Token + 100; // token after node1s token should be in cms
             cluster.get(4).nodetoolResult("move", String.valueOf(newNode4Token));

test/distributed/org/apache/cassandra/io/sstable/format/ForwardingSSTableReader.java

@@ -238,9 +238,9 @@ public abstract class ForwardingSSTableReader extends SSTableReader
     }
 
     @Override
-    public KeyReader keyReader() throws IOException
+    public KeyReader keyReader(boolean detailed) throws IOException
     {
-        return delegate.keyReader();
+        return delegate.keyReader(detailed);
     }
 
     public KeyReader keyReader(PartitionPosition key) throws IOException

test/distributed/org/apache/cassandra/service/accord/AccordJournalBurnTest.java

@@ -304,7 +304,7 @@ public class AccordJournalBurnTest extends BurnTestBase
                                          while (ci.hasNext())
                                              writer.append(ci.next());
 
-                                         ci.setTargetDirectory(writer.getSStableDirectory().path());
+                                         ci.setTargetDirectory(writer.getSStableDirectoryPath());
                                          // point of no return
                                          newSStables = writer.finish();
                                      }

test/harry/main/org/apache/cassandra/harry/test/HarryCompactionTest.java

@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.util.BitSet;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+public class HarryCompactionTest extends CQLTester
+{
+    private static final AtomicInteger idGen = new AtomicInteger(0);
+    public static final int TEST_REPS = 100;
+    public static final int PARTITIONS_RANGE = 100;
+    public static final int COLUMNS_RANGE = 10;
+    private final Generator<BitSet> staticColumnsGenerator = BitSet.generator(3);
+    private final Generator<BitSet> regularColumnsGenerator = BitSet.generator(9);
+
+    private String keyspace;
+    private String table;
+    private String qualifiedTable;
+    private File dataDir;
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    public void perTestSetup() throws IOException
+    {
+        keyspace = "cql_keyspace" + idGen.incrementAndGet();
+        table = "table" + idGen.incrementAndGet();
+        qualifiedTable = keyspace + '.' + table;
+        dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+        assert dataDir.tryCreateDirectories();
+
+        ServerTestUtils.prepareServerNoRegister();
+        StorageService.instance.initServer();
+        requireNetwork();
+    }
+
+    private final Generator<SchemaSpec> simple_schema = rng -> {
+        return new SchemaSpec(rng.next(),
+                              1000,
+                              keyspace,
+                              table,
+                              Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType),
+                                            ColumnSpec.pk("pk2", ColumnSpec.int64Type)),
+                              Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, true),
+                                            ColumnSpec.ck("ck2", ColumnSpec.int64Type, true)),
+                              Arrays.asList(ColumnSpec.regularColumn("r1", ColumnSpec.asciiType),
+                                            ColumnSpec.regularColumn("r2", ColumnSpec.int64Type),
+                                            ColumnSpec.regularColumn("r3", ColumnSpec.int8Type),
+                                            ColumnSpec.regularColumn("r4", ColumnSpec.doubleType),
+                                            ColumnSpec.regularColumn("r5", ColumnSpec.floatType),
+                                            ColumnSpec.regularColumn("r6", ColumnSpec.int32Type),
+                                            ColumnSpec.regularColumn("r7", ColumnSpec.booleanType),
+                                            ColumnSpec.regularColumn("r8", ColumnSpec.int16Type),
+                                            ColumnSpec.regularColumn("r9", ColumnSpec.textType)),
+                              Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType),
+                                            ColumnSpec.staticColumn("s2", ColumnSpec.int64Type),
+                                            ColumnSpec.staticColumn("s3", ColumnSpec.asciiType)));
+    };
+
+    @Test
+    public void testFlushAndCompact1() throws IOException {
+        testFlushAndCompact(1);
+    }
+
+    @Test
+    public void testFlushAndCompact2() throws IOException {
+        testFlushAndCompact(2);
+    }
+
+    @Test
+    public void testFlushAndCompact3() throws IOException {
+        testFlushAndCompact(3);
+    }
+
+    @Test
+    public void testFlushAndCompact4() throws IOException {
+        testFlushAndCompact(4);
+    }
+
+    @Test
+    public void testFlushAndCompact5() throws IOException
+    {
+        testFlushAndCompact(5);
+    }
+
+    public void testFlushAndCompact(int flushcount) throws IOException
+    {
+        for (int i = 0; i < TEST_REPS; i++)
+            testFlushAndCompactOnce(flushcount);
+    }
+
+    public void testFlushAndCompactOnce(int flushcount) throws IOException
+    {
+        perTestSetup();
+        withRandom( rng -> {
+
+            SchemaSpec schema = simple_schema.generate(rng);
+            schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", schema.keyspace));
+            createTable(schema.compile());
+
+            HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+            history.customThrowing(() -> {
+                ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+                cfs.disableAutoCompaction();
+            }, "disable compaction");
+
+            AtomicReference<HarrySSTableWriter> sstableWriter = new AtomicReference<>();
+            ThrowingRunnable flushAndChangeWriter = () -> {
+                HarrySSTableWriter prev = sstableWriter.get();
+                if (prev != null)
+                {
+                    prev.close();
+                    StorageService.instance.bulkLoad(dataDir.absolutePath());
+                    dataDir.forEach(file -> file.delete());
+                }
+
+                Invariants.require(sstableWriter.getAndSet(HarrySSTableWriter.builder()
+                                                                                   .forTable(schema.compile())
+                                                                                   .inDirectory(dataDir)
+                                                                                   .build()) == prev);
+            };
+            flushAndChangeWriter.run();
+
+            for (int sstablesFlushed = 0; sstablesFlushed < flushcount; sstablesFlushed++)
+            {
+                for (int i = 0; i < PARTITIONS_RANGE; i++)
+                {
+                    for (int j = 0; j < COLUMNS_RANGE; j++)
+                    {
+                        history.insert(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 2 * COLUMNS_RANGE)); // some overlap, but not all
+                        history.deleteRow(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 2 * COLUMNS_RANGE)); // some overlap, but not all
+                        history.deleteColumns(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 20),
+                                              regularColumnsGenerator.generate(rng),
+                                              staticColumnsGenerator.generate(rng));
+                    }
+                    history.deletePartition(rng.nextInt(0, 2 * PARTITIONS_RANGE)); // some overlap, but not all
+                }
+
+                history.customThrowing(flushAndChangeWriter, "flush sstable" + sstablesFlushed);
+            }
+
+            history.customThrowing(() -> {
+                ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+                cfs.forceMajorCompaction();
+            }, "major compaction");
+
+            for (int i = 0; i < 2*PARTITIONS_RANGE; i++)
+                history.selectPartition(i);
+
+            replay(schema, history, sstableWriter::get);
+        });
+    }
+
+    public void replay(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier<HarrySSTableWriter> writer)
+    {
+        CQLVisitExecutor executor = create(schema, historyBuilder, writer);
+        for (Visit visit : historyBuilder)
+            executor.execute(visit);
+    }
+
+    public CQLVisitExecutor create(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier<HarrySSTableWriter> writer)
+    {
+        DataTracker tracker = new DataTracker.SequentialDataTracker();
+        return new CQLTesterVisitExecutor(schema, tracker,
+                                          new QuiescentChecker(schema.valueGenerators, tracker, historyBuilder),
+                                          statement -> {
+                                              if (logger.isTraceEnabled())
+                                                  logger.trace(statement.toString());
+                                              return execute(statement.cql(), statement.bindings());
+                                          })
+        {
+            @Override
+            protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+            {
+                try
+                {
+                    writer.get().addRow(statement.cql(), statement.bindings());
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+            }
+
+            @Override
+            protected void executeValidatingVisit(Visit visit, List<Operations.SelectStatement> selects, CompiledStatement compiledStatement)
+            {
+                super.executeValidatingVisit(visit, selects, compiledStatement);
+            }
+
+            @Override
+            public void execute(Visit visit)
+            {
+                if (visit.visitedPartitions.size() > 1)
+                    throw new IllegalStateException("SSTable Generator does not support batch statements and transactions");
+
+                super.execute(visit);
+            }
+        };
+    }
+}

test/harry/main/org/apache/cassandra/harry/test/HarryCompactionWithRangeDeletionsTest.java

@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.util.BitSet;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+public class HarryCompactionWithRangeDeletionsTest extends CQLTester
+{
+    private static final AtomicInteger idGen = new AtomicInteger(0);
+    public static final int TEST_REPS = 100;
+    public static final int PARTITIONS_RANGE = 100;
+    public static final int ROWS_RANGE = 10;
+    public static final int STATIC_COLS = 3;
+    public static final int REG_COLS = 9;
+    private final Generator<BitSet> staticColumnsGenerator = BitSet.generator(STATIC_COLS);
+    private final Generator<BitSet> regularColumnsGenerator = BitSet.generator(REG_COLS);
+
+    private String keyspace;
+    private String table;
+    private String qualifiedTable;
+    private File dataDir;
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    public void perTestSetup() throws IOException
+    {
+        keyspace = "cql_keyspace" + idGen.incrementAndGet();
+        table = "table" + idGen.incrementAndGet();
+        qualifiedTable = keyspace + '.' + table;
+        dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+        assert dataDir.tryCreateDirectories();
+
+        ServerTestUtils.prepareServerNoRegister();
+        StorageService.instance.initServer();
+        requireNetwork();
+    }
+
+    private final Generator<SchemaSpec> schemaSpecGenerator = rng -> {
+        return new SchemaSpec(rng.next(),
+                              1000,
+                              keyspace,
+                              table,
+                              Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType),
+                                            ColumnSpec.pk("pk2", ColumnSpec.int64Type)),
+                              Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, true),
+                                            ColumnSpec.ck("ck2", ColumnSpec.int64Type, true)),
+                              Arrays.asList(ColumnSpec.regularColumn("r1", ColumnSpec.asciiType),
+                                            ColumnSpec.regularColumn("r2", ColumnSpec.int64Type),
+                                            ColumnSpec.regularColumn("r3", ColumnSpec.int8Type),
+                                            ColumnSpec.regularColumn("r4", ColumnSpec.doubleType),
+                                            ColumnSpec.regularColumn("r5", ColumnSpec.floatType),
+                                            ColumnSpec.regularColumn("r6", ColumnSpec.int32Type),
+                                            ColumnSpec.regularColumn("r7", ColumnSpec.booleanType),
+                                            ColumnSpec.regularColumn("r8", ColumnSpec.int16Type),
+                                            ColumnSpec.regularColumn("r9", ColumnSpec.textType)),
+                              Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType),
+                                            ColumnSpec.staticColumn("s2", ColumnSpec.int64Type),
+                                            ColumnSpec.staticColumn("s3", ColumnSpec.asciiType)));
+    };
+
+    @Test
+    public void testFlushAndCompact1() throws IOException {
+        testFlushAndCompact(1);
+    }
+
+    @Test
+    public void testFlushAndCompact2() throws IOException {
+        testFlushAndCompact(2);
+    }
+
+    @Test
+    public void testFlushAndCompact3() throws IOException {
+        testFlushAndCompact(3);
+    }
+
+    @Test
+    public void testFlushAndCompact4() throws IOException {
+        testFlushAndCompact(4);
+    }
+
+    @Test
+    public void testFlushAndCompact5() throws IOException
+    {
+        testFlushAndCompact(5);
+    }
+
+    public void testFlushAndCompact(int flushcount) throws IOException
+    {
+        for (int i = 0; i < TEST_REPS; i++)
+            testFlushAndCompactOnce(flushcount);
+    }
+
+    public void testFlushAndCompactOnce(int flushcount) throws IOException
+    {
+        perTestSetup();
+        withRandom(205413964293041L, rng -> {
+
+            SchemaSpec schema = schemaSpecGenerator.generate(rng);
+            schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", schema.keyspace));
+            createTable(schema.compile());
+
+            HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+            history.customThrowing(() -> {
+                ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+                cfs.disableAutoCompaction();
+            }, "disable compaction");
+
+            AtomicReference<HarrySSTableWriter> sstableWriter = new AtomicReference<>();
+            ThrowingRunnable flushAndChangeWriter = () -> {
+                HarrySSTableWriter prev = sstableWriter.get();
+                if (prev != null)
+                {
+                    prev.close();
+                    StorageService.instance.bulkLoad(dataDir.absolutePath());
+                    dataDir.forEach(file -> file.delete());
+                }
+
+                Invariants.require(sstableWriter.getAndSet(HarrySSTableWriter.builder()
+                                                                                   .forTable(schema.compile())
+                                                                                   .inDirectory(dataDir)
+                                                                                   .build()) == prev);
+            };
+            flushAndChangeWriter.run();
+
+            for (int sstablesFlushed = 0; sstablesFlushed < flushcount; sstablesFlushed++)
+            {
+                for (int i = 0; i < PARTITIONS_RANGE; i++)
+                {
+                    for (int j = 0; j < ROWS_RANGE; j++)
+                    {
+                        history.insert(rng.nextInt(0, 2 * PARTITIONS_RANGE), rng.nextInt(0, 2 * ROWS_RANGE)); // some overlap, but not all
+                    }
+                }
+                int lowerBoundRowIdx = rng.nextInt(ROWS_RANGE);
+                int upperBoundRowIdx = rng.nextInt(lowerBoundRowIdx, 2 * ROWS_RANGE);
+                history.deleteRowRange(rng.nextInt(0, 2 * PARTITIONS_RANGE),
+                                       lowerBoundRowIdx,
+                                       upperBoundRowIdx,
+                                       rng.nextInt(REG_COLS),
+                                       rng.nextBoolean(),
+                                       rng.nextBoolean());
+
+                history.customThrowing(flushAndChangeWriter, "flush sstable" + sstablesFlushed);
+            }
+
+            history.customThrowing(() -> {
+                ColumnFamilyStore cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+                cfs.forceMajorCompaction();
+            }, "major compaction");
+
+            for (int i = 0; i < 2 * PARTITIONS_RANGE; i++)
+                history.selectPartition(i);
+
+            replay(schema, history, sstableWriter::get);
+        });
+    }
+
+    public void replay(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier<HarrySSTableWriter> writer)
+    {
+        CQLVisitExecutor executor = create(schema, historyBuilder, writer);
+        for (Visit visit : historyBuilder)
+            executor.execute(visit);
+    }
+
+    public CQLVisitExecutor create(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier<HarrySSTableWriter> writer)
+    {
+        DataTracker tracker = new DataTracker.SequentialDataTracker();
+        return new CQLTesterVisitExecutor(schema, tracker,
+                                          new QuiescentChecker(schema.valueGenerators, tracker, historyBuilder),
+                                          statement -> {
+                                              if (logger.isTraceEnabled())
+                                                  logger.trace(statement.toString());
+                                              return execute(statement.cql(), statement.bindings());
+                                          })
+        {
+            @Override
+            protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+            {
+                try
+                {
+                    writer.get().addRow(statement.cql(), statement.bindings());
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+            }
+
+            @Override
+            protected void executeValidatingVisit(Visit visit, List<Operations.SelectStatement> selects, CompiledStatement compiledStatement)
+            {
+                super.executeValidatingVisit(visit, selects, compiledStatement);
+            }
+
+            @Override
+            public void execute(Visit visit)
+            {
+                if (visit.visitedPartitions.size() > 1)
+                    throw new IllegalStateException("SSTable Generator does not support batch statements and transactions");
+
+                super.execute(visit);
+            }
+        };
+    }
+}

test/harry/main/org/apache/cassandra/harry/test/HarrySSTableWriterTest.java

@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.harry.test;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Supplier;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import accord.utils.Invariants;
+import org.apache.cassandra.ServerTestUtils;
+import org.apache.cassandra.cql3.CQLTester;
+import org.apache.cassandra.harry.ColumnSpec;
+import org.apache.cassandra.harry.SchemaSpec;
+import org.apache.cassandra.harry.dsl.HistoryBuilder;
+import org.apache.cassandra.harry.execution.CQLTesterVisitExecutor;
+import org.apache.cassandra.harry.execution.CQLVisitExecutor;
+import org.apache.cassandra.harry.execution.CompiledStatement;
+import org.apache.cassandra.harry.execution.DataTracker;
+import org.apache.cassandra.harry.gen.Generator;
+import org.apache.cassandra.harry.model.QuiescentChecker;
+import org.apache.cassandra.harry.op.Operations;
+import org.apache.cassandra.harry.op.Visit;
+import org.apache.cassandra.harry.util.ThrowingRunnable;
+import org.apache.cassandra.io.sstable.HarrySSTableWriter;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.service.StorageService;
+
+import static org.apache.cassandra.harry.checker.TestHelper.withRandom;
+
+public class HarrySSTableWriterTest extends CQLTester
+{
+    private static final AtomicInteger idGen = new AtomicInteger(0);
+    private static final int NUMBER_WRITES_IN_RUNNABLE = 10;
+
+    private String keyspace;
+    private String table;
+    private String qualifiedTable;
+    private File dataDir;
+
+    @Rule
+    public TemporaryFolder tempFolder = new TemporaryFolder();
+
+    @Before
+    public void perTestSetup() throws IOException
+    {
+        keyspace = "cql_keyspace" + idGen.incrementAndGet();
+        table = "table" + idGen.incrementAndGet();
+        qualifiedTable = keyspace + '.' + table;
+        dataDir = new File(tempFolder.newFolder().getAbsolutePath() + File.pathSeparator() + keyspace + File.pathSeparator() + table);
+        assert dataDir.tryCreateDirectories();
+
+        ServerTestUtils.prepareServerNoRegister();
+        StorageService.instance.initServer();
+        requireNetwork();
+    }
+
+    private final Generator<SchemaSpec> simple_schema = rng -> {
+        return new SchemaSpec(rng.next(),
+                              1000,
+                              keyspace,
+                              table,
+                              Arrays.asList(ColumnSpec.pk("pk1", ColumnSpec.asciiType),
+                                            ColumnSpec.pk("pk2", ColumnSpec.int64Type)),
+                              Arrays.asList(ColumnSpec.ck("ck1", ColumnSpec.asciiType, false),
+                                            ColumnSpec.ck("ck2", ColumnSpec.int64Type, false)),
+                              Arrays.asList(ColumnSpec.regularColumn("r1", ColumnSpec.asciiType),
+                                            ColumnSpec.regularColumn("r2", ColumnSpec.int64Type),
+                                            ColumnSpec.regularColumn("r3", ColumnSpec.asciiType)),
+                              Arrays.asList(ColumnSpec.staticColumn("s1", ColumnSpec.asciiType),
+                                            ColumnSpec.staticColumn("s2", ColumnSpec.int64Type),
+                                            ColumnSpec.staticColumn("s3", ColumnSpec.asciiType)));
+    };
+
+    @Test
+    public void generateSSTableTest()
+    {
+        withRandom(rng -> {
+
+            SchemaSpec schema = simple_schema.generate(rng);
+            schemaChange(String.format("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}", schema.keyspace));
+            createTable(schema.compile());
+
+            HistoryBuilder history = new HistoryBuilder(schema.valueGenerators);
+            for (int i = 0; i < 100; i++)
+                history.insert(1);
+
+            AtomicReference<HarrySSTableWriter> sstableWriter = new AtomicReference<>();
+            ThrowingRunnable resetWriter = () -> {
+                HarrySSTableWriter prev = sstableWriter.get();
+                if (prev != null)
+                {
+                    prev.close();
+                    StorageService.instance.bulkLoad(dataDir.absolutePath());
+                }
+
+                Invariants.require(sstableWriter.getAndSet(HarrySSTableWriter.builder()
+                                                                                   .forTable(schema.compile())
+                                                                                   .inDirectory(dataDir)
+                                                                                   .build()) == prev);
+            };
+            resetWriter.run();
+
+            for (int i = 0; i < 100; i++)
+            {
+                for (int j = 0; j < 10; j++)
+                    history.insert(i, j);
+            }
+
+            history.customThrowing(resetWriter, "flush sstable");
+
+            for (int i = 0; i < 100; i++)
+                history.selectPartition(i);
+
+            replay(schema, history, sstableWriter::get);
+        });
+    }
+
+    public void replay(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier<HarrySSTableWriter> writer)
+    {
+        CQLVisitExecutor executor = create(schema, historyBuilder, writer);
+        for (Visit visit : historyBuilder)
+            executor.execute(visit);
+    }
+
+    public CQLVisitExecutor create(SchemaSpec schema, HistoryBuilder historyBuilder, Supplier<HarrySSTableWriter> writer)
+    {
+        DataTracker tracker = new DataTracker.SequentialDataTracker();
+        return new CQLTesterVisitExecutor(schema, tracker,
+                                          new QuiescentChecker(schema.valueGenerators, tracker, historyBuilder),
+                                          statement -> {
+                                              if (logger.isTraceEnabled())
+                                                  logger.trace(statement.toString());
+                                              return execute(statement.cql(), statement.bindings());
+                                          })
+        {
+            @Override
+            protected void executeMutatingVisit(Visit visit, CompiledStatement statement)
+            {
+                try
+                {
+                    writer.get().addRow(statement.cql(), statement.bindings());
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+            }
+
+            @Override
+            protected void executeValidatingVisit(Visit visit, List<Operations.SelectStatement> selects, CompiledStatement compiledStatement)
+            {
+                super.executeValidatingVisit(visit, selects, compiledStatement);
+            }
+
+            @Override
+            public void execute(Visit visit)
+            {
+                if (visit.visitedPartitions.size() > 1)
+                    throw new IllegalStateException("SSTable Generator does not support batch statements and transactions");
+
+                super.execute(visit);
+            }
+        };
+    }
+}

test/harry/main/org/apache/cassandra/io/sstable/HarrySSTableWriter.java

@@ -0,0 +1,656 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.io.sstable;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.file.NoSuchFileException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.NavigableSet;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.cql3.ColumnSpecification;
+import org.apache.cassandra.cql3.QueryOptions;
+import org.apache.cassandra.cql3.QueryProcessor;
+import org.apache.cassandra.cql3.UpdateParameters;
+import org.apache.cassandra.cql3.functions.types.TypeCodec;
+import org.apache.cassandra.cql3.statements.ModificationStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateIndexStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTableStatement;
+import org.apache.cassandra.cql3.statements.schema.CreateTypeStatement;
+import org.apache.cassandra.db.Clustering;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.Slice;
+import org.apache.cassandra.db.Slices;
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Murmur3Partitioner;
+import org.apache.cassandra.exceptions.InvalidRequestException;
+import org.apache.cassandra.exceptions.SyntaxException;
+import org.apache.cassandra.index.sai.StorageAttachedIndexGroup;
+import org.apache.cassandra.io.sstable.format.SSTableFormat;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.schema.KeyspaceMetadata;
+import org.apache.cassandra.schema.KeyspaceParams;
+import org.apache.cassandra.schema.Keyspaces;
+import org.apache.cassandra.schema.Schema;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.schema.SchemaTransformation;
+import org.apache.cassandra.schema.SchemaTransformations;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.schema.TableMetadataRef;
+import org.apache.cassandra.schema.Tables;
+import org.apache.cassandra.schema.Types;
+import org.apache.cassandra.schema.UserFunctions;
+import org.apache.cassandra.schema.Views;
+import org.apache.cassandra.service.ClientState;
+import org.apache.cassandra.tcm.ClusterMetadata;
+import org.apache.cassandra.tcm.ClusterMetadataService;
+import org.apache.cassandra.tcm.transformations.AlterSchema;
+import org.apache.cassandra.transport.ProtocolVersion;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.cassandra.utils.JavaDriverUtils;
+
+import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis;
+
+public class HarrySSTableWriter implements Closeable
+{
+    public static final ByteBuffer UNSET_VALUE = ByteBufferUtil.UNSET_BYTE_BUFFER;
+
+    static
+    {
+        CassandraRelevantProperties.FORCE_LOAD_LOCAL_KEYSPACES.setBoolean(true);
+        DatabaseDescriptor.clientInitialization(false);
+        // Partitioner is not set in client mode.
+        if (DatabaseDescriptor.getPartitioner() == null)
+            DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance);
+        ClusterMetadataService.initializeForClients();
+    }
+
+    private final AbstractSSTableSimpleWriter writer;
+
+    private HarrySSTableWriter(AbstractSSTableSimpleWriter writer)
+    {
+        this.writer = writer;
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    public HarrySSTableWriter addRow(String cql, Object... values) throws IOException
+    {
+        ModificationStatement statement = prepare(cql);
+        List<ColumnSpecification> boundNames = statement.getBindVariables();
+        // TODO: avoid materializing this
+        List<TypeCodec<Object>> typeCodecs = boundNames.stream()
+                                                       .map(bn -> JavaDriverUtils.codecFor(JavaDriverUtils.driverType(bn.type)))
+                                                       .collect(Collectors.toList());
+
+        int size = Math.min(values.length, boundNames.size());
+        List<ByteBuffer> rawValues = new ArrayList<>(size);
+        for (int i = 0; i < size; i++)
+        {
+            Object value = values[i];
+            rawValues.add(serialize(value, typeCodecs.get(i), boundNames.get(i)));
+        }
+
+        return rawAddRow(statement, rawValues, boundNames);
+    }
+
+    private ModificationStatement prepare(String cql)
+    {
+        ModificationStatement.Parsed statement = QueryProcessor.parseStatement(cql,
+                                                                               ModificationStatement.Parsed.class,
+                                                                               "INSERT/UPDATE/DELETE");
+        ClientState state = ClientState.forInternalCalls();
+        ModificationStatement preparedModificationStatement = statement.prepare(state);
+        preparedModificationStatement.validate(state);
+
+        if (preparedModificationStatement.hasConditions())
+            throw new IllegalArgumentException("Conditional statements are not supported");
+        if (preparedModificationStatement.isCounter())
+            throw new IllegalArgumentException("Counter modification statements are not supported");
+        if (preparedModificationStatement.getBindVariables().isEmpty())
+            throw new IllegalArgumentException("Provided preparedModificationStatement statement has no bind variables");
+
+        return preparedModificationStatement;
+    }
+
+    /**
+     * Adds a new row to the writer given already serialized values.
+     * <p>
+     * This is a shortcut for {@code rawAddRow(Arrays.asList(values))}.
+     *
+     * @param values the row values (corresponding to the bind variables of the
+     *               modification statement used when creating by this writer) as binary.
+     * @return this writer.
+     */
+    public HarrySSTableWriter rawAddRow(ModificationStatement modificationStatement, List<ByteBuffer> values, List<ColumnSpecification> boundNames) throws InvalidRequestException, IOException
+    {
+        if (values.size() != boundNames.size())
+            throw new InvalidRequestException(String.format("Invalid number of arguments, expecting %d values but got %d", boundNames.size(), values.size()));
+
+        QueryOptions options = QueryOptions.forInternalCalls(null, values);
+        ClientState state = ClientState.forInternalCalls();
+        List<ByteBuffer> keys = modificationStatement.buildPartitionKeyNames(options, state);
+
+        long now = currentTimeMillis();
+        // Note that we asks indexes to not validate values (the last 'false' arg below) because that triggers a 'Keyspace.open'
+        // and that forces a lot of initialization that we don't want.
+        UpdateParameters params = new UpdateParameters(modificationStatement.metadata,
+                                                       ClientState.forInternalCalls(),
+                                                       options,
+                                                       modificationStatement.getTimestamp(TimeUnit.MILLISECONDS.toMicros(now), options),
+                                                       options.getNowInSec((int) TimeUnit.MILLISECONDS.toSeconds(now)),
+                                                       modificationStatement.getTimeToLive(options),
+                                                       Collections.emptyMap());
+
+        try
+        {
+            if (modificationStatement.hasSlices())
+            {
+                Slices slices = modificationStatement.createSlices(options);
+
+                for (ByteBuffer key : keys)
+                {
+                    for (Slice slice : slices)
+                        modificationStatement.addUpdateForKey(writer.getUpdateFor(key), slice, params);
+                }
+            }
+            else
+            {
+                NavigableSet<Clustering<?>> clusterings = modificationStatement.createClustering(options, state);
+
+                for (ByteBuffer key : keys)
+                {
+                    for (Clustering clustering : clusterings)
+                        modificationStatement.addUpdateForKey(writer.getUpdateFor(key), clustering, params);
+                }
+            }
+            return this;
+        }
+        catch (SSTableSimpleUnsortedWriter.SyncException e)
+        {
+            // If we use a BufferedWriter and had a problem writing to disk, the IOException has been
+            // wrapped in a SyncException (see BufferedWriter below). We want to extract that IOE.
+            throw (IOException) e.getCause();
+        }
+    }
+
+    /**
+     * Close this writer.
+     * <p>
+     * This method should be called, otherwise the produced sstables are not
+     * guaranteed to be complete (and won't be in practice).
+     */
+    public void close() throws IOException
+    {
+        writer.close();
+    }
+
+    private ByteBuffer serialize(Object value, TypeCodec codec, ColumnSpecification columnSpecification)
+    {
+        if (value == null || value == UNSET_VALUE)
+            return (ByteBuffer) value;
+
+        try
+        {
+            return codec.serialize(value, ProtocolVersion.CURRENT);
+        }
+        catch (ClassCastException cce)
+        {
+            // For backwards-compatibility with consumers that may be passing
+            // an Integer for a Date field, for example.
+            return ((AbstractType) columnSpecification.type).decompose(value);
+        }
+    }
+
+    /**
+     * A Builder for a CQLSSTableWriter object.
+     */
+    public static class Builder
+    {
+        private static final Logger logger = LoggerFactory.getLogger(Builder.class);
+        private static final long DEFAULT_BUFFER_SIZE_IN_MIB_FOR_UNSORTED = 128L;
+
+        protected SSTableFormat<?, ?> format = null;
+
+        private final List<CreateTypeStatement.Raw> typeStatements;
+        private final List<CreateIndexStatement.Raw> indexStatements;
+
+        private File directory;
+        private CreateTableStatement.Raw schemaStatement;
+        private IPartitioner partitioner;
+        private boolean sorted = false;
+        private long maxSSTableSizeInMiB = -1L;
+        private boolean buildIndexes = true;
+        private Consumer<Collection<SSTableReader>> sstableProducedListener;
+        private boolean openSSTableOnProduced = false;
+
+        protected Builder()
+        {
+            this.typeStatements = new ArrayList<>();
+            this.indexStatements = new ArrayList<>();
+        }
+
+        /**
+         * The directory where to write the sstables.
+         * <p>
+         * This is a mandatory option.
+         *
+         * @param directory the directory to use, which should exists and be writable.
+         * @return this builder.
+         * @throws IllegalArgumentException if {@code directory} doesn't exist or is not writable.
+         */
+        public Builder inDirectory(String directory)
+        {
+            return inDirectory(new File(directory));
+        }
+
+        /**
+         * The directory where to write the sstables (mandatory option).
+         * <p>
+         * This is a mandatory option.
+         *
+         * @param directory the directory to use, which should exists and be writable.
+         * @return this builder.
+         * @throws IllegalArgumentException if {@code directory} doesn't exist or is not writable.
+         */
+        public Builder inDirectory(File directory)
+        {
+            if (!directory.exists())
+                throw new IllegalArgumentException(directory + " doesn't exists");
+            if (!directory.isWritable())
+                throw new IllegalArgumentException(directory + " exists but is not writable");
+
+            this.directory = directory;
+            return this;
+        }
+
+        public Builder withType(String typeDefinition) throws SyntaxException
+        {
+            typeStatements.add(QueryProcessor.parseStatement(typeDefinition, CreateTypeStatement.Raw.class, "CREATE TYPE"));
+            return this;
+        }
+
+        /**
+         * The schema (CREATE TABLE statement) for the table for which sstable are to be created.
+         * <p>
+         * Please note that the provided CREATE TABLE statement <b>must</b> use a fully-qualified
+         * table name, one that include the keyspace name.
+         * <p>
+         * This is a mandatory option.
+         *
+         * @param schema the schema of the table for which sstables are to be created.
+         * @return this builder.
+         * @throws IllegalArgumentException if {@code schema} is not a valid CREATE TABLE statement
+         *                                  or does not have a fully-qualified table name.
+         */
+        public Builder forTable(String schema)
+        {
+            this.schemaStatement = QueryProcessor.parseStatement(schema, CreateTableStatement.Raw.class, "CREATE TABLE");
+            return this;
+        }
+
+        /**
+         * The schema (CREATE INDEX statement) for index to be created for the table. Only SAI indexes are supported.
+         *
+         * @param indexes CQL statements representing SAI indexes to be created.
+         * @return this builder
+         */
+        public Builder withIndexes(String... indexes)
+        {
+            for (String index : indexes)
+                indexStatements.add(QueryProcessor.parseStatement(index, CreateIndexStatement.Raw.class, "CREATE INDEX"));
+
+            return this;
+        }
+
+        /**
+         * The partitioner to use.
+         * <p>
+         * By default, {@code Murmur3Partitioner} will be used. If this is not the partitioner used
+         * by the cluster for which the SSTables are created, you need to use this method to
+         * provide the correct partitioner.
+         *
+         * @param partitioner the partitioner to use.
+         * @return this builder.
+         */
+        public Builder withPartitioner(IPartitioner partitioner)
+        {
+            this.partitioner = partitioner;
+            return this;
+        }
+
+        /**
+         * Defines the maximum SSTable size in mebibytes when using the sorted writer.
+         * By default, i.e. not specified, there is no maximum size limit for the produced SSTable
+         *
+         * @param size the maximum sizein mebibytes of each individual SSTable allowed
+         * @return this builder
+         */
+        public Builder withMaxSSTableSizeInMiB(int size)
+        {
+            if (size <= 0)
+            {
+                logger.warn("A non-positive value for maximum SSTable size is specified, " +
+                            "which disables the size limiting effectively. Please supply a positive value in order " +
+                            "to enforce size limiting for the produced SSTables.");
+            }
+            this.maxSSTableSizeInMiB = size;
+            return this;
+        }
+
+        /**
+         * The size of the buffer to use.
+         * <p>
+         * This defines how much data will be buffered before being written as
+         * a new SSTable. This corresponds roughly to the data size that will have the created
+         * sstable.
+         * <p>
+         * The default is 128MiB, which should be reasonable for a 1GiB heap. If you experience
+         * OOM while using the writer, you should lower this value.
+         *
+         * @param size the size to use in MiB.
+         * @return this builder.
+         * @deprecated This method is deprecated in favor of the new withMaxSSTableSizeInMiB(int size)
+         */
+        @Deprecated(since = "5.0")
+        public Builder withBufferSizeInMiB(int size)
+        {
+            return withMaxSSTableSizeInMiB(size);
+        }
+
+        /**
+         * The size of the buffer to use.
+         * <p>
+         * This defines how much data will be buffered before being written as
+         * a new SSTable. This corresponds roughly to the data size that will have the created
+         * sstable.
+         * <p>
+         * The default is 128MiB, which should be reasonable for a 1GiB heap. If you experience
+         * OOM while using the writer, you should lower this value.
+         *
+         * @param size the size to use in MiB.
+         * @return this builder.
+         * @deprecated This method is deprecated in favor of the new withBufferSizeInMiB(int size). See CASSANDRA-17675
+         */
+        @Deprecated(since = "4.1")
+        public Builder withBufferSizeInMB(int size)
+        {
+            return withBufferSizeInMiB(size);
+        }
+
+        /**
+         * Creates a CQLSSTableWriter that expects sorted inputs.
+         * <p>
+         * If this option is used, the resulting writer will expect rows to be
+         * added in SSTable sorted order (and an exception will be thrown if that
+         * is not the case during modification). The SSTable sorted order means that
+         * rows are added such that their partition key respect the partitioner
+         * order.
+         * <p>
+         * You should thus only use this option is you know that you can provide
+         * the rows in order, which is rarely the case. If you can provide the
+         * rows in order however, using this sorted might be more efficient.
+         * <p>
+         * Note that if used, some option like withBufferSizeInMiB will be ignored.
+         *
+         * @return this builder.
+         */
+        public Builder sorted()
+        {
+            this.sorted = true;
+            return this;
+        }
+
+        /**
+         * Whether indexes should be built and serialized to disk along data. Defaults to true.
+         *
+         * @param buildIndexes true if indexes should be built, false otherwise
+         * @return this builder
+         */
+        public Builder withBuildIndexes(boolean buildIndexes)
+        {
+            this.buildIndexes = buildIndexes;
+            return this;
+        }
+
+        /**
+         * Set the listener to receive notifications on sstable produced
+         * <p>
+         * Note that if listener is registered, the sstables are opened into {@link SSTableReader}.
+         * The consumer is responsible for releasing the {@link SSTableReader}
+         *
+         * @param sstableProducedListener receives the produced sstables
+         * @return this builder
+         */
+        public Builder withSSTableProducedListener(Consumer<Collection<SSTableReader>> sstableProducedListener)
+        {
+            this.sstableProducedListener = sstableProducedListener;
+            return this;
+        }
+
+        /**
+         * Whether the produced sstable should be open or not.
+         * By default, the writer does not open the produced sstables
+         *
+         * @return this builder
+         */
+        public Builder openSSTableOnProduced()
+        {
+            this.openSSTableOnProduced = true;
+            return this;
+        }
+
+        public HarrySSTableWriter build()
+        {
+            if (directory == null)
+                throw new IllegalStateException("No ouptut directory specified, you should provide a directory with inDirectory()");
+            if (schemaStatement == null)
+                throw new IllegalStateException("Missing schema, you should provide the schema for the SSTable to create with forTable()");
+
+            Preconditions.checkState(Sets.difference(SchemaConstants.LOCAL_SYSTEM_KEYSPACE_NAMES, Schema.instance.getKeyspaces()).isEmpty(),
+                                     "Local keyspaces were not loaded. If this is running as a client, please make sure to add %s=true system property.",
+                                     CassandraRelevantProperties.FORCE_LOAD_LOCAL_KEYSPACES.getKey());
+
+            // Assign the default max SSTable size if not defined in builder
+            if (isMaxSSTableSizeUnset())
+            {
+                maxSSTableSizeInMiB = sorted ? -1L : DEFAULT_BUFFER_SIZE_IN_MIB_FOR_UNSORTED;
+            }
+
+            synchronized (HarrySSTableWriter.class)
+            {
+                String keyspaceName = schemaStatement.keyspace();
+                String tableName = schemaStatement.table();
+
+                Schema.instance.submit(SchemaTransformations.addKeyspace(KeyspaceMetadata.create(keyspaceName,
+                                                                                                 KeyspaceParams.simple(1),
+                                                                                                 Tables.none(),
+                                                                                                 Views.none(),
+                                                                                                 Types.none(),
+                                                                                                 UserFunctions.none()), true));
+
+                KeyspaceMetadata ksm = KeyspaceMetadata.create(keyspaceName,
+                                                               KeyspaceParams.simple(1),
+                                                               Tables.none(),
+                                                               Views.none(),
+                                                               Types.none(),
+                                                               UserFunctions.none());
+
+                TableMetadata tableMetadata = Schema.instance.getTableMetadata(keyspaceName, tableName);
+                if (tableMetadata == null)
+                {
+                    Types types = createTypes(keyspaceName);
+                    Schema.instance.submit(SchemaTransformations.addTypes(types, true));
+                    tableMetadata = createTable(types, ksm.userFunctions);
+                    Schema.instance.submit(SchemaTransformations.addTable(tableMetadata, true));
+
+                    if (buildIndexes && !indexStatements.isEmpty())
+                    {
+                        // we need to commit keyspace metadata first so applyIndexes sees that keyspace from TCM
+                        commitKeyspaceMetadata(ksm.withSwapped(ksm.tables.with(tableMetadata)));
+                        applyIndexes(keyspaceName);
+                    }
+
+                    KeyspaceMetadata keyspaceMetadata = ClusterMetadata.current().schema.getKeyspaceMetadata(keyspaceName);
+                    tableMetadata = keyspaceMetadata.tables.getNullable(tableName);
+
+                    Schema.instance.submit(SchemaTransformations.addTable(tableMetadata, true));
+                }
+
+                KeyspaceMetadata keyspaceMetadata = ClusterMetadata.current().schema.getKeyspaceMetadata(keyspaceName);
+                Keyspace keyspace = Keyspace.mockKS(keyspaceMetadata);
+                Directories directories = new Directories(tableMetadata, Collections.singleton(new Directories.DataDirectory(new File(directory.toPath()))));
+                ColumnFamilyStore cfs = ColumnFamilyStore.createColumnFamilyStore(keyspace,
+                                                                tableName,
+                                                                tableMetadata,
+                                                                directories,
+                                                                false,
+                                                                false);
+
+                keyspace.initCfCustom(cfs);
+
+                // this is the empty directory / leftover from times we initialized ColumnFamilyStore
+                // it will automatically create directories for keyspace and table on disk after initialization
+                // we set that directory to the destination of generated SSTables so we just remove empty directories here
+                try
+                {
+                    new File(directory, keyspaceName).deleteRecursive();
+                }
+                catch (UncheckedIOException ex)
+                {
+                    if (!(ex.getCause() instanceof NoSuchFileException))
+                    {
+                        throw ex;
+                    }
+                }
+
+                TableMetadataRef ref = tableMetadata.ref;
+                AbstractSSTableSimpleWriter writer = sorted
+                                                     ? new SSTableSimpleWriter(directory, ref, cfs.metadata.get().regularAndStaticColumns(), maxSSTableSizeInMiB)
+                                                     : new SSTableSimpleUnsortedWriter(directory, ref, cfs.metadata.get().regularAndStaticColumns(), maxSSTableSizeInMiB);
+
+                if (format != null)
+                    writer.setSSTableFormatType(format);
+
+                if (buildIndexes && !indexStatements.isEmpty() && cfs != null)
+                {
+                    StorageAttachedIndexGroup saiGroup = StorageAttachedIndexGroup.getIndexGroup(cfs);
+                    if (saiGroup != null)
+                        writer.addIndexGroup(saiGroup);
+                }
+
+                if (sstableProducedListener != null)
+                    writer.setSSTableProducedListener(sstableProducedListener);
+
+                writer.setShouldOpenProducedSSTable(openSSTableOnProduced);
+
+                return new HarrySSTableWriter(writer);
+            }
+        }
+
+        private boolean isMaxSSTableSizeUnset()
+        {
+            return maxSSTableSizeInMiB <= 0;
+        }
+
+        private Types createTypes(String keyspace)
+        {
+            Types.RawBuilder builder = Types.rawBuilder(keyspace);
+            for (CreateTypeStatement.Raw st : typeStatements)
+                st.addToRawBuilder(builder);
+            return builder.build();
+        }
+
+        /**
+         * Applies any provided index definitions to the target table
+         *
+         * @param keyspaceName name of the keyspace to apply indexes for
+         * @return table metadata reflecting applied indexes
+         */
+        private void applyIndexes(String keyspaceName)
+        {
+            ClientState state = ClientState.forInternalCalls();
+
+            for (CreateIndexStatement.Raw statement : indexStatements)
+            {
+                Keyspaces keyspaces = statement.prepare(state).apply(ClusterMetadata.current());
+                commitKeyspaceMetadata(keyspaces.getNullable(keyspaceName));
+            }
+        }
+
+        private void commitKeyspaceMetadata(KeyspaceMetadata keyspaceMetadata)
+        {
+            SchemaTransformation schemaTransformation = new SchemaTransformation()
+            {
+                @Override
+                public Keyspaces apply(ClusterMetadata metadata)
+                {
+                    return metadata.schema.getKeyspaces().withAddedOrUpdated(keyspaceMetadata);
+                }
+
+                @Override
+                public boolean compatibleWith(ClusterMetadata metadata)
+                {
+                    return true;
+                }
+            };
+            ClusterMetadataService.instance().commit(new AlterSchema(schemaTransformation));
+        }
+
+        /**
+         * Creates the table according to schema statement
+         *
+         * @param types types this table should be created with
+         */
+        private TableMetadata createTable(Types types, UserFunctions functions)
+        {
+            ClientState state = ClientState.forInternalCalls();
+            CreateTableStatement statement = schemaStatement.prepare(state);
+            statement.validate(ClientState.forInternalCalls());
+
+            TableMetadata.Builder builder = statement.builder(types, functions);
+            if (partitioner != null)
+                builder.partitioner(partitioner);
+
+            return builder.build();
+        }
+    }
+}

test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java

@@ -23,64 +23,87 @@ import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.*;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.CQLTester;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.service.snapshot.SnapshotManager;
+import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.openjdk.jmh.annotations.*;
 
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
-@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
 @Fork(value = 1)
 @Threads(1)
 @State(Scope.Benchmark)
 public class CompactionBench extends CQLTester
 {
-    static String keyspace;
-    String table;
-    String writeStatement;
-    String readStatement;
-    ColumnFamilyStore cfs;
-    List<File> snapshotFiles;
-    List<Descriptor> liveFiles;
+    protected static String keyspace;
+    protected String table;
+    protected String writeStatement;
+    protected ColumnFamilyStore cfs;
+    protected List<File> snapshotFiles;
+
+    @Param("2")
+    protected int sstableCount = 2;
+
+    @Param("50000")
+    protected int rowCount = 50000;
+
+    @Param("NONE")
+    protected String overlap = "NONE";
+
+    @Param("true")
+    protected boolean isCursor = true;
 
     @Setup(Level.Trial)
     public void setup() throws Throwable
     {
         CQLTester.prepareServer();
+        DatabaseDescriptor.setCursorCompactionEnabled(isCursor);
+        DatabaseDescriptor.setCompactionThroughputMebibytesPerSec(10*1024); // no rate limiting
+        createSStables();
+        takeSnapshot();
+    }
+
+    protected void createSStables()
+    {
         keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
         table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))");
         execute("use "+keyspace+";");
         writeStatement = "INSERT INTO "+table+"(userid,picid,commentid)VALUES(?,?,?)";
-        readStatement = "SELECT * from "+table+" limit 100";
 
         Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
 
         cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
         cfs.disableAutoCompaction();
 
-        //Warm up
-        System.err.println("Writing 50k");
-        for (long i = 0; i < 50000; i++)
-            execute(writeStatement, i, i, i );
-
-
-        cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
-
-        System.err.println("Writing 50k again...");
-        for (long i = 0; i < 50000; i++)
-            execute(writeStatement, i, i, i );
+        for (int j = 0; j < sstableCount; j++)
+        {
+            int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+            int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+            for (long i = 0; i < rowCount; i++)
+            {
+                execute(writeStatement, (pPrefix + i), (rPrefix + i), j * rowCount + i);
+            }
 
             cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        }
+    }
 
+    private void takeSnapshot()
+    {
         SnapshotManager.instance.takeSnapshot("originals", cfs.getKeyspaceTableName());
         snapshotFiles = cfs.getDirectories().sstableLister(Directories.OnTxnErr.IGNORE).snapshots("originals").listFiles();
+        long[] sum = new long[1];
+        snapshotFiles.forEach(f -> sum[0] += f.length());
+        System.out.println("Total input size: " + sum[0]);
     }
 
     @TearDown(Level.Trial)
@@ -95,28 +118,36 @@ public class CompactionBench extends CQLTester
                 System.err.println("Thread "+t.getName());
         }
 
+        CommitLog.instance.shutdownBlocking();
+        ClusterMetadataService.instance().log().close();
+        CQLTester.tearDownClass();
         CQLTester.cleanup();
     }
 
 
     @TearDown(Level.Invocation)
-    public void resetSnapshot()
+    public void resetSnapshot() throws IOException, InterruptedException
     {
         cfs.truncateBlocking();
 
         List<File> directories = cfs.getDirectories().getCFDirectories();
-
+        // Sometimes deletes are unreliable...
+        int deleted = 0;
+        do
+        {
+            deleted = 0;
             for (File file : directories)
             {
                 for (File f : file.tryList())
                 {
                     if (f.isDirectory())
                         continue;
-
-                FileUtils.delete(f);
+                    f.tryDelete();
+                    deleted++;
                 }
             }
-
+            Thread.sleep(10);
+        } while (deleted != 0);
 
         for (File file : snapshotFiles)
             FileUtils.createHardLink(file, new File(new File(file.toPath().getParent().getParent().getParent()), file.name()));

test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionLargeCellBench.java

@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.test.microbench.CompactionBench;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class CompactionLargeCellBench extends CompactionBench
+{
+    @Param("128")
+    int blobSize = 128;
+
+    protected void createSStables()
+    {
+        keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+        table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, b blob, PRIMARY KEY(userid, picid))");
+        execute("use "+keyspace+";");
+        writeStatement = "INSERT INTO "+table+"(userid,picid,b)VALUES(?,?,?)";
+
+        Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+        cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        cfs.disableAutoCompaction();
+
+        byte[] blob = new byte[blobSize];
+        for (int j = 0; j < sstableCount; j++)
+        {
+            int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+            int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+            for (long i = 0; i < rowCount; i++)
+            {
+                ThreadLocalRandom.current().nextBytes(blob);
+                execute(writeStatement, (pPrefix + i), (rPrefix + i), ByteBuffer.wrap(blob));
+            }
+
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        }
+    }
+}

test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionStringsBench.java

@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.test.microbench.CompactionBench;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class CompactionStringsBench extends CompactionBench
+{
+    @Param("128")
+    int stringSizeMin = 128;
+    @Param("256")
+    int stringSizeMax = 256;
+
+    protected void createSStables()
+    {
+        keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+        table = createTable(keyspace, "CREATE TABLE %s ( pk ascii, ck ascii, c ascii, PRIMARY KEY(pk, ck))");
+        execute("use "+keyspace+";");
+        writeStatement = "INSERT INTO "+table+"(pk,ck,c)VALUES(?,?,?)";
+
+        Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+        cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        cfs.disableAutoCompaction();
+        for (int j = 0; j < sstableCount; j++)
+        {
+            Random r = new Random(42L);
+            int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+            int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+            for (long i = 0; i < rowCount; i++)
+            {
+                execute(writeStatement, (pPrefix + i) + getNextString(r), (rPrefix + i) + getNextString(r), getNextString(r));
+            }
+
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        }
+    }
+
+    private String getNextString(Random r)
+    {
+        int rangeDelta = stringSizeMax - stringSizeMin;
+        int rangeRandom = rangeDelta > 0 ? r.nextInt(rangeDelta) : 0;
+        byte[] blob = new byte[stringSizeMin + rangeRandom];
+        r.nextBytes(blob);
+        for (int i = 0; i < blob.length; i++)
+        {
+            if (blob[i] < 0)
+                blob[i] = 0;
+        }
+        String nextString = new String(blob);
+        return nextString;
+    }
+}

test/microbench/org/apache/cassandra/test/microbench/sstable/CompactionWideRowBench.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.test.microbench.sstable;
+
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.test.microbench.CompactionBench;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+@Threads(1)
+@State(Scope.Benchmark)
+public class CompactionWideRowBench extends CompactionBench
+{
+    @Param("1")
+    int rowPerPkCount = 1;
+
+    @Param("1")
+    int ckCount = 1;
+
+    @Param("1")
+    int colCount = 1;
+
+    protected void createSStables()
+    {
+        keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false");
+        String tableCreate = "CREATE TABLE %s ( userid bigint";
+        for (int i=0;i<ckCount;i++)
+            tableCreate += ",ck"+i+ " bigint";
+        for (int i=0;i<colCount;i++)
+            tableCreate += ",c"+i+ " bigint";
+        tableCreate += ", PRIMARY KEY(userid";
+        for (int i=0;i<ckCount;i++)
+            tableCreate += ",ck"+i;
+        tableCreate +="))";
+        table = createTable(keyspace, tableCreate);
+        execute("use "+keyspace+";");
+        writeStatement = "INSERT INTO "+table+"(userid";
+        for (int i=0;i<ckCount;i++)
+            writeStatement += ",ck"+i;
+        for (int i=0;i<colCount;i++)
+            writeStatement += ",c"+i;
+        writeStatement += ")VALUES(?";
+        for (int i=0;i<ckCount+colCount;i++)
+            writeStatement += ",?";
+        writeStatement += ")";
+
+        Object[] values = new Object[1+ckCount+colCount];
+        Keyspace.system().forEach(k -> k.getColumnFamilyStores().forEach(c -> c.disableAutoCompaction()));
+
+        cfs = Keyspace.open(keyspace).getColumnFamilyStore(table);
+        cfs.disableAutoCompaction();
+        int pkCount = rowCount/rowPerPkCount;
+        for (int j = 0; j < sstableCount; j++)
+        {
+            int pPrefix = overlap.startsWith("PK") ? 0 : j * rowCount;
+            int rPrefix = overlap.startsWith("PK.ROW") ? 0 : j * rowCount;
+            for (long pkIndex = 0; pkIndex < pkCount; pkIndex++)
+            {
+                for (long rowIndex = 0; rowIndex < rowPerPkCount; rowIndex++)
+                {
+                    values[0] = (pPrefix + pkIndex);
+                    Arrays.fill(values, 1, ckCount + 1, (rPrefix + rowIndex));
+                    Arrays.fill(values, 1 + ckCount, values.length, j * rowCount + pkIndex);
+                    execute(writeStatement, values);
+                }
+            }
+
+            cfs.forceBlockingFlush(ColumnFamilyStore.FlushReason.USER_FORCED);
+        }
+    }
+}