Implementation of the Unified Compaction Strategy as described in CEP-26

The approach is documented in the included UnifiedCompactionStrategy.md.

Also included:
- Generalize prettyPrint to prettyPrintBinary (with 2^10 multiplier,
  e.g. MiB) and prettyPrintDecimal (with 1000 multiplier, e.g. ms)
  that cover the full range of double, and add a parseHumanReadable
  method that can read both.

- CASSANDRA-18123: Fix invalid reuse of metadata collector during flushing

- Fix invalid token range splitting with non-unit weights

- Add flushSizeOnDisk metric

- Add ability to change compaction default in YAML

patch by Alex Sorokoumov, Branimir Lambov, Dimitar Dimitrov and Stefania Alborghetti
reviewed by Alex Sorokoumov, Jaroslaw Grabowski and Maxwell Guo for CASSANDRA-18397

CHANGES.txt

@@ -1,4 +1,5 @@
 5.0
+ * Implementation of the Unified Compaction Strategy as described in CEP-26 (CASSANDRA-18397)
  * Upgrade commons cli to 1.5.0 (CASSANDRA-18659)
  * Disable the deprecated keyspace/table thresholds and convert them to guardrails (CASSANDRA-18617)
  * Deprecate CloudstackSnitch and remove duplicate code in snitches (CASSANDRA-18438)

NEWS.txt

@@ -71,6 +71,10 @@ using the provided 'sstableupgrade' tool.
 
 New features
 ------------
+    - Added a new "unified" compaction strategy that supports the use cases of the legacy compaction strategies, with
+      low space overhead, high parallelism and flexible configuration. Implemented by the UnifiedCompactionStrategy
+      class. Further details and documentation can be found in
+      src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md
     - New `VectorType` (cql `vector<element_type, dimension>`) which adds new fixed-length element arrays. See CASSANDRA-18504
     - Removed UDT type migration logic for 3.6+ clusters upgrading to 4.0.  If migration has been disabled, it must be
       enabled before upgrading to 5.0 if the cluster used UDTs. See CASSANDRA-18504

conf/cassandra.yaml

@@ -1002,6 +1002,20 @@ snapshot_links_per_second: 0
 # Min unit: KiB
 column_index_cache_size: 2KiB
 
+# Default compaction strategy, applied when a table's parameters do not
+# specify compaction.
+# The selected compaction strategy will also apply to system tables.
+#
+# The default is to use SizeTieredCompactionStrategy, with its default
+# compaction parameters.
+#
+# default_compaction:
+#   class_name: UnifiedCompactionStrategy
+#   parameters:
+#     scaling_parameters: T4
+#     target_sstable_size: 1GiB
+
+
 # Number of simultaneous compactions to allow, NOT including
 # validation "compactions" for anti-entropy repair.  Simultaneous
 # compactions can help preserve read performance in a mixed read/write

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -29,6 +29,7 @@ import com.google.common.primitives.Ints;
 import org.apache.cassandra.db.virtual.LogMessagesTable;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.service.FileSystemOwnershipCheck;
+import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.StorageCompatibilityMode;
 
 // checkstyle: suppress below 'blockSystemPropertyUsage'
@@ -516,6 +517,12 @@ public enum CassandraRelevantProperties
     TRIGGERS_DIR("cassandra.triggers_dir"),
     TRUNCATE_BALLOT_METADATA("cassandra.truncate_ballot_metadata"),
     TYPE_UDT_CONFLICT_BEHAVIOR("cassandra.type.udt.conflict_behavior"),
+    // See org.apache.cassandra.db.compaction.unified.Controller for the definition of the UCS parameters
+    UCS_BASE_SHARD_COUNT("unified_compaction.base_shard_count", "4"),
+    UCS_OVERLAP_INCLUSION_METHOD("unified_compaction.overlap_inclusion_method"),
+    UCS_SCALING_PARAMETER("unified_compaction.scaling_parameters", "T4"),
+    UCS_SURVIVAL_FACTOR("unified_compaction.survival_factor", "1"),
+    UCS_TARGET_SSTABLE_SIZE("unified_compaction.target_sstable_size", "1GiB"),
     UDF_EXECUTOR_THREAD_KEEPALIVE_MS("cassandra.udf_executor_thread_keepalive_ms", "30000"),
     UNSAFE_SYSTEM("cassandra.unsafesystem"),
     /** User's home directory. */
@@ -725,6 +732,56 @@ public enum CassandraRelevantProperties
         return LONG_CONVERTER.convert(value);
     }
 
+    /**
+     * Gets the value of a system property as a double.
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public double getDouble()
+    {
+        String value = System.getProperty(key);
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return DOUBLE_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
+    /**
+     * Gets the value of a system property as a double.
+     * @return system property value if it exists, defaultValue otherwise.
+     */
+    public double getDouble(double overrideDefaultValue)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            return overrideDefaultValue;
+
+        return DOUBLE_CONVERTER.convert(value);
+    }
+
+    /**
+     * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
+     * @return System property value if it exists, defaultValue otherwise. Throws an exception if no default value is set.
+     */
+    public long getSizeInBytes()
+    {
+        String value = System.getProperty(key);
+        if (value == null && defaultVal == null)
+            throw new ConfigurationException("Missing property value or default value is not set: " + key);
+        return SIZE_IN_BYTES_CONVERTER.convert(value == null ? defaultVal : value);
+    }
+
+    /**
+     * Gets the value of a system property, given as a human-readable size in bytes (e.g. 100MiB, 10GB, 500B).
+     * @return System property value if it exists, defaultValue otherwise.
+     */
+    public long getSizeInBytes(long overrideDefaultValue)
+    {
+        String value = System.getProperty(key);
+        if (value == null)
+            return overrideDefaultValue;
+
+        return SIZE_IN_BYTES_CONVERTER.convert(value);
+    }
+
     /**
      * Gets the value of a system property as an int.
      * @return system property int value if it exists, overrideDefaultValue otherwise.
@@ -847,6 +904,32 @@ public enum CassandraRelevantProperties
         }
     };
 
+    private static final PropertyConverter<Long> SIZE_IN_BYTES_CONVERTER = value ->
+    {
+        try
+        {
+            return FBUtilities.parseHumanReadableBytes(value);
+        }
+        catch (ConfigurationException e)
+        {
+            throw new ConfigurationException(String.format("Invalid value for system property: " +
+                                                           "expected size in bytes with unit but got '%s'\n%s", value, e));
+        }
+    };
+
+    private static final PropertyConverter<Double> DOUBLE_CONVERTER = value ->
+    {
+        try
+        {
+            return Double.parseDouble(value);
+        }
+        catch (NumberFormatException e)
+        {
+            throw new ConfigurationException(String.format("Invalid value for system property: " +
+                                                           "expected floating point value but got '%s'", value));
+        }
+    };
+
     /**
      * @return whether a system property is present or not.
      */

src/java/org/apache/cassandra/config/Config.java

@@ -1098,6 +1098,11 @@ public class Config
     public volatile long min_tracked_partition_tombstone_count = 5000;
     public volatile boolean top_partitions_enabled = true;
 
+    /**
+     * Default compaction configuration, used if a table does not specify any.
+     */
+    public ParameterizedClass default_compaction = null;
+
     public static Supplier<Config> getOverrideLoadConfig()
     {
         return overrideLoadConfig;

src/java/org/apache/cassandra/config/DatabaseDescriptor.java

@@ -4751,4 +4751,9 @@ public class DatabaseDescriptor
         else
             return conf.storage_compatibility_mode;
     }
+
+    public static ParameterizedClass getDefaultCompaction()
+    {
+        return conf != null ? conf.default_compaction : null;
+    }
 }

src/java/org/apache/cassandra/db/AbstractCompactionController.java

@@ -44,7 +44,7 @@ public abstract class AbstractCompactionController implements AutoCloseable
 
     public String getKeyspace()
     {
-        return cfs.keyspace.getName();
+        return cfs.getKeyspaceName();
     }
 
     public String getColumnFamily()

src/java/org/apache/cassandra/db/ColumnFamilyStore.java

@@ -64,7 +64,6 @@ import com.google.common.base.Predicate;
 import com.google.common.base.Predicates;
 import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
@@ -83,6 +82,7 @@ import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.DurationSpec;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.CompactionInfo;
 import org.apache.cassandra.db.compaction.CompactionManager;
@@ -129,7 +129,6 @@ import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.format.Version;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.io.util.FileOutputStreamPlus;
 import org.apache.cassandra.metrics.Sampler;
@@ -260,6 +259,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     public static final String SNAPSHOT_DROP_PREFIX = "dropped";
     static final String TOKEN_DELIMITER = ":";
 
+    /** Special values used when the local ranges are not changed with ring changes (e.g. local tables). */
+    public static final int RING_VERSION_IRRELEVANT = -1;
+
     static
     {
         try
@@ -347,7 +349,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
                 if (history != null)
                     return history;
 
-                history = TablePaxosRepairHistory.load(keyspace.getName(), name);
+                history = TablePaxosRepairHistory.load(getKeyspaceName(), name);
                 return history;
             }
         }
@@ -488,7 +490,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         additionalWriteLatencyMicros = DatabaseDescriptor.getWriteRpcTimeout(TimeUnit.MICROSECONDS) / 2;
         memtableFactory = metadata.get().params.memtable.factory();
 
-        logger.info("Initializing {}.{}", keyspace.getName(), name);
+        logger.info("Initializing {}.{}", getKeyspaceName(), name);
 
         // Create Memtable and its metrics object only on online
         Memtable initialMemtable = null;
@@ -539,8 +541,8 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         if (registerBookeeping)
         {
             // register the mbean
-            mbeanName = getTableMBeanName(keyspace.getName(), name, isIndex());
-            oldMBeanName = getColumnFamilieMBeanName(keyspace.getName(), name, isIndex());
+            mbeanName = getTableMBeanName(getKeyspaceName(), name, isIndex());
+            oldMBeanName = getColumnFamilieMBeanName(getKeyspaceName(), name, isIndex());
 
             String[] objectNames = {mbeanName, oldMBeanName};
             for (String objectName : objectNames)
@@ -556,7 +558,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         repairManager = new CassandraTableRepairManager(this);
         sstableImporter = new SSTableImporter(this);
 
-        if (SchemaConstants.isSystemKeyspace(keyspace.getName()))
+        if (SchemaConstants.isSystemKeyspace(getKeyspaceName()))
             topPartitions = null;
         else
             topPartitions = new TopPartitionTracker(metadata());
@@ -646,15 +648,19 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         return memtableFactory.streamFromMemtable();
     }
 
-    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
+    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
     {
-        MetadataCollector collector = new MetadataCollector(metadata().comparator).sstableLevel(sstableLevel);
-        return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, collector, header, lifecycleNewTracker);
+        return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, null, 0, header, lifecycleNewTracker);
     }
 
-    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, MetadataCollector metadataCollector, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
+    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet<CommitLogPosition> commitLogPositions, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
     {
-        return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadataCollector, header, indexManager.listIndexes(), lifecycleNewTracker);
+        return createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, 0, header, lifecycleNewTracker);
+    }
+
+    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, IntervalSet<CommitLogPosition> commitLogPositions, int sstableLevel, SerializationHeader header, LifecycleNewTracker lifecycleNewTracker)
+    {
+        return getCompactionStrategyManager().createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, commitLogPositions, sstableLevel, header, indexManager.listIndexes(), lifecycleNewTracker);
     }
 
     public boolean supportsEarlyOpen()
@@ -887,7 +893,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
     public void rebuildSecondaryIndex(String idxName)
     {
-        rebuildSecondaryIndex(keyspace.getName(), metadata.name, idxName);
+        rebuildSecondaryIndex(getKeyspaceName(), metadata.name, idxName);
     }
 
     public static void rebuildSecondaryIndex(String ksName, String cfName, String... idxNames)
@@ -923,6 +929,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         return name;
     }
 
+    public String getKeyspaceName()
+    {
+        return keyspace.getName();
+    }
+
     public Descriptor newSSTableDescriptor(File directory)
     {
         return newSSTableDescriptor(directory, DatabaseDescriptor.getSelectedSSTableFormat().getLatestVersion());
@@ -937,7 +948,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     {
         Descriptor newDescriptor = new Descriptor(version,
                                                   directory,
-                                                  keyspace.getName(),
+                                                  getKeyspaceName(),
                                                   name,
                                                   sstableIdGenerator.get());
         assert !newDescriptor.fileFor(Components.DATA).exists();
@@ -1003,7 +1014,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         for (ColumnFamilyStore indexCfs : indexManager.getAllIndexColumnFamilyStores())
             indexCfs.getTracker().getView().getCurrentMemtable().addMemoryUsageTo(usage);
 
-        logger.info("Enqueuing flush of {}.{}, Reason: {}, Usage: {}", keyspace.getName(), name, reason, usage);
+        logger.info("Enqueuing flush of {}.{}, Reason: {}, Usage: {}", getKeyspaceName(), name, reason, usage);
     }
 
 
@@ -1250,7 +1261,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
                 {
                     // flush the memtable
                     flushRunnables = Flushing.flushRunnables(cfs, memtable, txn);
-                    ExecutorPlus[] executors = perDiskflushExecutors.getExecutorsFor(keyspace.getName(), name);
+                    ExecutorPlus[] executors = perDiskflushExecutors.getExecutorsFor(getKeyspaceName(), name);
 
                     for (int i = 0; i < flushRunnables.size(); i++)
                         futures.add(executors[i].submit(flushRunnables.get(i)));
@@ -1281,7 +1292,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
                     {
                         @SuppressWarnings("resource")
                         SSTableMultiWriter writer = writerIterator.next();
-                        if (writer.getFilePointer() > 0)
+                        if (writer.getBytesWritten() > 0)
                         {
                             writer.setOpenResult(true).prepareToCommit();
                         }
@@ -1305,7 +1316,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
                 Throwable accumulate = null;
                 for (SSTableMultiWriter writer : flushResults)
+                {
                     accumulate = writer.commit(accumulate);
+                    metric.flushSizeOnDisk.update(writer.getOnDiskBytesWritten());
+                }
 
                 maybeFail(txn.commit(accumulate));
 
@@ -1440,7 +1454,53 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         {
             throw new RuntimeException(e.getMessage()
                                        + " for ks: "
-                                       + keyspace.getName() + ", table: " + name, e);
+                                       + getKeyspaceName() + ", table: " + name, e);
+        }
+    }
+
+    public static class VersionedLocalRanges extends ArrayList<Splitter.WeightedRange>
+    {
+        public final long ringVersion;
+
+        public VersionedLocalRanges(long ringVersion, int initialSize)
+        {
+            super(initialSize);
+            this.ringVersion = ringVersion;
+        }
+    }
+
+    public VersionedLocalRanges localRangesWeighted()
+    {
+        if (!SchemaConstants.isLocalSystemKeyspace(getKeyspaceName())
+            && getPartitioner() == StorageService.instance.getTokenMetadata().partitioner)
+        {
+            DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
+            Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
+            long ringVersion = versionedLocalRanges.ringVersion;
+
+            if (!localRanges.isEmpty())
+            {
+                VersionedLocalRanges weightedRanges = new VersionedLocalRanges(ringVersion, localRanges.size());
+                for (Range<Token> r : localRanges)
+                {
+                    // WeightedRange supports only unwrapped ranges as it relies
+                    // on right - left == num tokens equality
+                    for (Range<Token> u: r.unwrap())
+                        weightedRanges.add(new Splitter.WeightedRange(1.0, u));
+                }
+                weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
+                return weightedRanges;
+            }
+            else
+            {
+                return fullWeightedRange(ringVersion, getPartitioner());
+            }
+        }
+        else
+        {
+            // Local tables need to cover the full token range and don't care about ring changes.
+            // We also end up here if the table's partitioner is not the database's, which can happen in tests.
+            return fullWeightedRange(RING_VERSION_IRRELEVANT, getPartitioner());
         }
     }
 
@@ -1454,54 +1514,26 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
         if (shardBoundaries == null ||
             shardBoundaries.shardCount() != shardCount ||
-            shardBoundaries.ringVersion != -1 && shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion())
+            (shardBoundaries.ringVersion != RING_VERSION_IRRELEVANT &&
+             shardBoundaries.ringVersion != StorageService.instance.getTokenMetadata().getRingVersion()))
         {
-            List<Splitter.WeightedRange> weightedRanges;
-            long ringVersion;
-            if (!SchemaConstants.isLocalSystemKeyspace(keyspace.getName())
-                && getPartitioner() == StorageService.instance.getTokenMetadata().partitioner)
-            {
-                DiskBoundaryManager.VersionedRangesAtEndpoint versionedLocalRanges = DiskBoundaryManager.getVersionedLocalRanges(this);
-                Set<Range<Token>> localRanges = versionedLocalRanges.rangesAtEndpoint.ranges();
-                ringVersion = versionedLocalRanges.ringVersion;
-
-                if (!localRanges.isEmpty())
-                {
-                    weightedRanges = new ArrayList<>(localRanges.size());
-                    for (Range<Token> r : localRanges)
-                    {
-                        // WeightedRange supports only unwrapped ranges as it relies
-                        // on right - left == num tokens equality
-                        for (Range<Token> u: r.unwrap())
-                            weightedRanges.add(new Splitter.WeightedRange(1.0, u));
-                    }
-                    weightedRanges.sort(Comparator.comparing(Splitter.WeightedRange::left));
-                }
-                else
-                {
-                    weightedRanges = fullWeightedRange();
-                }
-            }
-            else
-            {
-                // Local tables need to cover the full token range and don't care about ring changes.
-                // We also end up here if the table's partitioner is not the database's, which can happen in tests.
-                weightedRanges = fullWeightedRange();
-                ringVersion = -1;
-            }
+            VersionedLocalRanges weightedRanges = localRangesWeighted();
 
             List<Token> boundaries = getPartitioner().splitter().get().splitOwnedRanges(shardCount, weightedRanges, false);
             shardBoundaries = new ShardBoundaries(boundaries.subList(0, boundaries.size() - 1),
-                                                  ringVersion);
+                                                  weightedRanges.ringVersion);
             cachedShardBoundaries = shardBoundaries;
-            logger.debug("Memtable shard boundaries for {}.{}: {}", keyspace.getName(), getTableName(), boundaries);
+            logger.debug("Memtable shard boundaries for {}.{}: {}", getKeyspaceName(), getTableName(), boundaries);
         }
         return shardBoundaries;
     }
 
-    private ImmutableList<Splitter.WeightedRange> fullWeightedRange()
+    @VisibleForTesting
+    public static VersionedLocalRanges fullWeightedRange(long ringVersion, IPartitioner partitioner)
     {
-        return ImmutableList.of(new Splitter.WeightedRange(1.0, new Range<>(getPartitioner().getMinimumToken(), getPartitioner().getMaximumToken())));
+        VersionedLocalRanges ranges = new VersionedLocalRanges(ringVersion, 1);
+        ranges.add(new Splitter.WeightedRange(1.0, new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken())));
+        return ranges;
     }
 
     /**
@@ -1521,7 +1553,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         View view = data.getView();
 
         List<SSTableReader> sortedByFirst = Lists.newArrayList(sstables);
-        sortedByFirst.sort(SSTableReader.sstableComparator);
+        sortedByFirst.sort(SSTableReader.firstKeyComparator);
 
         List<AbstractBounds<PartitionPosition>> bounds = new ArrayList<>();
         DecoratedKey first = null, last = null;
@@ -1622,7 +1654,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
         // cleanup size estimation only counts bytes for keys local to this node
         long expectedFileSize = 0;
-        Collection<Range<Token>> ranges = StorageService.instance.getLocalReplicas(keyspace.getName()).ranges();
+        Collection<Range<Token>> ranges = StorageService.instance.getLocalReplicas(getKeyspaceName()).ranges();
         for (SSTableReader sstable : sstables)
         {
             List<SSTableReader.PartitionPositionBounds> positions = sstable.getPositionsForRanges(ranges);
@@ -1989,7 +2021,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
             //Not duplicating the buffer for safety because AbstractSerializer and ByteBufferUtil.bytesToHex
             //don't modify position or limit
             result.add(new CompositeDataSupport(COUNTER_COMPOSITE_TYPE, COUNTER_NAMES, new Object[] {
-                    keyspace.getName() + "." + name,
+                    getKeyspaceName() + "." + name,
                     counter.count,
                     counter.error,
                     samplerImpl.toString(counter.value) })); // string
@@ -2011,7 +2043,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
     public void cleanupCache()
     {
-        Collection<Range<Token>> ranges = StorageService.instance.getLocalReplicas(keyspace.getName()).ranges();
+        Collection<Range<Token>> ranges = StorageService.instance.getLocalReplicas(getKeyspaceName()).ranges();
 
         for (Iterator<RowCacheKey> keyIter = CacheService.instance.rowCache.keyIterator();
              keyIter.hasNext(); )
@@ -2543,11 +2575,13 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
             return null;
 
         List<Memtable.FlushablePartitionSet<?>> dataSets = new ArrayList<>(ranges.size());
+        IntervalSet.Builder<CommitLogPosition> commitLogIntervals = new IntervalSet.Builder();
         long keys = 0;
         for (Range<PartitionPosition> range : ranges)
         {
             Memtable.FlushablePartitionSet<?> dataSet = current.getFlushSet(range.left, range.right);
             dataSets.add(dataSet);
+            commitLogIntervals.add(dataSet.commitLogLowerBound(), dataSet.commitLogUpperBound());
             keys += dataSet.partitionCount();
         }
         if (keys == 0)
@@ -2560,7 +2594,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
                                                              0,
                                                              repairSessionID,
                                                              false,
-                                                             0,
+                                                             commitLogIntervals.build(),
                                                              new SerializationHeader(true,
                                                                                      firstDataSet.metadata(),
                                                                                      firstDataSet.columns(),
@@ -2643,7 +2677,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         // beginning if we restart before they [the CL segments] are discarded for
         // normal reasons post-truncate.  To prevent this, we store truncation
         // position in the System keyspace.
-        logger.info("Truncating {}.{}", keyspace.getName(), name);
+        logger.info("Truncating {}.{}", getKeyspaceName(), name);
 
         viewManager.stopBuild();
 
@@ -2677,7 +2711,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
         {
             public void run()
             {
-                logger.info("Truncating {}.{} with truncatedAt={}", keyspace.getName(), getTableName(), truncatedAt);
+                logger.info("Truncating {}.{} with truncatedAt={}", getKeyspaceName(), getTableName(), truncatedAt);
                 // since truncation can happen at different times on different nodes, we need to make sure
                 // that any repairs are aborted, otherwise we might clear the data on one node and then
                 // stream in data that is actually supposed to have been deleted
@@ -2704,7 +2738,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
 
         viewManager.build();
 
-        logger.info("Truncate of {}.{} is complete", keyspace.getName(), name);
+        logger.info("Truncate of {}.{} is complete", getKeyspaceName(), name);
     }
 
     /**
@@ -2866,7 +2900,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     public String toString()
     {
         return "CFS(" +
-               "Keyspace='" + keyspace.getName() + '\'' +
+               "Keyspace='" + getKeyspaceName() + '\'' +
                ", ColumnFamily='" + name + '\'' +
                ')';
     }
@@ -3306,9 +3340,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean, Memtable.Owner
     public void setNeverPurgeTombstones(boolean value)
     {
         if (neverPurgeTombstones != value)
-            logger.info("Changing neverPurgeTombstones for {}.{} from {} to {}", keyspace.getName(), getTableName(), neverPurgeTombstones, value);
+            logger.info("Changing neverPurgeTombstones for {}.{} from {} to {}", getKeyspaceName(), getTableName(), neverPurgeTombstones, value);
         else
-            logger.info("Not changing neverPurgeTombstones for {}.{}, it is {}", keyspace.getName(), getTableName(), neverPurgeTombstones);
+            logger.info("Not changing neverPurgeTombstones for {}.{}, it is {}", getKeyspaceName(), getTableName(), neverPurgeTombstones);
 
         neverPurgeTombstones = value;
     }

src/java/org/apache/cassandra/db/DiskBoundaries.java

@@ -109,7 +109,7 @@ public class DiskBoundaries
             return getBoundariesFromSSTableDirectory(sstable.descriptor);
         }
 
-        int pos = Collections.binarySearch(positions, sstable.first);
+        int pos = Collections.binarySearch(positions, sstable.getFirst());
         assert pos < 0; // boundaries are .minkeybound and .maxkeybound so they should never be equal
         return -pos - 1;
     }
@@ -146,7 +146,7 @@ public class DiskBoundaries
     {
         int diskIndex = getDiskIndex(sstable);
         PartitionPosition diskLast = positions.get(diskIndex);
-        return directories.get(diskIndex).equals(currentLocation) && sstable.last.compareTo(diskLast) <= 0;
+        return directories.get(diskIndex).equals(currentLocation) && sstable.getLast().compareTo(diskLast) <= 0;
     }
 
     private int getDiskIndex(DecoratedKey key)

src/java/org/apache/cassandra/db/DiskBoundaryManager.java

@@ -51,10 +51,10 @@ public class DiskBoundaryManager
             {
                 if (diskBoundaries == null || diskBoundaries.isOutOfDate())
                 {
-                    logger.debug("Refreshing disk boundary cache for {}.{}", cfs.keyspace.getName(), cfs.getTableName());
+                    logger.debug("Refreshing disk boundary cache for {}.{}", cfs.getKeyspaceName(), cfs.getTableName());
                     DiskBoundaries oldBoundaries = diskBoundaries;
                     diskBoundaries = getDiskBoundaryValue(cfs);
-                    logger.debug("Updating boundaries from {} to {} for {}.{}", oldBoundaries, diskBoundaries, cfs.keyspace.getName(), cfs.getTableName());
+                    logger.debug("Updating boundaries from {} to {} for {}.{}", oldBoundaries, diskBoundaries, cfs.getKeyspaceName(), cfs.getTableName());
                 }
             }
         }
@@ -128,7 +128,7 @@ public class DiskBoundaryManager
         && !StorageService.isReplacingSameAddress()) // When replacing same address, the node marks itself as UN locally
         {
             PendingRangeCalculatorService.instance.blockUntilFinished();
-            localRanges = tmd.getPendingRanges(cfs.keyspace.getName(), FBUtilities.getBroadcastAddressAndPort());
+            localRanges = tmd.getPendingRanges(cfs.getKeyspaceName(), FBUtilities.getBroadcastAddressAndPort());
         }
         else
         {

src/java/org/apache/cassandra/db/SSTableImporter.java

@@ -73,7 +73,7 @@ public class SSTableImporter
     synchronized List<String> importNewSSTables(Options options)
     {
         UUID importID = UUID.randomUUID();
-        logger.info("[{}] Loading new SSTables for {}/{}: {}", importID, cfs.keyspace.getName(), cfs.getTableName(), options);
+        logger.info("[{}] Loading new SSTables for {}/{}: {}", importID, cfs.getKeyspaceName(), cfs.getTableName(), options);
 
         List<Pair<Directories.SSTableLister, String>> listers = getSSTableListers(options.srcPaths);
 
@@ -175,11 +175,11 @@ public class SSTableImporter
 
         if (newSSTables.isEmpty())
         {
-            logger.info("[{}] No new SSTables were found for {}/{}", importID, cfs.keyspace.getName(), cfs.getTableName());
+            logger.info("[{}] No new SSTables were found for {}/{}", importID, cfs.getKeyspaceName(), cfs.getTableName());
             return failedDirectories;
         }
 
-        logger.info("[{}] Loading new SSTables and building secondary indexes for {}/{}: {}", importID, cfs.keyspace.getName(), cfs.getTableName(), newSSTables);
+        logger.info("[{}] Loading new SSTables and building secondary indexes for {}/{}: {}", importID, cfs.getKeyspaceName(), cfs.getTableName(), newSSTables);
         if (logger.isTraceEnabled())
             logLeveling(importID, newSSTables);
 
@@ -199,7 +199,7 @@ public class SSTableImporter
             throw new RuntimeException("Failed adding SSTables", t);
         }
 
-        logger.info("[{}] Done loading load new SSTables for {}/{}", importID, cfs.keyspace.getName(), cfs.getTableName());
+        logger.info("[{}] Done loading load new SSTables for {}/{}", importID, cfs.getKeyspaceName(), cfs.getTableName());
         return failedDirectories;
     }
 

src/java/org/apache/cassandra/db/compaction/AbstractCompactionStrategy.java

@@ -34,6 +34,8 @@ import org.slf4j.LoggerFactory;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.dht.Range;
@@ -530,7 +532,7 @@ public abstract class AbstractCompactionStrategy
     {
         int groupSize = 2;
         List<SSTableReader> sortedSSTablesToGroup = new ArrayList<>(sstablesToGroup);
-        Collections.sort(sortedSSTablesToGroup, SSTableReader.sstableComparator);
+        Collections.sort(sortedSSTablesToGroup, SSTableReader.firstKeyComparator);
 
         Collection<Collection<SSTableReader>> groupedSSTables = new ArrayList<>();
         Collection<SSTableReader> currGroup = new ArrayList<>(groupSize);
@@ -560,12 +562,23 @@ public abstract class AbstractCompactionStrategy
                                                        long repairedAt,
                                                        TimeUUID pendingRepair,
                                                        boolean isTransient,
-                                                       MetadataCollector meta,
+                                                       IntervalSet<CommitLogPosition> commitLogPositions,
+                                                       int sstableLevel,
                                                        SerializationHeader header,
                                                        Collection<Index> indexes,
                                                        LifecycleNewTracker lifecycleNewTracker)
     {
-        return SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, cfs.metadata, meta, header, indexes, lifecycleNewTracker, cfs);
+        return SimpleSSTableMultiWriter.create(descriptor,
+                                               keyCount,
+                                               repairedAt,
+                                               pendingRepair,
+                                               isTransient,
+                                               cfs.metadata,
+                                               commitLogPositions,
+                                               sstableLevel,
+                                               header,
+                                               indexes,
+                                               lifecycleNewTracker, cfs);
     }
 
     public boolean supportsEarlyOpen()

src/java/org/apache/cassandra/db/compaction/AbstractStrategyHolder.java

@@ -29,6 +29,8 @@ import com.google.common.base.Preconditions;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -37,7 +39,6 @@ import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.utils.TimeUUID;
 
@@ -194,7 +195,8 @@ public abstract class AbstractStrategyHolder
                                                                 long repairedAt,
                                                                 TimeUUID pendingRepair,
                                                                 boolean isTransient,
-                                                                MetadataCollector collector,
+                                                                IntervalSet<CommitLogPosition> commitLogPositions,
+                                                                int sstableLevel,
                                                                 SerializationHeader header,
                                                                 Collection<Index> indexes,
                                                                 LifecycleNewTracker lifecycleNewTracker);

src/java/org/apache/cassandra/db/compaction/CompactionController.java

@@ -109,7 +109,7 @@ public class CompactionController extends AbstractCompactionController
 
         if (cfs.getNeverPurgeTombstones())
         {
-            logger.debug("not refreshing overlaps for {}.{} - neverPurgeTombstones is enabled", cfs.keyspace.getName(), cfs.getTableName());
+            logger.debug("not refreshing overlaps for {}.{} - neverPurgeTombstones is enabled", cfs.getKeyspaceName(), cfs.getTableName());
             return;
         }
 
@@ -173,11 +173,11 @@ public class CompactionController extends AbstractCompactionController
             Set<SSTableReader> fullyExpired = new HashSet<>();
             for (SSTableReader candidate : compacting)
             {
-                if (candidate.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+                if (candidate.getMaxLocalDeletionTime() < gcBefore)
                 {
                     fullyExpired.add(candidate);
                     logger.trace("Dropping overlap ignored expired SSTable {} (maxLocalDeletionTime={}, gcBefore={})",
-                                 candidate, candidate.getSSTableMetadata().maxLocalDeletionTime, gcBefore);
+                                 candidate, candidate.getMaxLocalDeletionTime(), gcBefore);
                 }
             }
             return fullyExpired;
@@ -190,13 +190,13 @@ public class CompactionController extends AbstractCompactionController
         {
             // Overlapping might include fully expired sstables. What we care about here is
             // the min timestamp of the overlapping sstables that actually contain live data.
-            if (sstable.getSSTableMetadata().maxLocalDeletionTime >= gcBefore)
+            if (sstable.getMaxLocalDeletionTime() >= gcBefore)
                 minTimestamp = Math.min(minTimestamp, sstable.getMinTimestamp());
         }
 
         for (SSTableReader candidate : compacting)
         {
-            if (candidate.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+            if (candidate.getMaxLocalDeletionTime() < gcBefore)
                 candidates.add(candidate);
             else
                 minTimestamp = Math.min(minTimestamp, candidate.getMinTimestamp());
@@ -224,7 +224,7 @@ public class CompactionController extends AbstractCompactionController
             else
             {
                logger.trace("Dropping expired SSTable {} (maxLocalDeletionTime={}, gcBefore={})",
-                        candidate, candidate.getSSTableMetadata().maxLocalDeletionTime, gcBefore);
+                        candidate, candidate.getMaxLocalDeletionTime(), gcBefore);
             }
         }
         return new HashSet<>(candidates);

src/java/org/apache/cassandra/db/compaction/CompactionIterator.java

@@ -743,6 +743,6 @@ public class CompactionIterator extends CompactionInfo.Holder implements Unfilte
 
     private static boolean isPaxos(ColumnFamilyStore cfs)
     {
-        return cfs.name.equals(SystemKeyspace.PAXOS) && cfs.keyspace.getName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
+        return cfs.name.equals(SystemKeyspace.PAXOS) && cfs.getKeyspaceName().equals(SchemaConstants.SYSTEM_KEYSPACE_NAME);
     }
 }

src/java/org/apache/cassandra/db/compaction/CompactionLogger.java

@@ -228,7 +228,7 @@ public class CompactionLogger
         ColumnFamilyStore cfs = cfsRef.get();
         if (cfs == null)
             return;
-        node.put("keyspace", cfs.keyspace.getName());
+        node.put("keyspace", cfs.getKeyspaceName());
         node.put("table", cfs.getTableName());
         node.put("time", currentTimeMillis());
     }

src/java/org/apache/cassandra/db/compaction/CompactionManager.java

@@ -243,12 +243,12 @@ public class CompactionManager implements CompactionManagerMBean
         if (count > 0 && executor.getActiveTaskCount() >= executor.getMaximumPoolSize())
         {
             logger.trace("Background compaction is still running for {}.{} ({} remaining). Skipping",
-                         cfs.keyspace.getName(), cfs.name, count);
+                         cfs.getKeyspaceName(), cfs.name, count);
             return Collections.emptyList();
         }
 
         logger.trace("Scheduling a background task check for {}.{} with {}",
-                     cfs.keyspace.getName(),
+                     cfs.getKeyspaceName(),
                      cfs.name,
                      cfs.getCompactionStrategyManager().getName());
 
@@ -353,7 +353,7 @@ public class CompactionManager implements CompactionManagerMBean
             boolean ranCompaction = false;
             try
             {
-                logger.trace("Checking {}.{}", cfs.keyspace.getName(), cfs.name);
+                logger.trace("Checking {}.{}", cfs.getKeyspaceName(), cfs.name);
                 if (!cfs.isValid())
                 {
                     logger.trace("Aborting compaction for dropped CF");
@@ -383,7 +383,7 @@ public class CompactionManager implements CompactionManagerMBean
 
         boolean maybeRunUpgradeTask(CompactionStrategyManager strategy)
         {
-            logger.debug("Checking for upgrade tasks {}.{}", cfs.keyspace.getName(), cfs.getTableName());
+            logger.debug("Checking for upgrade tasks {}.{}", cfs.getKeyspaceName(), cfs.getTableName());
             try
             {
                 if (currentlyBackgroundUpgrading.incrementAndGet() <= DatabaseDescriptor.maxConcurrentAutoUpgradeTasks())
@@ -426,10 +426,10 @@ public class CompactionManager implements CompactionManagerMBean
                                                            OperationType operationType)
     {
         String operationName = operationType.name();
-        String keyspace = cfs.keyspace.getName();
+        String keyspace = cfs.getKeyspaceName();
         String table = cfs.getTableName();
         return cfs.withAllSSTables(operationType, (compacting) -> {
-            logger.info("Starting {} for {}.{}", operationType, cfs.keyspace.getName(), cfs.getTableName());
+            logger.info("Starting {} for {}.{}", operationType, cfs.getKeyspaceName(), cfs.getTableName());
             List<LifecycleTransaction> transactions = new ArrayList<>();
             List<Future<?>> futures = new ArrayList<>();
             try
@@ -657,7 +657,7 @@ public class CompactionManager implements CompactionManagerMBean
                     }
                 }
                 logger.info("Skipping cleanup for {}/{} sstables for {}.{} since they are fully contained in owned ranges (full ranges: {}, transient ranges: {})",
-                            skippedSStables, totalSSTables, cfStore.keyspace.getName(), cfStore.getTableName(), fullRanges, transientRanges);
+                            skippedSStables, totalSSTables, cfStore.getKeyspaceName(), cfStore.getTableName(), fullRanges, transientRanges);
                 sortedSSTables.sort(SSTableReader.sizeComparator);
                 return sortedSSTables;
             }
@@ -744,7 +744,7 @@ public class CompactionManager implements CompactionManagerMBean
             return AllSSTableOpStatus.ABORTED;
         }
 
-        if (StorageService.instance.getLocalReplicas(cfs.keyspace.getName()).isEmpty())
+        if (StorageService.instance.getLocalReplicas(cfs.getKeyspaceName()).isEmpty())
         {
             logger.info("Relocate cannot run before a node has joined the ring");
             return AllSSTableOpStatus.ABORTED;
@@ -903,7 +903,7 @@ public class CompactionManager implements CompactionManagerMBean
             Preconditions.checkArgument(!replicas.isEmpty(), "No ranges to anti-compact");
 
             if (logger.isInfoEnabled())
-                logger.info("{} Starting anticompaction for {}.{} on {}/{} sstables", PreviewKind.NONE.logPrefix(sessionID), cfs.keyspace.getName(), cfs.getTableName(), validatedForRepair.size(), cfs.getLiveSSTables().size());
+                logger.info("{} Starting anticompaction for {}.{} on {}/{} sstables", PreviewKind.NONE.logPrefix(sessionID), cfs.getKeyspaceName(), cfs.getTableName(), validatedForRepair.size(), cfs.getLiveSSTables().size());
             if (logger.isTraceEnabled())
                 logger.trace("{} Starting anticompaction for ranges {}", PreviewKind.NONE.logPrefix(sessionID), replicas);
 
@@ -1661,7 +1661,7 @@ public class CompactionManager implements CompactionManagerMBean
                          .setPendingRepair(pendingRepair)
                          .setTransientSSTable(isTransient)
                          .setTableMetadataRef(cfs.metadata)
-                         .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator, minLevel))
+                         .setMetadataCollector(new MetadataCollector(sstables, cfs.metadata().comparator).sstableLevel(minLevel))
                          .setSerializationHeader(SerializationHeader.make(cfs.metadata(), sstables))
                          .addDefaultComponents()
                          .addFlushObserversForSecondaryIndexes(cfs.indexManager.listIndexes(), txn.opType())
@@ -1736,7 +1736,7 @@ public class CompactionManager implements CompactionManagerMBean
             return 0;
         }
 
-        logger.info("Anticompacting {} in {}.{} for {}", txn.originals(), cfs.keyspace.getName(), cfs.getTableName(), pendingRepair);
+        logger.info("Anticompacting {} in {}.{} for {}", txn.originals(), cfs.getKeyspaceName(), cfs.getTableName(), pendingRepair);
         Set<SSTableReader> sstableAsSet = txn.originals();
 
         File destination = cfs.getDirectories().getWriteableLocationAsFile(cfs.getExpectedCompactedFileSize(sstableAsSet, OperationType.ANTICOMPACTION));
@@ -1846,7 +1846,7 @@ public class CompactionManager implements CompactionManagerMBean
             txn.commit();
             logger.info("Anticompacted {} in {}.{} to full = {}, transient = {}, unrepaired = {} for {}",
                         sstableAsSet,
-                        cfs.keyspace.getName(),
+                        cfs.getKeyspaceName(),
                         cfs.getTableName(),
                         fullSSTables,
                         transSSTables,

src/java/org/apache/cassandra/db/compaction/CompactionStrategyHolder.java

@@ -27,6 +27,8 @@ import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -35,7 +37,6 @@ import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.TimeUUID;
@@ -220,7 +221,8 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
                                                        long repairedAt,
                                                        TimeUUID pendingRepair,
                                                        boolean isTransient,
-                                                       MetadataCollector collector,
+                                                       IntervalSet<CommitLogPosition> commitLogPositions,
+                                                       int sstableLevel,
                                                        SerializationHeader header,
                                                        Collection<Index> indexes,
                                                        LifecycleNewTracker lifecycleNewTracker)
@@ -244,7 +246,8 @@ public class CompactionStrategyHolder extends AbstractStrategyHolder
                                                  repairedAt,
                                                  pendingRepair,
                                                  isTransient,
-                                                 collector,
+                                                 commitLogPositions,
+                                                 sstableLevel,
                                                  header,
                                                  indexes,
                                                  lifecycleNewTracker);

src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java

@@ -50,6 +50,8 @@ import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.DiskBoundaries;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.compaction.AbstractStrategyHolder.TaskSupplier;
 import org.apache.cassandra.db.compaction.PendingRepairManager.CleanupTask;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
@@ -64,7 +66,6 @@ import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
 import org.apache.cassandra.io.util.File;
 import org.apache.cassandra.notifications.INotification;
@@ -485,7 +486,7 @@ public class CompactionStrategyManager implements INotificationConsumer
     private void reloadParamsFromSchema(CompactionParams newParams)
     {
         logger.debug("Recreating compaction strategy for {}.{} - compaction parameters changed via CQL",
-                     cfs.keyspace.getName(), cfs.getTableName());
+                     cfs.getKeyspaceName(), cfs.getTableName());
 
         /*
          * It's possible for compaction to be explicitly enabled/disabled
@@ -532,7 +533,7 @@ public class CompactionStrategyManager implements INotificationConsumer
     private void reloadParamsFromJMX(CompactionParams newParams)
     {
         logger.debug("Recreating compaction strategy for {}.{} - compaction parameters changed via JMX",
-                     cfs.keyspace.getName(), cfs.getTableName());
+                     cfs.getKeyspaceName(), cfs.getTableName());
 
         setStrategy(newParams);
 
@@ -587,12 +588,12 @@ public class CompactionStrategyManager implements INotificationConsumer
         if (newBoundaries.isEquivalentTo(oldBoundaries))
         {
             logger.debug("Not recreating compaction strategy for {}.{} - disk boundaries are equivalent",
-                         cfs.keyspace.getName(), cfs.getTableName());
+                         cfs.getKeyspaceName(), cfs.getTableName());
             return;
         }
 
         logger.debug("Recreating compaction strategy for {}.{} - disk boundaries are out of date",
-                     cfs.keyspace.getName(), cfs.getTableName());
+                     cfs.getKeyspaceName(), cfs.getTableName());
         setStrategy(params);
         startup();
     }
@@ -1073,7 +1074,7 @@ public class CompactionStrategyManager implements INotificationConsumer
             {
                 for (AbstractStrategyHolder holder : holders)
                 {
-                    for (AbstractCompactionTask task: holder.getMaximalTasks(gcBefore, splitOutput)) 
+                    for (AbstractCompactionTask task: holder.getMaximalTasks(gcBefore, splitOutput))
                     {
                         tasks.add(task.setCompactionType(operationType));
                     }
@@ -1233,7 +1234,8 @@ public class CompactionStrategyManager implements INotificationConsumer
                                                        long repairedAt,
                                                        TimeUUID pendingRepair,
                                                        boolean isTransient,
-                                                       MetadataCollector collector,
+                                                       IntervalSet<CommitLogPosition> commitLogPositions,
+                                                       int sstableLevel,
                                                        SerializationHeader header,
                                                        Collection<Index> indexes,
                                                        LifecycleNewTracker lifecycleNewTracker)
@@ -1248,7 +1250,8 @@ public class CompactionStrategyManager implements INotificationConsumer
                                                                                               repairedAt,
                                                                                               pendingRepair,
                                                                                               isTransient,
-                                                                                              collector,
+                                                                                              commitLogPositions,
+                                                                                              sstableLevel,
                                                                                               header,
                                                                                               indexes,
                                                                                               lifecycleNewTracker);

src/java/org/apache/cassandra/db/compaction/CompactionTask.java

@@ -252,7 +252,7 @@ public class CompactionTask extends AbstractCompactionTask
             for (int i = 0; i < mergedRowCounts.length; i++)
                 totalSourceRows += mergedRowCounts[i] * (i + 1);
 
-            String mergeSummary = updateCompactionHistory(taskId, cfs.keyspace.getName(), cfs.getTableName(), mergedRowCounts, startsize, endsize,
+            String mergeSummary = updateCompactionHistory(taskId, cfs.getKeyspaceName(), cfs.getTableName(), mergedRowCounts, startsize, endsize,
                                                           ImmutableMap.of(COMPACTION_TYPE_PROPERTY, compactionType.type));
 
             logger.info(String.format("Compacted (%s) %d sstables to [%s] to level=%d.  %s to %s (~%d%% of original) in %,dms.  Read Throughput = %s, Write Throughput = %s, Row Throughput = ~%,d/s.  %,d total partitions merged to %,d.  Partition merge counts were {%s}. Time spent writing keys = %,dms",
@@ -417,7 +417,7 @@ public class CompactionTask extends AbstractCompactionTask
 
                 String msg = String.format("Not enough space for compaction (%s) of %s.%s, estimated sstables = %d, expected write size = %d",
                                            taskId,
-                                           cfs.keyspace.getName(),
+                                           cfs.getKeyspaceName(),
                                            cfs.name,
                                            Math.max(1, writeSize / strategy.getMaxSSTableBytes()),
                                            writeSize);

src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java

@@ -428,7 +428,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
 
             totalLength = length;
             compressedLength = cLength;
-            Collections.sort(this.sstables, SSTableReader.sstableComparator);
+            Collections.sort(this.sstables, SSTableReader.firstKeyComparator);
             sstableIterator = this.sstables.iterator();
             assert sstableIterator.hasNext(); // caller should check intersecting first
             SSTableReader currentSSTable = sstableIterator.next();
@@ -446,7 +446,7 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
             {
                 for (SSTableReader sstable : sstables)
                 {
-                    Range<Token> sstableRange = new Range<>(sstable.first.getToken(), sstable.last.getToken());
+                    Range<Token> sstableRange = new Range<>(sstable.getFirst().getToken(), sstable.getLast().getToken());
                     if (range == null || sstableRange.intersects(range))
                         filtered.add(sstable);
                 }
@@ -556,8 +556,8 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy
             {
                 ObjectNode node = JsonNodeFactory.instance.objectNode();
                 node.put("level", sstable.getSSTableLevel());
-                node.put("min_token", sstable.first.getToken().toString());
-                node.put("max_token", sstable.last.getToken().toString());
+                node.put("min_token", sstable.getFirst().getToken().toString());
+                node.put("max_token", sstable.getLast().getToken().toString());
                 return node;
             }
 

src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java

@@ -75,7 +75,7 @@ class LeveledGenerations
     private final TreeSet<SSTableReader> [] levels = new TreeSet[MAX_LEVEL_COUNT - 1];
 
     private static final Comparator<SSTableReader> nonL0Comparator = (o1, o2) -> {
-        int cmp = SSTableReader.sstableComparator.compare(o1, o2);
+        int cmp = SSTableReader.firstKeyComparator.compare(o1, o2);
         if (cmp == 0)
             cmp = SSTableIdFactory.COMPARATOR.compare(o1.descriptor.id, o2.descriptor.id);
         return cmp;
@@ -154,8 +154,8 @@ class LeveledGenerations
             SSTableReader after = level.ceiling(sstable);
             SSTableReader before = level.floor(sstable);
 
-            if (before != null && before.last.compareTo(sstable.first) >= 0 ||
-                after != null && after.first.compareTo(sstable.last) <= 0)
+            if (before != null && before.getLast().compareTo(sstable.getFirst()) >= 0 ||
+                after != null && after.getFirst().compareTo(sstable.getLast()) <= 0)
             {
                 sendToL0(sstable);
             }
@@ -264,7 +264,7 @@ class LeveledGenerations
         while (tail.hasNext())
         {
             SSTableReader potentialPivot = tail.peek();
-            if (potentialPivot.first.compareTo(lastCompactedSSTable.last) > 0)
+            if (potentialPivot.getFirst().compareTo(lastCompactedSSTable.getLast()) > 0)
             {
                 pivot = potentialPivot;
                 break;
@@ -322,7 +322,7 @@ class LeveledGenerations
             for (SSTableReader sstable : get(i))
             {
                 // no overlap:
-                assert prev == null || prev.last.compareTo(sstable.first) < 0;
+                assert prev == null || prev.getLast().compareTo(sstable.getFirst()) < 0;
                 prev = sstable;
                 // make sure it does not exist in any other level:
                 for (int j = 0; j < levelCount(); j++)

src/java/org/apache/cassandra/db/compaction/LeveledManifest.java

@@ -159,7 +159,7 @@ public class LeveledManifest
         if (logger.isTraceEnabled())
             logger.trace("Adding [{}]", toString(added));
         generations.addAll(added);
-        lastCompactedSSTables[minLevel] = SSTableReader.sstableOrdering.max(added);
+        lastCompactedSSTables[minLevel] = SSTableReader.firstKeyOrdering.max(added);
     }
 
     private String toString(Collection<SSTableReader> sstables)
@@ -366,10 +366,10 @@ public class LeveledManifest
                     PartitionPosition min = null;
                     for (SSTableReader candidate : candidates)
                     {
-                        if (min == null || candidate.first.compareTo(min) < 0)
-                            min = candidate.first;
-                        if (max == null || candidate.last.compareTo(max) > 0)
-                            max = candidate.last;
+                        if (min == null || candidate.getFirst().compareTo(min) < 0)
+                            min = candidate.getFirst();
+                        if (max == null || candidate.getLast().compareTo(max) > 0)
+                            max = candidate.getLast();
                     }
                     if (min == null || max == null || min.equals(max)) // single partition sstables - we cannot include a high level sstable.
                         return candidates;
@@ -377,7 +377,7 @@ public class LeveledManifest
                     Range<PartitionPosition> boundaries = new Range<>(min, max);
                     for (SSTableReader sstable : generations.get(i))
                     {
-                        Range<PartitionPosition> r = new Range<>(sstable.first, sstable.last);
+                        Range<PartitionPosition> r = new Range<>(sstable.getFirst(), sstable.getLast());
                         if (boundaries.contains(r) && !compacting.contains(sstable))
                         {
                             logger.info("Adding high-level (L{}) {} to candidates", sstable.getSSTableLevel(), sstable);
@@ -438,20 +438,20 @@ public class LeveledManifest
          */
         Iterator<SSTableReader> iter = candidates.iterator();
         SSTableReader sstable = iter.next();
-        Token first = sstable.first.getToken();
-        Token last = sstable.last.getToken();
+        Token first = sstable.getFirst().getToken();
+        Token last = sstable.getLast().getToken();
         while (iter.hasNext())
         {
             sstable = iter.next();
-            first = first.compareTo(sstable.first.getToken()) <= 0 ? first : sstable.first.getToken();
-            last = last.compareTo(sstable.last.getToken()) >= 0 ? last : sstable.last.getToken();
+            first = first.compareTo(sstable.getFirst().getToken()) <= 0 ? first : sstable.getFirst().getToken();
+            last = last.compareTo(sstable.getLast().getToken()) >= 0 ? last : sstable.getLast().getToken();
         }
         return overlapping(first, last, others);
     }
 
     private static Set<SSTableReader> overlappingWithBounds(SSTableReader sstable, Map<SSTableReader, Bounds<Token>> others)
     {
-        return overlappingWithBounds(sstable.first.getToken(), sstable.last.getToken(), others);
+        return overlappingWithBounds(sstable.getFirst().getToken(), sstable.getLast().getToken(), others);
     }
 
     /**
@@ -482,7 +482,7 @@ public class LeveledManifest
         Map<SSTableReader, Bounds<Token>> boundsMap = new HashMap<>();
         for (SSTableReader sstable : ssTableReaders)
         {
-            boundsMap.put(sstable, new Bounds<>(sstable.first.getToken(), sstable.last.getToken()));
+            boundsMap.put(sstable, new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()));
         }
         return boundsMap;
     }
@@ -507,10 +507,10 @@ public class LeveledManifest
             PartitionPosition firstCompactingKey = null;
             for (SSTableReader candidate : compactingL0)
             {
-                if (firstCompactingKey == null || candidate.first.compareTo(firstCompactingKey) < 0)
-                    firstCompactingKey = candidate.first;
-                if (lastCompactingKey == null || candidate.last.compareTo(lastCompactingKey) > 0)
-                    lastCompactingKey = candidate.last;
+                if (firstCompactingKey == null || candidate.getFirst().compareTo(firstCompactingKey) < 0)
+                    firstCompactingKey = candidate.getFirst();
+                if (lastCompactingKey == null || candidate.getLast().compareTo(lastCompactingKey) > 0)
+                    lastCompactingKey = candidate.getLast();
             }
 
             // L0 is the dumping ground for new sstables which thus may overlap each other.
@@ -660,7 +660,7 @@ public class LeveledManifest
         }
 
         logger.trace("Estimating {} compactions to do for {}.{}",
-                     Arrays.toString(estimated), cfs.keyspace.getName(), cfs.name);
+                     Arrays.toString(estimated), cfs.getKeyspaceName(), cfs.name);
         return Ints.checkedCast(tasks);
     }
 

src/java/org/apache/cassandra/db/compaction/PendingRepairHolder.java

@@ -28,6 +28,8 @@ import com.google.common.collect.Iterables;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
@@ -36,7 +38,6 @@ import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.ISSTableScanner;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.schema.CompactionParams;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.TimeUUID;
@@ -238,7 +239,8 @@ public class PendingRepairHolder extends AbstractStrategyHolder
                                                        long repairedAt,
                                                        TimeUUID pendingRepair,
                                                        boolean isTransient,
-                                                       MetadataCollector collector,
+                                                       IntervalSet<CommitLogPosition> commitLogPositions,
+                                                       int sstableLevel,
                                                        SerializationHeader header,
                                                        Collection<Index> indexes,
                                                        LifecycleNewTracker lifecycleNewTracker)
@@ -254,7 +256,8 @@ public class PendingRepairHolder extends AbstractStrategyHolder
                                                  repairedAt,
                                                  pendingRepair,
                                                  isTransient,
-                                                 collector,
+                                                 commitLogPositions,
+                                                 sstableLevel,
                                                  header,
                                                  indexes,
                                                  lifecycleNewTracker);

src/java/org/apache/cassandra/db/compaction/ShardManager.java

@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableList;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public interface ShardManager
+{
+    /**
+     * Single-partition, and generally sstables with very few partitions, can cover very small sections of the token
+     * space, resulting in very high densities.
+     * Additionally, sstables that have completely fallen outside of the local token ranges will end up with a zero
+     * coverage.
+     * To avoid problems with both we check if coverage is below the minimum, and replace it with 1.
+     */
+    static final double MINIMUM_TOKEN_COVERAGE = Math.scalb(1.0, -48);
+
+    static ShardManager create(ColumnFamilyStore cfs)
+    {
+        final ImmutableList<PartitionPosition> diskPositions = cfs.getDiskBoundaries().positions;
+        ColumnFamilyStore.VersionedLocalRanges localRanges = cfs.localRangesWeighted();
+        IPartitioner partitioner = cfs.getPartitioner();
+
+        if (diskPositions != null && diskPositions.size() > 1)
+            return new ShardManagerDiskAware(localRanges, diskPositions.stream()
+                                                                       .map(PartitionPosition::getToken)
+                                                                       .collect(Collectors.toList()));
+        else if (partitioner.splitter().isPresent())
+            return new ShardManagerNoDisks(localRanges);
+        else
+            return new ShardManagerTrivial(partitioner);
+    }
+
+    boolean isOutOfDate(long ringVersion);
+
+    /**
+     * The token range fraction spanned by the given range, adjusted for the local range ownership.
+     */
+    double rangeSpanned(Range<Token> tableRange);
+
+    /**
+     * The total fraction of the token space covered by the local ranges.
+     */
+    double localSpaceCoverage();
+
+    /**
+     * The fraction of the token space covered by a shard set, i.e. the space that is split in the requested number of
+     * shards.
+     * If no disks are defined, this is the same as localSpaceCoverage(). Otherwise, it is the token coverage of a disk.
+     */
+    double shardSetCoverage();
+
+    /**
+     * Construct a boundary/shard iterator for the given number of shards.
+     *
+     * Note: This does not offer a method of listing the shard boundaries it generates, just to advance to the
+     * corresponding one for a given token.  The only usage for listing is currently in tests. Should a need for this
+     * arise, see {@link CompactionSimulationTest} for a possible implementation.
+     */
+    ShardTracker boundaries(int shardCount);
+
+    static Range<Token> coveringRange(SSTableReader sstable)
+    {
+        return coveringRange(sstable.getFirst(), sstable.getLast());
+    }
+
+    static Range<Token> coveringRange(PartitionPosition first, PartitionPosition last)
+    {
+        // To include the token of last, the range's upper bound must be increased.
+        return new Range<>(first.getToken(), last.getToken().nextValidToken());
+    }
+
+
+    /**
+     * Return the token space share that the given SSTable spans, excluding any non-locally owned space.
+     * Returns a positive floating-point number between 0 and 1.
+     */
+    default double rangeSpanned(SSTableReader rdr)
+    {
+        double reported = rdr.tokenSpaceCoverage();
+        double span;
+        if (reported > 0)   // also false for NaN
+            span = reported;
+        else
+            span = rangeSpanned(rdr.getFirst(), rdr.getLast());
+
+        if (span >= MINIMUM_TOKEN_COVERAGE)
+            return span;
+
+        // Too small ranges are expected to be the result of either a single-partition sstable or falling outside
+        // of the local token ranges. In these cases we substitute it with 1 because for them sharding and density
+        // tiering does not make sense.
+        return 1.0;  // This will be chosen if span is NaN too.
+    }
+
+    default double rangeSpanned(PartitionPosition first, PartitionPosition last)
+    {
+        return rangeSpanned(ShardManager.coveringRange(first, last));
+    }
+
+    /**
+     * Return the density of an SSTable, i.e. its size divided by the covered token space share.
+     * This is an improved measure of the compaction age of an SSTable that grows both with STCS-like full-SSTable
+     * compactions (where size grows, share is constant), LCS-like size-threshold splitting (where size is constant
+     * but share shrinks), UCS-like compactions (where size may grow and covered shards i.e. share may decrease)
+     * and can reproduce levelling structure that corresponds to all, including their mixtures.
+     */
+    default double density(SSTableReader rdr)
+    {
+        return rdr.onDiskLength() / rangeSpanned(rdr);
+    }
+
+    default int compareByDensity(SSTableReader a, SSTableReader b)
+    {
+        return Double.compare(density(a), density(b));
+    }
+
+    /**
+     * Estimate the density of the sstable that will be the result of compacting the given sources.
+     */
+    default double calculateCombinedDensity(Set<? extends SSTableReader> sstables)
+    {
+        if (sstables.isEmpty())
+            return 0;
+        long onDiskLength = 0;
+        PartitionPosition min = null;
+        PartitionPosition max = null;
+        for (SSTableReader sstable : sstables)
+        {
+            onDiskLength += sstable.onDiskLength();
+            min = min == null || min.compareTo(sstable.getFirst()) > 0 ? sstable.getFirst() : min;
+            max = max == null || max.compareTo(sstable.getLast()) < 0 ? sstable.getLast() : max;
+        }
+        double span = rangeSpanned(min, max);
+        if (span >= MINIMUM_TOKEN_COVERAGE)
+            return onDiskLength / span;
+        else
+            return onDiskLength;
+    }
+}

src/java/org/apache/cassandra/db/compaction/ShardManagerDiskAware.java

@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Collections;
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
+import org.apache.cassandra.dht.Token;
+
+public class ShardManagerDiskAware extends ShardManagerNoDisks
+{
+    /**
+     * Positions for the disk boundaries, in covered token range. The last number defines the total token
+     * share owned by the node.
+     */
+    private final double[] diskBoundaryPositions;
+    private final int[] diskStartRangeIndex;
+    private final List<Token> diskBoundaries;
+
+    public ShardManagerDiskAware(ColumnFamilyStore.VersionedLocalRanges localRanges, List<Token> diskBoundaries)
+    {
+        super(localRanges);
+        assert diskBoundaries != null && !diskBoundaries.isEmpty();
+        this.diskBoundaries = diskBoundaries;
+
+        double position = 0;
+        final List<Splitter.WeightedRange> ranges = localRanges;
+        int diskIndex = 0;
+        diskBoundaryPositions = new double[diskBoundaries.size()];
+        diskStartRangeIndex = new int[diskBoundaryPositions.length];
+        diskStartRangeIndex[0] = 0;
+
+        for (int i = 0; i < localRangePositions.length; ++i)
+        {
+            Range<Token> range = ranges.get(i).range();
+            double weight = ranges.get(i).weight();
+            double span = localRangePositions[i] - position;
+
+            Token diskBoundary = diskBoundaries.get(diskIndex);
+            while (diskIndex < diskBoundaryPositions.length - 1 && (range.right.isMinimum() || diskBoundary.compareTo(range.right) < 0))
+            {
+                double leftPart = range.left.size(diskBoundary) * weight;
+                if (leftPart > span)    // if the boundary falls on left or before it
+                    leftPart = 0;
+                diskBoundaryPositions[diskIndex] = position + leftPart;
+                diskStartRangeIndex[diskIndex + 1] = i;
+                ++diskIndex;
+                diskBoundary = diskBoundaries.get(diskIndex);
+            }
+
+            position += span;
+        }
+        diskBoundaryPositions[diskIndex] = position;
+        assert diskIndex + 1 == diskBoundaryPositions.length : "Disk boundaries are not within local ranges";
+    }
+
+    @Override
+    public double shardSetCoverage()
+    {
+        return localSpaceCoverage() / diskBoundaryPositions.length;
+        // The above is an approximation that works correctly for the normal allocation of disks.
+        // This can be properly calculated if a contained token is supplied as argument and the diskBoundaryPosition
+        // difference is retrieved for the disk containing that token.
+        // Unfortunately we don't currently have a way to get a representative position when an sstable writer is
+        // constructed for flushing.
+    }
+
+    /**
+     * Construct a boundary/shard iterator for the given number of shards.
+     */
+    public ShardTracker boundaries(int shardCount)
+    {
+        return new BoundaryTrackerDiskAware(shardCount);
+    }
+
+    public class BoundaryTrackerDiskAware implements ShardTracker
+    {
+        private final int countPerDisk;
+        private double shardStep;
+        private double diskStart;
+        private int diskIndex;
+        private int nextShardIndex;
+        private int currentRange;
+        private Token currentStart;
+        @Nullable
+        private Token currentEnd;   // null for the last shard
+
+        public BoundaryTrackerDiskAware(int countPerDisk)
+        {
+            this.countPerDisk = countPerDisk;
+            currentStart = localRanges.get(0).left();
+            diskIndex = -1;
+        }
+
+        void enterDisk(int diskIndex)
+        {
+            this.diskIndex = diskIndex;
+            currentRange = 0;
+            diskStart = diskIndex > 0 ? diskBoundaryPositions[diskIndex - 1] : 0;
+            shardStep = (diskBoundaryPositions[diskIndex] - diskStart) / countPerDisk;
+            nextShardIndex = 1;
+        }
+
+        private Token getEndToken(double toPos)
+        {
+            double left = currentRange > 0 ? localRangePositions[currentRange - 1] : 0;
+            double right = localRangePositions[currentRange];
+            while (toPos > right)
+            {
+                left = right;
+                right = localRangePositions[++currentRange];
+            }
+
+            final Range<Token> range = localRanges.get(currentRange).range();
+            return currentStart.getPartitioner().split(range.left, range.right, (toPos - left) / (right - left));
+        }
+
+        public Token shardStart()
+        {
+            return currentStart;
+        }
+
+        public Token shardEnd()
+        {
+            return currentEnd;
+        }
+
+        public Range<Token> shardSpan()
+        {
+            return new Range<>(currentStart, currentEnd != null ? currentEnd : currentStart.minValue());
+        }
+
+        public double shardSpanSize()
+        {
+            return shardStep;
+        }
+
+        /**
+         * Advance to the given token (e.g. before writing a key). Returns true if this resulted in advancing to a new
+         * shard, and false otherwise.
+         */
+        public boolean advanceTo(Token nextToken)
+        {
+            if (diskIndex < 0)
+            {
+                int search = Collections.binarySearch(diskBoundaries, nextToken);
+                if (search < 0)
+                    search = -1 - search;
+                // otherwise (on equal) we are good as ranges are end-inclusive
+                enterDisk(search);
+                setEndToken();
+            }
+
+            if (currentEnd == null || nextToken.compareTo(currentEnd) <= 0)
+                return false;
+            do
+            {
+                currentStart = currentEnd;
+                if (nextShardIndex == countPerDisk)
+                    enterDisk(diskIndex + 1);
+                else
+                    ++nextShardIndex;
+
+                setEndToken();
+            }
+            while (!(currentEnd == null || nextToken.compareTo(currentEnd) <= 0));
+            return true;
+        }
+
+        private void setEndToken()
+        {
+            if (nextShardIndex == countPerDisk)
+            {
+                if (diskIndex + 1 == diskBoundaryPositions.length)
+                    currentEnd = null;
+                else
+                    currentEnd = diskBoundaries.get(diskIndex);
+            }
+            else
+                currentEnd = getEndToken(diskStart + shardStep * nextShardIndex);
+        }
+
+        public int count()
+        {
+            return countPerDisk;
+        }
+
+        /**
+         * Returns the fraction of the given token range's coverage that falls within this shard.
+         * E.g. if the span covers two shards exactly and the current shard is one of them, it will return 0.5.
+         */
+        public double fractionInShard(Range<Token> targetSpan)
+        {
+            Range<Token> shardSpan = shardSpan();
+            Range<Token> covered = targetSpan.intersectionNonWrapping(shardSpan);
+            if (covered == null)
+                return 0;
+            if (covered == targetSpan)
+                return 1;
+            double inShardSize = covered == shardSpan ? shardSpanSize() : ShardManagerDiskAware.this.rangeSpanned(covered);
+            double totalSize = ShardManagerDiskAware.this.rangeSpanned(targetSpan);
+            return inShardSize / totalSize;
+        }
+
+        public double rangeSpanned(PartitionPosition first, PartitionPosition last)
+        {
+            return ShardManagerDiskAware.this.rangeSpanned(first, last);
+        }
+
+        public int shardIndex()
+        {
+            return nextShardIndex - 1;
+        }
+    }
+}

src/java/org/apache/cassandra/db/compaction/ShardManagerNoDisks.java

@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.List;
+
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Splitter;
+import org.apache.cassandra.dht.Token;
+
+public class ShardManagerNoDisks implements ShardManager
+{
+    final ColumnFamilyStore.VersionedLocalRanges localRanges;
+
+    /**
+     * Ending positions for the local token ranges, in covered token range; in other words, the accumulated share of
+     * the local ranges up and including the given index.
+     * The last number defines the total token share owned by the node.
+     */
+    final double[] localRangePositions;
+
+    public ShardManagerNoDisks(ColumnFamilyStore.VersionedLocalRanges localRanges)
+    {
+        this.localRanges = localRanges;
+        double position = 0;
+        final List<Splitter.WeightedRange> ranges = localRanges;
+        localRangePositions = new double[ranges.size()];
+        for (int i = 0; i < localRangePositions.length; ++i)
+        {
+            double span = ranges.get(i).size();
+            position += span;
+            localRangePositions[i] = position;
+        }
+    }
+
+    public boolean isOutOfDate(long ringVersion)
+    {
+        return ringVersion != localRanges.ringVersion &&
+               localRanges.ringVersion != ColumnFamilyStore.RING_VERSION_IRRELEVANT;
+    }
+
+    @Override
+    public double rangeSpanned(Range<Token> tableRange)
+    {
+        assert !tableRange.isTrulyWrapAround();
+        return rangeSizeNonWrapping(tableRange);
+    }
+
+    private double rangeSizeNonWrapping(Range<Token> tableRange)
+    {
+        double size = 0;
+        for (Splitter.WeightedRange range : localRanges)
+        {
+            Range<Token> ix = range.range().intersectionNonWrapping(tableRange); // local and table ranges are non-wrapping
+            if (ix == null)
+                continue;
+            size += ix.left.size(ix.right) * range.weight();
+        }
+        return size;
+    }
+
+    @Override
+    public double localSpaceCoverage()
+    {
+        return localRangePositions[localRangePositions.length - 1];
+    }
+
+    @Override
+    public double shardSetCoverage()
+    {
+        return localSpaceCoverage();
+    }
+
+    @Override
+    public ShardTracker boundaries(int shardCount)
+    {
+        return new BoundaryTracker(shardCount);
+    }
+
+    public class BoundaryTracker implements ShardTracker
+    {
+        private final double rangeStep;
+        private final int count;
+        private int nextShardIndex;
+        private int currentRange;
+        private Token currentStart;
+        @Nullable
+        private Token currentEnd;   // null for the last shard
+
+        public BoundaryTracker(int count)
+        {
+            this.count = count;
+            rangeStep = localSpaceCoverage() / count;
+            currentStart = localRanges.get(0).left();
+            currentRange = 0;
+            nextShardIndex = 1;
+            if (nextShardIndex == count)
+                currentEnd = null;
+            else
+                currentEnd = getEndToken(rangeStep * nextShardIndex);
+        }
+
+        private Token getEndToken(double toPos)
+        {
+            double left = currentRange > 0 ? localRangePositions[currentRange - 1] : 0;
+            double right = localRangePositions[currentRange];
+            while (toPos > right)
+            {
+                left = right;
+                right = localRangePositions[++currentRange];
+            }
+
+            final Range<Token> range = localRanges.get(currentRange).range();
+            return currentStart.getPartitioner().split(range.left, range.right, (toPos - left) / (right - left));
+        }
+
+        @Override
+        public Token shardStart()
+        {
+            return currentStart;
+        }
+
+        @Override
+        public Token shardEnd()
+        {
+            return currentEnd;
+        }
+
+        @Override
+        public Range<Token> shardSpan()
+        {
+            return new Range<>(currentStart, currentEnd != null ? currentEnd
+                                                                : currentStart.getPartitioner().getMinimumToken());
+        }
+
+        @Override
+        public double shardSpanSize()
+        {
+            return rangeStep;
+        }
+
+        @Override
+        public boolean advanceTo(Token nextToken)
+        {
+            if (currentEnd == null || nextToken.compareTo(currentEnd) <= 0)
+                return false;
+            do
+            {
+                currentStart = currentEnd;
+                if (++nextShardIndex == count)
+                    currentEnd = null;
+                else
+                    currentEnd = getEndToken(rangeStep * nextShardIndex);
+            }
+            while (!(currentEnd == null || nextToken.compareTo(currentEnd) <= 0));
+            return true;
+        }
+
+        @Override
+        public int count()
+        {
+            return count;
+        }
+
+        @Override
+        public double fractionInShard(Range<Token> targetSpan)
+        {
+            Range<Token> shardSpan = shardSpan();
+            Range<Token> covered = targetSpan.intersectionNonWrapping(shardSpan);
+            if (covered == null)
+                return 0;
+            // If one of the ranges is completely subsumed in the other, intersectionNonWrapping returns that range.
+            // We take advantage of this in the shortcuts below (note that if they are equal but not the same, the
+            // path below will still return the expected result).
+            if (covered == targetSpan)
+                return 1;
+            double inShardSize = covered == shardSpan ? shardSpanSize() : ShardManagerNoDisks.this.rangeSpanned(covered);
+            double totalSize = ShardManagerNoDisks.this.rangeSpanned(targetSpan);
+            return inShardSize / totalSize;
+        }
+
+        @Override
+        public double rangeSpanned(PartitionPosition first, PartitionPosition last)
+        {
+            return ShardManagerNoDisks.this.rangeSpanned(first, last);
+        }
+
+        @Override
+        public int shardIndex()
+        {
+            return nextShardIndex - 1;
+        }
+    }
+}

src/java/org/apache/cassandra/db/compaction/ShardManagerTrivial.java

@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Set;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.IPartitioner;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+public class ShardManagerTrivial implements ShardManager
+{
+    private final IPartitioner partitioner;
+
+    public ShardManagerTrivial(IPartitioner partitioner)
+    {
+        this.partitioner = partitioner;
+    }
+
+    public boolean isOutOfDate(long ringVersion)
+    {
+        // We don't do any routing, always up to date
+        return false;
+    }
+
+    @Override
+    public double rangeSpanned(Range<Token> tableRange)
+    {
+        return 1;
+    }
+
+    @Override
+    public double rangeSpanned(SSTableReader rdr)
+    {
+        return 1;
+    }
+
+    @Override
+    public double calculateCombinedDensity(Set<? extends SSTableReader> sstables)
+    {
+        double totalSize = 0;
+        for (SSTableReader sstable : sstables)
+            totalSize += sstable.onDiskLength();
+        return totalSize;
+    }
+
+    @Override
+    public double localSpaceCoverage()
+    {
+        return 1;
+    }
+
+    @Override
+    public double shardSetCoverage()
+    {
+        return 1;
+    }
+
+    ShardTracker iterator = new ShardTracker()
+    {
+        @Override
+        public Token shardStart()
+        {
+            return partitioner.getMinimumToken();
+        }
+
+        @Override
+        public Token shardEnd()
+        {
+            return partitioner.getMinimumToken();
+        }
+
+        @Override
+        public Range<Token> shardSpan()
+        {
+            return new Range<>(partitioner.getMinimumToken(), partitioner.getMinimumToken());
+        }
+
+        @Override
+        public double shardSpanSize()
+        {
+            return 1;
+        }
+
+        @Override
+        public boolean advanceTo(Token nextToken)
+        {
+            return false;
+        }
+
+        @Override
+        public int count()
+        {
+            return 1;
+        }
+
+        @Override
+        public double fractionInShard(Range<Token> targetSpan)
+        {
+            return 1;
+        }
+
+        @Override
+        public double rangeSpanned(PartitionPosition first, PartitionPosition last)
+        {
+            return 1;
+        }
+
+        @Override
+        public int shardIndex()
+        {
+            return 0;
+        }
+
+        @Override
+        public long shardAdjustedKeyCount(Set<SSTableReader> sstables)
+        {
+            long shardAdjustedKeyCount = 0;
+            for (SSTableReader sstable : sstables)
+                shardAdjustedKeyCount += sstable.estimatedKeys();
+            return shardAdjustedKeyCount;
+        }
+    };
+
+    @Override
+    public ShardTracker boundaries(int shardCount)
+    {
+        return iterator;
+    }
+}

src/java/org/apache/cassandra/db/compaction/ShardTracker.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.Set;
+import javax.annotation.Nullable;
+
+import org.apache.cassandra.db.PartitionPosition;
+import org.apache.cassandra.dht.Range;
+import org.apache.cassandra.dht.Token;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+
+public interface ShardTracker
+{
+    Token shardStart();
+
+    @Nullable
+    Token shardEnd();
+
+    Range<Token> shardSpan();
+
+    double shardSpanSize();
+
+    /**
+     * Advance to the given token (e.g. before writing a key). Returns true if this resulted in advancing to a new
+     * shard, and false otherwise.
+     */
+    boolean advanceTo(Token nextToken);
+
+    int count();
+
+    /**
+     * Returns the fraction of the given token range's coverage that falls within this shard.
+     * E.g. if the span covers two shards exactly and the current shard is one of them, it will return 0.5.
+     */
+    double fractionInShard(Range<Token> targetSpan);
+
+    double rangeSpanned(PartitionPosition first, PartitionPosition last);
+
+    int shardIndex();
+
+    default long shardAdjustedKeyCount(Set<SSTableReader> sstables)
+    {
+        // Note: computationally non-trivial; can be optimized if we save start/stop shards and size per table.
+        long shardAdjustedKeyCount = 0;
+        for (SSTableReader sstable : sstables)
+            shardAdjustedKeyCount += sstable.estimatedKeys() * fractionInShard(ShardManager.coveringRange(sstable));
+        return shardAdjustedKeyCount;
+    }
+
+    default void applyTokenSpaceCoverage(SSTableWriter writer)
+    {
+        if (writer.getFirst() != null)
+            writer.setTokenSpaceCoverage(rangeSpanned(writer.getFirst(), writer.getLast()));
+    }
+}

src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java

@@ -0,0 +1,891 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.compaction.unified.Controller;
+import org.apache.cassandra.db.compaction.unified.ShardedMultiWriter;
+import org.apache.cassandra.db.compaction.unified.UnifiedCompactionTask;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.schema.TableMetadata;
+import org.apache.cassandra.service.StorageService;
+import org.apache.cassandra.utils.Clock;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.Overlaps;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * The design of the unified compaction strategy is described in the accompanying UnifiedCompactionStrategy.md.
+ *
+ * See CEP-26: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-26%3A+Unified+Compaction+Strategy
+ */
+public class UnifiedCompactionStrategy extends AbstractCompactionStrategy
+{
+    private static final Logger logger = LoggerFactory.getLogger(UnifiedCompactionStrategy.class);
+
+    static final int MAX_LEVELS = 32;   // This is enough for a few petabytes of data (with the worst case fan factor
+    // at W=0 this leaves room for 2^32 sstables, presumably of at least 1MB each).
+
+    private static final Pattern SCALING_PARAMETER_PATTERN = Pattern.compile("(N)|L(\\d+)|T(\\d+)|([+-]?\\d+)");
+    private static final String SCALING_PARAMETER_PATTERN_SIMPLIFIED = SCALING_PARAMETER_PATTERN.pattern()
+                                                                                                .replaceAll("[()]", "")
+                                                                                                .replace("\\d", "[0-9]");
+
+    private final Controller controller;
+
+    private volatile ShardManager shardManager;
+
+    private long lastExpiredCheck;
+
+    protected volatile int estimatedRemainingTasks;
+    @VisibleForTesting
+    protected final Set<SSTableReader> sstables = new HashSet<>();
+
+    public UnifiedCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options)
+    {
+        this(cfs, options, Controller.fromOptions(cfs, options));
+    }
+
+    public UnifiedCompactionStrategy(ColumnFamilyStore cfs, Map<String, String> options, Controller controller)
+    {
+        super(cfs, options);
+        this.controller = controller;
+        estimatedRemainingTasks = 0;
+    }
+
+    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    {
+        return Controller.validateOptions(AbstractCompactionStrategy.validateOptions(options));
+    }
+
+    public static int fanoutFromScalingParameter(int w)
+    {
+        return w < 0 ? 2 - w : 2 + w; // see formula in design doc
+    }
+
+    public static int thresholdFromScalingParameter(int w)
+    {
+        return w <= 0 ? 2 : 2 + w; // see formula in design doc
+    }
+
+    public static int parseScalingParameter(String value)
+    {
+        Matcher m = SCALING_PARAMETER_PATTERN.matcher(value);
+        if (!m.matches())
+            throw new ConfigurationException("Scaling parameter " + value + " must match " + SCALING_PARAMETER_PATTERN_SIMPLIFIED);
+
+        if (m.group(1) != null)
+            return 0;
+        else if (m.group(2) != null)
+            return 2 - atLeast2(Integer.parseInt(m.group(2)), value);
+        else if (m.group(3) != null)
+            return atLeast2(Integer.parseInt(m.group(3)), value) - 2;
+        else
+            return Integer.parseInt(m.group(4));
+    }
+
+    private static int atLeast2(int value, String str)
+    {
+        if (value < 2)
+            throw new ConfigurationException("Fan factor cannot be lower than 2 in " + str);
+        return value;
+    }
+
+    public static String printScalingParameter(int w)
+    {
+        if (w < 0)
+            return "L" + Integer.toString(2 - w);
+        else if (w > 0)
+            return "T" + Integer.toString(w + 2);
+        else
+            return "N";
+    }
+
+    @Override
+    public synchronized Collection<AbstractCompactionTask> getMaximalTask(long gcBefore, boolean splitOutput)
+    {
+        maybeUpdateShardManager();
+        // The tasks are split by repair status and disk, as well as in non-overlapping sections to enable some
+        // parallelism (to the amount that L0 sstables are split, i.e. at least base_shard_count). The result will be
+        // split across shards according to its density. Depending on the parallelism, the operation may require up to
+        // 100% extra space to complete.
+        List<AbstractCompactionTask> tasks = new ArrayList<>();
+        List<Set<SSTableReader>> nonOverlapping = splitInNonOverlappingSets(filterSuspectSSTables(getSSTables()));
+        for (Set<SSTableReader> set : nonOverlapping)
+        {
+            @SuppressWarnings("resource")   // closed by the returned task
+            LifecycleTransaction txn = cfs.getTracker().tryModify(set, OperationType.COMPACTION);
+            if (txn != null)
+                tasks.add(createCompactionTask(txn, gcBefore));
+        }
+        return tasks;
+    }
+
+    private static List<Set<SSTableReader>> splitInNonOverlappingSets(Collection<SSTableReader> sstables)
+    {
+        List<Set<SSTableReader>> overlapSets = Overlaps.constructOverlapSets(new ArrayList<>(sstables),
+                                                                             UnifiedCompactionStrategy::startsAfter,
+                                                                             SSTableReader.firstKeyComparator,
+                                                                             SSTableReader.lastKeyComparator);
+        if (overlapSets.isEmpty())
+            return overlapSets;
+
+        Set<SSTableReader> group = overlapSets.get(0);
+        List<Set<SSTableReader>> groups = new ArrayList<>();
+        for (int i = 1; i < overlapSets.size(); ++i)
+        {
+            Set<SSTableReader> current = overlapSets.get(i);
+            if (Sets.intersection(current, group).isEmpty())
+            {
+                groups.add(group);
+                group = current;
+            }
+            else
+            {
+                group.addAll(current);
+            }
+        }
+        groups.add(group);
+        return groups;
+    }
+
+    @Override
+    @SuppressWarnings("resource")   // transaction closed by the returned task
+    public AbstractCompactionTask getUserDefinedTask(Collection<SSTableReader> sstables, final long gcBefore)
+    {
+        assert !sstables.isEmpty(); // checked for by CM.submitUserDefined
+
+        LifecycleTransaction transaction = cfs.getTracker().tryModify(sstables, OperationType.COMPACTION);
+        if (transaction == null)
+        {
+            logger.trace("Unable to mark {} for compaction; probably a background compaction got to it first.  You can disable background compactions temporarily if this is a problem", sstables);
+            return null;
+        }
+
+        return createCompactionTask(transaction, gcBefore).setUserDefined(true);
+    }
+
+    /**
+     * Returns a compaction task to run next.
+     *
+     * This method is synchronized because task creation is significantly more expensive in UCS; the strategy is
+     * stateless, therefore it has to compute the shard/bucket structure on each call.
+     *
+     * @param gcBefore throw away tombstones older than this
+     */
+    @Override
+    public synchronized UnifiedCompactionTask getNextBackgroundTask(long gcBefore)
+    {
+        while (true)
+        {
+            CompactionPick pick = getNextCompactionPick(gcBefore);
+            if (pick == null)
+                return null;
+            UnifiedCompactionTask task = createCompactionTask(pick, gcBefore);
+            if (task != null)
+                return task;
+        }
+    }
+
+    @SuppressWarnings("resource")   // transaction closed by the returned task
+    private UnifiedCompactionTask createCompactionTask(CompactionPick pick, long gcBefore)
+    {
+        Preconditions.checkNotNull(pick);
+        Preconditions.checkArgument(!pick.isEmpty());
+
+        LifecycleTransaction transaction = cfs.getTracker().tryModify(pick,
+                                                                      OperationType.COMPACTION);
+        if (transaction != null)
+        {
+            return createCompactionTask(transaction, gcBefore);
+        }
+        else
+        {
+            // This can happen e.g. due to a race with upgrade tasks.
+            logger.warn("Failed to submit compaction {} because a transaction could not be created. If this happens frequently, it should be reported", pick);
+            // This may be an indication of an SSTableReader reference leak. See CASSANDRA-18342.
+            return null;
+        }
+    }
+
+    /**
+     * Create the sstable writer used for flushing.
+     *
+     * @return an sstable writer that will split sstables into a number of shards as calculated by the controller for
+     *         the expected flush density.
+     */
+    @Override
+    public SSTableMultiWriter createSSTableMultiWriter(Descriptor descriptor,
+                                                       long keyCount,
+                                                       long repairedAt,
+                                                       TimeUUID pendingRepair,
+                                                       boolean isTransient,
+                                                       IntervalSet<CommitLogPosition> commitLogPositions,
+                                                       int sstableLevel,
+                                                       SerializationHeader header,
+                                                       Collection<Index> indexes,
+                                                       LifecycleNewTracker lifecycleNewTracker)
+    {
+        ShardManager shardManager = getShardManager();
+        double flushDensity = cfs.metric.flushSizeOnDisk.get() * shardManager.shardSetCoverage() / shardManager.localSpaceCoverage();
+        ShardTracker boundaries = shardManager.boundaries(controller.getNumShards(flushDensity));
+        return new ShardedMultiWriter(cfs,
+                                      descriptor,
+                                      keyCount,
+                                      repairedAt,
+                                      pendingRepair,
+                                      isTransient,
+                                      commitLogPositions,
+                                      header,
+                                      indexes,
+                                      lifecycleNewTracker,
+                                      boundaries);
+    }
+
+    /**
+     * Create the task that in turns creates the sstable writer used for compaction.
+     *
+     * @return a sharded compaction task that in turn will create a sharded compaction writer.
+     */
+    private UnifiedCompactionTask createCompactionTask(LifecycleTransaction transaction, long gcBefore)
+    {
+        return new UnifiedCompactionTask(cfs, this, transaction, gcBefore, getShardManager());
+    }
+
+    private void maybeUpdateShardManager()
+    {
+        if (shardManager != null && !shardManager.isOutOfDate(StorageService.instance.getTokenMetadata().getRingVersion()))
+            return; // the disk boundaries (and thus the local ranges too) have not changed since the last time we calculated
+
+        synchronized (this)
+        {
+            // Recheck after entering critical section, another thread may have beaten us to it.
+            while (shardManager == null || shardManager.isOutOfDate(StorageService.instance.getTokenMetadata().getRingVersion()))
+                shardManager = ShardManager.create(cfs);
+            // Note: this can just as well be done without the synchronization (races would be benign, just doing some
+            // redundant work). For the current usages of this blocking is fine and expected to perform no worse.
+        }
+    }
+
+    @VisibleForTesting
+    ShardManager getShardManager()
+    {
+        maybeUpdateShardManager();
+        return shardManager;
+    }
+
+    /**
+     * Selects a compaction to run next.
+     */
+    @VisibleForTesting
+    CompactionPick getNextCompactionPick(long gcBefore)
+    {
+        SelectionContext context = new SelectionContext(controller);
+        List<SSTableReader> suitable = getCompactableSSTables(getSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction);
+        Set<SSTableReader> expired = maybeGetExpiredSSTables(gcBefore, suitable);
+        suitable.removeAll(expired);
+
+        CompactionPick selected = chooseCompactionPick(suitable, context);
+        estimatedRemainingTasks = context.estimatedRemainingTasks;
+        if (selected == null)
+        {
+            if (expired.isEmpty())
+                return null;
+            else
+                return new CompactionPick(-1, -1, expired);
+        }
+
+        selected.addAll(expired);
+        return selected;
+    }
+
+    private Set<SSTableReader> maybeGetExpiredSSTables(long gcBefore, List<SSTableReader> suitable)
+    {
+        Set<SSTableReader> expired;
+        long ts = Clock.Global.currentTimeMillis();
+        if (ts - lastExpiredCheck > controller.getExpiredSSTableCheckFrequency())
+        {
+            lastExpiredCheck = ts;
+            expired = CompactionController.getFullyExpiredSSTables(cfs,
+                                                                   suitable,
+                                                                   cfs.getOverlappingLiveSSTables(suitable),
+                                                                   gcBefore,
+                                                                   controller.getIgnoreOverlapsInExpirationCheck());
+            if (logger.isTraceEnabled() && !expired.isEmpty())
+                logger.trace("Expiration check for {}.{} found {} fully expired SSTables",
+                             cfs.getKeyspaceName(),
+                             cfs.getTableName(),
+                             expired.size());
+        }
+        else
+            expired = Collections.emptySet();
+        return expired;
+    }
+
+    private CompactionPick chooseCompactionPick(List<SSTableReader> suitable, SelectionContext context)
+    {
+        // Select the level with the highest overlap; when multiple levels have the same overlap, prefer the lower one
+        // (i.e. reduction of RA for bigger token coverage).
+        int maxOverlap = -1;
+        CompactionPick selected = null;
+        for (Level level : formLevels(suitable))
+        {
+            CompactionPick pick = level.getCompactionPick(context);
+            int levelOverlap = level.maxOverlap;
+            if (levelOverlap > maxOverlap)
+            {
+                maxOverlap = levelOverlap;
+                selected = pick;
+            }
+        }
+        if (logger.isDebugEnabled() && selected != null)
+            logger.debug("Selected compaction on level {} overlap {} sstables {}",
+                         selected.level, selected.overlap, selected.size());
+
+        return selected;
+    }
+
+    @Override
+    public int getEstimatedRemainingTasks()
+    {
+        return estimatedRemainingTasks;
+    }
+
+    @Override
+    public long getMaxSSTableBytes()
+    {
+        return Long.MAX_VALUE;
+    }
+
+    @VisibleForTesting
+    public Controller getController()
+    {
+        return controller;
+    }
+
+    public static boolean isSuitableForCompaction(SSTableReader rdr)
+    {
+        return !rdr.isMarkedSuspect() && rdr.openReason != SSTableReader.OpenReason.EARLY;
+    }
+
+    @Override
+    public synchronized void addSSTable(SSTableReader added)
+    {
+        sstables.add(added);
+    }
+
+    @Override
+    public synchronized void removeSSTable(SSTableReader sstable)
+    {
+        sstables.remove(sstable);
+    }
+
+    @Override
+    protected synchronized Set<SSTableReader> getSSTables()
+    {
+        // Filter the set of sstables through the live set. This is to ensure no zombie sstables are picked for
+        // compaction (see CASSANDRA-18342).
+        return ImmutableSet.copyOf(Iterables.filter(cfs.getLiveSSTables(), sstables::contains));
+    }
+
+    /**
+     * @return a list of the levels in the compaction hierarchy
+     */
+    @VisibleForTesting
+    List<Level> getLevels()
+    {
+        return getLevels(getSSTables(), UnifiedCompactionStrategy::isSuitableForCompaction);
+    }
+
+    /**
+     * Groups the sstables passed in into levels. This is used by the strategy to determine
+     * new compactions, and by external tools to analyze the strategy decisions.
+     *
+     * @param sstables a collection of the sstables to be assigned to levels
+     * @param compactionFilter a filter to exclude CompactionSSTables,
+     *                         e.g., {@link #isSuitableForCompaction}
+     *
+     * @return a list of the levels in the compaction hierarchy
+     */
+    public List<Level> getLevels(Collection<SSTableReader> sstables,
+                                 Predicate<SSTableReader> compactionFilter)
+    {
+        List<SSTableReader> suitable = getCompactableSSTables(sstables, compactionFilter);
+        return formLevels(suitable);
+    }
+
+    private List<Level> formLevels(List<SSTableReader> suitable)
+    {
+        maybeUpdateShardManager();
+        List<Level> levels = new ArrayList<>(MAX_LEVELS);
+        suitable.sort(shardManager::compareByDensity);
+
+        double maxDensity = controller.getMaxLevelDensity(0, controller.getBaseSstableSize(controller.getFanout(0)) / shardManager.localSpaceCoverage());
+        int index = 0;
+        Level level = new Level(controller, index, 0, maxDensity);
+        for (SSTableReader candidate : suitable)
+        {
+            final double density = shardManager.density(candidate);
+            if (density < level.max)
+            {
+                level.add(candidate);
+                continue;
+            }
+
+            level.complete();
+            levels.add(level); // add even if empty
+
+            while (true)
+            {
+                ++index;
+                double minDensity = maxDensity;
+                maxDensity = controller.getMaxLevelDensity(index, minDensity);
+                level = new Level(controller, index, minDensity, maxDensity);
+                if (density < level.max)
+                {
+                    level.add(candidate);
+                    break;
+                }
+                else
+                {
+                    levels.add(level); // add the empty level
+                }
+            }
+        }
+
+        if (!level.sstables.isEmpty())
+        {
+            level.complete();
+            levels.add(level);
+        }
+
+        return levels;
+    }
+
+    private List<SSTableReader> getCompactableSSTables(Collection<SSTableReader> sstables,
+                                                       Predicate<SSTableReader> compactionFilter)
+    {
+        Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
+        List<SSTableReader> suitable = new ArrayList<>(sstables.size());
+        for (SSTableReader rdr : sstables)
+        {
+            if (compactionFilter.test(rdr) && !compacting.contains(rdr))
+                suitable.add(rdr);
+        }
+        return suitable;
+    }
+
+    public TableMetadata getMetadata()
+    {
+        return cfs.metadata();
+    }
+
+    private static boolean startsAfter(SSTableReader a, SSTableReader b)
+    {
+        // Strict comparison because the span is end-inclusive.
+        return a.getFirst().compareTo(b.getLast()) > 0;
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Unified strategy %s", getMetadata());
+    }
+
+    /**
+     * A level: index, sstables and some properties.
+     */
+    public static class Level
+    {
+        final List<SSTableReader> sstables;
+        final int index;
+        final double survivalFactor;
+        final int scalingParameter; // scaling parameter used to calculate fanout and threshold
+        final int fanout; // fanout factor between levels
+        final int threshold; // number of SSTables that trigger a compaction
+        final double min; // min density of sstables for this level
+        final double max; // max density of sstables for this level
+        int maxOverlap = -1; // maximum number of overlapping sstables, i.e. maximum number of sstables that need
+                             // to be queried on this level for any given key
+
+        Level(Controller controller, int index, double minSize, double maxSize)
+        {
+            this.index = index;
+            this.survivalFactor = controller.getSurvivalFactor(index);
+            this.scalingParameter = controller.getScalingParameter(index);
+            this.fanout = controller.getFanout(index);
+            this.threshold = controller.getThreshold(index);
+            this.sstables = new ArrayList<>(threshold);
+            this.min = minSize;
+            this.max = maxSize;
+        }
+
+        public Collection<SSTableReader> getSSTables()
+        {
+            return sstables;
+        }
+
+        public int getIndex()
+        {
+            return index;
+        }
+
+        void add(SSTableReader sstable)
+        {
+            this.sstables.add(sstable);
+        }
+
+        void complete()
+        {
+            if (logger.isTraceEnabled())
+                logger.trace("Level: {}", this);
+        }
+
+        /**
+         * Return the compaction pick for this level.
+         * <p>
+         * This is done by splitting the level into buckets that we can treat as independent regions for compaction.
+         * We then use the maxOverlap value (i.e. the maximum number of sstables that can contain data for any covered
+         * key) of each bucket to determine if compactions are needed, and to prioritize the buckets that contribute
+         * most to the complexity of queries: if maxOverlap is below the level's threshold, no compaction is needed;
+         * otherwise, we choose one from the buckets that have the highest maxOverlap.
+         */
+        CompactionPick getCompactionPick(SelectionContext context)
+        {
+            List<Bucket> buckets = getBuckets(context);
+            if (buckets == null)
+            {
+                if (logger.isDebugEnabled())
+                    logger.debug("Level {} sstables {} max overlap {} buckets with compactions {} tasks {}",
+                                 index, sstables.size(), maxOverlap, 0, 0);
+                return null;    // nothing crosses the threshold in this level, nothing to do
+            }
+
+            int estimatedRemainingTasks = 0;
+            int overlapMatchingCount = 0;
+            Bucket selectedBucket = null;
+            Controller controller = context.controller;
+            for (Bucket bucket : buckets)
+            {
+                // We can have just one pick in each level. Pick one bucket randomly out of the ones with
+                // the highest overlap.
+                // The random() part below implements reservoir sampling with size 1, giving us a uniformly random selection.
+                if (bucket.maxOverlap == maxOverlap && controller.random().nextInt(++overlapMatchingCount) == 0)
+                    selectedBucket = bucket;
+                // The estimated remaining tasks is a measure of the remaining amount of work, thus we prefer to
+                // calculate the number of tasks we would do in normal operation, even though we may compact in bigger
+                // chunks when we are late.
+                estimatedRemainingTasks += bucket.maxOverlap / threshold;
+            }
+            context.estimatedRemainingTasks += estimatedRemainingTasks;
+            assert selectedBucket != null;
+
+            if (logger.isDebugEnabled())
+                logger.debug("Level {} sstables {} max overlap {} buckets with compactions {} tasks {}",
+                             index, sstables.size(), maxOverlap, buckets.size(), estimatedRemainingTasks);
+
+            CompactionPick selected = selectedBucket.constructPick(controller);
+
+            if (logger.isTraceEnabled())
+                logger.trace("Returning compaction pick with selected compaction {}",
+                             selected);
+            return selected;
+        }
+
+        /**
+         * Group the sstables in this level into buckets.
+         * <p>
+         * The buckets are formed by grouping sstables that overlap at some key together, and then expanded to cover
+         * any overlapping sstable according to the overlap inclusion method. With the usual TRANSITIVE method this
+         * results into non-overlapping buckets that can't affect one another and can be compacted in parallel without
+         * any loss of efficiency.
+         * <p>
+         * Other overlap inclusion methods are provided to cover situations where we may be okay with compacting
+         * sstables partially and doing more than the strictly necessary amount of compaction to solve a problem: e.g.
+         * after an upgrade from LCS where transitive overlap may cause a complete level to be compacted together
+         * (creating an operation that will take a very long time to complete) and we want to make some progress as
+         * quickly as possible at the cost of redoing some work.
+         * <p>
+         * The number of sstables that overlap at some key defines the "overlap" of a set of sstables. The maximum such
+         * value in the bucket is its "maxOverlap", i.e. the highest number of sstables we need to read to find the
+         * data associated with a given key.
+         */
+        @VisibleForTesting
+        List<Bucket> getBuckets(SelectionContext context)
+        {
+            List<SSTableReader> liveSet = sstables;
+
+            if (logger.isTraceEnabled())
+                logger.trace("Creating compaction pick with live set {}", liveSet);
+
+            List<Set<SSTableReader>> overlaps = Overlaps.constructOverlapSets(liveSet,
+                                                                              UnifiedCompactionStrategy::startsAfter,
+                                                                              SSTableReader.firstKeyComparator,
+                                                                              SSTableReader.lastKeyComparator);
+            for (Set<SSTableReader> overlap : overlaps)
+                maxOverlap = Math.max(maxOverlap, overlap.size());
+            if (maxOverlap < threshold)
+                return null;
+
+            List<Bucket> buckets = Overlaps.assignOverlapsIntoBuckets(threshold,
+                                                                      context.controller.overlapInclusionMethod(),
+                                                                      overlaps,
+                                                                      this::makeBucket);
+            return buckets;
+        }
+
+        private Bucket makeBucket(List<Set<SSTableReader>> overlaps, int startIndex, int endIndex)
+        {
+            return endIndex == startIndex + 1
+                   ? new SimpleBucket(this, overlaps.get(startIndex))
+                   : new MultiSetBucket(this, overlaps.subList(startIndex, endIndex));
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("W: %d, T: %d, F: %d, index: %d, min: %s, max %s, %d sstables, overlap %s",
+                                 scalingParameter,
+                                 threshold,
+                                 fanout,
+                                 index,
+                                 densityAsString(min),
+                                 densityAsString(max),
+                                 sstables.size(),
+                                 maxOverlap);
+        }
+
+        private String densityAsString(double density)
+        {
+            return FBUtilities.prettyPrintBinary(density, "B", " ");
+        }
+    }
+
+
+    /**
+     * A compaction bucket, i.e. a selection of overlapping sstables from which a compaction should be selected.
+     */
+    static abstract class Bucket
+    {
+        final Level level;
+        final List<SSTableReader> allSSTablesSorted;
+        final int maxOverlap;
+
+        Bucket(Level level, Collection<SSTableReader> allSSTablesSorted, int maxOverlap)
+        {
+            // single section
+            this.level = level;
+            this.allSSTablesSorted = new ArrayList<>(allSSTablesSorted);
+            this.allSSTablesSorted.sort(SSTableReader.maxTimestampDescending);  // we remove entries from the back
+            this.maxOverlap = maxOverlap;
+        }
+
+        Bucket(Level level, List<Set<SSTableReader>> overlapSections)
+        {
+            // multiple sections
+            this.level = level;
+            int maxOverlap = 0;
+            Set<SSTableReader> all = new HashSet<>();
+            for (Set<SSTableReader> section : overlapSections)
+            {
+                maxOverlap = Math.max(maxOverlap, section.size());
+                all.addAll(section);
+            }
+            this.allSSTablesSorted = new ArrayList<>(all);
+            this.allSSTablesSorted.sort(SSTableReader.maxTimestampDescending);  // we remove entries from the back
+            this.maxOverlap = maxOverlap;
+        }
+
+        /**
+         * Select compactions from this bucket. Normally this would form a compaction out of all sstables in the
+         * bucket, but if compaction is very late we may prefer to act more carefully:
+         * - we should not use more inputs than the permitted maximum
+         * - we should select SSTables in a way that preserves the structure of the compaction hierarchy
+         * These impose a limit on the size of a compaction; to make sure we always reduce the read amplification by
+         * this much, we treat this number as a limit on overlapping sstables, i.e. if A and B don't overlap with each
+         * other but both overlap with C and D, all four will be selected to form a limit-three compaction. A limit-two
+         * one may choose CD, ABC or ABD.
+         * Also, the subset is selected by max timestamp order, oldest first, to avoid violating sstable time order. In
+         * the example above, if B is oldest and C is older than D, the limit-two choice would be ABC (if A is older
+         * than D) or BC (if A is younger, avoiding combining C with A skipping D).
+         *
+         * @param controller The compaction controller.
+         * @return A compaction pick to execute next.
+         */
+        CompactionPick constructPick(Controller controller)
+        {
+            int count = maxOverlap;
+            int threshold = level.threshold;
+            int fanout = level.fanout;
+            int index = level.index;
+            int maxSSTablesToCompact = Math.max(fanout, controller.maxSSTablesToCompact());
+
+            assert count >= threshold;
+            if (count <= fanout)
+            {
+                /**
+                 * Happy path. We are not late or (for levelled) we are only so late that a compaction now will
+                 * have the same effect as doing levelled compactions one by one. Compact all. We do not cap
+                 * this pick at maxSSTablesToCompact due to an assumption that maxSSTablesToCompact is much
+                 * greater than F. See {@link Controller#MAX_SSTABLES_TO_COMPACT_OPTION} for more details.
+                 */
+                return new CompactionPick(index, count, allSSTablesSorted);
+            }
+            else if (count <= fanout * controller.getFanout(index + 1) || maxSSTablesToCompact == fanout)
+            {
+                // Compaction is a bit late, but not enough to jump levels via layout compactions. We need a special
+                // case to cap compaction pick at maxSSTablesToCompact.
+                if (count <= maxSSTablesToCompact)
+                    return new CompactionPick(index, count, allSSTablesSorted);
+
+                return new CompactionPick(index, maxSSTablesToCompact, pullOldestSSTables(maxSSTablesToCompact));
+            }
+            else
+            {
+                // We may, however, have accumulated a lot more than T if compaction is very late.
+                // In this case we pick a compaction in such a way that the result of doing it spreads the data in
+                // a similar way to how compaction would lay them if it was able to keep up. This means:
+                // - for tiered compaction (w >= 0), compact in sets of as many as required to get to a level.
+                //   for example, for w=2 and 55 sstables, pick a compaction of 16 sstables (on the next calls, given no
+                //   new files, 2 more of 16, 1 of 4, and leaving the other 3 sstables alone).
+                // - for levelled compaction (w < 0), compact all that would reach a level.
+                //   for w=-2 and 55, this means pick a compaction of 48 (on the next calls, given no new files, one of
+                //   4, and one of 3 sstables).
+                int pickSize = selectPickSize(controller, maxSSTablesToCompact);
+                return new CompactionPick(index, pickSize, pullOldestSSTables(pickSize));
+            }
+        }
+
+        private int selectPickSize(Controller controller, int maxSSTablesToCompact)
+        {
+            int pickSize;
+            int fanout = level.fanout;
+            int nextStep = fanout;
+            int index = level.index;
+            int limit = Math.min(maxSSTablesToCompact, maxOverlap);
+            do
+            {
+                pickSize = nextStep;
+                fanout = controller.getFanout(++index);
+                nextStep *= fanout;
+            }
+            while (nextStep <= limit);
+
+            if (level.scalingParameter < 0)
+            {
+                // For levelled compaction all the sstables that would reach this level need to be compacted to one,
+                // so select the highest multiple of step that fits.
+                pickSize *= limit / pickSize;
+                assert pickSize > 0;
+            }
+            return pickSize;
+        }
+
+        /**
+         * Pull the oldest sstables to get at most limit-many overlapping sstables to compact in each overlap section.
+         */
+        abstract Collection<SSTableReader> pullOldestSSTables(int overlapLimit);
+    }
+
+    public static class SimpleBucket extends Bucket
+    {
+        public SimpleBucket(Level level, Collection<SSTableReader> sstables)
+        {
+            super(level, sstables, sstables.size());
+        }
+
+        Collection<SSTableReader> pullOldestSSTables(int overlapLimit)
+        {
+            if (allSSTablesSorted.size() <= overlapLimit)
+                return allSSTablesSorted;
+            return Overlaps.pullLast(allSSTablesSorted, overlapLimit);
+        }
+    }
+
+    public static class MultiSetBucket extends Bucket
+    {
+        final List<Set<SSTableReader>> overlapSets;
+
+        public MultiSetBucket(Level level, List<Set<SSTableReader>> overlapSets)
+        {
+            super(level, overlapSets);
+            this.overlapSets = overlapSets;
+        }
+
+        Collection<SSTableReader> pullOldestSSTables(int overlapLimit)
+        {
+            return Overlaps.pullLastWithOverlapLimit(allSSTablesSorted, overlapSets, overlapLimit);
+        }
+    }
+
+    /**
+     * Utility class holding a collection of sstables for compaction.
+     */
+    static class CompactionPick extends ArrayList<SSTableReader>
+    {
+        final int level;
+        final int overlap;
+
+        CompactionPick(int level, int overlap, Collection<SSTableReader> sstables)
+        {
+            super(sstables);
+            this.level = level;
+            this.overlap = overlap;
+        }
+    }
+
+    static class SelectionContext
+    {
+        final Controller controller;
+        int estimatedRemainingTasks = 0;
+
+        SelectionContext(Controller controller)
+        {
+            this.controller = controller;
+        }
+    }
+}

src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md

@@ -0,0 +1,375 @@
+<!--
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+
+# Unified compaction strategy (UCS)
+
+This is a new compaction strategy that unifies tiered and leveled compaction strategies, adds sharding, lends itself to
+be reconfigured at any time and forms the basis for future compaction improvements including automatic adaptation to the
+workload.
+
+The strategy is based on two observations:
+
+- that tiered and levelled compaction can be generalized as the same thing if one observes that both form
+  exponentially-growing levels based on the size of sstables (or non-overlapping sstable runs) and trigger a
+  compaction when more than a given number of sstables are present on one level;
+- that instead of "size" in the description above we can use "density", i.e. the size of an sstable divided by
+  the width of the token range it covers, which permits sstables to be split at arbitrary points when the output
+  of a compaction is written and still produce a levelled hierarchy.
+
+UCS groups sstables in levels based on the logarithm of the sstable density, with
+the fanout factor $f$ as the base of the logarithm, and with each level triggering a compaction as soon as it has
+$t$ overlapping sstables. The choice of the parameters $f$ and $t$, and of a minimum sstable size, determines the
+behaviour of the strategy. This allows users to choose a levelled strategy by setting $t=2$, or a tiered strategy by
+choosing $t=f$. Because the two options are mutually exclusive, meet at $f=2$ and form a space of options for choosing
+different ratios of read amplification (RA) vs write amplification (WA) (where levelled compaction improves reads at
+the expense of writes and approaches a sorted array as $f$ increases, and tiered compaction favors writes at the
+expense of reads and approaches an unsorted log as $f$ increases), we combine the two parameters into one integer
+value, $w$, and set them to be:
+
+* If $w < 0$ then $f = 2 - w$ and $t = 2$. This means leveled compactions, high WA but low RA.
+  We write this as L*f* (e.g. L10 for $w = -8$).
+* If $w > 0$ then $f = 2 + w$ and $t = f$. This means tiered compactions, low WA but high RA.
+  We write this as T*f* (e.g. T4 for $w = 2$).
+* If $w = 0$ then $f = t = 2$. This is the middle ground, leveled and tiered compactions behave identically.
+  We write this as N.
+
+Further, UCS permits the value of $w$ to be defined separately for each level, and thus levels can have different
+behaviours. For example level zero could use tiered compaction (STCS-like) but higher levels could switch to levelled
+(LCS-like) with increasing levels of read optimization.
+
+The strategy splits sstables at specific shard boundaries whose number grows with the density of an sstable, and
+uses the non-overlap between sstables created by this splitting to be able to perform compactions concurrently.
+
+## Size-based levels
+
+Let's ignore density and splitting for a while and explore more closely how sstables are grouped into levels if
+they are never split.
+
+For a fixed fanout factor $f$ and a memtable flush size $m$, calculated as the average size of the runs of sstables
+written when a memtable is flushed and intended to form a base of the hierarchy where all newly-flushed sstables end
+up, the level $L$ for an sstable of size $s$ is calculated as follows:
+
+$$
+L = 
+\begin{cases}
+\left \lfloor \log_f {\frac s m} \right \rfloor & \text{if } s \ge m \\
+0 & \text{otherwise}
+\end{cases}
+$$
+
+This means that sstables are assigned to levels as follows:
+
+
+| Level | Min sstable size | Max sstable size  |
+| ----- | ---------------- | ----------------- |
+| 0     | 0                | $m \cdot f$       |
+| 1     | $m \cdot f$      | $m \cdot f^2$     |
+| 2     | $m \cdot f^2$    | $m \cdot f^3$     |
+| 3     | $m \cdot f^3$    | $m \cdot f^4$     |
+| ...   | ...              | ...               |
+| n     | $m \cdot f^n$    | $m \cdot f^{n+1}$ |
+
+If we define $t$ as the number of sstables in a level that triggers a compaction, then:
+
+* $t = 2$ means the strategy is using a leveled merged policy. An sstable enters level $n$ with size $\ge mf^n$.
+  When another sstable enters (also with size $\ge mf^n$) they compact and form a new table with size
+  $\sim 2mf^n$, which keeps the result in the same level for $f > 2$. After this repeats at least $f-2$
+  more times (i.e. $f$ tables enter the level altogether), the compaction result grows to $\ge  mf^{n+1}$
+  and enters the next level.
+* $t = f$ means the strategy is using a tiered merge policy. After $f$ sstables enter level $n$, each of size
+  $\ge mf^n$, they are compacted together, resulting in an sstable of size $\ge mf^{n+1}$ which belongs to the next
+  level.
+
+Note that the above ignores overwrites and deletions. Given knowledge of the expected proportion of overwrites/deletion,
+they can also be accounted for (this is implemented but not exposed at this time).
+
+For leveled strategies, the write amplification will be proportional to $f-1$ times the number of levels whilst
+for tiered strategies it will be proportional only to the number of levels. On the other hand, the read
+amplification will be proportional to the number of levels for leveled strategies and to $f-1$ times the number
+of levels for tiered strategies.
+
+The number of levels for our size based scheme can be calculated by substituting the maximal dataset size $D$ in our
+equation above, giving a maximal number of levels inversely proportional to the logarithm of $f$.
+
+Therefore when we try to control the overheads of compaction on the database, we have a space of choices for the strategy
+that range from:
+
+* leveled compaction ( $t=2$ ) with high $f$ &mdash; low number of levels, high read efficiency, high write cost,
+  moving closer to the behaviour of a sorted array as $f$ increases;
+* compaction with $t = f = 2$ where leveled is the same as tiered and we have a middle ground with logarithmically
+  increasing read and write costs;
+* tiered compaction ( $t=f$ ) with high $f$ &mdash; very high number of sstables, low read efficiency and low write cost,
+  moving closer to an unsorted log as $f$ increases.
+
+This can be easily generalised to varying fan factors, by replacing the exponentiation with the product of the fan
+factors for all lower levels:
+
+
+| Level | Min sstable size            | Max sstable size                  |
+| ----- | --------------------------- | --------------------------------- |
+| 0     | 0                           | $m \cdot f_0$                     |
+| 1     | $m \cdot f_0$               | $m \cdot f_0 \cdot f_1$           |
+| 2     | $m \cdot f_0 \cdot f_1$     | $m \cdot f_0 \cdot f_1 \cdot f_2$ |
+| ...   | ...                         | ...                               |
+| n     | $m \cdot \prod_{i < n} f_i$ | $m \cdot \prod_{i\le n} f_i$      |
+
+## Density levelling
+
+If we replace the size $s$ in the previous paragraph with the density measure $d = s / v$ where $v$ is the fraction of
+the token space that the sstable covers, all formulae and conclusions remain valid. However, we can now split the
+output at arbitrary points and still make use of the results. For example, if we start with four sstables each spanning
+a shard that covers 1/10 of the token space on a T4 level and compact them, splitting the output equally into four
+sstables, the resulting sstables (provided no overwrite/deletion) will be of the same size as the input sstables, but
+will now cover 1/40 of the token share each. As a result, they will now be interpreted to be four times as dense and
+thus fall on the next level of the hierarchy (recall that the upper density limit for a level is $f$ times the lower).
+If we can ensure that the split points are fixed (see below), when this repeats enough times for the next level to
+receive sufficiently many sstables, we can start 4 independent compactions concurrently.
+
+It is important to account for locally-owned token share when calculating $v$. Because vnodes mean that the local
+token ownership of a node is not contiguous, the difference between the first and last token is not sufficient to
+calculate token share &mdash; any non-locally-owned ranges must be excluded.
+
+Using the density measure allows us to control the size of sstables through sharding, as well as to execute
+compactions in parallel. With size levelling we could achieve parallelisation by pre-splitting the data in a fixed
+number of compaction arenas (e.g. by using the data directories mechanism), but this requires the number of shards to be
+predetermined and equal for all levels of the hierarchy, which still permits sstables to become too small or too large.
+Large sstables complicate streaming and repair and increase the duration of compaction operations, pinning resources to
+long-running operations and making it more likely that too many sstables will accumulate on lower levels of the
+hierarchy.
+
+Density levelling permits a much wider variety of splitting options including ones where the size of sstables can
+be kept close to a selected target, and also allows UCS to understand the levelling structure of STCS (where size grows
+with each level) as well as LCS (where token share shrinks with each level).
+
+## Sharding
+
+Once density levelling is in place, we have a range of choices for splitting sstables. One is to simply split
+when a certain output size is reached (like LCS), forming non-overlapping sstable runs instead of individual
+sstables. Another is to split the token space into shards at predefined boundary points. A third hybrid option is
+to split at predefined boundaries, but only if a certain minimum size has been reached.
+
+Splitting only by size has the problem that individual sstables start at positions that vary, and if we need to
+compact sstables split in this way we must either always start from the beginning and proceed to process the whole
+level sequentially, or have some part of the data compacted/copied more times than necessary as any smaller selection
+of sstables has to exclude some overlapping sstable. The other side of the latter problem is that some section of the
+compacted token range will include fewer inputs, and will thus be sparser than the rest of the compaction output;
+this will skew the density of the result, or need to be controlled by further splitting of the output. In the hybrid
+option the same problem occurs less frequently but is still present.
+
+To avoid these and permit concurrent compactions of all levels of the compaction hierarchy, we choose to predefine
+boundary points for every compaction and always split sstables on these points. The number of the boundaries is
+determined based on the density of the inputs and the estimated density of the result &mdash; as it grows higher
+the number of boundaries is increased to keep the size of individual sstables close to a predefined target. By
+only using power-of-two multiples of a specified base count (in other words, by only splitting shards in the
+middle), we also ensure that any boundary that applies to a given output density also applies to all higher
+densities.
+
+More precisely, the user specifies two sharding parameters:
+
+- base shard count $b$
+- target sstable size $t$
+
+At the start of every compaction, we estimate the density of the output $d$ and calculate a number of shards
+$S$ to split the local token space into to be
+
+$$
+S = 
+\begin{cases}
+b 
+  & \text{if } d < t b\\
+2^{\left\lfloor \log_2 \left( {\frac d t \cdot \frac 1 b}\right)\right\rceil} \cdot b
+  & \text{otherwise}
+\end{cases}
+$$
+
+(where $\lfloor x \rceil$ stands for $x$ rounded to the nearest integer, i.e. $\lfloor x + 0.5 \rfloor$)
+
+That is, we divide the density by the target size and round this to a power-of-two multiple of $b$.
+We then generate $S - 1$ boundaries that split the local token space equally into $S$ shards, and split the result
+of the compaction on these boundaries to form a separate sstable for each shard. This aims to produce sstable sizes that
+fall between $t/\sqrt 2$ and $t\cdot \sqrt 2$.
+
+For example, for a target sstable size of 100MiB and 4 base shards, a 200 MiB memtable will be split in four L0 shards
+of roughly 50 MiB each, because ${\frac{200}{100} \cdot \frac 1 4} < 1$ and thus we get
+the minimum of 4 shards, each spanning 1/4 of the token space. If in one of these shards we compact 6 of these 50 MiB
+sstables, the estimated density of the output would be 1200 MiB $({6 \cdot 50 \mathrm{MiB}} / (1/4))$, which results in
+a target ratio of $\frac{1200}{100} \cdot \frac 1 4 = 2^{\log_2 3}$, rounded to $2^2 \cdot 4$ shards for the whole
+local token space, thus 4 for the 1/4 span that the compaction covers. Assuming no overwrites and
+deletions, the resulting sstables will be of size 75 MiB, token share 1/16 and density 1200 MiB.
+
+This sharding mechanism is independent of the compaction specification.
+
+## Choosing sstables to compact
+
+The density levelling lets us separate sstables in levels defined by the compaction configuration's fan factors.
+However, unlike in the size levelling case where sstables are expected to cover the full token space, we cannot use the
+number of sstables on a level as a trigger as many of these sstables may be non-overlapping, i.e. not making read
+queries less efficient. To deal with this, take advantage of sharding to perform multiple compactions on a level
+concurrently, and reduce the size of individual compaction operations, we also need to separate non-overlapping
+sections in different buckets, and decide what to do based on the number of overlapping sstables in a bucket.
+
+To do this, we first form a minimal list of overlap sets that satisfy the following requirements:
+
+- two sstables that do not overlap are never put in the same set;
+- if two sstables overlap, there is a set in the list that contains both;
+- sstables are placed in consecutive positions in the list.
+
+The second condition can also be rephrased to say that for any point in the token range, there is a set in the list
+that contains all sstables whose range covers that point. In other words, the overlap sets give us the maximum number
+of sstables that need to be consulted to read any key, i.e. the read amplification that our trigger $t$ aims to
+control. We don't calculate or store the exact spans the overlapping sets cover, only the participating sstables.
+The sets can be obtained in $O(n\log n)$ time.
+
+For example, if sstables A, B, C and D cover, respectively, tokens 0-3, 2-7, 6-9 and 1-8, the overlap sets we compute
+are ABD and BCD. A and C don't overlap, so they must be in separate sets. A, B and D overlap at token 2 and must thus
+be present in at least one set, and similarly for B, C and D at 7. Only A and D overlap at 1, but the set ABD already
+includes this combination.
+
+These overlap sets are sufficient to decide whether or not a compaction should be carried out &mdash; if and only if the
+number of elements in a set is at least as large as $t$. However, we may need to include more sstables in the compaction
+than this set alone.
+
+It is possible for our sharding scheme to end up constructing sstables spanning differently-sized shards for the same
+level. One clear example is the case of levelled compaction, where, for example, sstables enter at some density, and
+after the first compaction the result &mdash; being 2x bigger than that density &mdash; is split in the middle because 
+it has double the density. As another sstable enters the same level, we will have separate overlap sets for the first 
+and second half of that older sstable; to be efficient, the compaction that is triggered next needs to select both.
+
+To deal with this and any other cases of partial overlap, the compaction strategy will transitively extend
+the overlap sets with all neighboring ones that share some sstable, constructing the set of all sstables that have some
+chain of overlapping ones that connects it to the initial set[^1]. This extended set forms the compaction bucket.
+
+In normal operation we compact all sstables in the compaction bucket. If compaction is very late we may apply a limit
+on the number of overlapping sources we compact; in that case we use the collection of oldest sstables that would
+select at most limit-many in any included overlap set, making sure that if an sstable is included in this compaction,
+all older ones are also included to maintain time order.
+
+## Selecting the compaction to run
+
+Compaction strategies aim to minimize the read amplification of queries, which is defined by the number of sstables
+that overlap on any given key. In order to do this most efficiently in situations where compaction is late, we select
+a compaction bucket whose overlap is the highest among the possible choices. If there are multiple such choices, we 
+choose one uniformly randomly within each level, and between the levels we prefer the lowest level (as this is expected 
+to cover a larger fraction of the token space for the same amount of work).
+
+Under sustained load, this mechanism prevents the accumulation of sstables on some level that could sometimes happen 
+with legacy strategies (e.g. all resources consumed by L0 and sstables accumulating on L1) and can lead to a 
+steady state where compactions always use more sstables than the assigned threshold and fan factor and maintain a tiered
+hierarchy based on the lowest overlap they are able to maintain for the load.
+
+## Major compaction
+
+Under the working principles of UCS, a major compaction is an operation which compacts together all sstables that have
+(transitive) overlap, and where the output is split on shard boundaries appropriate for the expected result density.
+
+In other words, it is expected that a major compaction will result in $b$ concurrent compactions, each containing all
+sstables covered in each of the base shards, and that the result will be split on shard boundaries whose number 
+depends on the total size of data contained in the shard.
+
+## Differences with STCS and LCS
+
+Note that there are some differences between the tiered flavors of UCS (UCS-tiered) and STCS, and between the leveled
+flavors of UCS (UCS-leveled) and LCS.
+
+#### UCS-tiered vs STCS
+
+SizeTieredCompactionStrategy is pretty close to UCS. However, it defines buckets/levels by looking for sstables of
+similar size rather than a predefined banding of sizes. This can result in some odd selections of buckets, possibly
+spanning sstables of wildly different sizes, while UCS's selection is more stable and predictable.
+
+STCS triggers a compaction when it finds at least `min_threshold` sstables on some bucket, and it compacts between
+`min_threshold` and `max_threshold` sstables from that bucket at a time. `min_threshold` is equivalent to UCS's
+$t = f = w + 2$. UCS drops the upper limit as we have seen that compaction is still efficient with very large numbers of
+sstables.
+
+UCS makes use of the density measure to split results in order to keep the size of sstables and the length of
+compactions low. Within a level it will only consider overlapping sstables when deciding whether the threshold is hit, 
+and will independently compact sets of sstables that do not overlap.
+
+If there are multiple choices to pick SSTables within a bucket, STCS groups them by size while UCS groups them by
+timestamp. Because of that, STCS easily loses time order which makes whole table expiration less efficient.
+
+#### UCS-leveled vs LCS
+
+On a first glance LeveledCompactionStrategy looks very different in behaviour compared to UCS.
+
+LCS keeps multiple sstables per level which form a sorted run of non-overlapping sstables of small fixed size. So
+physical sstables on increasing levels increase in number (by a factor of `fanout_size`) instead of size. LCS does that
+to reduce space amplification and to ensure shorter compaction operations. When it finds that the combined size of a
+run on a level is higher than expected, it selects some sstables to compact with overlapping ones from the next level
+of the hierarchy. This eventually pushes the size of the next level over its size limit and triggers higher-level
+operations.
+
+In UCS sstables on increasing levels increase in density (by a factor of $f$, see the **Size based levels** section
+above). UCS-leveled triggers a compaction when it finds a second overlapping sstable on some sharded level. It compacts
+the overlapping bucket on that level, and the result most often ends up on that level too, but eventually it reaches
+sufficient size for the next level. Given an even data spread, this is the same time as a run in LCS would outgrow its
+size, thus compactions are in effect triggered at the same time as LCS would trigger them.
+
+The two approaches end up with a very similar effect, with the added benefits for UCS that compactions cannot affect
+other levels like e.g. L0-to-L1 compactions in LCS can prevent any concurrent L1-to-L2 compactions, and that sstables
+are structured in a way that can be easily switched to UCS-tiered or a different set of values for the UCS parameters.
+
+Because the split positions of LCS sstables are based on size only and thus vary, when LCS selects sstables on the next
+level to compact with, it must include some that only partially overlap, which tends to cause these sstables to be
+compacted more often than strictly necessary. This is not acceptable if we need tight write amplification control (i.e.
+this solution suits UCS-leveled, but not UCS-tiered and is thus not general enough for UCS). UCS deals with this by
+splitting the run on specific boundaries selected before the compaction starts based on a file's density. As the
+boundaries for a specific density are also boundaries for the next ones, whenever we select sstables to compact some
+shard boundaries are shared, which guarantees that we can efficiently select higher-density sstables that exactly match
+the span of the lower-density ones.
+
+## Configuration
+
+UCS accepts these compaction strategy parameters:
+
+* **scaling_parameters**. A list of per-level scaling parameters, specified as L*f*, T*f*, N, or an integer value
+  specifying $w$ directly. If more levels are present than the length of this list, the last value is used for all
+  higher levels. Often this will be a single parameter, specifying the behaviour for all levels of the
+  hierarchy.  
+  Levelled compaction, specified as L*f*, is preferable for read-heavy workloads, especially if bloom filters are
+  not effective (e.g. with wide partitions); higher levelled fan factors improve read amplification (and hence latency,
+  as well as throughput for read-dominated workloads) at the expense of increased write costs.  
+  Tiered compaction, specified as T*f*, is preferable for write-heavy workloads, or ones where bloom filters or
+  time order can be exploited; higher tiered fan factors improve the cost of writes (and hence throughput) at the
+  expense of making reads more difficult.  
+  N is the middle ground that has the features of levelled (one sstable run per level) as well as tiered (one
+  compaction to be promoted to the next level) and a fan factor of 2. This can also be specified as T2 or L2.  
+  The default value is T4, matching the default STCS behaviour with threshold 4. To select an equivalent of LCS
+  with its default fan factor 10, use L10.
+* **target_sstable_size**. The target sstable size $t$, specified as a human-friendly size in bytes (e.g. 100 MiB =
+  $100\cdot 2^{20}$ B or (10 MB = 10,000,000 B)). The strategy will split data in shards that aim to produce sstables
+  of size between $t / \sqrt 2$ and $t \cdot \sqrt 2$.  
+  Smaller sstables improve streaming and repair, and make compactions shorter. On the other hand, each sstable
+  on disk has a non-trivial in-memory footprint that also affects garbage collection times.  
+  Increase this if the memory pressure from the number of sstables in the system becomes too high.  
+  The default value is 1 GiB.
+* **base_shard_count**. The minimum number of shards $b$, used for levels with the smallest density. This gives the
+  minimum compaction concurrency for the lowest levels. A low number would result in larger L0 sstables but may limit
+  the overall maximum write throughput (as every piece of data has to go through L0).  
+  The default value is 4 (1 for system tables, or when multiple data locations are defined).
+* **expired_sstable_check_frequency_seconds**. Determines how often to check for expired SSTables.  
+  The default value is 10 minutes.
+
+In **cassandra.yaml**:
+
+* **concurrent_compactors**. The number of compaction threads available. Higher values increase compaction performance
+  but may increase read and write latencies.
+
+[^1]: Note: in addition to TRANSITIVE, "overlap inclusion methods" of NONE and SINGLE are also implemented for
+    experimentation, but they are not recommended for the UCS sharding scheme.

src/java/org/apache/cassandra/db/compaction/unified/Controller.java

@@ -0,0 +1,563 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.cassandra.config.CassandraRelevantProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.exceptions.ConfigurationException;
+import org.apache.cassandra.schema.SchemaConstants;
+import org.apache.cassandra.utils.Overlaps;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.MonotonicClock;
+
+/**
+* The controller provides compaction parameters to the unified compaction strategy
+*/
+public class Controller
+{
+    protected static final Logger logger = LoggerFactory.getLogger(Controller.class);
+
+    /**
+     * The scaling parameters W, one per bucket index and separated by a comma.
+     * Higher indexes will use the value of the last index with a W specified.
+     */
+    final static String SCALING_PARAMETERS_OPTION = "scaling_parameters";
+    private final static String DEFAULT_SCALING_PARAMETERS =
+        CassandraRelevantProperties.UCS_SCALING_PARAMETER.getString();
+
+    /**
+     * Override for the flush size in MB. The database should be able to calculate this from executing flushes, this
+     * should only be necessary in rare cases.
+     */
+    static final String FLUSH_SIZE_OVERRIDE_OPTION = "flush_size_override";
+
+    static final String BASE_SHARD_COUNT_OPTION = "base_shard_count";
+    /**
+     * Default base shard count, used when a base count is not explicitly supplied. This value applies as long as the
+     * table is not a system one, and directories are not defined.
+     *
+     * For others a base count of 1 is used as system tables are usually small and do not need as much compaction
+     * parallelism, while having directories defined provides for parallelism in a different way.
+     */
+    public static final int DEFAULT_BASE_SHARD_COUNT =
+        CassandraRelevantProperties.UCS_BASE_SHARD_COUNT.getInt();
+
+    static final String TARGET_SSTABLE_SIZE_OPTION = "target_sstable_size";
+    public static final long DEFAULT_TARGET_SSTABLE_SIZE =
+        CassandraRelevantProperties.UCS_TARGET_SSTABLE_SIZE.getSizeInBytes();
+    static final long MIN_TARGET_SSTABLE_SIZE = 1L << 20;
+
+    /**
+     * This parameter is intended to modify the shape of the LSM by taking into account the survival ratio of data, for now it is fixed to one.
+     */
+    static final double DEFAULT_SURVIVAL_FACTOR =
+        CassandraRelevantProperties.UCS_SURVIVAL_FACTOR.getDouble();
+    static final double[] DEFAULT_SURVIVAL_FACTORS = new double[] { DEFAULT_SURVIVAL_FACTOR };
+
+    /**
+     * The maximum number of sstables to compact in one operation.
+     *
+     * The default is 32, which aims to keep the length of operations under control and prevent accummulation of
+     * sstables while compactions are taking place.
+     *
+     * If the fanout factor is larger than the maximum number of sstables, the strategy will ignore the latter.
+     */
+    static final String MAX_SSTABLES_TO_COMPACT_OPTION = "max_sstables_to_compact";
+
+    static final String ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION = "unsafe_aggressive_sstable_expiration";
+    static final boolean ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION =
+        CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.getBoolean();
+    static final boolean DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION = false;
+
+    static final int DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS = 60 * 10;
+    static final String EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION = "expired_sstable_check_frequency_seconds";
+
+    /** The maximum splitting factor for shards. The maximum number of shards is this number multiplied by the base count. */
+    static final double MAX_SHARD_SPLIT = 1048576;
+
+    /**
+     * Overlap inclusion method. NONE for participating sstables only (not recommended), SINGLE to only include sstables
+     * that overlap with participating (LCS-like, higher concurrency during upgrades but some double compaction),
+     * TRANSITIVE to include overlaps of overlaps (likely to trigger whole level compactions, safest).
+     */
+    static final String OVERLAP_INCLUSION_METHOD_OPTION = "overlap_inclusion_method";
+    static final Overlaps.InclusionMethod DEFAULT_OVERLAP_INCLUSION_METHOD =
+        CassandraRelevantProperties.UCS_OVERLAP_INCLUSION_METHOD.getEnum(Overlaps.InclusionMethod.TRANSITIVE);
+
+    protected final ColumnFamilyStore cfs;
+    protected final MonotonicClock clock;
+    private final int[] scalingParameters;
+    protected final double[] survivalFactors;
+    protected final long flushSizeOverride;
+    protected volatile long currentFlushSize;
+    protected final int maxSSTablesToCompact;
+    protected final long expiredSSTableCheckFrequency;
+    protected final boolean ignoreOverlapsInExpirationCheck;
+
+    protected final int baseShardCount;
+
+    protected final double targetSSTableSize;
+
+    static final double INVERSE_SQRT_2 = Math.sqrt(0.5);
+
+    protected final Overlaps.InclusionMethod overlapInclusionMethod;
+
+    Controller(ColumnFamilyStore cfs,
+               MonotonicClock clock,
+               int[] scalingParameters,
+               double[] survivalFactors,
+               long flushSizeOverride,
+               int maxSSTablesToCompact,
+               long expiredSSTableCheckFrequency,
+               boolean ignoreOverlapsInExpirationCheck,
+               int baseShardCount,
+               double targetSStableSize,
+               Overlaps.InclusionMethod overlapInclusionMethod)
+    {
+        this.cfs = cfs;
+        this.clock = clock;
+        this.scalingParameters = scalingParameters;
+        this.survivalFactors = survivalFactors;
+        this.flushSizeOverride = flushSizeOverride;
+        this.currentFlushSize = flushSizeOverride;
+        this.expiredSSTableCheckFrequency = TimeUnit.MILLISECONDS.convert(expiredSSTableCheckFrequency, TimeUnit.SECONDS);
+        this.baseShardCount = baseShardCount;
+        this.targetSSTableSize = targetSStableSize;
+        this.overlapInclusionMethod = overlapInclusionMethod;
+
+        if (maxSSTablesToCompact <= 0)
+            maxSSTablesToCompact = Integer.MAX_VALUE;
+
+        this.maxSSTablesToCompact = maxSSTablesToCompact;
+
+        if (ignoreOverlapsInExpirationCheck && !ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION)
+        {
+            logger.warn("Not enabling aggressive SSTable expiration, as the system property '" +
+                        CassandraRelevantProperties.ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION.name() +
+                        "' is set to 'false'. " +
+                        "Set it to 'true' to enable aggressive SSTable expiration.");
+        }
+        this.ignoreOverlapsInExpirationCheck = ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION && ignoreOverlapsInExpirationCheck;
+    }
+
+    /**
+     * @return the scaling parameter W
+     * @param index
+     */
+    public int getScalingParameter(int index)
+    {
+        if (index < 0)
+            throw new IllegalArgumentException("Index should be >= 0: " + index);
+
+        return index < scalingParameters.length ? scalingParameters[index] : scalingParameters[scalingParameters.length - 1];
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Controller, m: %s, o: %s, Ws: %s",
+                             FBUtilities.prettyPrintBinary(targetSSTableSize, "B", ""),
+                             Arrays.toString(survivalFactors),
+                             printScalingParameters(scalingParameters));
+    }
+
+    public int getFanout(int index) {
+        int W = getScalingParameter(index);
+        return UnifiedCompactionStrategy.fanoutFromScalingParameter(W);
+    }
+
+    public int getThreshold(int index) {
+        int W = getScalingParameter(index);
+        return UnifiedCompactionStrategy.thresholdFromScalingParameter(W);
+    }
+
+    /**
+     * Calculate the number of shards to split the local token space in for the given sstable density.
+     * This is calculated as a power-of-two multiple of baseShardCount, so that the expected size of resulting sstables
+     * is between sqrt(0.5) * targetSSTableSize and sqrt(2) * targetSSTableSize, with a minimum of baseShardCount shards
+     * for smaller sstables.
+     *
+     * Note that to get the sstables resulting from this splitting within the bounds, the density argument must be
+     * normalized to the span that is being split. In other words, if no disks are defined, the density should be
+     * scaled by the token coverage of the locally-owned ranges. If multiple data directories are defined, the density
+     * should be scaled by the token coverage of the respective data directory. That is localDensity = size / span,
+     * where the span is normalized so that span = 1 when the data covers the range that is being split.
+     */
+    public int getNumShards(double localDensity)
+    {
+        // How many we would have to aim for the target size. Divided by the base shard count, so that we can ensure
+        // the result is a multiple of it by multiplying back below.
+        double count = localDensity / (targetSSTableSize * INVERSE_SQRT_2 * baseShardCount);
+        if (count > MAX_SHARD_SPLIT)
+            count = MAX_SHARD_SPLIT;
+        assert !(count < 0);    // Must be positive, 0 or NaN, which should translate to baseShardCount
+
+        // Make it a power of two multiple of the base count so that split points for lower levels remain split points
+        // for higher.
+        // The conversion to int and highestOneBit round down, for which we compensate by using the sqrt(0.5) multiplier
+        // applied above.
+        // Setting the bottom bit to 1 ensures the result is at least baseShardCount.
+        int shards = baseShardCount * Integer.highestOneBit((int) count | 1);
+        logger.debug("Shard count {} for density {}, {} times target {}",
+                     shards,
+                     FBUtilities.prettyPrintBinary(localDensity, "B", " "),
+                     localDensity / targetSSTableSize,
+                     FBUtilities.prettyPrintBinary(targetSSTableSize, "B", " "));
+        return shards;
+    }
+
+    /**
+     * @return the survival factor o
+     * @param index
+     */
+    public double getSurvivalFactor(int index)
+    {
+        if (index < 0)
+            throw new IllegalArgumentException("Index should be >= 0: " + index);
+
+        return index < survivalFactors.length ? survivalFactors[index] : survivalFactors[survivalFactors.length - 1];
+    }
+
+    /**
+     * Return the flush sstable size in bytes.
+     *
+     * This is usually obtained from the observed sstable flush sizes, refreshed when it differs significantly
+     * from the current values.
+     * It can also be set by the user in the options.
+     *
+     * @return the flush size in bytes.
+     */
+    public long getFlushSizeBytes()
+    {
+        if (flushSizeOverride > 0)
+            return flushSizeOverride;
+
+        double envFlushSize = cfs.metric.flushSizeOnDisk.get();
+        if (currentFlushSize == 0 || Math.abs(1 - (currentFlushSize / envFlushSize)) > 0.5)
+        {
+            // The current size is not initialized, or it differs by over 50% from the observed.
+            // Use the observed size rounded up to a whole megabyte.
+            currentFlushSize = ((long) (Math.ceil(Math.scalb(envFlushSize, -20)))) << 20;
+        }
+        return currentFlushSize;
+    }
+
+    /**
+     * @return whether is allowed to drop expired SSTables without checking if partition keys appear in other SSTables.
+     * Same behavior as in TWCS.
+     */
+    public boolean getIgnoreOverlapsInExpirationCheck()
+    {
+        return ignoreOverlapsInExpirationCheck;
+    }
+
+    public long getExpiredSSTableCheckFrequency()
+    {
+        return expiredSSTableCheckFrequency;
+    }
+
+    public static Controller fromOptions(ColumnFamilyStore cfs, Map<String, String> options)
+    {
+        int[] Ws = parseScalingParameters(options.getOrDefault(SCALING_PARAMETERS_OPTION, DEFAULT_SCALING_PARAMETERS));
+
+        long flushSizeOverride = FBUtilities.parseHumanReadableBytes(options.getOrDefault(FLUSH_SIZE_OVERRIDE_OPTION,
+                                                                                          "0MiB"));
+        int maxSSTablesToCompact = Integer.parseInt(options.getOrDefault(MAX_SSTABLES_TO_COMPACT_OPTION, "0"));
+        long expiredSSTableCheckFrequency = options.containsKey(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION)
+                ? Long.parseLong(options.get(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION))
+                : DEFAULT_EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS;
+        boolean ignoreOverlapsInExpirationCheck = options.containsKey(ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION)
+                ? Boolean.parseBoolean(options.get(ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION))
+                : DEFAULT_ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION;
+
+        int baseShardCount;
+        if (options.containsKey(BASE_SHARD_COUNT_OPTION))
+        {
+            baseShardCount = Integer.parseInt(options.get(BASE_SHARD_COUNT_OPTION));
+        }
+        else
+        {
+            if (SchemaConstants.isSystemKeyspace(cfs.getKeyspaceName())
+                || (cfs.getDiskBoundaries().positions != null && cfs.getDiskBoundaries().positions.size() > 1))
+                baseShardCount = 1;
+            else
+                baseShardCount = DEFAULT_BASE_SHARD_COUNT;
+        }
+
+        long targetSStableSize = options.containsKey(TARGET_SSTABLE_SIZE_OPTION)
+                ? FBUtilities.parseHumanReadableBytes(options.get(TARGET_SSTABLE_SIZE_OPTION))
+                : DEFAULT_TARGET_SSTABLE_SIZE;
+
+        Overlaps.InclusionMethod inclusionMethod = options.containsKey(OVERLAP_INCLUSION_METHOD_OPTION)
+                ? Overlaps.InclusionMethod.valueOf(options.get(OVERLAP_INCLUSION_METHOD_OPTION).toUpperCase())
+                : DEFAULT_OVERLAP_INCLUSION_METHOD;
+
+        return new Controller(cfs,
+                              MonotonicClock.Global.preciseTime,
+                              Ws,
+                              DEFAULT_SURVIVAL_FACTORS,
+                              flushSizeOverride,
+                              maxSSTablesToCompact,
+                              expiredSSTableCheckFrequency,
+                              ignoreOverlapsInExpirationCheck,
+                              baseShardCount,
+                              targetSStableSize,
+                              inclusionMethod);
+    }
+
+    public static Map<String, String> validateOptions(Map<String, String> options) throws ConfigurationException
+    {
+        options = new HashMap<>(options);
+        String s;
+
+        s = options.remove(SCALING_PARAMETERS_OPTION);
+        if (s != null)
+            parseScalingParameters(s);
+
+        s = options.remove(BASE_SHARD_COUNT_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                int numShards = Integer.parseInt(s);
+                if (numShards <= 0)
+                    throw new ConfigurationException(String.format("Invalid configuration, %s should be positive: %d",
+                                                                   BASE_SHARD_COUNT_OPTION,
+                                                                   numShards));
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s",
+                                                               s,
+                                                               BASE_SHARD_COUNT_OPTION), e);
+            }
+        }
+
+        s = options.remove(TARGET_SSTABLE_SIZE_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                long targetSSTableSize = FBUtilities.parseHumanReadableBytes(s);
+                if (targetSSTableSize < MIN_TARGET_SSTABLE_SIZE)
+                {
+                    throw new ConfigurationException(String.format("%s %s is not acceptable, size must be at least %s",
+                                                                   TARGET_SSTABLE_SIZE_OPTION,
+                                                                   s,
+                                                                   FBUtilities.prettyPrintMemory(MIN_TARGET_SSTABLE_SIZE)));
+                }
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s %s is not a valid size in bytes: %s",
+                                                               TARGET_SSTABLE_SIZE_OPTION,
+                                                               s,
+                                                               e.getMessage()),
+                                                 e);
+            }
+        }
+
+        s = options.remove(FLUSH_SIZE_OVERRIDE_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                long flushSize = FBUtilities.parseHumanReadableBytes(s);
+                if (flushSize < MIN_TARGET_SSTABLE_SIZE)
+                    throw new ConfigurationException(String.format("%s %s is not acceptable, size must be at least %s",
+                                                                   FLUSH_SIZE_OVERRIDE_OPTION,
+                                                                   s,
+                                                                   FBUtilities.prettyPrintMemory(MIN_TARGET_SSTABLE_SIZE)));
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s %s is not a valid size in bytes: %s",
+                                                               FLUSH_SIZE_OVERRIDE_OPTION,
+                                                               s,
+                                                               e.getMessage()),
+                                                 e);
+            }
+        }
+
+        s = options.remove(MAX_SSTABLES_TO_COMPACT_OPTION);
+        if (s != null)
+        {
+             try
+             {
+                 Integer.parseInt(s); // values less than or equal to 0 enable the default
+             }
+             catch (NumberFormatException e)
+             {
+                 throw new ConfigurationException(String.format("%s is not a parsable int (base10) for %s",
+                                                                s,
+                                                                MAX_SSTABLES_TO_COMPACT_OPTION),
+                                                  e);
+             }
+        }
+        s = options.remove(EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                long expiredSSTableCheckFrequency = Long.parseLong(s);
+                if (expiredSSTableCheckFrequency <= 0)
+                    throw new ConfigurationException(String.format("Invalid configuration, %s should be positive: %d",
+                                                                   EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION,
+                                                                   expiredSSTableCheckFrequency));
+            }
+            catch (NumberFormatException e)
+            {
+                throw new ConfigurationException(String.format("%s is not a parsable long (base10) for %s",
+                                                               s,
+                                                               EXPIRED_SSTABLE_CHECK_FREQUENCY_SECONDS_OPTION),
+                                                 e);
+            }
+        }
+
+        s = options.remove(ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION);
+        if (s != null && !s.equalsIgnoreCase("true") && !s.equalsIgnoreCase("false"))
+        {
+            throw new ConfigurationException(String.format("%s should either be 'true' or 'false', not %s",
+                                                           ALLOW_UNSAFE_AGGRESSIVE_SSTABLE_EXPIRATION_OPTION, s));
+        }
+
+        s = options.remove(OVERLAP_INCLUSION_METHOD_OPTION);
+        if (s != null)
+        {
+            try
+            {
+                Overlaps.InclusionMethod.valueOf(s.toUpperCase());
+            }
+            catch (IllegalArgumentException e)
+            {
+                throw new ConfigurationException(String.format("Invalid overlap inclusion method %s. The valid options are %s.",
+                                                               s,
+                                                               Arrays.toString(Overlaps.InclusionMethod.values())));
+            }
+        }
+
+        return options;
+    }
+
+    // The methods below are implemented here (rather than directly in UCS) to aid testability.
+
+    public double getBaseSstableSize(int F)
+    {
+        // The compaction hierarchy should start at a minimum size which is close to the typical flush size, with
+        // some leeway to make sure we don't overcompact when flushes end up a little smaller.
+        // The leeway should be less than 1/F, though, to make sure we don't overshoot the boundary combining F-1
+        // sources instead of F.
+        // Note that while we have not had flushes, the size will be 0 and we will use 1MB as the flush size. With
+        // fixed and positive W this should not hurt us, as the hierarchy will be in multiples of F and will still
+        // result in the same buckets, but for negative W or hybrid strategies this may cause temporary overcompaction.
+        // If this is a concern, the flush size override should be used to avoid it until DB-4401.
+        return Math.max(1 << 20, getFlushSizeBytes()) * (1.0 - 0.9 / F);
+    }
+
+    public double getMaxLevelDensity(int index, double minSize)
+    {
+        return Math.floor(minSize * getFanout(index) * getSurvivalFactor(index));
+    }
+
+    public double maxThroughput()
+    {
+        double compactionThroughputMbPerSec = DatabaseDescriptor.getCompactionThroughputMebibytesPerSec();
+        if (compactionThroughputMbPerSec <= 0)
+            return Double.MAX_VALUE;
+        return Math.scalb(compactionThroughputMbPerSec, 20);
+    }
+
+    public int maxConcurrentCompactions()
+    {
+        return DatabaseDescriptor.getConcurrentCompactors();
+    }
+
+    public int maxSSTablesToCompact()
+    {
+        return maxSSTablesToCompact;
+    }
+
+    /**
+     * Random number generator to be used for the selection of tasks.
+     * Replaced by some tests.
+     */
+    public Random random()
+    {
+        return ThreadLocalRandom.current();
+    }
+
+    /**
+     * Return the overlap inclusion method to use when combining overlap sections into a bucket. For example, with
+     * SSTables A(0, 5), B(2, 9), C(6, 12), D(10, 12) whose overlap sections calculation returns [AB, BC, CD],
+     *   - NONE means no sections are to be merged. AB, BC and CD will be separate buckets, compactions AB, BC and CD
+     *     will be added separately, thus some SSTables will be partially used / single-source compacted, likely
+     *     to be recompacted again with the next selected bucket.
+     *   - SINGLE means only overlaps of the sstables in the selected bucket will be added. AB+BC will be one bucket,
+     *     and CD will be another (as BC is already used). A middle ground of sorts, should reduce overcompaction but
+     *     still has some.
+     *   - TRANSITIVE means a transitive closure of overlapping sstables will be selected. AB+BC+CD will be in the same
+     *     bucket, selected compactions will apply to all overlapping sstables and no overcompaction will be done, at
+     *     the cost of reduced compaction parallelism and increased length of the operation.
+     * TRANSITIVE is the default and makes most sense. NONE is a closer approximation to operation of legacy UCS.
+     * The option is exposed for experimentation.
+     */
+    public Overlaps.InclusionMethod overlapInclusionMethod()
+    {
+        return overlapInclusionMethod;
+    }
+
+    public static int[] parseScalingParameters(String str)
+    {
+        String[] vals = str.split(",");
+        int[] ret = new int[vals.length];
+        for (int i = 0; i < vals.length; i++)
+        {
+            String value = vals[i].trim();
+            int W = UnifiedCompactionStrategy.parseScalingParameter(value);
+            ret[i] = W;
+        }
+
+        return ret;
+    }
+
+    public static String printScalingParameters(int[] parameters)
+    {
+        StringBuilder builder = new StringBuilder();
+        int i;
+        for (i = 0; i < parameters.length - 1; ++i)
+        {
+            builder.append(UnifiedCompactionStrategy.printScalingParameter(parameters[i]));
+            builder.append(", ");
+        }
+        builder.append(UnifiedCompactionStrategy.printScalingParameter(parameters[i]));
+        return builder.toString();
+    }
+}

src/java/org/apache/cassandra/db/compaction/unified/ShardedCompactionWriter.java

@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.ShardTracker;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.utils.FBUtilities;
+
+/**
+ * A {@link CompactionAwareWriter} that splits the output sstable at the partition boundaries of the compaction
+ * shards used by {@link org.apache.cassandra.db.compaction.UnifiedCompactionStrategy}.
+ */
+public class ShardedCompactionWriter extends CompactionAwareWriter
+{
+    protected final static Logger logger = LoggerFactory.getLogger(ShardedCompactionWriter.class);
+
+    private final double uniqueKeyRatio;
+
+    private final ShardTracker boundaries;
+
+    public ShardedCompactionWriter(ColumnFamilyStore cfs,
+                                   Directories directories,
+                                   LifecycleTransaction txn,
+                                   Set<SSTableReader> nonExpiredSSTables,
+                                   boolean keepOriginals,
+                                   ShardTracker boundaries)
+    {
+        super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
+
+        this.boundaries = boundaries;
+        long totalKeyCount = nonExpiredSSTables.stream()
+                                               .mapToLong(SSTableReader::estimatedKeys)
+                                               .sum();
+        this.uniqueKeyRatio = 1.0 * SSTableReader.getApproximateKeyCount(nonExpiredSSTables) / totalKeyCount;
+    }
+
+    @Override
+    protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
+    {
+        // If we have written anything and cross a shard boundary, switch to a new writer. We use the uncompressed
+        // file pointer here because there may be writes that are not yet reflected in the on-disk size, and we want
+        // to split as soon as there is content, regardless how small.
+        final long uncompressedBytesWritten = sstableWriter.currentWriter().getFilePointer();
+        if (boundaries.advanceTo(key.getToken()) && uncompressedBytesWritten > 0)
+        {
+            logger.debug("Switching writer at boundary {}/{} index {}, with uncompressed size {} for {}.{}",
+                         key.getToken(), boundaries.shardStart(),
+                         boundaries.shardIndex(),
+                         FBUtilities.prettyPrintMemory(uncompressedBytesWritten),
+                         cfs.getKeyspaceName(), cfs.getTableName());
+            return true;
+        }
+
+        return false;
+    }
+
+    @Override
+    @SuppressWarnings("resource")
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+    {
+        if (nextKey != null)
+            boundaries.advanceTo(nextKey.getToken());
+        return super.sstableWriter(directory, nextKey);
+    }
+
+    protected long sstableKeyCount()
+    {
+        return shardAdjustedKeyCount(boundaries, nonExpiredSSTables, uniqueKeyRatio);
+    }
+
+    private static long shardAdjustedKeyCount(ShardTracker boundaries,
+                                              Set<SSTableReader> sstables,
+                                              double survivalRatio)
+    {
+        // Note: computationally non-trivial; can be optimized if we save start/stop shards and size per table.
+        return Math.round(boundaries.shardAdjustedKeyCount(sstables) * survivalRatio);
+    }
+
+    @Override
+    protected void doPrepare()
+    {
+        sstableWriter.forEachWriter(boundaries::applyTokenSpaceCoverage);
+        super.doPrepare();
+    }
+}

src/java/org/apache/cassandra/db/compaction/unified/ShardedMultiWriter.java

@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
+import org.apache.cassandra.db.compaction.ShardTracker;
+import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
+import org.apache.cassandra.db.rows.UnfilteredRowIterator;
+import org.apache.cassandra.index.Index;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableMultiWriter;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.SSTableWriter;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.schema.TableId;
+import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.TimeUUID;
+
+/**
+ * A {@link SSTableMultiWriter} that splits the output sstable at the partition boundaries of the compaction
+ * shards used by {@link org.apache.cassandra.db.compaction.UnifiedCompactionStrategy}.
+ * <p/>
+ * This is class is similar to {@link ShardedCompactionWriter} but for flushing. Unfortunately
+ * we currently have 2 separate writers hierarchy that are not compatible and so we must
+ * duplicate the functionality.
+ */
+public class ShardedMultiWriter implements SSTableMultiWriter
+{
+    protected final static Logger logger = LoggerFactory.getLogger(ShardedMultiWriter.class);
+
+    private final ColumnFamilyStore cfs;
+    private final Descriptor descriptor;
+    private final long keyCount;
+    private final long repairedAt;
+    private final TimeUUID pendingRepair;
+    private final boolean isTransient;
+    private final IntervalSet<CommitLogPosition> commitLogPositions;
+    private final SerializationHeader header;
+    private final Collection<Index> indexes;
+    private final LifecycleNewTracker lifecycleNewTracker;
+    private final ShardTracker boundaries;
+    private final SSTableWriter[] writers;
+    private int currentWriter;
+
+    public ShardedMultiWriter(ColumnFamilyStore cfs,
+                              Descriptor descriptor,
+                              long keyCount,
+                              long repairedAt,
+                              TimeUUID pendingRepair,
+                              boolean isTransient,
+                              IntervalSet<CommitLogPosition> commitLogPositions,
+                              SerializationHeader header,
+                              Collection<Index> indexes,
+                              LifecycleNewTracker lifecycleNewTracker,
+                              ShardTracker boundaries)
+    {
+        this.cfs = cfs;
+        this.descriptor = descriptor;
+        this.keyCount = keyCount;
+        this.repairedAt = repairedAt;
+        this.pendingRepair = pendingRepair;
+        this.isTransient = isTransient;
+        this.commitLogPositions = commitLogPositions;
+        this.header = header;
+        this.indexes = indexes;
+        this.lifecycleNewTracker = lifecycleNewTracker;
+        this.boundaries = boundaries;
+        this.writers = new SSTableWriter[this.boundaries.count()]; // at least one
+
+        this.currentWriter = 0;
+        this.writers[currentWriter] = createWriter(descriptor);
+    }
+
+    private SSTableWriter createWriter()
+    {
+        Descriptor newDesc = cfs.newSSTableDescriptor(descriptor.directory);
+        return createWriter(newDesc);
+    }
+
+    private SSTableWriter createWriter(Descriptor descriptor)
+    {
+        MetadataCollector metadataCollector = new MetadataCollector(cfs.metadata().comparator)
+                                              .commitLogIntervals(commitLogPositions != null ? commitLogPositions : IntervalSet.empty());
+        return descriptor.getFormat().getWriterFactory().builder(descriptor)
+                         .setKeyCount(forSplittingKeysBy(boundaries.count()))
+                         .setRepairedAt(repairedAt)
+                         .setPendingRepair(pendingRepair)
+                         .setTransientSSTable(isTransient)
+                         .setTableMetadataRef(cfs.metadata)
+                         .setMetadataCollector(metadataCollector)
+                         .setSerializationHeader(header)
+                         .addDefaultComponents()
+                         .addFlushObserversForSecondaryIndexes(indexes, lifecycleNewTracker.opType())
+                         .build(lifecycleNewTracker, cfs);
+    }
+
+    private long forSplittingKeysBy(long splits) {
+        return splits <= 1 ? keyCount : keyCount / splits;
+    }
+
+    @Override
+    public void append(UnfilteredRowIterator partition)
+    {
+        DecoratedKey key = partition.partitionKey();
+
+        // If we have written anything and cross a shard boundary, switch to a new writer.
+        final long currentUncompressedSize = writers[currentWriter].getFilePointer();
+        if (boundaries.advanceTo(key.getToken()) && currentUncompressedSize > 0)
+        {
+            logger.debug("Switching writer at boundary {}/{} index {}, with uncompressed size {} for {}.{}",
+                         key.getToken(), boundaries.shardStart(), currentWriter,
+                         FBUtilities.prettyPrintMemory(currentUncompressedSize),
+                         cfs.getKeyspaceName(), cfs.getTableName());
+
+            writers[++currentWriter] = createWriter();
+        }
+
+        writers[currentWriter].append(partition);
+    }
+
+    @Override
+    public Collection<SSTableReader> finish(boolean openResult)
+    {
+        List<SSTableReader> sstables = new ArrayList<>(writers.length);
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+            {
+                boundaries.applyTokenSpaceCoverage(writer);
+                sstables.add(writer.finish(openResult));
+            }
+        return sstables;
+    }
+
+    @Override
+    public Collection<SSTableReader> finished()
+    {
+        List<SSTableReader> sstables = new ArrayList<>(writers.length);
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+                sstables.add(writer.finished());
+        return sstables;
+    }
+
+    @Override
+    public SSTableMultiWriter setOpenResult(boolean openResult)
+    {
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+                writer.setOpenResult(openResult);
+        return this;
+    }
+
+    @Override
+    public String getFilename()
+    {
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+                return writer.getFilename();
+        return "";
+    }
+
+    @Override
+    public long getBytesWritten()
+    {
+        long bytesWritten = 0;
+        for (int i = 0; i <= currentWriter; ++i)
+            bytesWritten += writers[i].getFilePointer();
+        return bytesWritten;
+    }
+
+    @Override
+    public long getOnDiskBytesWritten()
+    {
+        long bytesWritten = 0;
+        for (int i = 0; i <= currentWriter; ++i)
+            bytesWritten += writers[i].getEstimatedOnDiskBytesWritten();
+        return bytesWritten;
+    }
+
+    @Override
+    public TableId getTableId()
+    {
+        return cfs.metadata().id;
+    }
+
+    @Override
+    public Throwable commit(Throwable accumulate)
+    {
+        Throwable t = accumulate;
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+                t = writer.commit(t);
+        return t;
+    }
+
+    @Override
+    public Throwable abort(Throwable accumulate)
+    {
+        Throwable t = accumulate;
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+            {
+                lifecycleNewTracker.untrackNew(writer);
+                t = writer.abort(t);
+            }
+        return t;
+    }
+
+    @Override
+    public void prepareToCommit()
+    {
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+            {
+                boundaries.applyTokenSpaceCoverage(writer);
+                writer.prepareToCommit();
+    }
+    }
+
+    @Override
+    public void close()
+    {
+        for (SSTableWriter writer : writers)
+            if (writer != null)
+                writer.close();
+    }
+}

src/java/org/apache/cassandra/db/compaction/unified/UnifiedCompactionTask.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.db.compaction.unified;
+
+import java.util.Set;
+
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.compaction.CompactionTask;
+import org.apache.cassandra.db.compaction.ShardManager;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
+import org.apache.cassandra.db.compaction.writers.CompactionAwareWriter;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+
+/**
+ * The sole purpose of this class is to currently create a {@link ShardedCompactionWriter}.
+ */
+public class UnifiedCompactionTask extends CompactionTask
+{
+    private final ShardManager shardManager;
+    private final Controller controller;
+
+    public UnifiedCompactionTask(ColumnFamilyStore cfs,
+                                 UnifiedCompactionStrategy strategy,
+                                 LifecycleTransaction txn,
+                                 long gcBefore,
+                                 ShardManager shardManager)
+    {
+        super(cfs, txn, gcBefore, strategy.getController().getIgnoreOverlapsInExpirationCheck());
+        this.controller = strategy.getController();
+        this.shardManager = shardManager;
+    }
+
+    @Override
+    public CompactionAwareWriter getCompactionAwareWriter(ColumnFamilyStore cfs,
+                                                          Directories directories,
+                                                          LifecycleTransaction txn,
+                                                          Set<SSTableReader> nonExpiredSSTables)
+    {
+        double density = shardManager.calculateCombinedDensity(nonExpiredSSTables);
+        int numShards = controller.getNumShards(density * shardManager.shardSetCoverage());
+        return new ShardedCompactionWriter(cfs, directories, txn, nonExpiredSSTables, keepOriginals, shardManager.boundaries(numShards));
+    }
+}

src/java/org/apache/cassandra/db/compaction/writers/CompactionAwareWriter.java

@@ -68,9 +68,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
     private final List<Directories.DataDirectory> locations;
     private final List<PartitionPosition> diskBoundaries;
     private int locationIndex;
-
-    // Keep targetDirectory for compactions, needed for `nodetool compactionstats`
-    protected Directories.DataDirectory sstableDirectory;
+    protected Directories.DataDirectory currentDirectory;
 
     public CompactionAwareWriter(ColumnFamilyStore cfs,
                                  Directories directories,
@@ -145,7 +143,7 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
 
     public final File getSStableDirectory() throws IOException
     {
-        return getDirectories().getLocationForDisk(sstableDirectory);
+        return getDirectories().getLocationForDisk(currentDirectory);
     }
 
     @Override
@@ -155,43 +153,102 @@ public abstract class CompactionAwareWriter extends Transactional.AbstractTransa
         return super.doPostCleanup(accumulate);
     }
 
-    protected abstract boolean realAppend(UnfilteredRowIterator partition);
+    protected boolean realAppend(UnfilteredRowIterator partition)
+    {
+        return sstableWriter.append(partition) != null;
+    }
 
     /**
+     * Switches the writer if necessary, i.e. if the new key should be placed in a different data directory, or if the
+     * specific strategy has decided a new sstable is needed.
      * Guaranteed to be called before the first call to realAppend.
-     * @param key
      */
     protected void maybeSwitchWriter(DecoratedKey key)
+    {
+        if (maybeSwitchLocation(key))
+            return;
+
+        if (shouldSwitchWriterInCurrentLocation(key))
+            switchCompactionWriter(currentDirectory, key);
+    }
+
+    /**
+     * Switches the file location and writer and returns true if the new key should be placed in a different data
+     * directory.
+     */
+    protected boolean maybeSwitchLocation(DecoratedKey key)
     {
         if (diskBoundaries == null)
         {
             if (locationIndex < 0)
             {
                 Directories.DataDirectory defaultLocation = getWriteDirectory(nonExpiredSSTables, getExpectedWriteSize());
-                switchCompactionLocation(defaultLocation);
+                switchCompactionWriter(defaultLocation, key);
                 locationIndex = 0;
+                return true;
             }
-            return;
+            return false;
         }
 
         if (locationIndex > -1 && key.compareTo(diskBoundaries.get(locationIndex)) < 0)
-            return;
+            return false;
 
         int prevIdx = locationIndex;
         while (locationIndex == -1 || key.compareTo(diskBoundaries.get(locationIndex)) > 0)
             locationIndex++;
+        Directories.DataDirectory newLocation = locations.get(locationIndex);
         if (prevIdx >= 0)
-            logger.debug("Switching write location from {} to {}", locations.get(prevIdx), locations.get(locationIndex));
-        switchCompactionLocation(locations.get(locationIndex));
+            logger.debug("Switching write location from {} to {}", locations.get(prevIdx), newLocation);
+        switchCompactionWriter(newLocation, key);
+        return true;
     }
 
     /**
-     * Implementations of this method should finish the current sstable writer and start writing to this directory.
-     *
-     * Called once before starting to append and then whenever we see a need to start writing to another directory.
-     * @param directory
+     * Returns true if the writer should be switched for reasons other than switching to a new data directory
+     * (e.g. because an sstable size limit has been reached).
      */
-    protected abstract void switchCompactionLocation(Directories.DataDirectory directory);
+    protected abstract boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key);
+
+    /**
+     * Implementations of this method should finish the current sstable writer and start writing to this directory.
+     * <p>
+     * Called once before starting to append and then whenever we see a need to start writing to another directory.
+     *
+     * @param directory
+     * @param nextKey
+     */
+    protected void switchCompactionWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+    {
+        currentDirectory = directory;
+        sstableWriter.switchWriter(sstableWriter(directory, nextKey));
+    }
+
+    @SuppressWarnings("resource")
+    protected SSTableWriter sstableWriter(Directories.DataDirectory directory, DecoratedKey nextKey)
+    {
+        Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
+        MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator)
+                                      .sstableLevel(sstableLevel());
+        SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
+
+        return newWriterBuilder(descriptor).setMetadataCollector(collector)
+                                           .setSerializationHeader(header)
+                                           .setKeyCount(sstableKeyCount())
+                                           .build(txn, cfs);
+    }
+
+    /**
+     * Returns the level that should be used when creating sstables.
+     */
+    protected int sstableLevel()
+    {
+        return 0;
+    }
+
+    /**
+     * Returns the key count with which created sstables should be set up.
+     */
+    abstract protected long sstableKeyCount();
 
     /**
      * The directories we can write to

src/java/org/apache/cassandra/db/compaction/writers/DefaultCompactionWriter.java

@@ -24,14 +24,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 /**
  * The default compaction writer - creates one output file in L0
@@ -60,30 +56,17 @@ public class DefaultCompactionWriter extends CompactionAwareWriter
     }
 
     @Override
-    public boolean realAppend(UnfilteredRowIterator partition)
+    protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
     {
-        return sstableWriter.append(partition) != null;
+        return false;
     }
 
-    @Override
-    public void switchCompactionLocation(Directories.DataDirectory directory)
+    protected int sstableLevel()
     {
-        sstableDirectory = directory;
-
-        Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(directory));
-        MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, sstableLevel);
-        SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
-
-        @SuppressWarnings("resource")
-        SSTableWriter writer = newWriterBuilder(descriptor).setMetadataCollector(collector)
-                                                              .setSerializationHeader(header)
-                                                              .setKeyCount(estimatedTotalKeys)
-                                                              .build(txn, cfs);
-        sstableWriter.switchWriter(writer);
+        return sstableLevel;
     }
 
-    @Override
-    public long estimatedKeys()
+    protected long sstableKeyCount()
     {
         return estimatedTotalKeys;
     }

src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java

@@ -20,16 +20,12 @@ package org.apache.cassandra.db.compaction.writers;
 import java.util.Set;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.compaction.LeveledManifest;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 public class MajorLeveledCompactionWriter extends CompactionAwareWriter
 {
@@ -67,11 +63,15 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     }
 
     @Override
-    @SuppressWarnings("resource")
     public boolean realAppend(UnfilteredRowIterator partition)
     {
-        AbstractRowIndexEntry rie = sstableWriter.append(partition);
         partitionsWritten++;
+        return super.realAppend(partition);
+    }
+
+    @Override
+    protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
+    {
         long totalWrittenInCurrentWriter = sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten();
         if (totalWrittenInCurrentWriter > maxSSTableSize)
         {
@@ -81,31 +81,29 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
                 totalWrittenInLevel = 0;
                 currentLevel++;
             }
-            switchCompactionLocation(sstableDirectory);
+            return true;
         }
-        return rie != null;
+        return false;
 
     }
 
     @Override
-    public void switchCompactionLocation(Directories.DataDirectory location)
+    public void switchCompactionWriter(Directories.DataDirectory location, DecoratedKey nextKey)
     {
-        sstableDirectory = location;
         averageEstimatedKeysPerSSTable = Math.round(((double) averageEstimatedKeysPerSSTable * sstablesWritten + partitionsWritten) / (sstablesWritten + 1));
-
-        Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory));
-        MetadataCollector collector = new MetadataCollector(txn.originals(), cfs.metadata().comparator, currentLevel);
-        SerializationHeader serializationHeader = SerializationHeader.make(cfs.metadata(), txn.originals());
-
-        @SuppressWarnings("resource")
-        SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(keysPerSSTable)
-                                                           .setSerializationHeader(serializationHeader)
-                                                           .setMetadataCollector(collector)
-                                                           .build(txn, cfs);
-
-        sstableWriter.switchWriter(writer);
         partitionsWritten = 0;
         sstablesWritten = 0;
+        super.switchCompactionWriter(location, nextKey);
+    }
+
+    protected int sstableLevel()
+    {
+        return currentLevel;
+    }
+
+    protected long sstableKeyCount()
+    {
+        return keysPerSSTable;
     }
 
     @Override

src/java/org/apache/cassandra/db/compaction/writers/MaxSSTableSizeWriter.java

@@ -20,23 +20,17 @@ package org.apache.cassandra.db.compaction.writers;
 import java.util.Set;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 public class MaxSSTableSizeWriter extends CompactionAwareWriter
 {
     private final long maxSSTableSize;
     private final int level;
     private final long estimatedSSTables;
-    private final Set<SSTableReader> allSSTables;
 
     public MaxSSTableSizeWriter(ColumnFamilyStore cfs,
                                 Directories directories,
@@ -57,7 +51,6 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
                                 boolean keepOriginals)
     {
         super(cfs, directories, txn, nonExpiredSSTables, keepOriginals);
-        this.allSSTables = txn.originals();
         this.level = level;
         this.maxSSTableSize = maxSSTableSize;
 
@@ -79,31 +72,20 @@ public class MaxSSTableSizeWriter extends CompactionAwareWriter
         return Math.round(estimatedCompactionRatio * cfs.getExpectedCompactedFileSize(nonExpiredSSTables, compactionType));
     }
 
-    protected boolean realAppend(UnfilteredRowIterator partition)
+    @Override
+    protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
     {
-        AbstractRowIndexEntry rie = sstableWriter.append(partition);
-        if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize)
-        {
-            switchCompactionLocation(sstableDirectory);
-        }
-        return rie != null;
+        return sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > maxSSTableSize;
     }
 
-    @Override
-    public void switchCompactionLocation(Directories.DataDirectory location)
+    protected int sstableLevel()
     {
-        sstableDirectory = location;
+        return level;
+    }
 
-        Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(sstableDirectory));
-        MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, level);
-        SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
-
-        @SuppressWarnings("resource")
-        SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(estimatedTotalKeys / estimatedSSTables)
-                                                              .setMetadataCollector(collector)
-                                                              .setSerializationHeader(header)
-                                                              .build(txn, cfs);
-        sstableWriter.switchWriter(writer);
+    protected long sstableKeyCount()
+    {
+        return estimatedTotalKeys / estimatedSSTables;
     }
 
     @Override

src/java/org/apache/cassandra/db/compaction/writers/SplittingSizeTieredCompactionWriter.java

@@ -24,15 +24,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.DecoratedKey;
 import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.rows.UnfilteredRowIterator;
-import org.apache.cassandra.io.sstable.AbstractRowIndexEntry;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.SSTableWriter;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 
 /**
  * CompactionAwareWriter that splits input in differently sized sstables
@@ -84,36 +79,28 @@ public class SplittingSizeTieredCompactionWriter extends CompactionAwareWriter
     }
 
     @Override
-    public boolean realAppend(UnfilteredRowIterator partition)
+    protected boolean shouldSwitchWriterInCurrentLocation(DecoratedKey key)
     {
-        AbstractRowIndexEntry rie = sstableWriter.append(partition);
         if (sstableWriter.currentWriter().getEstimatedOnDiskBytesWritten() > currentBytesToWrite && currentRatioIndex < ratios.length - 1) // if we underestimate how many keys we have, the last sstable might get more than we expect
         {
             currentRatioIndex++;
-
             currentBytesToWrite = getExpectedWriteSize();
-            switchCompactionLocation(sstableDirectory);
             logger.debug("Switching writer, currentBytesToWrite = {}", currentBytesToWrite);
+            return true;
         }
-        return rie != null;
+        return false;
     }
 
-    @Override
-    public void switchCompactionLocation(Directories.DataDirectory location)
+    protected int sstableLevel()
     {
-        sstableDirectory = location;
-        long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys);
-        Descriptor descriptor = cfs.newSSTableDescriptor(getDirectories().getLocationForDisk(location));
-        MetadataCollector collector = new MetadataCollector(allSSTables, cfs.metadata().comparator, 0);
-        SerializationHeader header = SerializationHeader.make(cfs.metadata(), nonExpiredSSTables);
+        return 0;
+    }
 
-        @SuppressWarnings("resource")
-        SSTableWriter writer = newWriterBuilder(descriptor).setKeyCount(currentPartitionsToWrite)
-                                                              .setMetadataCollector(collector)
-                                                              .setSerializationHeader(header)
-                                                              .build(txn, cfs);
+    protected long sstableKeyCount()
+    {
+        long currentPartitionsToWrite = Math.round(ratios[currentRatioIndex] * estimatedTotalKeys);
         logger.trace("Switching writer, currentPartitionsToWrite = {}", currentPartitionsToWrite);
-        sstableWriter.switchWriter(writer);
+        return currentPartitionsToWrite;
     }
 
     @Override

src/java/org/apache/cassandra/db/lifecycle/SSTableIntervalTree.java

@@ -54,7 +54,7 @@ public class SSTableIntervalTree extends IntervalTree<PartitionPosition, SSTable
     {
         List<Interval<PartitionPosition, SSTableReader>> intervals = new ArrayList<>(Iterables.size(sstables));
         for (SSTableReader sstable : sstables)
-            intervals.add(Interval.<PartitionPosition, SSTableReader>create(sstable.first, sstable.last, sstable));
+            intervals.add(Interval.<PartitionPosition, SSTableReader>create(sstable.getFirst(), sstable.getLast(), sstable));
         return intervals;
     }
 }

src/java/org/apache/cassandra/db/lifecycle/Tracker.java

@@ -176,7 +176,7 @@ public class Tracker
         for (SSTableReader sstable : newSSTables)
         {
             if (logger.isTraceEnabled())
-                logger.trace("adding {} to list of files tracked for {}.{}", sstable.descriptor, cfstore.keyspace.getName(), cfstore.name);
+                logger.trace("adding {} to list of files tracked for {}.{}", sstable.descriptor, cfstore.getKeyspaceName(), cfstore.name);
             try
             {
                 add += sstable.bytesOnDisk();
@@ -194,7 +194,7 @@ public class Tracker
         for (SSTableReader sstable : oldSSTables)
         {
             if (logger.isTraceEnabled())
-                logger.trace("removing {} from list of files tracked for {}.{}", sstable.descriptor, cfstore.keyspace.getName(), cfstore.name);
+                logger.trace("removing {} from list of files tracked for {}.{}", sstable.descriptor, cfstore.getKeyspaceName(), cfstore.name);
             try
             {
                 subtract += sstable.bytesOnDisk();

src/java/org/apache/cassandra/db/memtable/Flushing.java

@@ -41,7 +41,6 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.SSTableMultiWriter;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.metrics.TableMetrics;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.utils.FBUtilities;
@@ -170,7 +169,7 @@ public class Flushing
 
             if (logCompletion)
             {
-                long bytesFlushed = writer.getFilePointer();
+                long bytesFlushed = writer.getBytesWritten();
                 logger.info("Completed flushing {} ({}) for commitlog position {}",
                             writer.getFilename(),
                             FBUtilities.prettyPrintMemory(bytesFlushed),
@@ -201,16 +200,13 @@ public class Flushing
                                                        Descriptor descriptor,
                                                        long partitionCount)
     {
-        MetadataCollector sstableMetadataCollector = new MetadataCollector(flushSet.metadata().comparator)
-                                                     .commitLogIntervals(new IntervalSet<>(flushSet.commitLogLowerBound(),
-                                                                                           flushSet.commitLogUpperBound()));
-
         return cfs.createSSTableMultiWriter(descriptor,
                                             partitionCount,
                                             ActiveRepairService.UNREPAIRED_SSTABLE,
                                             ActiveRepairService.NO_PENDING_REPAIR,
                                             false,
-                                            sstableMetadataCollector,
+                                            new IntervalSet<>(flushSet.commitLogLowerBound(),
+                                                              flushSet.commitLogUpperBound()),
                                             new SerializationHeader(true,
                                                                     flushSet.metadata(),
                                                                     flushSet.columns(),

src/java/org/apache/cassandra/db/repair/CassandraTableRepairManager.java

@@ -79,7 +79,7 @@ public class CassandraTableRepairManager implements TableRepairManager
                         {
                             return sstable != null &&
                                    !sstable.metadata().isIndex() && // exclude SSTables from 2i
-                                   new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges);
+                                   new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()).intersects(ranges);
                         }
                     }, true, false); //ephemeral snapshot, if repair fails, it will be cleaned next startup
                 }

src/java/org/apache/cassandra/db/repair/CassandraValidationIterator.java

@@ -140,7 +140,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
         {
             for (SSTableReader sstable : sstableCandidates.sstables)
             {
-                if (new Bounds<>(sstable.first.getToken(), sstable.last.getToken()).intersects(ranges) && predicate.apply(sstable))
+                if (new Bounds<>(sstable.getFirst().getToken(), sstable.getLast().getToken()).intersects(ranges) && predicate.apply(sstable))
                 {
                     sstablesToValidate.add(sstable);
                 }
@@ -213,7 +213,7 @@ public class CassandraValidationIterator extends ValidationPartitionIterator
                     prs.previewKind.logPrefix(sessionID),
                     parentId,
                     sstables.size(),
-                    cfs.keyspace.getName(),
+                    cfs.getKeyspaceName(),
                     cfs.getTableName());
 
         controller = new ValidationCompactionController(cfs, getDefaultGcBefore(cfs, nowInSec));

src/java/org/apache/cassandra/db/streaming/CassandraCompressedStreamReader.java

@@ -67,7 +67,7 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader
         }
 
         logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', pendingRepair = '{}', table = '{}'.",
-                     session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(), pendingRepair,
+                     session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.getKeyspaceName(), pendingRepair,
                      cfs.getTableName());
 
         StreamDeserializer deserializer = null;
@@ -110,7 +110,7 @@ public class CassandraCompressedStreamReader extends CassandraStreamReader
         {
             Object partitionKey = deserializer != null ? deserializer.partitionKey() : "";
             logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.",
-                        session.planId(), partitionKey, cfs.keyspace.getName(), cfs.getTableName());
+                        session.planId(), partitionKey, cfs.getKeyspaceName(), cfs.getTableName());
             if (writer != null)
                 e = writer.abort(e);
             throw e;

src/java/org/apache/cassandra/db/streaming/CassandraOutgoingFile.java

@@ -89,7 +89,7 @@ public class CassandraOutgoingFile implements OutgoingStream
                                     .withSerializationHeader(sstable.header.toComponent())
                                     .isEntireSSTable(shouldStreamEntireSSTable)
                                     .withComponentManifest(manifest)
-                                    .withFirstKey(sstable.first)
+                                    .withFirstKey(sstable.getFirst())
                                     .withTableId(sstable.metadata().id)
                                     .build();
     }

src/java/org/apache/cassandra/db/streaming/CassandraStreamReader.java

@@ -114,7 +114,7 @@ public class CassandraStreamReader implements IStreamReader
             throw new IllegalStateException("Table " + tableId + " was dropped during streaming");
 
         logger.debug("[Stream #{}] Start receiving file #{} from {}, repairedAt = {}, size = {}, ks = '{}', table = '{}', pendingRepair = '{}'.",
-                     session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.keyspace.getName(),
+                     session.planId(), fileSeqNum, session.peer, repairedAt, totalSize, cfs.getKeyspaceName(),
                      cfs.getTableName(), pendingRepair);
 
         StreamDeserializer deserializer = null;
@@ -143,7 +143,7 @@ public class CassandraStreamReader implements IStreamReader
         {
             Object partitionKey = deserializer != null ? deserializer.partitionKey() : "";
             logger.warn("[Stream {}] Error while reading partition {} from stream on ks='{}' and table='{}'.",
-                        session.planId(), partitionKey, cfs.keyspace.getName(), cfs.getTableName(), e);
+                        session.planId(), partitionKey, cfs.getKeyspaceName(), cfs.getTableName(), e);
             if (writer != null)
                 e = writer.abort(e);
             throw e;

src/java/org/apache/cassandra/db/streaming/CassandraStreamReceiver.java

@@ -252,7 +252,7 @@ public class CassandraStreamReceiver implements StreamReceiver
                 if (cfs.isRowCacheEnabled() || cfs.metadata().isCounter())
                 {
                     List<Bounds<Token>> boundsToInvalidate = new ArrayList<>(readers.size());
-                    readers.forEach(sstable -> boundsToInvalidate.add(new Bounds<Token>(sstable.first.getToken(), sstable.last.getToken())));
+                    readers.forEach(sstable -> boundsToInvalidate.add(new Bounds<Token>(sstable.getFirst().getToken(), sstable.getLast().getToken())));
                     Set<Bounds<Token>> nonOverlappingBounds = Bounds.getNonOverlappingBounds(boundsToInvalidate);
 
                     if (cfs.isRowCacheEnabled())
@@ -261,7 +261,7 @@ public class CassandraStreamReceiver implements StreamReceiver
                         if (invalidatedKeys > 0)
                             logger.debug("[Stream #{}] Invalidated {} row cache entries on table {}.{} after stream " +
                                          "receive task completed.", session.planId(), invalidatedKeys,
-                                         cfs.keyspace.getName(), cfs.getTableName());
+                                         cfs.getKeyspaceName(), cfs.getTableName());
                     }
 
                     if (cfs.metadata().isCounter())
@@ -270,7 +270,7 @@ public class CassandraStreamReceiver implements StreamReceiver
                         if (invalidatedKeys > 0)
                             logger.debug("[Stream #{}] Invalidated {} counter cache entries on table {}.{} after stream " +
                                          "receive task completed.", session.planId(), invalidatedKeys,
-                                         cfs.keyspace.getName(), cfs.getTableName());
+                                         cfs.getKeyspaceName(), cfs.getTableName());
                     }
                 }
             }

src/java/org/apache/cassandra/db/view/View.java

@@ -169,7 +169,7 @@ public class View
                                                false);
 
             SelectStatement.RawStatement rawSelect =
-                new SelectStatement.RawStatement(new QualifiedName(baseCfs.keyspace.getName(), baseCfs.name),
+                new SelectStatement.RawStatement(new QualifiedName(baseCfs.getKeyspaceName(), baseCfs.name),
                                                  parameters,
                                                  selectClause(),
                                                  definition.whereClause,

src/java/org/apache/cassandra/db/view/ViewBuilderTask.java

@@ -135,7 +135,7 @@ public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<L
          */
         boolean schemaConverged = Gossiper.instance.waitForSchemaAgreement(10, TimeUnit.SECONDS, () -> this.isStopped);
         if (!schemaConverged)
-            logger.warn("Failed to get schema to converge before building view {}.{}", baseCfs.keyspace.getName(), view.name);
+            logger.warn("Failed to get schema to converge before building view {}.{}", baseCfs.getKeyspaceName(), view.name);
 
         Function<org.apache.cassandra.db.lifecycle.View, Iterable<SSTableReader>> function;
         function = org.apache.cassandra.db.lifecycle.View.select(SSTableSet.CANONICAL, s -> range.intersects(s.getBounds()));
@@ -175,7 +175,7 @@ public class ViewBuilderTask extends CompactionInfo.Holder implements Callable<L
 
     private void finish()
     {
-        String ksName = baseCfs.keyspace.getName();
+        String ksName = baseCfs.getKeyspaceName();
         if (!isStopped)
         {
             // Save the completed status using the end of the range as last token. This way it will be possible for

src/java/org/apache/cassandra/db/virtual/TableMetricTables.java

@@ -195,7 +195,7 @@ public class TableMetricTables
                 Metric metric = func.apply(cfs.metric);
 
                 // set new partition for this table
-                result.row(cfs.keyspace.getName(), cfs.name);
+                result.row(cfs.getKeyspaceName(), cfs.name);
 
                 // extract information by metric type and put it in row based on implementation of `add`
                 if (metric instanceof Counting)

src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java

@@ -136,7 +136,7 @@ public class ByteOrderedPartitioner implements IPartitioner
         }
 
         @Override
-        public Token increaseSlightly()
+        public Token nextValidToken()
         {
             throw new UnsupportedOperationException(String.format("Token type %s does not support token allocation.",
                                                                   getClass().getSimpleName()));

src/java/org/apache/cassandra/dht/ComparableObjectToken.java

@@ -75,7 +75,7 @@ abstract class ComparableObjectToken<C extends Comparable<C>> extends Token
     }
 
     @Override
-    public Token increaseSlightly()
+    public Token nextValidToken()
     {
         throw new UnsupportedOperationException(String.format("Token type %s does not support token allocation.",
                                                               getClass().getSimpleName()));

src/java/org/apache/cassandra/dht/Murmur3Partitioner.java

@@ -213,7 +213,7 @@ public class Murmur3Partitioner implements IPartitioner
         }
 
         @Override
-        public LongToken increaseSlightly()
+        public LongToken nextValidToken()
         {
             return new LongToken(token + 1);
         }

src/java/org/apache/cassandra/dht/RandomPartitioner.java

@@ -271,7 +271,7 @@ public class RandomPartitioner implements IPartitioner
             return HEAP_SIZE;
         }
 
-        public Token increaseSlightly()
+        public Token nextValidToken()
         {
             return new BigIntegerToken(token.add(BigInteger.ONE));
         }

src/java/org/apache/cassandra/dht/Range.java

@@ -221,6 +221,39 @@ public class Range<T extends RingPosition<T>> extends AbstractBounds<T> implemen
         return Collections.unmodifiableSet(intersection);
     }
 
+    /**
+     * Returns the intersection of this range with the provided one, assuming neither are wrapping.
+     *
+     * @param that the other range to return the intersection with. It must not be wrapping.
+     * @return the intersection of {@code this} and {@code that}, or {@code null} if both ranges don't intersect.
+     */
+    public Range<T> intersectionNonWrapping(Range<T> that)
+    {
+        assert !isTrulyWrapAround() : "wraparound " + this;
+        assert !that.isTrulyWrapAround() : "wraparound " + that;
+
+        if (left.compareTo(that.left) < 0)
+        {
+            if (right.isMinimum() || (!that.right.isMinimum() && right.compareTo(that.right) >= 0))
+                return that;  // this contains that.
+
+            if (right.compareTo(that.left) <= 0)
+                return null;  // this is fully before that.
+
+            return new Range<>(that.left, right);
+        }
+        else
+        {
+            if (that.right.isMinimum() || (!right.isMinimum() && that.right.compareTo(right) >= 0))
+                return this;  // that contains this.
+
+            if (that.right.compareTo(left) <= 0)
+                return null;  // that is fully before this.
+
+            return new Range<>(left, that.right);
+        }
+    }
+
     public Pair<AbstractBounds<T>, AbstractBounds<T>> split(T position)
     {
         assert contains(position) || left.equals(position);
@@ -262,6 +295,29 @@ public class Range<T extends RingPosition<T>> extends AbstractBounds<T> implemen
        return left.compareTo(right) >= 0;
     }
 
+    /**
+     * Checks if the range truly wraps around.
+     *
+     * This exists only because {@link #isWrapAround()} is a tad dumb and return true if right is the minimum token,
+     * no matter what left is, but for most intent and purposes, such range doesn't truly warp around (unwrap produces
+     * the identity in this case).
+     * <p>
+     * Also note that it could be that the remaining uses of {@link #isWrapAround()} could be replaced by this method,
+     * but that is to be checked carefully at some other time (Sylvain).
+     * <p>
+     * The one thing this method guarantees is that if it's true, then {@link #unwrap()} will return a list with
+     * exactly 2 ranges, never one.
+     */
+    public boolean isTrulyWrapAround()
+    {
+        return isTrulyWrapAround(left, right);
+    }
+
+    public static <T extends RingPosition<T>> boolean isTrulyWrapAround(T left, T right)
+    {
+        return isWrapAround(left, right) && !right.isMinimum();
+    }
+
     /**
      * Tells if the given range covers the entire ring
      */

src/java/org/apache/cassandra/dht/Splitter.java

@@ -144,10 +144,11 @@ public abstract class Splitter
         {
             BigInteger currentRangeWidth = weightedRange.totalTokens(this);
             BigInteger left = valueForToken(weightedRange.left());
+            BigInteger currentRangeFactor = BigInteger.valueOf(Math.max(1, (long) (1 / weightedRange.weight)));
             while (sum.add(currentRangeWidth).compareTo(perPart) >= 0)
             {
                 BigInteger withinRangeBoundary = perPart.subtract(sum);
-                left = left.add(withinRangeBoundary);
+                left = left.add(withinRangeBoundary.multiply(currentRangeFactor));
                 boundaries.add(tokenForValue(left));
                 tokensLeft = tokensLeft.subtract(perPart);
                 currentRangeWidth = currentRangeWidth.subtract(withinRangeBoundary);
@@ -282,6 +283,14 @@ public abstract class Splitter
             return size.abs().divide(factor);
         }
 
+        /**
+         * A less precise version of the above, returning the size of the span as a double approximation.
+         */
+        public double size()
+        {
+            return left().size(right()) * weight;
+        }
+
         public Token left()
         {
             return range.left;
@@ -297,6 +306,11 @@ public abstract class Splitter
             return range;
         }
 
+        public double weight()
+        {
+            return weight;
+        }
+
         public String toString()
         {
             return "WeightedRange{" +

src/java/org/apache/cassandra/dht/Token.java

@@ -144,11 +144,18 @@ public abstract class Token implements RingPosition<Token>, Serializable
      */
     abstract public double size(Token next);
     /**
-     * Returns a token that is slightly greater than this. Used to avoid clashes
-     * between nodes in separate datacentres trying to use the same token via
-     * the token allocation algorithm.
+     * Returns the next possible token in the token space, one that compares
+     * greater than this and such that there is no other token that sits
+     * between this token and it in the token order.
+     *
+     * This is not possible for all token types, esp. for comparison-based
+     * tokens such as the LocalPartioner used for classic secondary indexes.
+     *
+     * Used to avoid clashes between nodes in separate datacentres trying to
+     * use the same token via the token allocation algorithm, as well as in
+     * constructing token ranges for sstables.
      */
-    abstract public Token increaseSlightly();
+    abstract public Token nextValidToken();
 
     public Token getToken()
     {

src/java/org/apache/cassandra/dht/tokenallocator/TokenAllocation.java

@@ -161,7 +161,7 @@ public class TokenAllocation
                     InetAddressAndPort other = tokenMetadata.getEndpoint(t);
                     if (inAllocationRing(other))
                         throw new ConfigurationException(String.format("Allocated token %s already assigned to node %s. Is another node also allocating tokens?", t, other));
-                    t = t.increaseSlightly();
+                    t = t.nextValidToken();
                 }
                 filtered.add(t);
             }

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -622,7 +622,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      */
     private synchronized void markIndexesBuilding(Set<Index> indexes, boolean isFullRebuild, boolean isNewCF)
     {
-        String keyspaceName = baseCfs.keyspace.getName();
+        String keyspaceName = baseCfs.getKeyspaceName();
 
         // First step is to validate against concurrent rebuilds; it would be more optimized to do everything on a single
         // step, but we're not really expecting a very high number of indexes, and this isn't on any hot path, so
@@ -677,7 +677,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             {
                 inProgressBuilds.remove(indexName);
                 if (!needsFullRebuild.contains(indexName) && DatabaseDescriptor.isDaemonInitialized() && Keyspace.isInitialized())
-                    SystemKeyspace.setIndexBuilt(baseCfs.keyspace.getName(), indexName);
+                    SystemKeyspace.setIndexBuilt(baseCfs.getKeyspaceName(), indexName);
             }
         }
     }
@@ -701,7 +701,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
             counter.decrementAndGet();
 
             if (DatabaseDescriptor.isDaemonInitialized())
-                SystemKeyspace.setIndexRemoved(baseCfs.keyspace.getName(), indexName);
+                SystemKeyspace.setIndexRemoved(baseCfs.getKeyspaceName(), indexName);
 
             needsFullRebuild.add(indexName);
 
@@ -730,7 +730,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
      */
     private synchronized void markIndexRemoved(String indexName)
     {
-        SystemKeyspace.setIndexRemoved(baseCfs.keyspace.getName(), indexName);
+        SystemKeyspace.setIndexRemoved(baseCfs.getKeyspaceName(), indexName);
         queryableIndexes.remove(indexName);
         writableIndexes.remove(indexName);
         needsFullRebuild.remove(indexName);
@@ -863,7 +863,7 @@ public class SecondaryIndexManager implements IndexRegistry, INotificationConsum
         indexes.values().stream()
                .map(i -> i.getIndexMetadata().name)
                .forEach(allIndexNames::add);
-        return SystemKeyspace.getBuiltIndexes(baseCfs.keyspace.getName(), allIndexNames);
+        return SystemKeyspace.getBuiltIndexes(baseCfs.getKeyspaceName(), allIndexNames);
     }
 
     /**

src/java/org/apache/cassandra/index/internal/CassandraIndex.java

@@ -667,7 +667,7 @@ public abstract class CassandraIndex implements Index
 
     private boolean isBuilt()
     {
-        return SystemKeyspace.isIndexBuilt(baseCfs.keyspace.getName(), metadata.name);
+        return SystemKeyspace.isIndexBuilt(baseCfs.getKeyspaceName(), metadata.name);
     }
 
     private boolean isPrimaryKeyIndex()

src/java/org/apache/cassandra/index/sasi/plan/QueryController.java

@@ -254,7 +254,7 @@ public class QueryController
     {
         return Sets.filter(indexes, index -> {
             SSTableReader sstable = index.getSSTable();
-            return range.startKey().compareTo(sstable.last) <= 0 && (range.stopKey().isMinimum() || sstable.first.compareTo(range.stopKey()) <= 0);
+            return range.startKey().compareTo(sstable.getLast()) <= 0 && (range.stopKey().isMinimum() || sstable.getFirst().compareTo(range.stopKey()) <= 0);
         });
     }
 }

src/java/org/apache/cassandra/io/sstable/AbstractSSTableSimpleWriter.java

@@ -72,7 +72,7 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
         SerializationHeader header = new SerializationHeader(true, metadata.get(), columns, EncodingStats.NO_STATS);
 
         if (makeRangeAware)
-            return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, 0, header);
+            return SSTableTxnWriter.createRangeAware(metadata, 0, ActiveRepairService.UNREPAIRED_SSTABLE, ActiveRepairService.NO_PENDING_REPAIR, false, format, header);
 
         return SSTableTxnWriter.create(metadata,
                                        createDescriptor(directory, metadata.keyspace, metadata.name, format),
@@ -80,7 +80,6 @@ abstract class AbstractSSTableSimpleWriter implements Closeable
                                        ActiveRepairService.UNREPAIRED_SSTABLE,
                                        ActiveRepairService.NO_PENDING_REPAIR,
                                        false,
-                                       0,
                                        header,
                                        Collections.emptySet(),
                                        owner);

src/java/org/apache/cassandra/io/sstable/RangeAwareSSTableWriter.java

@@ -75,7 +75,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
                 throw new IOException(String.format("Insufficient disk space to store %s",
                                                     FBUtilities.prettyPrintMemory(totalSize)));
             Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(localDir), format);
-            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
+            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, lifecycleNewTracker);
         }
     }
 
@@ -97,30 +97,14 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
                 finishedWriters.add(currentWriter);
 
             Descriptor desc = cfs.newSSTableDescriptor(cfs.getDirectories().getLocationForDisk(directories.get(currentIndex)), format);
-            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, sstableLevel, header, lifecycleNewTracker);
+            currentWriter = cfs.createSSTableMultiWriter(desc, estimatedKeys, repairedAt, pendingRepair, isTransient, null, sstableLevel, header, lifecycleNewTracker);
         }
     }
 
-    public boolean append(UnfilteredRowIterator partition)
+    public void append(UnfilteredRowIterator partition)
     {
         maybeSwitchWriter(partition.partitionKey());
-        return currentWriter.append(partition);
-    }
-
-    @Override
-    public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
-    {
-        if (currentWriter != null)
-            finishedWriters.add(currentWriter);
-        currentWriter = null;
-        for (SSTableMultiWriter writer : finishedWriters)
-        {
-            if (writer.getFilePointer() > 0)
-                finishedReaders.addAll(writer.finish(repairedAt, maxDataAge, openResult));
-            else
-                SSTableMultiWriter.abortOrDie(writer);
-        }
-        return finishedReaders;
+        currentWriter.append(partition);
     }
 
     @Override
@@ -131,7 +115,7 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
         currentWriter = null;
         for (SSTableMultiWriter writer : finishedWriters)
         {
-            if (writer.getFilePointer() > 0)
+            if (writer.getBytesWritten() > 0)
                 finishedReaders.addAll(writer.finish(openResult));
             else
                 SSTableMultiWriter.abortOrDie(writer);
@@ -155,13 +139,19 @@ public class RangeAwareSSTableWriter implements SSTableMultiWriter
 
     public String getFilename()
     {
-        return String.join("/", cfs.keyspace.getName(), cfs.getTableName());
+        return String.join("/", cfs.getKeyspaceName(), cfs.getTableName());
     }
 
     @Override
-    public long getFilePointer()
+    public long getBytesWritten()
     {
-       return currentWriter != null ? currentWriter.getFilePointer() : 0L;
+       return currentWriter != null ? currentWriter.getBytesWritten() : 0L;
+    }
+
+    @Override
+    public long getOnDiskBytesWritten()
+    {
+        return currentWriter != null ? currentWriter.getOnDiskBytesWritten() : 0L;
     }
 
     @Override

src/java/org/apache/cassandra/io/sstable/SSTableMultiWriter.java

@@ -34,16 +34,16 @@ public interface SSTableMultiWriter extends Transactional
      * @param partition the partition to append
      * @return true if the partition was written, false otherwise
      */
-    boolean append(UnfilteredRowIterator partition);
+    void append(UnfilteredRowIterator partition);
 
-    Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult);
     Collection<SSTableReader> finish(boolean openResult);
     Collection<SSTableReader> finished();
 
     SSTableMultiWriter setOpenResult(boolean openResult);
 
     String getFilename();
-    long getFilePointer();
+    long getBytesWritten();
+    long getOnDiskBytesWritten();
     TableId getTableId();
 
     static void abortOrDie(SSTableMultiWriter writer)

src/java/org/apache/cassandra/io/sstable/SSTableRewriter.java

@@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Consumer;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -59,6 +60,7 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme
     private final List<SSTableReader> preparedForCommit = new ArrayList<>();
 
     private long currentlyOpenedEarlyAt; // the position (in MiB) in the target file we last (re)opened at
+    private long bytesWritten; // the bytes written by previous writers, or zero if the current writer is the first writer
 
     private final List<SSTableWriter> writers = new ArrayList<>();
     private final boolean keepOriginals; // true if we do not want to obsolete the originals
@@ -112,6 +114,19 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme
         return writer;
     }
 
+    public long bytesWritten()
+    {
+        return bytesWritten + (writer == null ? 0 : writer.getFilePointer());
+    }
+
+    public void forEachWriter(Consumer<SSTableWriter> op)
+    {
+        for (SSTableWriter writer : writers)
+            op.accept(writer);
+        if (writer != null)
+            op.accept(writer);
+    }
+
     public AbstractRowIndexEntry append(UnfilteredRowIterator partition)
     {
         // we do this before appending to ensure we can resetAndTruncate() safely if appending fails
@@ -152,7 +167,7 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme
                 writer.openEarly(reader -> {
                     transaction.update(reader, false);
                     currentlyOpenedEarlyAt = writer.getFilePointer();
-                    moveStarts(reader.last);
+                    moveStarts(reader.getLast());
                     transaction.checkpoint();
                 });
             }
@@ -202,10 +217,10 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme
             final SSTableReader latest = transaction.current(sstable);
 
             // skip any sstables that we know to already be shadowed
-            if (latest.first.compareTo(lowerbound) > 0)
+            if (latest.getFirst().compareTo(lowerbound) > 0)
                 continue;
 
-            if (lowerbound.compareTo(latest.last) >= 0)
+            if (lowerbound.compareTo(latest.getLast()) >= 0)
             {
                 if (!transaction.isObsolete(latest))
                     transaction.obsolete(latest);
@@ -255,11 +270,12 @@ public class SSTableRewriter extends Transactional.AbstractTransactional impleme
             writer.setMaxDataAge(maxAge);
             SSTableReader reader = writer.openFinalEarly();
             transaction.update(reader, false);
-            moveStarts(reader.last);
+            moveStarts(reader.getLast());
             transaction.checkpoint();
         }
 
         currentlyOpenedEarlyAt = 0;
+        bytesWritten += writer.getFilePointer();
         writer = newWriter;
     }
 

src/java/org/apache/cassandra/io/sstable/SSTableTxnWriter.java

@@ -30,7 +30,6 @@ import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.index.Index;
 import org.apache.cassandra.io.sstable.format.SSTableFormat;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
 import org.apache.cassandra.schema.TableMetadataRef;
 import org.apache.cassandra.utils.TimeUUID;
 import org.apache.cassandra.utils.concurrent.Transactional;
@@ -51,9 +50,9 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
         this.writer = writer;
     }
 
-    public boolean append(UnfilteredRowIterator iterator)
+    public void append(UnfilteredRowIterator iterator)
     {
-        return writer.append(iterator);
+        writer.append(iterator);
     }
 
     public String getFilename()
@@ -63,7 +62,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
 
     public long getFilePointer()
     {
-        return writer.getFilePointer();
+        return writer.getBytesWritten();
     }
 
     protected Throwable doCommit(Throwable accumulate)
@@ -98,10 +97,10 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
     }
 
     @SuppressWarnings("resource") // log and writer closed during doPostCleanup
-    public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, int sstableLevel, SerializationHeader header)
+    public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor descriptor, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header)
     {
         LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
-        SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, sstableLevel, header, txn);
+        SSTableMultiWriter writer = cfs.createSSTableMultiWriter(descriptor, keyCount, repairedAt, pendingRepair, isTransient, header, txn);
         return new SSTableTxnWriter(txn, writer);
     }
 
@@ -113,7 +112,6 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
                                                     TimeUUID pendingRepair,
                                                     boolean isTransient,
                                                     SSTableFormat<?, ?> type,
-                                                    int sstableLevel,
                                                     SerializationHeader header)
     {
 
@@ -122,7 +120,7 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
         SSTableMultiWriter writer;
         try
         {
-            writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, sstableLevel, 0, txn, header);
+            writer = new RangeAwareSSTableWriter(cfs, keyCount, repairedAt, pendingRepair, isTransient, type, 0, 0, txn, header);
         }
         catch (IOException e)
         {
@@ -141,20 +139,13 @@ public class SSTableTxnWriter extends Transactional.AbstractTransactional implem
                                           long repairedAt,
                                           TimeUUID pendingRepair,
                                           boolean isTransient,
-                                          int sstableLevel,
                                           SerializationHeader header,
                                           Collection<Index> indexes,
                                           SSTable.Owner owner)
     {
         // if the column family store does not exist, we create a new default SSTableMultiWriter to use:
         LifecycleTransaction txn = LifecycleTransaction.offline(OperationType.WRITE);
-        MetadataCollector collector = new MetadataCollector(metadata.get().comparator).sstableLevel(sstableLevel);
-        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, collector, header, indexes, txn, owner);
+        SSTableMultiWriter writer = SimpleSSTableMultiWriter.create(descriptor, keyCount, repairedAt, pendingRepair, isTransient, metadata, null, 0, header, indexes, txn, owner);
         return new SSTableTxnWriter(txn, writer);
     }
-
-    public static SSTableTxnWriter create(ColumnFamilyStore cfs, Descriptor desc, long keyCount, long repairedAt, TimeUUID pendingRepair, boolean isTransient, SerializationHeader header)
-    {
-        return create(cfs, desc, keyCount, repairedAt, pendingRepair, isTransient, 0, header);
-    }
 }

src/java/org/apache/cassandra/io/sstable/SSTableZeroCopyWriter.java

@@ -116,17 +116,11 @@ public class SSTableZeroCopyWriter extends SSTable implements SSTableMultiWriter
     }
 
     @Override
-    public boolean append(UnfilteredRowIterator partition)
+    public void append(UnfilteredRowIterator partition)
     {
         throw new UnsupportedOperationException();
     }
 
-    @Override
-    public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
-    {
-        return finish(openResult);
-    }
-
     @Override
     public Collection<SSTableReader> finish(boolean openResult)
     {
@@ -154,7 +148,13 @@ public class SSTableZeroCopyWriter extends SSTable implements SSTableMultiWriter
     }
 
     @Override
-    public long getFilePointer()
+    public long getBytesWritten()
+    {
+        return 0;
+    }
+
+    @Override
+    public long getOnDiskBytesWritten()
     {
         return 0;
     }

src/java/org/apache/cassandra/io/sstable/SimpleSSTableMultiWriter.java

@@ -21,6 +21,8 @@ import java.util.Collection;
 import java.util.Collections;
 
 import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.commitlog.CommitLogPosition;
+import org.apache.cassandra.db.commitlog.IntervalSet;
 import org.apache.cassandra.db.lifecycle.LifecycleNewTracker;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.index.Index;
@@ -42,17 +44,9 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
         this.writer = writer;
     }
 
-    public boolean append(UnfilteredRowIterator partition)
+    public void append(UnfilteredRowIterator partition)
     {
-        AbstractRowIndexEntry indexEntry = writer.append(partition);
-        return indexEntry != null;
-    }
-
-    public Collection<SSTableReader> finish(long repairedAt, long maxDataAge, boolean openResult)
-    {
-        writer.setRepairedAt(repairedAt);
-        writer.setMaxDataAge(maxDataAge);
-        return Collections.singleton(writer.finish(openResult));
+        writer.append(partition);
     }
 
     public Collection<SSTableReader> finish(boolean openResult)
@@ -76,11 +70,16 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
         return writer.getFilename();
     }
 
-    public long getFilePointer()
+    public long getBytesWritten()
     {
         return writer.getFilePointer();
     }
 
+    public long getOnDiskBytesWritten()
+    {
+        return writer.getEstimatedOnDiskBytesWritten();
+    }
+
     public TableId getTableId()
     {
         return writer.metadata().id;
@@ -114,12 +113,16 @@ public class SimpleSSTableMultiWriter implements SSTableMultiWriter
                                             TimeUUID pendingRepair,
                                             boolean isTransient,
                                             TableMetadataRef metadata,
-                                            MetadataCollector metadataCollector,
+                                            IntervalSet<CommitLogPosition> commitLogPositions,
+                                            int sstableLevel,
                                             SerializationHeader header,
                                             Collection<Index> indexes,
                                             LifecycleNewTracker lifecycleNewTracker,
                                             SSTable.Owner owner)
     {
+        MetadataCollector metadataCollector = new MetadataCollector(metadata.get().comparator)
+                                              .commitLogIntervals(commitLogPositions != null ? commitLogPositions : IntervalSet.empty())
+                                              .sstableLevel(sstableLevel);
         SSTableWriter writer = descriptor.getFormat().getWriterFactory().builder(descriptor)
                                             .setKeyCount(keyCount)
                                             .setRepairedAt(repairedAt)

src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java

@@ -183,8 +183,9 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
     }
     public final UniqueIdentifier instanceId = new UniqueIdentifier();
 
-    public static final Comparator<SSTableReader> sstableComparator = Comparator.comparing(o -> o.first);
-    public static final Ordering<SSTableReader> sstableOrdering = Ordering.from(sstableComparator);
+    public static final Comparator<SSTableReader> firstKeyComparator = (o1, o2) -> o1.getFirst().compareTo(o2.getFirst());
+    public static final Ordering<SSTableReader> firstKeyOrdering = Ordering.from(firstKeyComparator);
+    public static final Comparator<SSTableReader> lastKeyComparator = (o1, o2) -> o1.getLast().compareTo(o2.getLast());
 
     public static final Comparator<SSTableReader> idComparator = Comparator.comparing(t -> t.descriptor.id, SSTableIdFactory.COMPARATOR);
     public static final Comparator<SSTableReader> idReverseComparator = idComparator.reversed();
@@ -267,8 +268,8 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
 
     private volatile double crcCheckChance;
 
-    public final DecoratedKey first;
-    public final DecoratedKey last;
+    protected final DecoratedKey first;
+    protected final DecoratedKey last;
     public final AbstractBounds<Token> bounds;
 
     /**
@@ -842,6 +843,19 @@ public abstract class SSTableReader extends SSTable implements UnfilteredSource,
         return dfile.dataLength();
     }
 
+    /**
+     * @return the fraction of the token space for which this sstable has content. In the simplest case this is just the
+     * size of the interval returned by {@link #getBounds()}, but the sstable may contain "holes" when the locally-owned
+     * range is not contiguous (e.g. with vnodes).
+     * As this is affected by the local ranges which can change, the token space fraction is calculated at the time of
+     * writing the sstable and stored with its metadata.
+     * For older sstables that do not contain this metadata field, this method returns NaN.
+     */
+    public double tokenSpaceCoverage()
+    {
+        return sstableMetadata.tokenSpaceCoverage;
+    }
+
     /**
      * The length in bytes of the on disk size for this SSTable. For compressed files, this is not the same thing
      * as the data length (see {@link #uncompressedLength()}).

src/java/org/apache/cassandra/io/sstable/format/SSTableScanner.java

@@ -103,30 +103,30 @@ implements ISSTableScanner
 
     protected static AbstractBounds<PartitionPosition> fullRange(SSTableReader sstable)
     {
-        return new Bounds<>(sstable.first, sstable.last);
+        return new Bounds<>(sstable.getFirst(), sstable.getLast());
     }
 
     private static void addRange(SSTableReader sstable, AbstractBounds<PartitionPosition> requested, List<AbstractBounds<PartitionPosition>> boundsList)
     {
         if (requested instanceof Range && ((Range<?>) requested).isWrapAround())
         {
-            if (requested.right.compareTo(sstable.first) >= 0)
+            if (requested.right.compareTo(sstable.getFirst()) >= 0)
             {
                 // since we wrap, we must contain the whole sstable prior to stopKey()
-                Boundary<PartitionPosition> left = new Boundary<>(sstable.first, true);
+                Boundary<PartitionPosition> left = new Boundary<>(sstable.getFirst(), true);
                 Boundary<PartitionPosition> right;
                 right = requested.rightBoundary();
-                right = minRight(right, sstable.last, true);
+                right = minRight(right, sstable.getLast(), true);
                 if (!isEmpty(left, right))
                     boundsList.add(AbstractBounds.bounds(left, right));
             }
-            if (requested.left.compareTo(sstable.last) <= 0)
+            if (requested.left.compareTo(sstable.getLast()) <= 0)
             {
                 // since we wrap, we must contain the whole sstable after dataRange.startKey()
-                Boundary<PartitionPosition> right = new Boundary<>(sstable.last, true);
+                Boundary<PartitionPosition> right = new Boundary<>(sstable.getLast(), true);
                 Boundary<PartitionPosition> left;
                 left = requested.leftBoundary();
-                left = maxLeft(left, sstable.first, true);
+                left = maxLeft(left, sstable.getFirst(), true);
                 if (!isEmpty(left, right))
                     boundsList.add(AbstractBounds.bounds(left, right));
             }
@@ -137,10 +137,10 @@ implements ISSTableScanner
             Boundary<PartitionPosition> left, right;
             left = requested.leftBoundary();
             right = requested.rightBoundary();
-            left = maxLeft(left, sstable.first, true);
+            left = maxLeft(left, sstable.getFirst(), true);
             // apparently isWrapAround() doesn't count Bounds that extend to the limit (min) as wrapping
-            right = requested.right.isMinimum() ? new Boundary<>(sstable.last, true)
-                                                : minRight(right, sstable.last, true);
+            right = requested.right.isMinimum() ? new Boundary<>(sstable.getLast(), true)
+                                                : minRight(right, sstable.getLast(), true);
             if (!isEmpty(left, right))
                 boundsList.add(AbstractBounds.bounds(left, right));
         }

src/java/org/apache/cassandra/io/sstable/format/SSTableWriter.java

@@ -170,6 +170,12 @@ public abstract class SSTableWriter extends SSTable implements Transactional
         this.maxDataAge = maxDataAge;
     }
 
+    public SSTableWriter setTokenSpaceCoverage(double rangeSpanned)
+    {
+        metadataCollector.tokenSpaceCoverage(rangeSpanned);
+        return this;
+    }
+
     public void setOpenResult(boolean openResult)
     {
         txnProxy.openResult = openResult;

src/java/org/apache/cassandra/io/sstable/format/SortedTableScrubber.java

@@ -114,7 +114,7 @@ public abstract class SortedTableScrubber<R extends SSTableReaderWithFilter> imp
     {
         this.sstable = (R) transaction.onlyOne();
         Preconditions.checkNotNull(sstable.metadata());
-        assert sstable.metadata().keyspace.equals(cfs.keyspace.getName());
+        assert sstable.metadata().keyspace.equals(cfs.getKeyspaceName());
         if (!sstable.descriptor.cfname.equals(cfs.metadata().name))
         {
             logger.warn("Descriptor points to a different table {} than metadata {}", sstable.descriptor.cfname, cfs.metadata().name);

src/java/org/apache/cassandra/io/sstable/format/SortedTableVerifier.java

@@ -388,7 +388,7 @@ public abstract class SortedTableVerifier<R extends SSTableReaderWithFilter> imp
         {
             ByteBuffer last = it.key();
             while (it.advance()) last = it.key(); // no-op, just check if index is readable
-            if (!Objects.equals(last, sstable.last.getKey()))
+            if (!Objects.equals(last, sstable.getLast().getKey()))
                 throw new CorruptSSTableException(new IOException("Failed to read partition index"), it.toString());
         }
     }

src/java/org/apache/cassandra/io/sstable/format/Version.java

@@ -89,6 +89,14 @@ public abstract class Version
 
     public abstract boolean hasImprovedMinMax();
 
+    /**
+     * If the sstable has token space coverage data.
+     */
+    public abstract boolean hasTokenSpaceCoverage();
+
+    /**
+     * Records in th stats if the sstable has any partition deletions.
+     */
     public abstract boolean hasPartitionLevelDeletionsPresenceMarker();
 
     public abstract boolean hasKeyRange();

src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java

@@ -367,6 +367,7 @@ public class BigFormat extends AbstractSSTableFormat<BigTableReader, BigTableWri
         // nb (4.0.0): originating host id
         // nc (4.1): improved min/max, partition level deletion presence marker, key range (CASSANDRA-18134)
         // oa (5.0): Long deletionTime to prevent TTL overflow
+        //           token space coverage
         //
         // NOTE: When adding a new version:
         //  - Please add it to LegacySSTableTest
@@ -387,6 +388,7 @@ public class BigFormat extends AbstractSSTableFormat<BigTableReader, BigTableWri
         private final boolean hasPartitionLevelDeletionPresenceMarker;
         private final boolean hasKeyRange;
         private final boolean hasUintDeletionTime;
+        private final boolean hasTokenSpaceCoverage;
 
         /**
          * CASSANDRA-9067: 4.0 bloom filter representation changed (two longs just swapped)
@@ -416,6 +418,7 @@ public class BigFormat extends AbstractSSTableFormat<BigTableReader, BigTableWri
             hasPartitionLevelDeletionPresenceMarker = version.compareTo("nc") >= 0;
             hasKeyRange = version.compareTo("nc") >= 0;
             hasUintDeletionTime = version.compareTo("oa") >= 0;
+            hasTokenSpaceCoverage = version.compareTo("oa") >= 0;
         }
 
         @Override
@@ -496,6 +499,12 @@ public class BigFormat extends AbstractSSTableFormat<BigTableReader, BigTableWri
             return hasImprovedMinMax;
         }
 
+        @Override
+        public boolean hasTokenSpaceCoverage()
+        {
+            return hasTokenSpaceCoverage;
+        }
+
         @Override
         public boolean hasPartitionLevelDeletionsPresenceMarker()
         {

src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java

@@ -198,8 +198,8 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
     @Override
     public DecoratedKey firstKeyBeyond(PartitionPosition token)
     {
-        if (token.compareTo(first) < 0)
-            return first;
+        if (token.compareTo(getFirst()) < 0)
+            return getFirst();
 
         long sampledPosition = getIndexScanPosition(token);
 
@@ -263,7 +263,7 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
 
         // check the smallest and greatest keys in the sstable to see if it can't be present
         boolean skip = false;
-        if (key.compareTo(first) < 0)
+        if (key.compareTo(getFirst()) < 0)
         {
             if (searchOp == Operator.EQ)
             {
@@ -271,13 +271,13 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
             }
             else
             {
-                key = first;
+                key = getFirst();
                 searchOp = Operator.GE; // since op != EQ, bloom filter will be skipped; first key is included so no reason to check bloom filter
             }
         }
         else
         {
-            int l = last.compareTo(key);
+            int l = getLast().compareTo(key);
             skip = l < 0 // out of range, skip
                    || l == 0 && searchOp == Operator.GT; // search entry > key, but key is the last in range, so skip
             if (l == 0)
@@ -529,8 +529,8 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
      */
     long getIndexScanPosition(PartitionPosition key)
     {
-        if (openReason == OpenReason.MOVED_START && key.compareTo(first) < 0)
-            key = first;
+        if (openReason == OpenReason.MOVED_START && key.compareTo(getFirst()) < 0)
+            key = getFirst();
 
         return indexSummary.getScanPosition(key);
     }
@@ -599,7 +599,7 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
         return runWithLock(ignored -> {
             assert openReason != OpenReason.EARLY;
             // TODO: merge with caller's firstKeyBeyond() work,to save time
-            if (newStart.compareTo(first) > 0)
+            if (newStart.compareTo(getFirst()) > 0)
             {
                 Map<FileHandle, Long> handleAndPositions = new LinkedHashMap<>(2);
                 if (dfile != null)
@@ -654,8 +654,8 @@ public class BigTableReader extends SSTableReaderWithFilter implements IndexSumm
 
         // Always save the resampled index with lock to avoid racing with entire-sstable streaming
         return runWithLock(ignored -> {
-            new IndexSummaryComponent(newSummary, first, last).save(descriptor.fileFor(Components.SUMMARY), true);
-            return cloneAndReplace(first, OpenReason.METADATA_CHANGE, newSummary);
+            new IndexSummaryComponent(newSummary, getFirst(), getLast()).save(descriptor.fileFor(Components.SUMMARY), true);
+            return cloneAndReplace(getFirst(), OpenReason.METADATA_CHANGE, newSummary);
         });
     }
 

src/java/org/apache/cassandra/io/sstable/format/bti/BtiFormat.java

@@ -396,6 +396,12 @@ public class BtiFormat extends AbstractSSTableFormat<BtiTableReader, BtiTableWri
             return true;
         }
 
+        @Override
+        public boolean hasTokenSpaceCoverage()
+        {
+            return true;
+        }
+
         @Override
         public boolean hasPartitionLevelDeletionsPresenceMarker()
         {

src/java/org/apache/cassandra/io/sstable/format/bti/BtiTableReader.java

@@ -140,13 +140,13 @@ public class BtiTableReader extends SSTableReaderWithFilter
 
         if (operator == GT || operator == GE)
         {
-            if (filterLast() && last.compareTo(key) < 0)
+            if (filterLast() && getLast().compareTo(key) < 0)
             {
                 notifySkipped(SkippingReason.MIN_MAX_KEYS, listener, operator, updateStats);
                 return null;
             }
-            boolean filteredLeft = (filterFirst() && first.compareTo(key) > 0);
-            searchKey = filteredLeft ? first : key;
+            boolean filteredLeft = (filterFirst() && getFirst().compareTo(key) > 0);
+            searchKey = filteredLeft ? getFirst() : key;
             searchOp = filteredLeft ? GE : operator;
 
             try (PartitionIndex.Reader reader = partitionIndex.openReader())
@@ -226,7 +226,7 @@ public class BtiTableReader extends SSTableReaderWithFilter
                                     SSTableReadsListener listener,
                                     boolean updateStats)
     {
-        if ((filterFirst() && first.compareTo(dk) > 0) || (filterLast() && last.compareTo(dk) < 0))
+        if ((filterFirst() && getFirst().compareTo(dk) > 0) || (filterLast() && getLast().compareTo(dk) < 0))
         {
             notifySkipped(SkippingReason.MIN_MAX_KEYS, listener, EQ, updateStats);
             return null;
@@ -329,24 +329,24 @@ public class BtiTableReader extends SSTableReaderWithFilter
         for (Range<Token> range : Range.normalize(ranges))
         {
             PartitionPosition left = range.left.minKeyBound();
-            if (left.compareTo(first) <= 0)
+            if (left.compareTo(getFirst()) <= 0)
                 left = null;
-            else if (left.compareTo(last) > 0)
+            else if (left.compareTo(getLast()) > 0)
                 continue;   // no intersection
 
             PartitionPosition right = range.right.minKeyBound();
-            if (range.right.isMinimum() || right.compareTo(last) >= 0)
+            if (range.right.isMinimum() || right.compareTo(getLast()) >= 0)
                 right = null;
-            else if (right.compareTo(first) < 0)
+            else if (right.compareTo(getFirst()) < 0)
                 continue;   // no intersection
 
             if (left == null && right == null)
                 return partitionIndex.size();   // sstable is fully covered, return full partition count to avoid rounding errors
 
             if (left == null && filterFirst())
-                left = first;
+                left = getFirst();
             if (right == null && filterLast())
-                right = last;
+                right = getLast();
 
             long startPos = left != null ? getPosition(left, GE) : 0;
             long endPos = right != null ? getPosition(right, GE) : uncompressedLength();
@@ -421,7 +421,7 @@ public class BtiTableReader extends SSTableReaderWithFilter
     {
         return runWithLock(d -> {
             assert openReason != OpenReason.EARLY : "Cannot open early an early-open SSTable";
-            if (newStart.compareTo(first) > 0)
+            if (newStart.compareTo(getFirst()) > 0)
             {
                 final long dataStart = getPosition(newStart, Operator.EQ);
                 runOnClose(() -> dfile.dropPageCache(dataStart));

src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java

@@ -95,6 +95,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
                                  ActiveRepairService.UNREPAIRED_SSTABLE,
                                  -1,
                                  -1,
+                                 Double.NaN,
                                  null,
                                  null,
                                  false,
@@ -128,7 +129,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
      * be a corresponding end bound that is bigger).
      */
     private ClusteringPrefix<?> maxClustering = ClusteringBound.MIN_END;
-    private boolean clusteringInitialized = false;
 
     protected boolean hasLegacyCounterShards = false;
     private boolean hasPartitionLevelDeletions = false;
@@ -136,6 +136,8 @@ public class MetadataCollector implements PartitionStatisticsCollector
     protected long totalRows;
     public int totalTombstones;
 
+    protected double tokenSpaceCoverage = Double.NaN;
+
     /**
      * Default cardinality estimation method is to use HyperLogLog++.
      * Parameter here(p=13, sp=25) should give reasonable estimation
@@ -159,7 +161,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
         this.originatingHostId = originatingHostId;
     }
 
-    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
+    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator)
     {
         this(comparator);
 
@@ -173,7 +175,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
             }
         }
         commitLogIntervals(intervals.build());
-        sstableLevel(level);
     }
 
     public MetadataCollector addKey(ByteBuffer key)
@@ -292,6 +293,12 @@ public class MetadataCollector implements PartitionStatisticsCollector
         return this;
     }
 
+    public MetadataCollector tokenSpaceCoverage(double coverage)
+    {
+        tokenSpaceCoverage = coverage;
+        return this;
+    }
+
     public void updateClusteringValues(Clustering<?> clustering)
     {
         if (clustering == Clustering.STATIC_CLUSTERING)
@@ -372,6 +379,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
                                                              repairedAt,
                                                              totalColumnsSet,
                                                              totalRows,
+                                                             tokenSpaceCoverage,
                                                              originatingHostId,
                                                              pendingRepair,
                                                              isTransient,

src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java

@@ -73,6 +73,7 @@ public class StatsMetadata extends MetadataComponent
     public final int sstableLevel;
     public final Slice coveredClustering;
     public final boolean hasLegacyCounterShards;
+    public final double tokenSpaceCoverage;
     public final long repairedAt;
     public final long totalColumnsSet;
     public final long totalRows;
@@ -117,6 +118,7 @@ public class StatsMetadata extends MetadataComponent
                          long repairedAt,
                          long totalColumnsSet,
                          long totalRows,
+                         double tokenSpaceCoverage,
                          UUID originatingHostId,
                          TimeUUID pendingRepair,
                          boolean isTransient,
@@ -142,6 +144,7 @@ public class StatsMetadata extends MetadataComponent
         this.repairedAt = repairedAt;
         this.totalColumnsSet = totalColumnsSet;
         this.totalRows = totalRows;
+        this.tokenSpaceCoverage = tokenSpaceCoverage;
         this.originatingHostId = originatingHostId;
         this.pendingRepair = pendingRepair;
         this.isTransient = isTransient;
@@ -200,6 +203,7 @@ public class StatsMetadata extends MetadataComponent
                                  repairedAt,
                                  totalColumnsSet,
                                  totalRows,
+                                 tokenSpaceCoverage,
                                  originatingHostId,
                                  pendingRepair,
                                  isTransient,
@@ -228,6 +232,7 @@ public class StatsMetadata extends MetadataComponent
                                  newRepairedAt,
                                  totalColumnsSet,
                                  totalRows,
+                                 tokenSpaceCoverage,
                                  originatingHostId,
                                  newPendingRepair,
                                  newIsTransient,
@@ -261,6 +266,7 @@ public class StatsMetadata extends MetadataComponent
                        .append(hasLegacyCounterShards, that.hasLegacyCounterShards)
                        .append(totalColumnsSet, that.totalColumnsSet)
                        .append(totalRows, that.totalRows)
+                       .append(tokenSpaceCoverage, that.tokenSpaceCoverage)
                        .append(originatingHostId, that.originatingHostId)
                        .append(pendingRepair, that.pendingRepair)
                        .append(hasPartitionLevelDeletions, that.hasPartitionLevelDeletions)
@@ -290,6 +296,7 @@ public class StatsMetadata extends MetadataComponent
                        .append(hasLegacyCounterShards)
                        .append(totalColumnsSet)
                        .append(totalRows)
+                       .append(tokenSpaceCoverage)
                        .append(originatingHostId)
                        .append(pendingRepair)
                        .append(hasPartitionLevelDeletions)
@@ -375,6 +382,11 @@ public class StatsMetadata extends MetadataComponent
                 size += ByteBufferUtil.serializedSizeWithVIntLength(component.lastKey);
             }
 
+            if (version.hasTokenSpaceCoverage())
+            {
+                size += Double.BYTES;
+            }
+
             return size;
         }
 
@@ -493,6 +505,11 @@ public class StatsMetadata extends MetadataComponent
                 ByteBufferUtil.writeWithVIntLength(component.firstKey, out);
                 ByteBufferUtil.writeWithVIntLength(component.lastKey, out);
             }
+
+            if (version.hasTokenSpaceCoverage())
+            {
+                out.writeDouble(component.tokenSpaceCoverage);
+            }
         }
 
         private void serializeImprovedMinMax(Version version, StatsMetadata component, DataOutputPlus out) throws IOException
@@ -632,6 +649,12 @@ public class StatsMetadata extends MetadataComponent
                 lastKey = ByteBufferUtil.readWithVIntLength(in);
             }
 
+            double tokenSpaceCoverage = Double.NaN;
+            if (version.hasTokenSpaceCoverage())
+            {
+                tokenSpaceCoverage = in.readDouble();
+            }
+
             return new StatsMetadata(partitionSizes,
                                      columnCounts,
                                      commitLogIntervals,
@@ -650,6 +673,7 @@ public class StatsMetadata extends MetadataComponent
                                      repairedAt,
                                      totalColumnsSet,
                                      totalRows,
+                                     tokenSpaceCoverage,
                                      originatingHostId,
                                      pendingRepair,
                                      isTransient,

src/java/org/apache/cassandra/metrics/TableMetrics.java

@@ -58,6 +58,8 @@ import org.apache.cassandra.metrics.Sampler.SamplerType;
 import org.apache.cassandra.schema.Schema;
 import org.apache.cassandra.schema.SchemaConstants;
 import org.apache.cassandra.utils.EstimatedHistogram;
+import org.apache.cassandra.utils.ExpMovingAverage;
+import org.apache.cassandra.utils.MovingAverage;
 import org.apache.cassandra.utils.Pair;
 
 import static java.util.concurrent.TimeUnit.MICROSECONDS;
@@ -119,6 +121,8 @@ public class TableMetrics
     public final Counter pendingFlushes;
     /** Total number of bytes flushed since server [re]start */
     public final Counter bytesFlushed;
+    /** The average on-disk flushed size for sstables. */
+    public final MovingAverage flushSizeOnDisk;
     /** Total number of bytes written by compaction since server [re]start */
     public final Counter compactionBytesWritten;
     /** Estimate of number of pending compactios for this table */
@@ -623,6 +627,7 @@ public class TableMetrics
         rangeLatency = createLatencyMetrics("Range", cfs.keyspace.metric.rangeLatency, GLOBAL_RANGE_LATENCY);
         pendingFlushes = createTableCounter("PendingFlushes");
         bytesFlushed = createTableCounter("BytesFlushed");
+        flushSizeOnDisk = ExpMovingAverage.decayBy1000();
 
         compactionBytesWritten = createTableCounter("CompactionBytesWritten");
         pendingCompactions = createTableGauge("PendingCompactions", () -> cfs.getCompactionStrategyManager().getEstimatedRemainingTasks());
@@ -1260,7 +1265,7 @@ public class TableMetrics
 
         TableMetricNameFactory(ColumnFamilyStore cfs, String type)
         {
-            this.keyspaceName = cfs.keyspace.getName();
+            this.keyspaceName = cfs.getKeyspaceName();
             this.tableName = cfs.name;
             this.isIndex = cfs.isIndex();
             this.type = type;

src/java/org/apache/cassandra/repair/RepairRunnable.java

@@ -298,7 +298,7 @@ public class RepairRunnable implements Runnable, ProgressEventNotifier, RepairNo
 
         StringBuilder cfsb = new StringBuilder();
         for (ColumnFamilyStore cfs : columnFamilyStores)
-            cfsb.append(", ").append(cfs.keyspace.getName()).append(".").append(cfs.name);
+            cfsb.append(", ").append(cfs.getKeyspaceName()).append(".").append(cfs.name);
 
         TimeUUID sessionId = Tracing.instance.newSession(Tracing.TraceType.REPAIR);
         TraceState traceState = Tracing.instance.begin("repair", ImmutableMap.of("keyspace", state.keyspace, "columnFamilies",

src/java/org/apache/cassandra/repair/consistent/LocalSessions.java

@@ -301,7 +301,7 @@ public class LocalSessions
             }
         }
 
-        return new PendingStats(cfs.keyspace.getName(), cfs.name, pending.build(), finalized.build(), failed.build());
+        return new PendingStats(cfs.getKeyspaceName(), cfs.name, pending.build(), finalized.build(), failed.build());
     }
 
     public CleanupSummary cleanup(TableId tid, Collection<Range<Token>> ranges, boolean force)

src/java/org/apache/cassandra/repair/consistent/admin/CleanupSummary.java

@@ -76,7 +76,7 @@ public class CleanupSummary
 
     public CleanupSummary(ColumnFamilyStore cfs, Set<TimeUUID> successful, Set<TimeUUID> unsuccessful)
     {
-        this(cfs.keyspace.getName(), cfs.name, successful, unsuccessful);
+        this(cfs.getKeyspaceName(), cfs.name, successful, unsuccessful);
     }
 
     public static CleanupSummary add(CleanupSummary l, CleanupSummary r)

src/java/org/apache/cassandra/schema/CompactionParams.java

@@ -30,10 +30,13 @@ import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.ParameterizedClass;
 import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.LeveledCompactionStrategy;
 import org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy;
 import org.apache.cassandra.db.compaction.TimeWindowCompactionStrategy;
+import org.apache.cassandra.db.compaction.UnifiedCompactionStrategy;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.utils.FBUtilities;
 
@@ -84,8 +87,23 @@ public final class CompactionParams
         ImmutableMap.of(Option.MIN_THRESHOLD.toString(), Integer.toString(DEFAULT_MIN_THRESHOLD),
                         Option.MAX_THRESHOLD.toString(), Integer.toString(DEFAULT_MAX_THRESHOLD));
 
-    public static final CompactionParams DEFAULT =
-        new CompactionParams(SizeTieredCompactionStrategy.class, DEFAULT_THRESHOLDS, DEFAULT_ENABLED, DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE);
+    public static final CompactionParams DEFAULT;
+    static
+    {
+        ParameterizedClass defaultCompaction = DatabaseDescriptor.getDefaultCompaction();
+        if (defaultCompaction == null)
+        {
+            DEFAULT = new CompactionParams(SizeTieredCompactionStrategy.class,
+                                           DEFAULT_THRESHOLDS,
+                                           DEFAULT_ENABLED,
+                                           DEFAULT_PROVIDE_OVERLAPPING_TOMBSTONES_PROPERTY_VALUE);
+        }
+        else
+        {
+            DEFAULT = create(classFromName(defaultCompaction.class_name),
+                             defaultCompaction.parameters);
+        }
+    }
 
     private final Class<? extends AbstractCompactionStrategy> klass;
     private final ImmutableMap<String, String> options;
@@ -136,6 +154,11 @@ public final class CompactionParams
         return create(LeveledCompactionStrategy.class, options);
     }
 
+    public static CompactionParams ucs(Map<String, String> options)
+    {
+        return create(UnifiedCompactionStrategy.class, options);
+    }
+
     public static CompactionParams twcs(Map<String, String> options)
     {
         return create(TimeWindowCompactionStrategy.class, options);

src/java/org/apache/cassandra/service/ActiveRepairService.java

@@ -345,7 +345,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
 
         for (ColumnFamilyStore cfs : SchemaArgsParser.parse(schemaArgs))
         {
-            String keyspace = cfs.keyspace.getName();
+            String keyspace = cfs.getKeyspaceName();
             Collection<Range<Token>> ranges = userRanges != null
                                               ? userRanges
                                               : StorageService.instance.getLocalReplicas(keyspace).ranges();
@@ -365,7 +365,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                                               : null;
         for (ColumnFamilyStore cfs : SchemaArgsParser.parse(schemaArgs))
         {
-            String keyspace = cfs.keyspace.getName();
+            String keyspace = cfs.getKeyspaceName();
             Collection<Range<Token>> ranges = userRanges != null
                                               ? userRanges
                                               : StorageService.instance.getLocalReplicas(keyspace).ranges();
@@ -385,7 +385,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
                                               : null;
         for (ColumnFamilyStore cfs : SchemaArgsParser.parse(schemaArgs))
         {
-            String keyspace = cfs.keyspace.getName();
+            String keyspace = cfs.getKeyspaceName();
             Collection<Range<Token>> ranges = userRanges != null
                                               ? userRanges
                                               : StorageService.instance.getLocalReplicas(keyspace).ranges();

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -455,7 +455,7 @@ public class CassandraDaemon
                         }
                         else
                         {
-                            logger.info("Not enabling compaction for {}.{}; autocompaction_on_startup_enabled is set to false", store.keyspace.getName(), store.name);
+                            logger.info("Not enabling compaction for {}.{}; autocompaction_on_startup_enabled is set to false", store.getKeyspaceName(), store.name);
                         }
                     }
                 }

src/java/org/apache/cassandra/service/StorageService.java

@@ -1342,7 +1342,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
     private void executePreJoinTasks(boolean bootstrap)
     {
         StreamSupport.stream(ColumnFamilyStore.all().spliterator(), false)
-                .filter(cfs -> Schema.instance.getUserKeyspaces().names().contains(cfs.keyspace.getName()))
+                .filter(cfs -> Schema.instance.getUserKeyspaces().names().contains(cfs.getKeyspaceName()))
                 .forEach(cfs -> cfs.indexManager.executePreJoinTasksBlocking(bootstrap));
     }
 

src/java/org/apache/cassandra/service/paxos/uncommitted/UncommittedTableData.java

@@ -184,7 +184,7 @@ public class UncommittedTableData
             if (table == null)
                 return Range.normalize(FULL_RANGE);
 
-            String ksName = table.keyspace.getName();
+            String ksName = table.getKeyspaceName();
             List<Range<Token>> ranges = StorageService.instance.getLocalAndPendingRanges(ksName);
 
             // don't filter anything if we're not aware of any locally replicated ranges

src/java/org/apache/cassandra/streaming/StreamSession.java

@@ -881,7 +881,7 @@ public class StreamSession
             Set<FileStore> allWriteableFileStores = cfs.getDirectories().allFileStores(fileStoreMapper);
             if (allWriteableFileStores.isEmpty())
             {
-                logger.error("[Stream #{}] Could not get any writeable FileStores for {}.{}", planId, cfs.keyspace.getName(), cfs.getTableName());
+                logger.error("[Stream #{}] Could not get any writeable FileStores for {}.{}", planId, cfs.getKeyspaceName(), cfs.getTableName());
                 continue;
             }
             allFileStores.addAll(allWriteableFileStores);
@@ -906,7 +906,7 @@ public class StreamSession
                          newStreamBytesToWritePerFileStore,
                          perTableIdIncomingBytes.keySet().stream()
                                                 .map(ColumnFamilyStore::getIfExists).filter(Objects::nonNull)
-                                                .map(cfs -> cfs.keyspace.getName() + '.' + cfs.name)
+                                                .map(cfs -> cfs.getKeyspaceName() + '.' + cfs.name)
                                                 .collect(Collectors.joining(",")),
                          totalStreamRemaining,
                          totalCompactionWriteRemaining,
@@ -943,7 +943,7 @@ public class StreamSession
                     tasksStreamed = csm.getEstimatedRemainingTasks(perTableIdIncomingFiles.get(tableId),
                                                                    perTableIdIncomingBytes.get(tableId),
                                                                    isForIncremental);
-                    tables.add(String.format("%s.%s", cfs.keyspace.getName(), cfs.name));
+                    tables.add(String.format("%s.%s", cfs.getKeyspaceName(), cfs.name));
                 }
                 pendingCompactionsBeforeStreaming += tasksOther;
                 pendingCompactionsAfterStreaming += tasksStreamed;

src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java

@@ -108,13 +108,13 @@ public class SSTableExpiredBlockers
         Multimap<SSTableReader, SSTableReader> blockers = ArrayListMultimap.create();
         for (SSTableReader sstable : sstables)
         {
-            if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+            if (sstable.getMaxLocalDeletionTime() < gcBefore)
             {
                 for (SSTableReader potentialBlocker : sstables)
                 {
                     if (!potentialBlocker.equals(sstable) &&
                         potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp() &&
-                        potentialBlocker.getSSTableMetadata().maxLocalDeletionTime > gcBefore)
+                        potentialBlocker.getMaxLocalDeletionTime() > gcBefore)
                         blockers.put(potentialBlocker, sstable);
                 }
             }
@@ -127,7 +127,7 @@ public class SSTableExpiredBlockers
         StringBuilder sb = new StringBuilder();
 
         for (SSTableReader sstable : sstables)
-            sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable, sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(", ");
+            sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable, sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getMaxLocalDeletionTime())).append(", ");
 
         return sb.toString();
     }

src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java

@@ -392,6 +392,7 @@ public class SSTableMetadataViewer
                                                         String::valueOf,
                                                         String::valueOf);
             cellCount.printHistogram(out, color, unicode);
+            field("Local token space coverage", stats.tokenSpaceCoverage);
         }
         if (compaction != null)
         {

src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java

@@ -179,7 +179,7 @@ public class SSTableOfflineRelevel
                 @Override
                 public int compare(SSTableReader o1, SSTableReader o2)
                 {
-                    return o1.last.compareTo(o2.last);
+                    return o1.getLast().compareTo(o2.getLast());
                 }
             });
 
@@ -193,10 +193,10 @@ public class SSTableOfflineRelevel
                 while (it.hasNext())
                 {
                     SSTableReader sstable = it.next();
-                    if (lastLast == null || lastLast.compareTo(sstable.first) < 0)
+                    if (lastLast == null || lastLast.compareTo(sstable.getFirst()) < 0)
                     {
                         level.add(sstable);
-                        lastLast = sstable.last;
+                        lastLast = sstable.getLast();
                         it.remove();
                     }
                 }

src/java/org/apache/cassandra/tools/SSTablePartitions.java

@@ -427,7 +427,7 @@ public class SSTablePartitions
 
     private static String prettyPrintMemory(long bytes)
     {
-        return FBUtilities.prettyPrintMemory(bytes, true);
+        return FBUtilities.prettyPrintMemory(bytes, " ");
     }
 
     private static ISSTableScanner buildScanner(SSTableReader sstable,

src/java/org/apache/cassandra/utils/DiagnosticSnapshotService.java

@@ -210,8 +210,8 @@ public class DiagnosticSnapshotService
                 {
                     cfs.snapshot(command.snapshot_name,
                                  (sstable) -> checkIntersection(ranges,
-                                                                sstable.first.getToken(),
-                                                                sstable.last.getToken()),
+                                                                sstable.getFirst().getToken(),
+                                                                sstable.getLast().getToken()),
                                  false, false);
                 }
             }

src/java/org/apache/cassandra/utils/ExpMovingAverage.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import com.google.common.util.concurrent.AtomicDouble;
+
+/**
+ * Sample-based exponential moving average. On every update a fraction of the current average is replaced by the new
+ * sample. New values have greater representation in the average, and older samples' effect exponentially decays with
+ * new data.
+ */
+public class ExpMovingAverage implements MovingAverage
+{
+    /** The ratio of decay, between 0 and 1, where smaller alpha means values are averaged over more samples */
+    private final double alpha;
+
+    /** The long term average with exponential decay */
+    private final AtomicDouble average = new AtomicDouble(Double.NaN);
+
+    /**
+     * Create a {@link ExpMovingAverage} where older values have less than 1% effect after 1000 samples.
+     */
+    public static MovingAverage decayBy1000()
+    {
+        return new ExpMovingAverage(0.0046);
+    }
+
+    /**
+     * Create a {@link ExpMovingAverage} where older values have less than 1% effect after 100 samples.
+     */
+    public static ExpMovingAverage decayBy100()
+    {
+        return new ExpMovingAverage(0.045);
+    }
+
+    /**
+     * Create a {@link ExpMovingAverage} where older values have less than 1% effect after 10 samples.
+     */
+    public static ExpMovingAverage decayBy10()
+    {
+        return new ExpMovingAverage(0.37);
+    }
+
+    /**
+     * Create a {@link ExpMovingAverage} where older values have less effect than the given ratio after the given
+     * number of samples.
+     */
+    public static ExpMovingAverage withDecay(double ratio, int samples)
+    {
+        assert ratio > 0.0 && ratio < 1.0;
+        assert samples > 0;
+        return new ExpMovingAverage(1 - Math.pow(ratio, 1.0 / samples));
+    }
+
+    ExpMovingAverage(double alpha)
+    {
+        assert alpha > 0.0 && alpha <= 1.0;
+        this.alpha = alpha;
+    }
+
+    @Override
+    public MovingAverage update(double val)
+    {
+        double current, update;
+        do
+        {
+            current = average.get();
+
+            if (!Double.isNaN(current))
+                update = current + alpha * (val - current);
+            else
+                update = val;   // Not initialized yet. Incidentally, passing NaN will cause reinitialization on the
+                                // next update.
+        }
+        while (!average.compareAndSet(current, update));
+
+        return this;
+    }
+
+    @Override
+    public double get()
+    {
+        return average.get();
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("%.2f", get());
+    }
+}

src/java/org/apache/cassandra/utils/FBUtilities.java

@@ -50,6 +50,8 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 import javax.annotation.Nonnull;
@@ -827,38 +829,184 @@ public class FBUtilities
         return new WrappedCloseableIterator<T>(iterator);
     }
 
+    final static String UNIT_PREFIXES = "qryzafpnum KMGTPEZYRQ";
+    final static int UNIT_PREFIXES_BASE = UNIT_PREFIXES.indexOf(' ');
+    final static Pattern BASE_NUMBER_PATTERN = Pattern.compile("NaN|[+-]?Infinity|[+-]?\\d+(\\.\\d+)?([eE]([+-]?)\\d+)?");
+    final static Pattern BINARY_EXPONENT = Pattern.compile("\\*2\\^([+-]?\\d+)");
+
+    /**
+     * Convert the given size in bytes to a human-readable value using binary (i.e. 2^10-based) modifiers.
+     * For example, 1.000KiB, 2.100GiB etc., up to 8.000 EiB.
+     * @param size      Number to convert.
+     */
     public static String prettyPrintMemory(long size)
     {
-        return prettyPrintMemory(size, false);
+        return prettyPrintMemory(size, "");
     }
 
-    public static String prettyPrintMemory(long size, boolean includeSpace)
+    /**
+     * Convert the given size in bytes to a human-readable value using binary (i.e. 2^10-based) modifiers.
+     * For example, 1.000KiB, 2.100GiB etc., up to 8.000 EiB.
+     * @param size      Number to convert.
+     * @param separator Separator between the number and the (modified) unit.
+     */
+    public static String prettyPrintMemory(long size, String separator)
     {
-        if (size >= 1 << 30)
-            return String.format("%.3f%sGiB", size / (double) (1 << 30), includeSpace ? " " : "");
-        if (size >= 1 << 20)
-            return String.format("%.3f%sMiB", size / (double) (1 << 20), includeSpace ? " " : "");
-        return String.format("%.3f%sKiB", size / (double) (1 << 10), includeSpace ? " " : "");
+        int prefixIndex = (63 - Long.numberOfLeadingZeros(Math.abs(size))) / 10;
+        if (prefixIndex == 0)
+            return String.format("%d%sB", size, separator);
+        else
+            return String.format("%.3f%s%ciB",
+                                 Math.scalb(size, -prefixIndex * 10),
+                                 separator,
+                                 UNIT_PREFIXES.charAt(UNIT_PREFIXES_BASE + prefixIndex));
+    }
+
+    /**
+     * Convert the given value to a human-readable string using binary (i.e. 2^10-based) modifiers.
+     * If the number is outside the modifier range (i.e. < 1 qi or > 1 Qi), it will be printed as v*2^e where e is a
+     * multiple of 10 with sign.
+     * For example, 1.000KiB, 2.100 miB/s, 7.006*2^+150, -Infinity.
+     * @param value     Number to convert.
+     * @param separator Separator between the number and the (modified) unit.
+     */
+    public static String prettyPrintBinary(double value, String unit, String separator)
+    {
+        int prefixIndex = Math.floorDiv(Math.getExponent(value), 10);
+        if (prefixIndex == 0 || !Double.isFinite(value) || value == 0)
+            return String.format("%.3f%s%s", value, separator, unit);
+        else if (prefixIndex > UNIT_PREFIXES_BASE || prefixIndex < -UNIT_PREFIXES_BASE)
+            return String.format("%.3f*2^%+d%s%s",
+                                 Math.scalb(value, -prefixIndex * 10),
+                                 prefixIndex * 10,
+                                 separator,
+                                 unit);
+        else
+            return String.format("%.3f%s%ci%s",
+                                 Math.scalb(value, -prefixIndex * 10),
+                                 separator,
+                                 UNIT_PREFIXES.charAt(UNIT_PREFIXES_BASE + prefixIndex),
+                                 unit);
+    }
+
+    /**
+     * Convert the given value to a human-readable string using decimal (i.e. 10^3-based) modifiers.
+     * If the number is outside the modifier range (i.e. < 1 qi or > 1 Qi), it will be printed as vEe where e is a
+     * multiple of 3 with sign.
+     * For example, 1.000km, 2.100 ms, 10E+45, NaN.
+     * @param value     Number to convert.
+     * @param separator Separator between the number and the (modified) unit.
+     */
+    public static String prettyPrintDecimal(double value, String unit, String separator)
+    {
+        int prefixIndex = (int) Math.floor(Math.log10(Math.abs(value)) / 3);
+        double base = value * Math.pow(1000.0, -prefixIndex);
+        if (prefixIndex == 0 || !Double.isFinite(value) || !Double.isFinite(base) || value == 0)
+            return String.format("%.3f%s%s", value, separator, unit);
+        else if (prefixIndex > UNIT_PREFIXES_BASE || prefixIndex < -UNIT_PREFIXES_BASE)
+            return String.format("%.3fe%+d%s%s",
+                                 base,
+                                 prefixIndex * 3,
+                                 separator,
+                                 unit);
+        else
+            return String.format("%.3f%s%c%s",
+                                 base,
+                                 separator,
+                                 UNIT_PREFIXES.charAt(UNIT_PREFIXES_BASE + prefixIndex),
+                                 unit);
     }
 
     public static String prettyPrintMemoryPerSecond(long rate)
     {
-        if (rate >= 1 << 30)
-            return String.format("%.3fGiB/s", rate / (double) (1 << 30));
-        if (rate >= 1 << 20)
-            return String.format("%.3fMiB/s", rate / (double) (1 << 20));
-        return String.format("%.3fKiB/s", rate / (double) (1 << 10));
+        return prettyPrintMemory(rate) + "/s";
     }
 
     public static String prettyPrintMemoryPerSecond(long bytes, long timeInNano)
     {
-        // We can't sanely calculate a rate over 0 nanoseconds
-        if (timeInNano == 0)
-            return "NaN  KiB/s";
+        return prettyPrintBinary(bytes * 1.0e9 / timeInNano, "B/s", "");
+    }
 
-        long rate = (long) (((double) bytes / timeInNano) * 1000 * 1000 * 1000);
+    /**
+     * Parse a human-readable value printed using one of the methods above. Understands both binary and decimal
+     * modifiers, as well as decimal exponents using the E notation and binary exponents using *2^e.
+     *
+     * @param datum     The human-readable number.
+     * @param separator Expected separator, null to accept any amount of whitespace.
+     * @param unit      Expected unit. If null, the method will accept any string as unit, i.e. it will parse the number
+     *                  at the start of the supplied string and ignore any remainder.
+     * @return The parsed value.
+     */
+    public static double parseHumanReadable(String datum, String separator, String unit)
+    {
+        int end = datum.length();
+        if (unit != null)
+        {
+            if (!datum.endsWith(unit))
+                throw new NumberFormatException(datum + " does not end in unit " + unit);
+            end -= unit.length();
+        }
 
-        return prettyPrintMemoryPerSecond(rate);
+        Matcher m = BASE_NUMBER_PATTERN.matcher(datum);
+        m.region(0, end);
+        if (!m.lookingAt())
+            throw new NumberFormatException();
+        double v = Double.parseDouble(m.group(0));
+
+        int pos = m.end();
+        if (m.group(2) == null) // possible binary exponent, parse
+        {
+            m = BINARY_EXPONENT.matcher(datum);
+            m.region(pos, end);
+            if (m.lookingAt())
+            {
+                int power = Integer.parseInt(m.group(1));
+                v = Math.scalb(v, power);
+                pos = m.end();
+            }
+        }
+
+        if (separator != null)
+        {
+            if (!datum.startsWith(separator, pos))
+                throw new NumberFormatException("Missing separator " + separator + " in " + datum);
+            pos += separator.length();
+        }
+        else
+        {
+            while (pos < end && Character.isWhitespace(datum.charAt(pos)))
+                ++pos;
+        }
+
+        if (pos < end)
+        {
+            char prefixChar = datum.charAt(pos);
+            int prefixIndex = UNIT_PREFIXES.indexOf(prefixChar);
+            if (prefixIndex >= 0)
+            {
+                prefixIndex -= UNIT_PREFIXES_BASE;
+                ++pos;
+                if (pos < end && datum.charAt(pos) == 'i')
+                {
+                    ++pos;
+                    v = Math.scalb(v, prefixIndex * 10);
+                }
+                else
+                {
+                    v *= Math.exp(Math.log(1000.0) * prefixIndex);
+                }
+            }
+        }
+
+        if (pos != end && unit != null)
+            throw new NumberFormatException("Unexpected characters between pos " + pos + " and " + end + " in " + datum);
+
+        return v;
+    }
+
+    public static long parseHumanReadableBytes(String value)
+    {
+        return (long) parseHumanReadable(value, null, "B");
     }
 
     /**

src/java/org/apache/cassandra/utils/MovingAverage.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+public interface MovingAverage
+{
+    MovingAverage update(double val);
+
+    double get();
+}